From 7f0760e2d50673718a1d20742e6d5b5f557ed3b4 Mon Sep 17 00:00:00 2001 From: Hwuiwon Kim Date: Wed, 12 Aug 2026 09:03:16 -0400 Subject: [PATCH 01/21] add long-horizon execution with storage-only waits and a dispatch outbox --- .config/nextest.toml | 10 + crates/moa-artifacts/src/execution_plan.rs | 57 +- crates/moa-artifacts/src/validation.rs | 179 +- .../src/validation/execution_plan.rs | 253 + .../artifacts_offline/definition_roundtrip.rs | 9 + .../execution_plan_validation.rs | 402 +- crates/moa-auth/authz/src/poller.rs | 213 +- .../src/execution_planning/request.rs | 18 +- .../src/prompts/execution_planner.txt | 6 +- .../src/prompts/execution_router.txt | 2 + crates/moa-brain/tests/brain_turn_offline.rs | 30 +- crates/moa-config/src/env_overlay/mod.rs | 32 + crates/moa-config/src/env_overlay/tests.rs | 97 + crates/moa-config/src/execution.rs | 338 + crates/moa-config/src/lib.rs | 102 +- crates/moa-core/src/events.rs | 98 + crates/moa-core/src/traits/mod.rs | 1 + crates/moa-core/src/types/action_policy.rs | 12 +- .../moa-core/src/types/execution_planning.rs | 418 +- crates/moa-core/src/types/identifiers.rs | 17 +- .../moa-core/src/types/sandbox_workspace.rs | 69 +- crates/moa-core/src/types/tools.rs | 120 + crates/moa-db/src/lib.rs | 149 +- crates/moa-edge/README.md | 6 +- .../src/external_job_callback_proxy.rs | 326 + crates/moa-edge/src/lib.rs | 1 + crates/moa-edge/src/main.rs | 13 +- crates/moa-edge/src/proxy.rs | 5 +- crates/moa-edge/src/routes.rs | 12 + crates/moa-edge/src/routes/external_jobs.rs | 78 + crates/moa-edge/src/routes/session_stream.rs | 94 +- .../moa-edge/tests/direct_read_routes_db.rs | 4 + .../graceful_shutdown_db.rs | 5 +- .../session_message_attachments_docker.rs | 4 + .../examples/generate_execution_corpus.rs | 6 + .../execution/contract-recorded.jsonl | 160 +- .../scenarios/execution/manifest.toml | 2 +- crates/moa-eval/src/execution/snapshot.rs | 3 + .../tests/eval_offline/execution_snapshot.rs | 71 +- crates/moa-execution/src/capability.rs | 68 +- .../moa-execution/src/compiler/amendment.rs | 33 +- crates/moa-execution/src/compiler/estimate.rs | 15 +- crates/moa-execution/src/compiler/mod.rs | 240 +- crates/moa-execution/src/compiler/tests.rs | 158 +- .../moa-execution/src/compiler/validation.rs | 11 + .../compiler/validation/activation_bounds.rs | 87 + .../compiler/validation/schema_references.rs | 7 + crates/moa-execution/src/completion.rs | 64 +- crates/moa-execution/src/error.rs | 75 + .../src/interpreter/aggregate.rs | 11 +- .../src/interpreter/materialize.rs | 396 +- crates/moa-execution/src/interpreter/mod.rs | 136 +- .../src/interpreter/projection.rs | 1 + .../src/interpreter/reservation.rs | 1 + .../src/interpreter/temporal_wait.rs | 196 + .../moa-execution/src/interpreter/terminal.rs | 71 +- crates/moa-execution/src/interpreter/tests.rs | 13 +- crates/moa-execution/src/lib.rs | 5 +- .../moa-execution/src/repository/admission.rs | 233 +- .../moa-execution/src/repository/amendment.rs | 207 + crates/moa-execution/src/repository/audit.rs | 147 + .../src/repository/audit_codec.rs | 1 + .../moa-execution/src/repository/capacity.rs | 1563 +++++ .../src/repository/compensation.rs | 6074 ++++++++++++++--- .../src/repository/completion.rs | 1571 +++++ .../src/repository/external_job.rs | 2450 +++++++ .../src/repository/materialize.rs | 26 +- crates/moa-execution/src/repository/mod.rs | 773 ++- crates/moa-execution/src/repository/outbox.rs | 1731 +++++ .../moa-execution/src/repository/outcome.rs | 729 +- .../src/repository/outcome_support.rs | 11 + .../src/repository/projection.rs | 90 - crates/moa-execution/src/repository/ready.rs | 2678 ++++++++ .../src/repository/replan_stop.rs | 323 + .../moa-execution/src/repository/retention.rs | 1375 ++++ crates/moa-execution/src/repository/rows.rs | 97 +- crates/moa-execution/src/repository/run.rs | 1182 +++- .../moa-execution/src/repository/schedule.rs | 1463 ++++ crates/moa-execution/src/repository/sql.rs | 249 +- crates/moa-execution/src/repository/task.rs | 5115 +++++++++++++- .../moa-execution/src/repository/terminal.rs | 525 +- .../src/repository/transition.rs | 1195 ++++ .../moa-execution/src/repository/trigger.rs | 2450 +++++++ crates/moa-execution/src/state.rs | 148 +- crates/moa-execution/src/wire.rs | 684 +- crates/moa-execution/tests/compiler.rs | 285 +- crates/moa-execution/tests/completion.rs | 10 +- crates/moa-execution/tests/execution_db.rs | 18 + .../execution_db/active_run_capacity_db.rs | 1366 ++++ .../execution_db/amendment_projection_db.rs | 172 + .../execution_db/compensation_attempts_db.rs | 1962 ++++++ .../tests/execution_db/compensation_db.rs | 1213 +--- .../execution_db/completion_projection_db.rs | 665 ++ .../execution_db/execution_capacity_db.rs | 463 ++ .../execution_db/incremental_scheduler_db.rs | 1378 ++++ .../execution_db/long_horizon_state_db.rs | 257 + .../execution_db/outcomes_and_replan_db.rs | 233 +- .../execution_db/planning_and_audit_db.rs | 243 +- .../tests/execution_db/retention_db.rs | 344 + .../execution_db/scope_and_lifecycle_db.rs | 698 +- .../tests/execution_db/support.rs | 215 +- .../tests/execution_db/trigger_outbox_db.rs | 3441 ++++++++++ crates/moa-execution/tests/interpreter.rs | 429 +- .../moa-hands/src/adapters/daytona/tests.rs | 1 + .../src/adapters/daytona/workspace.rs | 69 +- crates/moa-hands/src/adapters/e2b/mod.rs | 13 + crates/moa-hands/src/adapters/e2b/tests.rs | 1 + .../moa-hands/src/adapters/e2b/workspace.rs | 22 +- crates/moa-hands/src/adapters/local/mod.rs | 13 + crates/moa-hands/src/adapters/local/tests.rs | 1 + .../moa-hands/src/adapters/local/workspace.rs | 25 +- crates/moa-hands/src/adapters/mcp/mod.rs | 23 +- crates/moa-hands/src/adapters/mcp/tests.rs | 54 + crates/moa-hands/src/core/construction.rs | 35 +- crates/moa-hands/src/core/dispatch.rs | 35 +- crates/moa-hands/src/core/leases.rs | 441 +- crates/moa-hands/src/core/lifecycle.rs | 406 +- crates/moa-hands/src/core/lifecycle/tests.rs | 100 +- .../core/maintenance_provider_inventory.rs | 177 + crates/moa-hands/src/core/mod.rs | 475 +- crates/moa-hands/src/core/reaper.rs | 318 +- crates/moa-hands/src/core/recovery.rs | 10 +- crates/moa-hands/src/core/registration.rs | 47 +- .../src/core/sandbox_workspace/capacity.rs | 816 ++- .../src/core/sandbox_workspace/lifecycle.rs | 828 ++- .../maintenance/inventory.rs | 337 +- .../core/sandbox_workspace/maintenance/mod.rs | 4 +- .../src/core/sandbox_workspace/model.rs | 86 +- .../src/core/sandbox_workspace/reaper.rs | 386 +- .../core/sandbox_workspace/repository/base.rs | 12 +- .../repository/checkpoints.rs | 206 + .../sandbox_workspace/repository/lifecycle.rs | 1171 ++++ .../core/sandbox_workspace/repository/mod.rs | 20 +- crates/moa-hands/src/lib.rs | 13 +- .../moa-hands/src/tools/sandbox_descriptor.rs | 1 + crates/moa-hands/tests/daytona_live.rs | 2 + crates/moa-hands/tests/e2b_live.rs | 1 + .../tests/hands_db/hand_lease_reaper_db.rs | 252 +- .../hands_db/sandbox_workspace/capacity_db.rs | 197 +- .../hands_db/sandbox_workspace/dispatch_db.rs | 127 + .../sandbox_workspace/lifecycle_db.rs | 792 ++- .../sandbox_workspace/reconciliation_db.rs | 211 +- .../sandbox_workspace/retention_db.rs | 22 + .../hands_db/sandbox_workspace/rls_db.rs | 8 +- crates/moa-hands/tests/hands_offline.rs | 2 + .../maintenance_provider_inventory_offline.rs | 333 + .../tests/sandbox_workspace_docker.rs | 1 + .../moa-migrations/migration-ownership.toml | 108 + .../V000059__long_horizon_execution.sql | 3071 +++++++++ ...00060__sandbox_active_compute_capacity.sql | 527 ++ .../execution_and_security_catalog.rs | 781 ++- .../execution_compensation.rs | 15 +- .../tests/run_idempotency_db/hand_leases.rs | 55 +- .../moa-observability/src/runtime_metrics.rs | 527 ++ .../src/action_reviews/app.rs | 3 +- .../src/action_reviews/store.rs | 83 +- .../src/authz_challenges/store.rs | 149 + .../src/external_job_ingress.rs | 1053 +++ crates/moa-orchestrator/src/lib.rs | 2 + crates/moa-orchestrator/src/main.rs | 696 +- .../moa-orchestrator/src/objects/cron_job.rs | 129 +- .../src/objects/execution_run_controller.rs | 131 + .../execution_run_controller/advance.rs | 1360 ++++ .../execution_run_controller/progress.rs | 106 + .../execution_run_controller/settlement.rs | 99 + .../objects/execution_run_controller/tests.rs | 339 + crates/moa-orchestrator/src/objects/mod.rs | 1 + .../src/objects/session/execution_runs.rs | 130 +- .../src/objects/session/handlers.rs | 2 +- .../session/handlers/execution_bridge.rs | 3 +- .../src/objects/session/handlers/lifecycle.rs | 1 + .../src/objects/session/state.rs | 34 +- .../src/objects/session/state/execution.rs | 22 +- crates/moa-orchestrator/src/runtime/deps.rs | 78 +- .../moa-orchestrator/src/runtime/endpoint.rs | 257 +- .../src/runtime/execution_dispatch.rs | 468 ++ crates/moa-orchestrator/src/runtime/jobs.rs | 295 +- crates/moa-orchestrator/src/runtime/mod.rs | 1 + .../src/services/action_policy.rs | 3 + .../src/services/action_review_dispatcher.rs | 184 +- .../src/services/action_reviews.rs | 264 +- .../src/services/action_reviews_reaper.rs | 352 +- .../src/services/authz_challenges.rs | 54 + .../src/services/authz_challenges_reaper.rs | 235 +- .../src/services/durable_timeout.rs | 329 + .../src/services/execution.rs | 92 +- .../services/execution/capability_catalog.rs | 6 + .../src/services/execution/handlers.rs | 700 +- .../services/execution/planning_context.rs | 67 +- .../src/services/execution/start.rs | 50 +- .../src/services/execution/support.rs | 83 +- .../src/services/execution/tests.rs | 90 +- .../src/services/execution_dispatcher.rs | 1076 +++ .../src/services/execution_retention.rs | 283 + .../src/services/execution_schedule.rs | 742 ++ .../src/services/execution_trigger.rs | 819 +++ .../src/services/llm_gateway.rs | 69 +- crates/moa-orchestrator/src/services/mod.rs | 5 + .../src/services/skill_regression/gate.rs | 3 +- .../src/services/tool_executor.rs | 2125 +++++- .../src/tool_invocation/governed.rs | 151 +- .../moa-orchestrator/src/workflows/errors.rs | 107 +- .../src/workflows/execution_compensation.rs | 975 --- .../execution_compensation_attempt.rs | 354 + .../execution_compensation_attempt/active.rs | 427 ++ .../external.rs | 136 + .../yielding.rs | 441 ++ .../src/workflows/execution_run.rs | 3428 ---------- .../src/workflows/execution_task.rs | 2400 ------- .../src/workflows/execution_task_attempt.rs | 419 ++ .../execution_task_attempt/active.rs | 1838 +++++ .../execution_task_attempt/external.rs | 118 + .../execution_task_attempt/watchdog.rs | 316 + .../execution_task_attempt/yielding.rs | 639 ++ .../experiment_trial_run/target_execution.rs | 18 +- crates/moa-orchestrator/src/workflows/mod.rs | 5 +- .../src/workflows/progress_delivery.rs | 31 + .../src/workflows/turn_execution/mod.rs | 20 +- .../src/workflows/turn_execution/tools.rs | 3 +- .../src/workflows/worker_turn_execution.rs | 3 +- ...oordinator_worker_behavior_provider_e2e.rs | 49 +- .../execution_execution_support/evaluation.rs | 155 +- .../tests/execution_run_service_e2e.rs | 305 +- .../admission_replay.rs | 6 + .../bulk_and_recovery.rs | 6 + .../compensation_recovery.rs | 193 + .../controller_activation.rs | 46 + .../observability.rs | 23 +- .../replan_and_completion.rs | 114 +- .../execution_run_service_e2e/routing.rs | 12 + .../task_lifecycle.rs | 314 +- .../terminal_matrix.rs | 1215 +--- .../integration/action_policy_flow_e2e.rs | 7 + .../long_horizon_execution_canary_live.rs | 267 + .../long_horizon_execution_service_e2e.rs | 963 +++ .../accelerated_week.rs | 313 + .../burst_admission.rs | 632 ++ .../deadline_and_waits.rs | 571 ++ .../deployment_drain.rs | 148 + .../disaster_recovery.rs | 397 ++ .../pause_and_external.rs | 1858 +++++ .../moa-orchestrator/tests/orchestrator_db.rs | 4 + .../action_reviews_reaper_db.rs | 97 +- .../orchestrator_db/analytics_export_db.rs | 6 + .../orchestrator_db/authz_challenges_db.rs | 95 +- .../execution_dispatch_reconciliation_db.rs | 318 + .../orchestrator_db/execution_schedule_db.rs | 1035 +++ .../orchestrator_db/execution_service_db.rs | 91 +- .../tests/orchestrator_offline/session_vo.rs | 179 +- .../orchestrator_offline/tool_executor.rs | 1 + ...ry_matrix_sandbox_workspace_service_e2e.rs | 4 +- .../sandbox_workspace_soak_service_e2e.rs | 133 +- crates/moa-session/src/store/dashboard.rs | 11 +- .../tests/session_db/execution_events_db.rs | 17 + .../learning_candidate_planning_audit_db.rs | 4 +- .../src/fixture_capability.rs | 2 + crates/moa-test-support/src/lib.rs | 14 +- .../src/orchestrator_fixture.rs | 440 +- .../src/orchestrator_fixture/external_job.rs | 786 +++ .../src/orchestrator_fixture/postgres.rs | 31 +- .../src/orchestrator_fixture/process.rs | 78 +- .../src/orchestrator_fixture/redis.rs | 18 +- .../src/orchestrator_fixture/restate.rs | 154 +- crates/moa-wire/src/turn.rs | 17 + .../check_architecture_boundaries/budgets.rs | 12 +- crates/xtask/src/execution_trace_manifest.rs | 1028 ++- docker-compose.yml | 2 +- docs/00-direction.md | 16 +- docs/01-architecture-overview.md | 83 +- docs/02-brain-orchestration.md | 134 +- docs/05-session-event-log.md | 49 +- docs/06-hands-and-mcp.md | 27 +- docs/10-technology-stack.md | 57 +- docs/12-restate-architecture.md | 233 +- docs/17-observability.md | 39 +- docs/19-data-operations.md | 83 +- docs/20-testing.md | 25 +- docs/22-load-and-chaos-testing.md | 107 + docs/23-environment-variables.md | 19 +- docs/25-sandbox-workspaces.md | 44 +- .../artifacts/damaged-food-order.skill.yaml | 6 + .../patterns/custom-logic.skill.yaml | 6 + .../patterns/human-approval.skill.yaml | 12 + .../patterns/parallel-review.skill.yaml | 12 + .../artifacts/patterns/react-agent.skill.yaml | 6 + .../artifacts/patterns/sequential.skill.yaml | 6 + docs/operations/edge-network-isolation.md | 4 +- docs/operations/restate-operations.md | 90 +- docs/schemas/moa-skill-v1.schema.json | 86 +- k8s/base/20-orchestrator-deployment.yaml | 28 +- k8s/base/25-maintenance-deployment.yaml | 118 + k8s/base/25-orchestrator-service.yaml | 6 +- k8s/base/50-edge-deployment.yaml | 2 +- k8s/base/kustomization.yaml | 1 + k8s/overlays/production/kustomization.yaml | 12 + .../patches/runtime-security-profile.yaml | 11 +- k8s/scripts/smoke.sh | 101 +- k8s/scripts/validate-observability.sh | 243 +- ops/prometheus/alerts/kustomization.yaml | 1 + .../alerts/moa-long-horizon-execution.yaml | 122 + ops/prometheus/alerts/moa-restate.yaml | 9 + ops/prometheus/alerts/sandbox-workspaces.yaml | 14 +- scripts/cutover-long-horizon-execution.sh | 410 ++ scripts/run-clean-e2e.sh | 44 +- 304 files changed, 88401 insertions(+), 15608 deletions(-) create mode 100644 crates/moa-artifacts/src/validation/execution_plan.rs create mode 100644 crates/moa-edge/src/external_job_callback_proxy.rs create mode 100644 crates/moa-edge/src/routes/external_jobs.rs create mode 100644 crates/moa-execution/src/compiler/validation/activation_bounds.rs create mode 100644 crates/moa-execution/src/interpreter/temporal_wait.rs create mode 100644 crates/moa-execution/src/repository/amendment.rs create mode 100644 crates/moa-execution/src/repository/capacity.rs create mode 100644 crates/moa-execution/src/repository/completion.rs create mode 100644 crates/moa-execution/src/repository/external_job.rs create mode 100644 crates/moa-execution/src/repository/outbox.rs create mode 100644 crates/moa-execution/src/repository/ready.rs create mode 100644 crates/moa-execution/src/repository/replan_stop.rs create mode 100644 crates/moa-execution/src/repository/retention.rs create mode 100644 crates/moa-execution/src/repository/schedule.rs create mode 100644 crates/moa-execution/src/repository/trigger.rs create mode 100644 crates/moa-execution/tests/execution_db/active_run_capacity_db.rs create mode 100644 crates/moa-execution/tests/execution_db/amendment_projection_db.rs create mode 100644 crates/moa-execution/tests/execution_db/compensation_attempts_db.rs create mode 100644 crates/moa-execution/tests/execution_db/completion_projection_db.rs create mode 100644 crates/moa-execution/tests/execution_db/execution_capacity_db.rs create mode 100644 crates/moa-execution/tests/execution_db/incremental_scheduler_db.rs create mode 100644 crates/moa-execution/tests/execution_db/long_horizon_state_db.rs create mode 100644 crates/moa-execution/tests/execution_db/retention_db.rs create mode 100644 crates/moa-execution/tests/execution_db/trigger_outbox_db.rs create mode 100644 crates/moa-hands/src/core/maintenance_provider_inventory.rs create mode 100644 crates/moa-hands/tests/hands_offline/maintenance_provider_inventory_offline.rs create mode 100644 crates/moa-migrations/migrations/postgres/V000059__long_horizon_execution.sql create mode 100644 crates/moa-migrations/migrations/postgres/V000060__sandbox_active_compute_capacity.sql create mode 100644 crates/moa-orchestrator/src/external_job_ingress.rs create mode 100644 crates/moa-orchestrator/src/objects/execution_run_controller.rs create mode 100644 crates/moa-orchestrator/src/objects/execution_run_controller/advance.rs create mode 100644 crates/moa-orchestrator/src/objects/execution_run_controller/progress.rs create mode 100644 crates/moa-orchestrator/src/objects/execution_run_controller/settlement.rs create mode 100644 crates/moa-orchestrator/src/objects/execution_run_controller/tests.rs create mode 100644 crates/moa-orchestrator/src/runtime/execution_dispatch.rs create mode 100644 crates/moa-orchestrator/src/services/durable_timeout.rs create mode 100644 crates/moa-orchestrator/src/services/execution_dispatcher.rs create mode 100644 crates/moa-orchestrator/src/services/execution_retention.rs create mode 100644 crates/moa-orchestrator/src/services/execution_schedule.rs create mode 100644 crates/moa-orchestrator/src/services/execution_trigger.rs delete mode 100644 crates/moa-orchestrator/src/workflows/execution_compensation.rs create mode 100644 crates/moa-orchestrator/src/workflows/execution_compensation_attempt.rs create mode 100644 crates/moa-orchestrator/src/workflows/execution_compensation_attempt/active.rs create mode 100644 crates/moa-orchestrator/src/workflows/execution_compensation_attempt/external.rs create mode 100644 crates/moa-orchestrator/src/workflows/execution_compensation_attempt/yielding.rs delete mode 100644 crates/moa-orchestrator/src/workflows/execution_run.rs delete mode 100644 crates/moa-orchestrator/src/workflows/execution_task.rs create mode 100644 crates/moa-orchestrator/src/workflows/execution_task_attempt.rs create mode 100644 crates/moa-orchestrator/src/workflows/execution_task_attempt/active.rs create mode 100644 crates/moa-orchestrator/src/workflows/execution_task_attempt/external.rs create mode 100644 crates/moa-orchestrator/src/workflows/execution_task_attempt/watchdog.rs create mode 100644 crates/moa-orchestrator/src/workflows/execution_task_attempt/yielding.rs create mode 100644 crates/moa-orchestrator/tests/execution_run_service_e2e/controller_activation.rs create mode 100644 crates/moa-orchestrator/tests/long_horizon_execution_canary_live.rs create mode 100644 crates/moa-orchestrator/tests/long_horizon_execution_service_e2e.rs create mode 100644 crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/accelerated_week.rs create mode 100644 crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/burst_admission.rs create mode 100644 crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/deadline_and_waits.rs create mode 100644 crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/deployment_drain.rs create mode 100644 crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/disaster_recovery.rs create mode 100644 crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/pause_and_external.rs create mode 100644 crates/moa-orchestrator/tests/orchestrator_db/execution_dispatch_reconciliation_db.rs create mode 100644 crates/moa-orchestrator/tests/orchestrator_db/execution_schedule_db.rs create mode 100644 crates/moa-test-support/src/orchestrator_fixture/external_job.rs create mode 100644 k8s/base/25-maintenance-deployment.yaml create mode 100644 ops/prometheus/alerts/moa-long-horizon-execution.yaml create mode 100755 scripts/cutover-long-horizon-execution.sh diff --git a/.config/nextest.toml b/.config/nextest.toml index e39b481b3..59f575f9f 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -97,6 +97,16 @@ binary(/^orchestrator_fixture_service_e2e$/) | binary(/^turn_terminal_failure_service_e2e$/) ''' +# Accelerated eight-logical-day execution validation. The binary owns one +# disposable Restate/Postgres/Valkey fixture and deliberately restarts those +# dependencies, so cases are serialized and have a hard ten-minute ceiling. +[profile.long-horizon-execution] +inherits = "default" +test-threads = 1 +junit.path = "target/nextest/long-horizon-execution/junit.xml" +default-filter = 'package(moa-orchestrator) & binary(/^long_horizon_execution_service_e2e$/)' +slow-timeout = { period = "60s", terminate-after = 10 } + # Small deterministic crash/replay and worker-deadline cases run serially on # pull requests. Large fan-out, A/B rollout, node-loss, and chaos scenarios # remain in their existing local/nightly lanes. diff --git a/crates/moa-artifacts/src/execution_plan.rs b/crates/moa-artifacts/src/execution_plan.rs index 5fb6901c4..566192fee 100644 --- a/crates/moa-artifacts/src/execution_plan.rs +++ b/crates/moa-artifacts/src/execution_plan.rs @@ -129,6 +129,8 @@ pub enum CompletionCheckKind { pub struct ExecutionPlanDefinition { /// Explicit policy for effects already committed when the run is cancelled. pub cancel_policy: ExecutionCancelPolicy, + /// Expiry behavior for runtime input requests returned by executable tasks. + pub input_wait_policy: ExecutionWaitPolicy, /// JSON Schema for run input. pub input_schema: Value, /// JSON Schema for terminal output. @@ -314,7 +316,7 @@ pub enum ExecutionReducer { }, } -/// The seven operations supported by the execution-plan DSL. +/// The eight operations supported by the execution-plan DSL. #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] pub enum ExecutionOperation { @@ -362,11 +364,22 @@ pub enum ExecutionOperation { Review { /// Review prompt shown to the tenant reviewer. prompt: String, + /// Exact expiry and settlement behavior for the review wait. + wait_policy: ExecutionWaitPolicy, }, /// Pause for one external or user signal. WaitSignal { /// Stable signal name awaited by the run. signal_name: String, + /// Exact expiry and settlement behavior for the signal wait. + wait_policy: ExecutionWaitPolicy, + }, + /// Park until an exact or wait-entry-relative time without retaining active compute. + WaitUntil { + /// Temporal target at which the node becomes ready to continue. + wake: ExecutionTemporalTarget, + /// Structured result made available when the timer fires. + result: Value, }, /// Resolve and validate the plan's terminal output. Output { @@ -375,6 +388,47 @@ pub enum ExecutionOperation { }, } +/// Exact expiry policy for a storage-only execution wait. +#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ExecutionWaitPolicy { + /// Temporal target at which the unresolved wait expires. + pub expiry: ExecutionTemporalTarget, + /// Deterministic settlement applied when the wait expires. + pub on_expiry: ExecutionWaitExpiryAction, +} + +/// Exact or wait-entry-relative target for a durable execution timer. +#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum ExecutionTemporalTarget { + /// Exact absolute UTC instant used by one-off generated and compiled plans. + At { + /// Absolute UTC instant at which the timer becomes due. + at: DateTime, + }, + /// Positive delay resolved when the owning wait state is entered. + After { + /// Number of seconds after wait entry at which the timer becomes due. + delay_seconds: u64, + }, +} + +/// Deterministic action applied when an execution wait expires. +#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum ExecutionWaitExpiryAction { + /// Fail only the waiting logical task. + FailTask, + /// Fail the complete execution run. + FailRun, + /// Settle the wait successfully with a declared structured output. + ContinueWith { + /// Structured output supplied to downstream nodes. + output: Value, + }, +} + /// Shared resource ceiling for an execution run or node. #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] #[serde(deny_unknown_fields)] @@ -647,6 +701,7 @@ impl ExecutionPlanDefinition { | ExecutionOperation::Reduce { .. } | ExecutionOperation::Review { .. } | ExecutionOperation::WaitSignal { .. } + | ExecutionOperation::WaitUntil { .. } | ExecutionOperation::Output { .. } => {} } } diff --git a/crates/moa-artifacts/src/validation.rs b/crates/moa-artifacts/src/validation.rs index 8f0861b19..b20274549 100644 --- a/crates/moa-artifacts/src/validation.rs +++ b/crates/moa-artifacts/src/validation.rs @@ -1,6 +1,7 @@ //! Semantic validation for artifact documents. mod connectors; +mod execution_plan; mod json; use std::collections::{HashMap, HashSet}; @@ -19,8 +20,7 @@ use crate::document::{ArtifactDefinition, ArtifactDocument, ArtifactKind, Artifa use crate::execution_plan::{ CapabilityReference, CompensationValueSource, CompletionCheckKind, ExecutionCondition, ExecutionGoalContract, ExecutionGoalTemplate, ExecutionNode, ExecutionOperation, - ExecutionPlanDefinition, ExecutionReducer, ExecutionTaskOutcome, MapTask, PlanAmendment, - PlanAmendmentOperation, + ExecutionPlanDefinition, ExecutionTaskOutcome, PlanAmendment, PlanAmendmentOperation, }; use crate::reference::{ArtifactRef, ReferenceResolution, ReferenceState}; use crate::simulation::{ @@ -220,7 +220,7 @@ pub fn validate_execution_plan_definition( definition: &ExecutionPlanDefinition, ) -> ValidationReport { let mut report = ValidationReport::default(); - validate_execution_plan_at("execution_plan", definition, &mut report); + validate_execution_plan_at("execution_plan", definition, true, &mut report); report } @@ -261,7 +261,7 @@ pub fn validate_plan_amendment(amendment: &PlanAmendment) -> ValidationReport { let root = format!("plan_amendment.operations[{index}]"); match operation { PlanAmendmentOperation::AddNode { node } => { - validate_execution_node(&format!("{root}.node"), node, None, &mut report); + validate_execution_node(&format!("{root}.node"), node, None, true, &mut report); } PlanAmendmentOperation::ReplacePendingNode { node_id, node } => { validate_stable_id( @@ -270,7 +270,7 @@ pub fn validate_plan_amendment(amendment: &PlanAmendment) -> ValidationReport { "pending node id", &mut report, ); - validate_execution_node(&format!("{root}.node"), node, None, &mut report); + validate_execution_node(&format!("{root}.node"), node, None, true, &mut report); } PlanAmendmentOperation::RemovePendingNode { node_id } => validate_stable_id( &format!("{root}.node_id"), @@ -856,6 +856,7 @@ fn validate_skill(definition: &SkillDefinition, report: &mut ValidationReport) { validate_execution_plan_at( "definition.spec.execution_plan.plan", &execution_plan.plan, + false, report, ); } @@ -930,8 +931,15 @@ fn validate_execution_goal_template( fn validate_execution_plan_at( root: &str, definition: &ExecutionPlanDefinition, + allow_absolute_temporal_targets: bool, report: &mut ValidationReport, ) { + execution_plan::validate_temporal_target( + &format!("{root}.input_wait_policy.expiry"), + &definition.input_wait_policy.expiry, + allow_absolute_temporal_targets, + report, + ); validate_json_schema( &format!("{root}.input_schema"), &definition.input_schema, @@ -957,6 +965,7 @@ fn validate_execution_plan_at( &format!("{root}.nodes[{index}]"), node, Some(&node_ids), + allow_absolute_temporal_targets, report, ); } @@ -969,6 +978,7 @@ fn validate_execution_node( root: &str, node: &ExecutionNode, known_node_ids: Option<&HashSet<&str>>, + allow_absolute_temporal_targets: bool, report: &mut ValidationReport, ) { validate_stable_id(&format!("{root}.id"), &node.id, "execution node id", report); @@ -1023,7 +1033,7 @@ fn validate_execution_node( report, ); validate_retry_policy(root, node, report); - validate_execution_operation(root, node, report); + execution_plan::validate_operation(root, node, allow_absolute_temporal_targets, report); validate_execution_compensation(root, node, report); } @@ -1132,163 +1142,6 @@ fn validate_retry_policy(root: &str, node: &ExecutionNode, report: &mut Validati } } -fn validate_execution_operation(root: &str, node: &ExecutionNode, report: &mut ValidationReport) { - let operation_root = format!("{root}.operation"); - match &node.operation { - ExecutionOperation::Capability { reference } => { - validate_capability_reference( - &format!("{operation_root}.reference"), - reference, - report, - ); - } - ExecutionOperation::Agent { - instructions, - skill_refs, - capability_refs, - max_turns, - } => validate_agent_operation( - &operation_root, - instructions, - skill_refs, - capability_refs, - *max_turns, - report, - ), - ExecutionOperation::Map { - items, - item_key, - max_items, - item_output_schema, - task, - } => { - validate_dynamic_value( - &format!("{operation_root}.items"), - items, - node, - false, - report, - ); - validate_json_pointer(&format!("{operation_root}.item_key"), item_key, report); - if *max_items == 0 { - report.push_error( - format!("{operation_root}.max_items"), - "map max_items must be at least one", - ); - } - if items.as_array().is_some_and(|items| { - u64::try_from(items.len()).map_or(true, |length| length > *max_items) - }) { - report.push_error( - format!("{operation_root}.items"), - "literal map items must not exceed max_items", - ); - } - validate_json_schema( - &format!("{operation_root}.item_output_schema"), - item_output_schema, - report, - ); - validate_static_map_keys(&operation_root, items, item_key, report); - match task { - MapTask::Capability { reference } => validate_capability_reference( - &format!("{operation_root}.task.reference"), - reference, - report, - ), - MapTask::Agent { - instructions, - skill_refs, - capability_refs, - max_turns, - } => validate_agent_operation( - &format!("{operation_root}.task"), - instructions, - skill_refs, - capability_refs, - *max_turns, - report, - ), - } - } - ExecutionOperation::Reduce { - items, - max_items, - reducer, - batch_size, - } => { - validate_dynamic_value( - &format!("{operation_root}.items"), - items, - node, - false, - report, - ); - if *max_items == 0 { - report.push_error( - format!("{operation_root}.max_items"), - "reduce max_items must be at least one", - ); - } - if items.as_array().is_some_and(|items| { - u64::try_from(items.len()).map_or(true, |length| length > *max_items) - }) { - report.push_error( - format!("{operation_root}.items"), - "literal reduce items must not exceed max_items", - ); - } - if *batch_size < 2 { - report.push_error( - format!("{operation_root}.batch_size"), - "reduce batch_size must be at least two", - ); - } - match reducer { - ExecutionReducer::Capability { reference } => validate_capability_reference( - &format!("{operation_root}.reducer.reference"), - reference, - report, - ), - ExecutionReducer::Agent { - instructions, - skill_refs, - capability_refs, - max_turns, - } => validate_agent_operation( - &format!("{operation_root}.reducer"), - instructions, - skill_refs, - capability_refs, - *max_turns, - report, - ), - } - } - ExecutionOperation::Review { prompt } => require_non_empty( - format!("{operation_root}.prompt"), - prompt, - "review prompt", - report, - ), - ExecutionOperation::WaitSignal { signal_name } => { - if !is_capability_component(signal_name, 64) { - report.push_error( - format!("{operation_root}.signal_name"), - "signal_name must be a non-empty ASCII name of at most 64 characters", - ); - } - } - ExecutionOperation::Output { value } => validate_dynamic_value( - &format!("{operation_root}.value"), - value, - node, - false, - report, - ), - } -} - fn validate_agent_operation( root: &str, instructions: &str, diff --git a/crates/moa-artifacts/src/validation/execution_plan.rs b/crates/moa-artifacts/src/validation/execution_plan.rs new file mode 100644 index 000000000..b5dd374bb --- /dev/null +++ b/crates/moa-artifacts/src/validation/execution_plan.rs @@ -0,0 +1,253 @@ +//! Execution-plan operation, wait-policy, and temporal-target validation. + +use crate::execution_plan::{ + ExecutionNode, ExecutionOperation, ExecutionReducer, ExecutionTemporalTarget, + ExecutionWaitExpiryAction, ExecutionWaitPolicy, MapTask, +}; + +use super::{ + ValidationReport, is_capability_component, require_non_empty, validate_agent_operation, + validate_capability_reference, validate_dynamic_value, validate_json_pointer, + validate_json_schema, validate_static_map_keys, +}; + +pub(super) fn validate_operation( + root: &str, + node: &ExecutionNode, + allow_absolute_temporal_targets: bool, + report: &mut ValidationReport, +) { + let operation_root = format!("{root}.operation"); + match &node.operation { + ExecutionOperation::Capability { reference } => { + validate_capability_reference( + &format!("{operation_root}.reference"), + reference, + report, + ); + } + ExecutionOperation::Agent { + instructions, + skill_refs, + capability_refs, + max_turns, + } => validate_agent_operation( + &operation_root, + instructions, + skill_refs, + capability_refs, + *max_turns, + report, + ), + ExecutionOperation::Map { + items, + item_key, + max_items, + item_output_schema, + task, + } => { + validate_dynamic_value( + &format!("{operation_root}.items"), + items, + node, + false, + report, + ); + validate_json_pointer(&format!("{operation_root}.item_key"), item_key, report); + if *max_items == 0 { + report.push_error( + format!("{operation_root}.max_items"), + "map max_items must be at least one", + ); + } + if items.as_array().is_some_and(|items| { + u64::try_from(items.len()).map_or(true, |length| length > *max_items) + }) { + report.push_error( + format!("{operation_root}.items"), + "literal map items must not exceed max_items", + ); + } + validate_json_schema( + &format!("{operation_root}.item_output_schema"), + item_output_schema, + report, + ); + validate_static_map_keys(&operation_root, items, item_key, report); + match task { + MapTask::Capability { reference } => validate_capability_reference( + &format!("{operation_root}.task.reference"), + reference, + report, + ), + MapTask::Agent { + instructions, + skill_refs, + capability_refs, + max_turns, + } => validate_agent_operation( + &format!("{operation_root}.task"), + instructions, + skill_refs, + capability_refs, + *max_turns, + report, + ), + } + } + ExecutionOperation::Reduce { + items, + max_items, + reducer, + batch_size, + } => { + validate_dynamic_value( + &format!("{operation_root}.items"), + items, + node, + false, + report, + ); + if *max_items == 0 { + report.push_error( + format!("{operation_root}.max_items"), + "reduce max_items must be at least one", + ); + } + if items.as_array().is_some_and(|items| { + u64::try_from(items.len()).map_or(true, |length| length > *max_items) + }) { + report.push_error( + format!("{operation_root}.items"), + "literal reduce items must not exceed max_items", + ); + } + if *batch_size < 2 { + report.push_error( + format!("{operation_root}.batch_size"), + "reduce batch_size must be at least two", + ); + } + match reducer { + ExecutionReducer::Capability { reference } => validate_capability_reference( + &format!("{operation_root}.reducer.reference"), + reference, + report, + ), + ExecutionReducer::Agent { + instructions, + skill_refs, + capability_refs, + max_turns, + } => validate_agent_operation( + &format!("{operation_root}.reducer"), + instructions, + skill_refs, + capability_refs, + *max_turns, + report, + ), + } + } + ExecutionOperation::Review { + prompt, + wait_policy, + } => { + require_non_empty( + format!("{operation_root}.prompt"), + prompt, + "review prompt", + report, + ); + validate_wait_policy( + &format!("{operation_root}.wait_policy"), + wait_policy, + node, + allow_absolute_temporal_targets, + report, + ); + } + ExecutionOperation::WaitSignal { + signal_name, + wait_policy, + } => { + if !is_capability_component(signal_name, 64) { + report.push_error( + format!("{operation_root}.signal_name"), + "signal_name must be a non-empty ASCII name of at most 64 characters", + ); + } + validate_wait_policy( + &format!("{operation_root}.wait_policy"), + wait_policy, + node, + allow_absolute_temporal_targets, + report, + ); + } + ExecutionOperation::WaitUntil { wake, result } => { + validate_temporal_target( + &format!("{operation_root}.wake"), + wake, + allow_absolute_temporal_targets, + report, + ); + validate_dynamic_value( + &format!("{operation_root}.result"), + result, + node, + false, + report, + ); + } + ExecutionOperation::Output { value } => validate_dynamic_value( + &format!("{operation_root}.value"), + value, + node, + false, + report, + ), + } +} + +fn validate_wait_policy( + root: &str, + policy: &ExecutionWaitPolicy, + node: &ExecutionNode, + allow_absolute_temporal_targets: bool, + report: &mut ValidationReport, +) { + validate_temporal_target( + &format!("{root}.expiry"), + &policy.expiry, + allow_absolute_temporal_targets, + report, + ); + if let ExecutionWaitExpiryAction::ContinueWith { output } = &policy.on_expiry { + validate_dynamic_value( + &format!("{root}.on_expiry.output"), + output, + node, + false, + report, + ); + } +} + +pub(super) fn validate_temporal_target( + root: &str, + target: &ExecutionTemporalTarget, + allow_absolute: bool, + report: &mut ValidationReport, +) { + match target { + ExecutionTemporalTarget::At { .. } if !allow_absolute => report.push_error( + root, + "reusable execution templates require an after temporal target", + ), + ExecutionTemporalTarget::After { delay_seconds: 0 } => { + report.push_error(root, "temporal delay_seconds must be at least one"); + } + ExecutionTemporalTarget::At { .. } | ExecutionTemporalTarget::After { .. } => {} + } +} diff --git a/crates/moa-artifacts/tests/artifacts_offline/definition_roundtrip.rs b/crates/moa-artifacts/tests/artifacts_offline/definition_roundtrip.rs index d0b45613d..1abf63963 100644 --- a/crates/moa-artifacts/tests/artifacts_offline/definition_roundtrip.rs +++ b/crates/moa-artifacts/tests/artifacts_offline/definition_roundtrip.rs @@ -527,6 +527,9 @@ definition: completion_checks: [] plan: cancel_policy: retain_effects + input_wait_policy: + expiry: { kind: after, delay_seconds: 3600 } + on_expiry: { kind: fail_run } input_schema: { type: object } output_schema: { type: object } nodes: @@ -616,6 +619,9 @@ definition: completion_checks: [] plan: cancel_policy: retain_effects + input_wait_policy: + expiry: { kind: after, delay_seconds: 3600 } + on_expiry: { kind: fail_run } input_schema: { type: object } output_schema: { type: object } nodes: @@ -690,6 +696,9 @@ definition: completion_checks: [] plan: cancel_policy: retain_effects + input_wait_policy: + expiry: { kind: after, delay_seconds: 3600 } + on_expiry: { kind: fail_run } input_schema: { type: object } output_schema: { type: object } nodes: diff --git a/crates/moa-artifacts/tests/artifacts_offline/execution_plan_validation.rs b/crates/moa-artifacts/tests/artifacts_offline/execution_plan_validation.rs index cb140e9c2..a8d55f91c 100644 --- a/crates/moa-artifacts/tests/artifacts_offline/execution_plan_validation.rs +++ b/crates/moa-artifacts/tests/artifacts_offline/execution_plan_validation.rs @@ -1,23 +1,25 @@ -use moa_artifacts::document::ArtifactDocument; +use chrono::{DateTime, Utc}; +use moa_artifacts::document::{ArtifactDocument, ArtifactStatus}; use moa_artifacts::execution_plan::{ CapabilityReference, CompensationInputBinding, CompensationInputMapping, CompensationValueSource, CompletionCheck, CompletionCheckKind, CoverageRequirement, ExecutionCancelPolicy, ExecutionCompensation, ExecutionCondition, ExecutionConstraint, ExecutionDeliverable, ExecutionGoalContract, ExecutionNode, ExecutionOperation, ExecutionPlanDefinition, ExecutionReducer, ExecutionReference, ExecutionRequirement, - ExecutionTaskOutcome, ExecutionTaskResult, ExecutionUsage, InputAudience, MapTask, - PlanAmendment, PlanAmendmentOperation, RetryPolicy, + ExecutionTaskOutcome, ExecutionTaskResult, ExecutionTemporalTarget, ExecutionUsage, + ExecutionWaitExpiryAction, ExecutionWaitPolicy, InputAudience, MapTask, PlanAmendment, + PlanAmendmentOperation, RetryPolicy, }; use moa_artifacts::reference::ArtifactRef; use moa_artifacts::validation::{ ValidationReport, validate_execution_goal_contract, validate_execution_plan_definition, - validate_execution_task_outcome, validate_plan_amendment, + validate_execution_task_outcome, validate_for_status, validate_plan_amendment, }; use serde_json::{Value, json}; #[test] -fn all_seven_execution_operations_round_trip_exact_json_and_yaml() { - // Pins: the public v1 wire shape has exactly the seven contract operations. +fn all_eight_execution_operations_round_trip_exact_json_and_yaml() { + // Pins: the public v1 wire shape has exactly the eight contract operations. let cases = [ ( ExecutionOperation::Capability { @@ -94,14 +96,43 @@ fn all_seven_execution_operations_round_trip_exact_json_and_yaml() { ( ExecutionOperation::Review { prompt: "Approve the report?".to_string(), + wait_policy: wait_policy(ExecutionWaitExpiryAction::FailRun), }, - json!({ "kind": "review", "prompt": "Approve the report?" }), + json!({ + "kind": "review", + "prompt": "Approve the report?", + "wait_policy": { + "expiry": { "kind": "after", "delay_seconds": 3600 }, + "on_expiry": { "kind": "fail_run" } + } + }), ), ( ExecutionOperation::WaitSignal { signal_name: "source_ready".to_string(), + wait_policy: wait_policy(ExecutionWaitExpiryAction::FailTask), }, - json!({ "kind": "wait_signal", "signal_name": "source_ready" }), + json!({ + "kind": "wait_signal", + "signal_name": "source_ready", + "wait_policy": { + "expiry": { "kind": "after", "delay_seconds": 3600 }, + "on_expiry": { "kind": "fail_task" } + } + }), + ), + ( + ExecutionOperation::WaitUntil { + wake: ExecutionTemporalTarget::At { + at: at("2030-01-02T02:00:00Z"), + }, + result: json!({ "window": "open" }), + }, + json!({ + "kind": "wait_until", + "wake": { "kind": "at", "at": "2030-01-02T02:00:00Z" }, + "result": { "window": "open" } + }), ), ( ExecutionOperation::Output { @@ -114,7 +145,7 @@ fn all_seven_execution_operations_round_trip_exact_json_and_yaml() { ), ]; - assert_eq!(cases.len(), 7); + assert_eq!(cases.len(), 8); for (operation, expected_json) in cases { let actual_json = serde_json::to_value(&operation).expect("serialize operation to JSON"); assert_eq!(actual_json, expected_json); @@ -133,6 +164,260 @@ fn all_seven_execution_operations_round_trip_exact_json_and_yaml() { } } +#[test] +fn wait_expiry_actions_round_trip_with_canonical_tagged_shapes() { + // Pins: wait expiry is explicit and every settlement action has one stable wire shape. + let cases = [ + ( + ExecutionWaitExpiryAction::FailTask, + json!({ "kind": "fail_task" }), + ), + ( + ExecutionWaitExpiryAction::FailRun, + json!({ "kind": "fail_run" }), + ), + ( + ExecutionWaitExpiryAction::ContinueWith { + output: json!({ "timed_out": true }), + }, + json!({ + "kind": "continue_with", + "output": { "timed_out": true } + }), + ), + ]; + + for (action, expected_json) in cases { + let actual_json = serde_json::to_value(&action).expect("serialize wait expiry action"); + assert_eq!(actual_json, expected_json); + assert_eq!( + serde_json::from_value::(actual_json) + .expect("deserialize exact wait expiry action"), + action + ); + } + + let missing_output = serde_json::from_value::(json!({ + "kind": "continue_with" + })); + assert!( + missing_output.is_err(), + "continue_with must declare its deterministic output" + ); + let unknown_action = serde_json::from_value::(json!({ + "kind": "skip" + })); + assert!( + unknown_action.is_err(), + "undeclared wait expiry actions must reject" + ); +} + +#[test] +fn temporal_targets_round_trip_and_reject_zero_relative_delay() { + // Pins: timers use one typed absolute-or-relative target and relative waits always advance time. + let cases = [ + ( + absolute_target(), + json!({ "kind": "at", "at": "2030-01-02T02:00:00Z" }), + ), + ( + ExecutionTemporalTarget::After { + delay_seconds: 3_600, + }, + json!({ "kind": "after", "delay_seconds": 3600 }), + ), + ]; + for (target, expected_json) in cases { + let actual_json = serde_json::to_value(&target).expect("serialize temporal target"); + assert_eq!(actual_json, expected_json); + assert_eq!( + serde_json::from_value::(actual_json) + .expect("deserialize exact temporal target"), + target + ); + } + + let mut zero_input_wait = valid_plan(); + zero_input_wait.input_wait_policy.expiry = ExecutionTemporalTarget::After { delay_seconds: 0 }; + assert_error( + &validate_execution_plan_definition(&zero_input_wait), + "execution_plan.input_wait_policy.expiry", + "temporal delay_seconds must be at least one", + ); + + let mut zero_timer = valid_plan(); + zero_timer.nodes[0].operation = ExecutionOperation::WaitUntil { + wake: ExecutionTemporalTarget::After { delay_seconds: 0 }, + result: json!({}), + }; + assert_error( + &validate_execution_plan_definition(&zero_timer), + "execution_plan.nodes[0].operation.wake", + "temporal delay_seconds must be at least one", + ); +} + +#[test] +fn reusable_plan_templates_reject_absolute_temporal_targets_at_every_wait_surface() { + // Pins: reusable skill templates stay valid over time by expressing waits relative to when + // each wait state is entered; exact UTC targets remain valid only for one-off plans. + let mut standalone = valid_plan(); + standalone.input_wait_policy.expiry = absolute_target(); + assert!( + validate_execution_plan_definition(&standalone).is_ok(), + "one-off plans may declare an exact UTC input-wait expiry" + ); + + let mut cases = Vec::new(); + cases.push(( + standalone, + "definition.spec.execution_plan.plan.input_wait_policy.expiry", + )); + + let mut review = valid_plan(); + review.nodes[0].operation = ExecutionOperation::Review { + prompt: "Approve?".to_string(), + wait_policy: ExecutionWaitPolicy { + expiry: absolute_target(), + on_expiry: ExecutionWaitExpiryAction::FailRun, + }, + }; + cases.push(( + review, + "definition.spec.execution_plan.plan.nodes[0].operation.wait_policy.expiry", + )); + + let mut signal = valid_plan(); + signal.nodes[0].operation = ExecutionOperation::WaitSignal { + signal_name: "ready".to_string(), + wait_policy: ExecutionWaitPolicy { + expiry: absolute_target(), + on_expiry: ExecutionWaitExpiryAction::FailTask, + }, + }; + cases.push(( + signal, + "definition.spec.execution_plan.plan.nodes[0].operation.wait_policy.expiry", + )); + + let mut timer = valid_plan(); + timer.nodes[0].operation = ExecutionOperation::WaitUntil { + wake: absolute_target(), + result: json!({ "ready": true }), + }; + cases.push(( + timer, + "definition.spec.execution_plan.plan.nodes[0].operation.wake", + )); + + for (plan, expected_path) in cases { + let document = skill_document_with_plan(plan); + assert_error( + &validate_for_status(&document, ArtifactStatus::Draft), + expected_path, + "reusable execution templates require an after temporal target", + ); + } +} + +#[test] +fn skill_schema_and_rust_types_require_the_same_wait_contract() { + // Pins: the hand-maintained skill schema cannot accept stale wait operations or plans that + // the canonical Rust artifact types reject. + let skill_schema: Value = serde_json::from_str(include_str!( + "../../../../docs/schemas/moa-skill-v1.schema.json" + )) + .expect("parse skill JSON schema"); + let operation_validator = + jsonschema::validator_for(&schema_entry_document(&skill_schema, "ExecutionOperation")) + .expect("compile execution operation schema"); + let plan_validator = jsonschema::validator_for(&schema_entry_document( + &skill_schema, + "ExecutionPlanDefinition", + )) + .expect("compile execution plan schema"); + + let wait_operations = [ + ExecutionOperation::Review { + prompt: "Approve?".to_string(), + wait_policy: wait_policy(ExecutionWaitExpiryAction::FailRun), + }, + ExecutionOperation::WaitSignal { + signal_name: "ready".to_string(), + wait_policy: wait_policy(ExecutionWaitExpiryAction::ContinueWith { + output: json!({ "ready": false }), + }), + }, + ExecutionOperation::WaitUntil { + wake: ExecutionTemporalTarget::At { + at: at("2030-01-02T02:00:00Z"), + }, + result: json!({ "ready": true }), + }, + ]; + for operation in wait_operations { + let encoded = serde_json::to_value(operation).expect("serialize wait operation"); + assert!( + operation_validator.is_valid(&encoded), + "skill schema must accept canonical operation: {encoded}" + ); + } + + let stale_review = json!({ + "kind": "review", + "prompt": "Approve?" + }); + assert!( + serde_json::from_value::(stale_review.clone()).is_err(), + "Rust type must reject a review without wait_policy" + ); + assert!( + !operation_validator.is_valid(&stale_review), + "skill schema must reject a review without wait_policy" + ); + let review_with_extra_expiry_field = json!({ + "kind": "review", + "prompt": "Approve?", + "wait_policy": { + "expiry": { "kind": "after", "delay_seconds": 3600 }, + "on_expiry": { "kind": "fail_task", "output": null } + } + }); + assert!( + !operation_validator.is_valid(&review_with_extra_expiry_field), + "skill schema must reject fields not declared by a fieldless expiry action" + ); + let zero_delay_timer = json!({ + "kind": "wait_until", + "wake": { "kind": "after", "delay_seconds": 0 }, + "result": {} + }); + assert!( + !operation_validator.is_valid(&zero_delay_timer), + "skill schema must reject a zero-second relative timer" + ); + + let plan_json = serde_json::to_value(valid_plan()).expect("serialize canonical plan"); + assert!( + plan_validator.is_valid(&plan_json), + "skill schema must accept the canonical Rust plan" + ); + let mut stale_plan = plan_json; + stale_plan + .as_object_mut() + .expect("plan is an object") + .remove("input_wait_policy"); + assert!( + serde_json::from_value::(stale_plan.clone()).is_err(), + "Rust type must reject a plan without input_wait_policy" + ); + assert!( + !plan_validator.is_valid(&stale_plan), + "skill schema must reject a plan without input_wait_policy" + ); +} + #[test] fn execution_plan_rejects_unknown_fields_old_operations_and_nested_maps() { // Pins: strict v1 model output cannot smuggle controls or revive old/recursive node kinds. @@ -215,6 +500,10 @@ fn skill_reference_paths_cover_agent_map_and_reducer_agents_only() { }, "plan": { "cancel_policy": "retain_effects", + "input_wait_policy": { + "expiry": { "kind": "after", "delay_seconds": 3600 }, + "on_expiry": { "kind": "fail_run" } + }, "input_schema": { "type": "object" }, "output_schema": { "type": "object" }, "nodes": [ @@ -335,6 +624,13 @@ fn execution_plan_round_trips_without_a_nested_version() { let encoded = serde_json::to_value(&plan).expect("serialize plan"); assert!(encoded.get("schema_version").is_none()); assert_eq!(encoded["cancel_policy"], json!("retain_effects")); + assert_eq!( + encoded["input_wait_policy"], + json!({ + "expiry": { "kind": "after", "delay_seconds": 3600 }, + "on_expiry": { "kind": "fail_run" } + }) + ); assert_eq!( serde_json::from_value::(encoded).expect("deserialize exact plan"), plan @@ -566,6 +862,7 @@ fn execution_plan_requires_one_terminal_output_with_all_nodes_as_ancestors() { let mut no_output = valid_plan(); no_output.nodes[1].operation = ExecutionOperation::Review { prompt: "Review".to_string(), + wait_policy: wait_policy(ExecutionWaitExpiryAction::FailTask), }; assert_error( &validate_execution_plan_definition(&no_output), @@ -587,6 +884,7 @@ fn execution_plan_requires_one_terminal_output_with_all_nodes_as_ancestors() { &["output"], ExecutionOperation::Review { prompt: "Impossible review".to_string(), + wait_policy: wait_policy(ExecutionWaitExpiryAction::FailTask), }, )); let report = validate_execution_plan_definition(&output_has_dependent); @@ -607,6 +905,7 @@ fn execution_plan_requires_one_terminal_output_with_all_nodes_as_ancestors() { &[], ExecutionOperation::Review { prompt: "Unused review".to_string(), + wait_policy: wait_policy(ExecutionWaitExpiryAction::FailTask), }, )); assert_error( @@ -749,6 +1048,7 @@ fn execution_plan_rejects_malformed_hidden_and_recursive_references() { &["lookup"], ExecutionOperation::Review { prompt: "Review".to_string(), + wait_policy: wait_policy(ExecutionWaitExpiryAction::FailTask), }, ), ); @@ -762,6 +1062,32 @@ fn execution_plan_rejects_malformed_hidden_and_recursive_references() { "execution reference may only read a declared dependency output", ); + let mut hidden_timer_result = valid_plan(); + hidden_timer_result.nodes[0].operation = ExecutionOperation::WaitUntil { + wake: ExecutionTemporalTarget::At { + at: at("2030-01-02T02:00:00Z"), + }, + result: json!({ "$ref": "$.nodes.output.output" }), + }; + assert_error( + &validate_execution_plan_definition(&hidden_timer_result), + "execution_plan.nodes[0].operation.result", + "execution reference may only read a declared dependency output", + ); + + let mut hidden_expiry_output = valid_plan(); + hidden_expiry_output.nodes[0].operation = ExecutionOperation::Review { + prompt: "Approve?".to_string(), + wait_policy: wait_policy(ExecutionWaitExpiryAction::ContinueWith { + output: json!({ "$ref": "$.nodes.output.output" }), + }), + }; + assert_error( + &validate_execution_plan_definition(&hidden_expiry_output), + "execution_plan.nodes[0].operation.wait_policy.on_expiry.output", + "execution reference may only read a declared dependency output", + ); + let mut recursive = valid_plan(); recursive.nodes[0].input = json!({ "$ref": "$.nodes.lookup.output" }); assert_error( @@ -1046,6 +1372,7 @@ fn task_outcome_variants_round_trip_without_extra_envelope_fields() { fn valid_plan() -> ExecutionPlanDefinition { ExecutionPlanDefinition { cancel_policy: ExecutionCancelPolicy::RetainEffects, + input_wait_policy: wait_policy(ExecutionWaitExpiryAction::FailRun), input_schema: json!({ "type": "object" }), output_schema: json!({ "type": "object" }), nodes: vec![ @@ -1067,6 +1394,63 @@ fn valid_plan() -> ExecutionPlanDefinition { } } +fn wait_policy(on_expiry: ExecutionWaitExpiryAction) -> ExecutionWaitPolicy { + ExecutionWaitPolicy { + expiry: ExecutionTemporalTarget::After { + delay_seconds: 3_600, + }, + on_expiry, + } +} + +fn at(value: &str) -> DateTime { + value.parse().expect("fixture timestamp must be RFC 3339") +} + +fn absolute_target() -> ExecutionTemporalTarget { + ExecutionTemporalTarget::At { + at: at("2030-01-02T02:00:00Z"), + } +} + +fn skill_document_with_plan(plan: ExecutionPlanDefinition) -> ArtifactDocument { + ArtifactDocument::from_json( + &json!({ + "api_version": "moa.artifact/v1", + "kind": "skill", + "metadata": { "name": "temporal-template" }, + "definition": { + "type": "skill", + "spec": { + "execution_plan": { + "goal": { + "requirements": [{ + "id": "req_one", + "description": "Complete the temporal work." + }], + "deliverables": [], + "coverage": [], + "constraints": [], + "completion_checks": [] + }, + "plan": plan + } + } + } + }) + .to_string(), + ) + .expect("parse temporal skill template fixture") +} + +fn schema_entry_document(schema: &Value, definition: &str) -> Value { + json!({ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": format!("#/$defs/{definition}"), + "$defs": schema["$defs"].clone() + }) +} + fn operation_node_json(id: &str, operation: Value) -> Value { json!({ "id": id, diff --git a/crates/moa-auth/authz/src/poller.rs b/crates/moa-auth/authz/src/poller.rs index 71ce6fb05..ae0207818 100644 --- a/crates/moa-auth/authz/src/poller.rs +++ b/crates/moa-auth/authz/src/poller.rs @@ -5,8 +5,14 @@ use crate::error::AuthzError; use crate::outbox::OutboxRow; use serde_json::json; use sqlx::PgPool; -use std::time::Duration; -use tokio::sync::oneshot; +use std::{ + sync::{ + Arc, RwLock, + atomic::{AtomicBool, Ordering}, + }, + time::{Duration, Instant}, +}; +use tokio::sync::watch; use tokio::time::sleep; use uuid::Uuid; @@ -56,36 +62,58 @@ impl OutboxPoller { /// Spawn the poller on the current Tokio runtime and return a shutdown handle. pub fn spawn(self) -> PollerHandle { - let (shutdown_tx, mut shutdown_rx) = oneshot::channel::<()>(); + let health = Arc::new(PollerHealth { + started_at: Instant::now(), + last_success: RwLock::new(None), + exited: AtomicBool::new(false), + }); + let (shutdown, mut shutdown_rx) = watch::channel(false); + let task_health = Arc::clone(&health); + let heartbeat_maximum_age = self + .cfg + .poll_interval + .saturating_mul(3) + .max( + self.cfg + .lease_duration + .saturating_add(self.cfg.poll_interval), + ) + .max(Duration::from_secs(5)); let task = tokio::spawn(async move { - loop { - tokio::select! { - biased; - _ = &mut shutdown_rx => { - tracing::info!("outbox poller received shutdown"); - break; - } - result = self.tick() => { - if let Err(error) = result { - tracing::error!(error = %error, "outbox poller tick failed"); + let result = async { + loop { + tokio::select! { + biased; + _ = shutdown_rx.changed() => { + tracing::info!("outbox poller received shutdown"); + return Ok(()); + } + result = self.tick() => { + result?; + set_poller_heartbeat(&task_health); } } - } - tokio::select! { - biased; - _ = &mut shutdown_rx => { - tracing::info!("outbox poller received shutdown"); - break; + tokio::select! { + biased; + _ = shutdown_rx.changed() => { + tracing::info!("outbox poller received shutdown"); + return Ok(()); + } + () = sleep(self.cfg.poll_interval) => {} } - _ = sleep(self.cfg.poll_interval) => {} } } + .await; + task_health.exited.store(true, Ordering::Release); + result }); PollerHandle { - shutdown: Some(shutdown_tx), + health, + shutdown, task, + heartbeat_maximum_age, } } @@ -513,18 +541,117 @@ impl ClaimedOutboxRow { /// Handle for cleanly shutting down a spawned outbox poller. pub struct PollerHandle { - shutdown: Option>, - task: tokio::task::JoinHandle<()>, + health: Arc, + shutdown: watch::Sender, + task: tokio::task::JoinHandle>, + heartbeat_maximum_age: Duration, } impl PollerHandle { - /// Signal the poller to stop and wait for its task to exit. - pub async fn shutdown(mut self) { - if let Some(shutdown) = self.shutdown.take() { - let _ = shutdown.send(()); + /// Returns a cloneable readiness projection for the supervised poller. + #[must_use] + pub fn readiness(&self) -> PollerReadiness { + PollerReadiness { + health: Arc::clone(&self.health), + heartbeat_maximum_age: self.heartbeat_maximum_age, } - let _ = self.task.await; } + + /// Waits for the poller task so unexpected failure can terminate its owner process. + pub async fn task_result(&mut self) -> Result<(), PollerTaskError> { + match (&mut self.task).await { + Ok(result) => result.map_err(PollerTaskError::Poll), + Err(error) => Err(PollerTaskError::Join(error.to_string())), + } + } + + /// Signals the poller to stop and waits for its task to exit. + pub async fn shutdown(mut self) -> Result<(), PollerTaskError> { + let _ = self.shutdown.send(true); + self.task_result().await + } +} + +impl Drop for PollerHandle { + fn drop(&mut self) { + self.health.exited.store(true, Ordering::Release); + let _ = self.shutdown.send(true); + self.task.abort(); + } +} + +/// Cloneable health projection for the supervised authorization outbox poller. +#[derive(Clone)] +pub struct PollerReadiness { + health: Arc, + heartbeat_maximum_age: Duration, +} + +impl PollerReadiness { + /// Returns the age of the most recent complete successful poll. + #[must_use] + pub fn heartbeat_age(&self) -> Duration { + let last_success = self + .health + .last_success + .read() + .ok() + .and_then(|heartbeat| *heartbeat); + last_success.map_or_else( + || self.health.started_at.elapsed(), + |heartbeat| heartbeat.elapsed(), + ) + } + + /// Returns why the poller must not be considered ready. + #[must_use] + pub fn unready_reason(&self) -> Option { + if self.health.exited.load(Ordering::Acquire) { + return Some("authorization outbox poller exited".to_string()); + } + let last_success = self + .health + .last_success + .read() + .ok() + .and_then(|heartbeat| *heartbeat); + let age = self.heartbeat_age(); + if last_success.is_none() { + return Some( + "authorization outbox poller has not completed its first pass".to_string(), + ); + } + (age > self.heartbeat_maximum_age).then(|| { + format!( + "authorization outbox poller heartbeat is stale by {:.3}s", + age.as_secs_f64() + ) + }) + } +} + +#[derive(Debug)] +struct PollerHealth { + started_at: Instant, + last_success: RwLock>, + exited: AtomicBool, +} + +fn set_poller_heartbeat(health: &PollerHealth) { + if let Ok(mut heartbeat) = health.last_success.write() { + *heartbeat = Some(Instant::now()); + } +} + +/// Failure returned by the supervised authorization outbox poller handle. +#[derive(Debug, thiserror::Error)] +pub enum PollerTaskError { + /// One poll pass failed. + #[error("authorization outbox poll failed: {0}")] + Poll(AuthzError), + /// The Tokio owner task could not be joined. + #[error("authorization outbox poller task join failed: {0}")] + Join(String), } fn limit_i64(limit: usize) -> i64 { @@ -541,7 +668,7 @@ fn missed_claim_is_satisfied(status: Option<&str>) -> bool { #[cfg(test)] mod tests { - use super::missed_claim_is_satisfied; + use super::*; #[test] fn missed_claim_accepts_only_same_generation_success() { @@ -557,4 +684,32 @@ mod tests { assert!(!missed_claim_is_satisfied(status), "status={status:?}"); } } + + #[test] + fn poller_readiness_requires_success_and_rejects_exit() { + // Pins: maintenance readiness never reports a poller ready before one + // complete pass or after its sole correctness owner exits. + let health = Arc::new(PollerHealth { + started_at: Instant::now(), + last_success: RwLock::new(None), + exited: AtomicBool::new(false), + }); + let readiness = PollerReadiness { + health: Arc::clone(&health), + heartbeat_maximum_age: Duration::from_secs(60), + }; + assert_eq!( + readiness.unready_reason().as_deref(), + Some("authorization outbox poller has not completed its first pass") + ); + + set_poller_heartbeat(&health); + assert_eq!(readiness.unready_reason(), None); + + health.exited.store(true, Ordering::Release); + assert_eq!( + readiness.unready_reason().as_deref(), + Some("authorization outbox poller exited") + ); + } } diff --git a/crates/moa-brain/src/execution_planning/request.rs b/crates/moa-brain/src/execution_planning/request.rs index dde36f5c3..f6a8c165a 100644 --- a/crates/moa-brain/src/execution_planning/request.rs +++ b/crates/moa-brain/src/execution_planning/request.rs @@ -11,7 +11,7 @@ use moa_core::types::{ }; use moa_execution::{ compiler::CanonicalExecutionPlan, - state::{ExecutionProjection, ExecutionTaskId}, + state::{ExecutionAmendmentProjection, ExecutionTaskId}, wire::ExecutionPlanningContextSnapshot, }; use schemars::schema_for; @@ -19,7 +19,7 @@ use serde::{Deserialize, Serialize}; use serde_json::Value; /// Stable execution-planner prompt identifier. -pub const EXECUTION_PLANNER_PROMPT_VERSION: &str = "execution-planner-v5"; +pub const EXECUTION_PLANNER_PROMPT_VERSION: &str = "execution-planner-v7"; /// Fixed maximum collected planner output tokens. pub const EXECUTION_PLANNER_MAX_OUTPUT_TOKENS: usize = 32_768; const EXECUTION_PLANNER_PROMPT: &str = include_str!("../prompts/execution_planner.txt"); @@ -51,8 +51,8 @@ pub struct AmendmentPlanningEvidence { pub goal: moa_artifacts::execution_plan::ExecutionGoalContract, /// Active immutable plan snapshot. pub active_plan: CanonicalExecutionPlan, - /// Current durable run projection and completed structured outputs. - pub projection: ExecutionProjection, + /// Compiler-bounded aggregate node state and exact replan origin. + pub projection: ExecutionAmendmentProjection, /// Structured failure evidence that caused WaitingReplan. pub failure_evidence: Value, /// Exact originating waiting task. @@ -295,10 +295,10 @@ mod tests { use super::*; #[test] - fn execution_planner_prompt_v5_pins_compensation_and_compiler_invariants() { + fn execution_planner_prompt_v7_pins_long_horizon_and_compiler_invariants() { // Pins: the current emitted prompt version and its compiler-facing guidance // change together so live planner provenance identifies this exact contract. - assert_eq!(EXECUTION_PLANNER_PROMPT_VERSION, "execution-planner-v5"); + assert_eq!(EXECUTION_PLANNER_PROMPT_VERSION, "execution-planner-v7"); assert_eq!( EXECUTION_PLANNER_PROMPT, concat!( @@ -307,7 +307,11 @@ mod tests { "Compiler invariants:\n", "- Set `goal.objective` to the frozen `objective` byte-for-byte.\n", "- Choose exactly one explicit `plan.cancel_policy`: `retain_effects` or `compensate_committed`.\n", - "- Set every node's `compensation` explicitly. Use `null` unless the node is a direct side-effecting `Capability` whose exact catalog entry advertises the same compensator and bounded input mapping. Never add compensation to reads, agents, maps, reduces, reviews, signals, or outputs, and never invent rollback authority.\n", + "- Treat the frozen `budget.deadline_at` as the absolute Durable-run deadline. Never emit a wait, retry window, or active task whose bound reaches or exceeds it.\n", + "- Use `WaitUntil` for an absolute calendar-time delay. Its `wake_at` must be before the run deadline, and its declared `result` is the structured value made available to downstream nodes after the timer fires.\n", + "- Give every `Review` and `WaitSignal` an explicit wait policy. Represent human and external waits only with these storage-backed wait operations; never keep an `Agent` or `Capability` active while waiting for a person, callback, schedule, or retry time.\n", + "- Decompose long work into bounded active tasks separated by durable nodes. Never plan a continuously running multi-hour or multi-day model call, tool call, shell process, network connection, or sandbox; use a registered asynchronous capability when the catalog explicitly provides one.\n", + "- Set every node's `compensation` explicitly. Use `null` unless the node is a direct side-effecting `Capability` whose exact catalog entry advertises the same compensator and bounded input mapping, and that compensator has `requires_sandbox=false`. Never add compensation to reads, agents, maps, reduces, reviews, signals, or outputs, never use a sandbox-backed compensator, and never invent rollback authority.\n", "- An amendment must preserve compensation for work that is running or committed and must not weaken the run's cancellation policy.\n", "- Every goal-entry ID, completion-check ID, execution-node ID, and every ID referenced from those structures must match `[a-z][a-z0-9_-]{0,63}`.\n", "- Link every requirement and every constraint to at least one completion check via `requirement_ids` and `constraint_ids`.\n", diff --git a/crates/moa-brain/src/prompts/execution_planner.txt b/crates/moa-brain/src/prompts/execution_planner.txt index 1099abc62..eb1615220 100644 --- a/crates/moa-brain/src/prompts/execution_planner.txt +++ b/crates/moa-brain/src/prompts/execution_planner.txt @@ -5,7 +5,11 @@ Preserve the user's objective, scope, definitions, time range, universe, output Compiler invariants: - Set `goal.objective` to the frozen `objective` byte-for-byte. - Choose exactly one explicit `plan.cancel_policy`: `retain_effects` or `compensate_committed`. -- Set every node's `compensation` explicitly. Use `null` unless the node is a direct side-effecting `Capability` whose exact catalog entry advertises the same compensator and bounded input mapping. Never add compensation to reads, agents, maps, reduces, reviews, signals, or outputs, and never invent rollback authority. +- Treat the frozen `budget.deadline_at` as the absolute Durable-run deadline. Never emit a wait, retry window, or active task whose bound reaches or exceeds it. +- Use `WaitUntil` for an absolute calendar-time delay. Its `wake_at` must be before the run deadline, and its declared `result` is the structured value made available to downstream nodes after the timer fires. +- Give every `Review` and `WaitSignal` an explicit wait policy. Represent human and external waits only with these storage-backed wait operations; never keep an `Agent` or `Capability` active while waiting for a person, callback, schedule, or retry time. +- Decompose long work into bounded active tasks separated by durable nodes. Never plan a continuously running multi-hour or multi-day model call, tool call, shell process, network connection, or sandbox; use a registered asynchronous capability when the catalog explicitly provides one. +- Set every node's `compensation` explicitly. Use `null` unless the node is a direct side-effecting `Capability` whose exact catalog entry advertises the same compensator and bounded input mapping, and that compensator has `requires_sandbox=false`. Never add compensation to reads, agents, maps, reduces, reviews, signals, or outputs, never use a sandbox-backed compensator, and never invent rollback authority. - An amendment must preserve compensation for work that is running or committed and must not weaken the run's cancellation policy. - Every goal-entry ID, completion-check ID, execution-node ID, and every ID referenced from those structures must match `[a-z][a-z0-9_-]{0,63}`. - Link every requirement and every constraint to at least one completion check via `requirement_ids` and `constraint_ids`. diff --git a/crates/moa-brain/src/prompts/execution_router.txt b/crates/moa-brain/src/prompts/execution_router.txt index ab23396db..0a8d5414a 100644 --- a/crates/moa-brain/src/prompts/execution_router.txt +++ b/crates/moa-brain/src/prompts/execution_router.txt @@ -8,6 +8,8 @@ When the user asks MOA to actually call an authorized tool, spawn or delegate to Choose execute with strategy durable when the work materially benefits from persistence, resumability, parallel or high-fan-out execution, long waits, external coordination, approval or signal handling, or a compiled plan. These are examples, not an exhaustive taxonomy. Task difficulty alone does not require Durable execution. +Calendar duration and active-compute duration are different. Route day/week work with explicit timers, human waits, callbacks, or resumable milestones to durable execution, but never imply that Durable keeps a model call, tool call, shell process, network connection, or sandbox live throughout that time. Continuously running long work is executable only when a registered asynchronous capability supports it; otherwise the planner must reject the unsupported requirement. + Parallelism alone does not require durable execution when the request is a bounded same-turn conversational-worker or tool loop. Choose needs_input only when a concrete input is genuinely missing and work cannot responsibly begin. List each missing input briefly. diff --git a/crates/moa-brain/tests/brain_turn_offline.rs b/crates/moa-brain/tests/brain_turn_offline.rs index de94425f2..0a14ef378 100644 --- a/crates/moa-brain/tests/brain_turn_offline.rs +++ b/crates/moa-brain/tests/brain_turn_offline.rs @@ -829,7 +829,7 @@ fn execution_planning_candidate(objective: &str, max_attempts: u32) -> String { fn execution_amendment_planning_request() -> moa_brain::execution_planning::ExecutionAmendmentPlanningRequest { - use std::collections::BTreeMap; + use std::collections::{BTreeMap, BTreeSet}; let objective = "Repair the durable report"; let mut initial = serde_json::from_str::< @@ -872,29 +872,6 @@ fn execution_amendment_planning_request() ) }); let run_uid = uuid::Uuid::from_u128(700); - let prepare_task = moa_execution::state::ExecutionTaskProjection { - task_id: moa_execution::state::ExecutionTaskId::derive(run_uid, "prepare", "") - .expect("prepare task id"), - node_id: "prepare".to_string(), - item_key: String::new(), - status: moa_execution::state::ExecutionTaskStatus::Completed, - attempt: 1, - generation: 1, - input: json!({}), - outcome: Some(moa_artifacts::execution_plan::ExecutionTaskOutcome { - schema_version: 1, - usage: moa_artifacts::execution_plan::ExecutionUsage { - cost_microusd: 0, - tokens: 0, - tool_calls: 0, - retrieved_bytes: 0, - }, - result: moa_artifacts::execution_plan::ExecutionTaskResult::Completed { - output: json!({"value": "completed-value"}), - citations: Vec::new(), - }, - }), - }; let waiting_task = moa_execution::state::ExecutionTaskProjection { task_id: moa_execution::state::ExecutionTaskId::derive(run_uid, "output", "") .expect("waiting task id"), @@ -926,7 +903,7 @@ fn execution_amendment_planning_request() evidence: moa_brain::execution_planning::AmendmentPlanningEvidence { goal: initial.goal, active_plan: compiled.plan, - projection: moa_execution::state::ExecutionProjection { + projection: moa_execution::state::ExecutionAmendmentProjection { plan_revision: 7, node_statuses: BTreeMap::from([ ( @@ -938,7 +915,8 @@ fn execution_amendment_planning_request() moa_execution::state::ExecutionNodeStatus::Waiting, ), ]), - tasks: vec![prepare_task, waiting_task], + started_node_ids: BTreeSet::from(["prepare".to_string(), "output".to_string()]), + replan_tasks: vec![waiting_task], }, failure_evidence: json!({"reason": "shape changed"}), waiting_task: waiting_task_id, diff --git a/crates/moa-config/src/env_overlay/mod.rs b/crates/moa-config/src/env_overlay/mod.rs index 2817c50ac..14f9c5783 100644 --- a/crates/moa-config/src/env_overlay/mod.rs +++ b/crates/moa-config/src/env_overlay/mod.rs @@ -730,6 +730,38 @@ pub struct EnvOverlay { pub execution_repeated_failure_limit: Option, /// `MOA_EXECUTION_MAX_IN_FLIGHT_TASKS`. pub execution_max_in_flight_tasks: Option, + /// `MOA_EXECUTION_MAXIMUM_HORIZON_SECONDS`. + pub execution_maximum_horizon_seconds: Option, + /// `MOA_EXECUTION_MAXIMUM_ACTIVATION_STEPS`. + pub execution_maximum_activation_steps: Option, + /// `MOA_EXECUTION_DISPATCH_BATCH_SIZE`. + pub execution_dispatch_batch_size: Option, + /// `MOA_EXECUTION_ACTIVE_ATTEMPT_TIMEOUT_SECONDS`. + pub execution_active_attempt_timeout_seconds: Option, + /// `MOA_EXECUTION_MAX_TENANT_ACTIVE_RUNS`. + pub execution_max_tenant_active_runs: Option, + /// `MOA_EXECUTION_MAX_FLEET_ACTIVE_RUNS`. + pub execution_max_fleet_active_runs: Option, + /// `MOA_EXECUTION_MAX_TENANT_ACTIVE_TASKS`. + pub execution_max_tenant_active_tasks: Option, + /// `MOA_EXECUTION_MAX_FLEET_ACTIVE_TASKS`. + pub execution_max_fleet_active_tasks: Option, + /// `MOA_EXECUTION_MAX_TENANT_PARKED_RUNS`. + pub execution_max_tenant_parked_runs: Option, + /// `MOA_EXECUTION_MAX_FLEET_PARKED_RUNS`. + pub execution_max_fleet_parked_runs: Option, + /// `MOA_EXECUTION_MAX_TENANT_SCHEDULED_TRIGGERS`. + pub execution_max_tenant_scheduled_triggers: Option, + /// `MOA_EXECUTION_MAX_FLEET_SCHEDULED_TRIGGERS`. + pub execution_max_fleet_scheduled_triggers: Option, + /// `MOA_EXECUTION_MAX_TENANT_EXTERNAL_JOBS`. + pub execution_max_tenant_external_jobs: Option, + /// `MOA_EXECUTION_MAX_FLEET_EXTERNAL_JOBS`. + pub execution_max_fleet_external_jobs: Option, + /// `MOA_EXECUTION_TRIGGER_RECONCILIATION_CADENCE_SECONDS`. + pub execution_trigger_reconciliation_cadence_seconds: Option, + /// `MOA_EXECUTION_TERMINAL_DETAIL_RETENTION_DAYS`. + pub execution_terminal_detail_retention_days: Option, /// `MOA_EXECUTION_MAX_TASKS`. pub execution_max_tasks: Option, /// `MOA_EXECUTION_MAX_TOKENS`. diff --git a/crates/moa-config/src/env_overlay/tests.rs b/crates/moa-config/src/env_overlay/tests.rs index d4a5723af..b03822c59 100644 --- a/crates/moa-config/src/env_overlay/tests.rs +++ b/crates/moa-config/src/env_overlay/tests.rs @@ -566,6 +566,22 @@ fn from_iter_applies_every_execution_resource_override() { ("MOA_EXECUTION_PLANNER_REPAIR_ATTEMPTS", "2"), ("MOA_EXECUTION_REPEATED_FAILURE_LIMIT", "4"), ("MOA_EXECUTION_MAX_IN_FLIGHT_TASKS", "96"), + ("MOA_EXECUTION_MAXIMUM_HORIZON_SECONDS", "1209600"), + ("MOA_EXECUTION_MAXIMUM_ACTIVATION_STEPS", "192"), + ("MOA_EXECUTION_DISPATCH_BATCH_SIZE", "48"), + ("MOA_EXECUTION_ACTIVE_ATTEMPT_TIMEOUT_SECONDS", "900"), + ("MOA_EXECUTION_MAX_TENANT_ACTIVE_RUNS", "120"), + ("MOA_EXECUTION_MAX_FLEET_ACTIVE_RUNS", "1200"), + ("MOA_EXECUTION_MAX_TENANT_ACTIVE_TASKS", "384"), + ("MOA_EXECUTION_MAX_FLEET_ACTIVE_TASKS", "6144"), + ("MOA_EXECUTION_MAX_TENANT_PARKED_RUNS", "12000"), + ("MOA_EXECUTION_MAX_FLEET_PARKED_RUNS", "120000"), + ("MOA_EXECUTION_MAX_TENANT_SCHEDULED_TRIGGERS", "60000"), + ("MOA_EXECUTION_MAX_FLEET_SCHEDULED_TRIGGERS", "600000"), + ("MOA_EXECUTION_MAX_TENANT_EXTERNAL_JOBS", "1200"), + ("MOA_EXECUTION_MAX_FLEET_EXTERNAL_JOBS", "12000"), + ("MOA_EXECUTION_TRIGGER_RECONCILIATION_CADENCE_SECONDS", "90"), + ("MOA_EXECUTION_TERMINAL_DETAIL_RETENTION_DAYS", "45"), ("MOA_EXECUTION_MAX_TASKS", "20000"), ("MOA_EXECUTION_MAX_TOKENS", "20000000"), ("MOA_EXECUTION_MAX_TOOL_CALLS", "200000"), @@ -591,6 +607,22 @@ fn from_iter_applies_every_execution_resource_override() { assert_eq!(config.execution.planner_repair_attempts, 2); assert_eq!(config.execution.repeated_failure_limit, 4); assert_eq!(config.execution.max_in_flight_tasks, 96); + assert_eq!(config.execution.maximum_horizon_seconds, 1_209_600); + assert_eq!(config.execution.maximum_activation_steps, 192); + assert_eq!(config.execution.dispatch_batch_size, 48); + assert_eq!(config.execution.active_attempt_timeout_seconds, 900); + assert_eq!(config.execution.max_tenant_active_runs, 120); + assert_eq!(config.execution.max_fleet_active_runs, 1_200); + assert_eq!(config.execution.max_tenant_active_tasks, 384); + assert_eq!(config.execution.max_fleet_active_tasks, 6_144); + assert_eq!(config.execution.max_tenant_parked_runs, 12_000); + assert_eq!(config.execution.max_fleet_parked_runs, 120_000); + assert_eq!(config.execution.max_tenant_scheduled_triggers, 60_000); + assert_eq!(config.execution.max_fleet_scheduled_triggers, 600_000); + assert_eq!(config.execution.max_tenant_external_jobs, 1_200); + assert_eq!(config.execution.max_fleet_external_jobs, 12_000); + assert_eq!(config.execution.trigger_reconciliation_cadence_seconds, 90); + assert_eq!(config.execution.terminal_detail_retention_days, 45); assert_eq!(config.execution.max_tasks, 20_000); assert_eq!(config.execution.max_tokens, 20_000_000); assert_eq!(config.execution.max_tool_calls, 200_000); @@ -614,6 +646,22 @@ fn from_iter_rejects_invalid_values_for_every_execution_override() { "MOA_EXECUTION_PLANNER_REPAIR_ATTEMPTS", "MOA_EXECUTION_REPEATED_FAILURE_LIMIT", "MOA_EXECUTION_MAX_IN_FLIGHT_TASKS", + "MOA_EXECUTION_MAXIMUM_HORIZON_SECONDS", + "MOA_EXECUTION_MAXIMUM_ACTIVATION_STEPS", + "MOA_EXECUTION_DISPATCH_BATCH_SIZE", + "MOA_EXECUTION_ACTIVE_ATTEMPT_TIMEOUT_SECONDS", + "MOA_EXECUTION_MAX_TENANT_ACTIVE_RUNS", + "MOA_EXECUTION_MAX_FLEET_ACTIVE_RUNS", + "MOA_EXECUTION_MAX_TENANT_ACTIVE_TASKS", + "MOA_EXECUTION_MAX_FLEET_ACTIVE_TASKS", + "MOA_EXECUTION_MAX_TENANT_PARKED_RUNS", + "MOA_EXECUTION_MAX_FLEET_PARKED_RUNS", + "MOA_EXECUTION_MAX_TENANT_SCHEDULED_TRIGGERS", + "MOA_EXECUTION_MAX_FLEET_SCHEDULED_TRIGGERS", + "MOA_EXECUTION_MAX_TENANT_EXTERNAL_JOBS", + "MOA_EXECUTION_MAX_FLEET_EXTERNAL_JOBS", + "MOA_EXECUTION_TRIGGER_RECONCILIATION_CADENCE_SECONDS", + "MOA_EXECUTION_TERMINAL_DETAIL_RETENTION_DAYS", "MOA_EXECUTION_MAX_TASKS", "MOA_EXECUTION_MAX_TOKENS", "MOA_EXECUTION_MAX_TOOL_CALLS", @@ -655,6 +703,55 @@ fn execution_max_in_flight_tasks_overlay_rejects_zero() { ); } +#[test] +fn execution_long_horizon_overlay_rejects_zero_and_inconsistent_limits() { + // Pins: bounded activations, recovery cadence, and tenant/fleet admission + // cannot be disabled or configured with a narrower fleet than tenant limit. + for name in [ + "MOA_EXECUTION_MAXIMUM_HORIZON_SECONDS", + "MOA_EXECUTION_MAXIMUM_ACTIVATION_STEPS", + "MOA_EXECUTION_DISPATCH_BATCH_SIZE", + "MOA_EXECUTION_ACTIVE_ATTEMPT_TIMEOUT_SECONDS", + "MOA_EXECUTION_MAX_TENANT_ACTIVE_RUNS", + "MOA_EXECUTION_MAX_FLEET_ACTIVE_RUNS", + "MOA_EXECUTION_MAX_TENANT_ACTIVE_TASKS", + "MOA_EXECUTION_MAX_FLEET_ACTIVE_TASKS", + "MOA_EXECUTION_MAX_TENANT_PARKED_RUNS", + "MOA_EXECUTION_MAX_FLEET_PARKED_RUNS", + "MOA_EXECUTION_MAX_TENANT_SCHEDULED_TRIGGERS", + "MOA_EXECUTION_MAX_FLEET_SCHEDULED_TRIGGERS", + "MOA_EXECUTION_MAX_TENANT_EXTERNAL_JOBS", + "MOA_EXECUTION_MAX_FLEET_EXTERNAL_JOBS", + "MOA_EXECUTION_TRIGGER_RECONCILIATION_CADENCE_SECONDS", + "MOA_EXECUTION_TERMINAL_DETAIL_RETENTION_DAYS", + ] { + let overlay = EnvOverlay::from_iter(env_pairs([(name, "0")])) + .expect("zero is syntactically a valid integer"); + let error = overlay + .apply_to(&mut MoaConfig::default()) + .expect_err("zero long-horizon limit must fail validation"); + assert!( + error.to_string().contains("must be greater than zero"), + "unexpected error for {name}: {error}" + ); + } + + let overlay = EnvOverlay::from_iter(env_pairs([ + ("MOA_EXECUTION_MAX_TENANT_ACTIVE_TASKS", "500"), + ("MOA_EXECUTION_MAX_FLEET_ACTIVE_TASKS", "499"), + ])) + .expect("capacity overlay should deserialize"); + let error = overlay + .apply_to(&mut MoaConfig::default()) + .expect_err("tenant capacity above the fleet capacity must fail validation"); + assert!( + error.to_string().contains( + "execution.max_tenant_active_tasks must not exceed execution.max_fleet_active_tasks" + ), + "unexpected error: {error}" + ); +} + #[test] fn mcp_servers_json_replaces_configured_servers() { // Pins: the production env seam accepts the complete typed MCP server array and replaces, diff --git a/crates/moa-config/src/execution.rs b/crates/moa-config/src/execution.rs index c3e6dac09..a77af8c5d 100644 --- a/crates/moa-config/src/execution.rs +++ b/crates/moa-config/src/execution.rs @@ -2,6 +2,10 @@ use serde::{Deserialize, Serialize}; +use moa_core::error::{MoaError, Result}; + +use super::require_positive_limit; + /// Provisional physical execution-task window pending the measured T3.3 default. const DEFAULT_MAX_IN_FLIGHT_TASKS: usize = 64; @@ -15,6 +19,38 @@ pub struct ExecutionConfig { pub repeated_failure_limit: u32, /// Maximum live execution-task invocations owned by one run. pub max_in_flight_tasks: usize, + /// Maximum accepted duration of a Durable run, in seconds. + pub maximum_horizon_seconds: u64, + /// Maximum scheduler transitions performed by one controller activation. + pub maximum_activation_steps: usize, + /// Maximum ready tasks dispatched by one controller activation. + pub dispatch_batch_size: usize, + /// Maximum duration of one active task attempt, in seconds. + pub active_attempt_timeout_seconds: u64, + /// Maximum non-parked execution runs admitted for one tenant. + pub max_tenant_active_runs: u32, + /// Maximum non-parked execution runs admitted across the fleet. + pub max_fleet_active_runs: u32, + /// Maximum active task attempts admitted for one tenant. + pub max_tenant_active_tasks: u32, + /// Maximum active task attempts admitted across the fleet. + pub max_fleet_active_tasks: u32, + /// Maximum combined active and parked run residency for one tenant. + pub max_tenant_parked_runs: u32, + /// Maximum combined active and parked run residency across the fleet. + pub max_fleet_parked_runs: u32, + /// Maximum pending scheduled execution triggers retained for one tenant. + pub max_tenant_scheduled_triggers: u32, + /// Maximum pending scheduled execution triggers retained across the fleet. + pub max_fleet_scheduled_triggers: u32, + /// Maximum nonterminal external execution jobs retained for one tenant. + pub max_tenant_external_jobs: u32, + /// Maximum nonterminal external execution jobs retained across the fleet. + pub max_fleet_external_jobs: u32, + /// Cadence for repairing due trigger delivery, in seconds. + pub trigger_reconciliation_cadence_seconds: u64, + /// Days to retain detailed terminal execution rows before bounded compaction. + pub terminal_detail_retention_days: u64, /// Default maximum logical tasks in one approved run. pub max_tasks: u64, /// Default maximum model tokens in one approved run. @@ -51,6 +87,22 @@ impl Default for ExecutionConfig { planner_repair_attempts: 1, repeated_failure_limit: 3, max_in_flight_tasks: DEFAULT_MAX_IN_FLIGHT_TASKS, + maximum_horizon_seconds: 30 * 24 * 60 * 60, + maximum_activation_steps: 128, + dispatch_batch_size: DEFAULT_MAX_IN_FLIGHT_TASKS, + active_attempt_timeout_seconds: 10 * 60, + max_tenant_active_runs: 100, + max_fleet_active_runs: 1_000, + max_tenant_active_tasks: 256, + max_fleet_active_tasks: 4_096, + max_tenant_parked_runs: 10_000, + max_fleet_parked_runs: 100_000, + max_tenant_scheduled_triggers: 50_000, + max_fleet_scheduled_triggers: 500_000, + max_tenant_external_jobs: 1_000, + max_fleet_external_jobs: 10_000, + trigger_reconciliation_cadence_seconds: 60, + terminal_detail_retention_days: 30, max_tasks: 10_000, max_tokens: 10_000_000, max_tool_calls: 100_000, @@ -69,6 +121,205 @@ impl Default for ExecutionConfig { } } +impl ExecutionConfig { + /// Validates execution envelopes, activation bounds, and admission capacities. + pub fn validate(&self) -> Result<()> { + for (name, value) in [ + ( + "execution.repeated_failure_limit", + u64::from(self.repeated_failure_limit), + ), + ( + "execution.max_in_flight_tasks", + usize_as_u64("execution.max_in_flight_tasks", self.max_in_flight_tasks)?, + ), + ( + "execution.maximum_horizon_seconds", + self.maximum_horizon_seconds, + ), + ( + "execution.maximum_activation_steps", + usize_as_u64( + "execution.maximum_activation_steps", + self.maximum_activation_steps, + )?, + ), + ( + "execution.dispatch_batch_size", + usize_as_u64("execution.dispatch_batch_size", self.dispatch_batch_size)?, + ), + ( + "execution.active_attempt_timeout_seconds", + self.active_attempt_timeout_seconds, + ), + ( + "execution.max_tenant_active_runs", + u64::from(self.max_tenant_active_runs), + ), + ( + "execution.max_fleet_active_runs", + u64::from(self.max_fleet_active_runs), + ), + ( + "execution.max_tenant_active_tasks", + u64::from(self.max_tenant_active_tasks), + ), + ( + "execution.max_fleet_active_tasks", + u64::from(self.max_fleet_active_tasks), + ), + ( + "execution.max_tenant_parked_runs", + u64::from(self.max_tenant_parked_runs), + ), + ( + "execution.max_fleet_parked_runs", + u64::from(self.max_fleet_parked_runs), + ), + ( + "execution.max_tenant_scheduled_triggers", + u64::from(self.max_tenant_scheduled_triggers), + ), + ( + "execution.max_fleet_scheduled_triggers", + u64::from(self.max_fleet_scheduled_triggers), + ), + ( + "execution.max_tenant_external_jobs", + u64::from(self.max_tenant_external_jobs), + ), + ( + "execution.max_fleet_external_jobs", + u64::from(self.max_fleet_external_jobs), + ), + ( + "execution.trigger_reconciliation_cadence_seconds", + self.trigger_reconciliation_cadence_seconds, + ), + ( + "execution.terminal_detail_retention_days", + self.terminal_detail_retention_days, + ), + ("execution.max_tasks", self.max_tasks), + ("execution.max_tokens", self.max_tokens), + ("execution.max_tool_calls", self.max_tool_calls), + ("execution.max_retrieved_bytes", self.max_retrieved_bytes), + ("execution.max_cost_microusd", self.max_cost_microusd), + ( + "execution.agent_turn_cost_microusd", + self.agent_turn_cost_microusd, + ), + ("execution.agent_turn_tokens", self.agent_turn_tokens), + ( + "execution.agent_turn_tool_calls", + self.agent_turn_tool_calls, + ), + ( + "execution.agent_turn_retrieved_bytes", + self.agent_turn_retrieved_bytes, + ), + ( + "execution.verifier_turn_cost_microusd", + self.verifier_turn_cost_microusd, + ), + ("execution.verifier_turn_tokens", self.verifier_turn_tokens), + ( + "execution.verifier_turn_tool_calls", + self.verifier_turn_tool_calls, + ), + ( + "execution.verifier_turn_retrieved_bytes", + self.verifier_turn_retrieved_bytes, + ), + ] { + require_positive_limit(name, value)?; + } + + if self.dispatch_batch_size > self.max_in_flight_tasks { + return Err(MoaError::ConfigError( + "execution.dispatch_batch_size must not exceed execution.max_in_flight_tasks" + .to_string(), + )); + } + if self.dispatch_batch_size < 3 { + return Err(MoaError::ConfigError( + "execution.dispatch_batch_size must be at least 3 so every reconciliation lane makes progress" + .to_string(), + )); + } + if self.active_attempt_timeout_seconds > self.maximum_horizon_seconds { + return Err(MoaError::ConfigError( + "execution.active_attempt_timeout_seconds must not exceed execution.maximum_horizon_seconds" + .to_string(), + )); + } + if self.trigger_reconciliation_cadence_seconds > self.active_attempt_timeout_seconds { + return Err(MoaError::ConfigError( + "execution.trigger_reconciliation_cadence_seconds must not exceed execution.active_attempt_timeout_seconds" + .to_string(), + )); + } + + for (tenant_name, tenant, fleet_name, fleet) in [ + ( + "execution.max_tenant_active_runs", + self.max_tenant_active_runs, + "execution.max_fleet_active_runs", + self.max_fleet_active_runs, + ), + ( + "execution.max_tenant_active_tasks", + self.max_tenant_active_tasks, + "execution.max_fleet_active_tasks", + self.max_fleet_active_tasks, + ), + ( + "execution.max_tenant_parked_runs", + self.max_tenant_parked_runs, + "execution.max_fleet_parked_runs", + self.max_fleet_parked_runs, + ), + ( + "execution.max_tenant_scheduled_triggers", + self.max_tenant_scheduled_triggers, + "execution.max_fleet_scheduled_triggers", + self.max_fleet_scheduled_triggers, + ), + ( + "execution.max_tenant_external_jobs", + self.max_tenant_external_jobs, + "execution.max_fleet_external_jobs", + self.max_fleet_external_jobs, + ), + ] { + if tenant > fleet { + return Err(MoaError::ConfigError(format!( + "{tenant_name} must not exceed {fleet_name}" + ))); + } + } + + if self.max_tenant_active_runs > self.max_tenant_parked_runs { + return Err(MoaError::ConfigError( + "execution.max_tenant_active_runs must not exceed execution.max_tenant_parked_runs because every admitted run needs parking entitlement" + .to_string(), + )); + } + if self.max_fleet_active_runs > self.max_fleet_parked_runs { + return Err(MoaError::ConfigError( + "execution.max_fleet_active_runs must not exceed execution.max_fleet_parked_runs because every admitted run needs parking entitlement" + .to_string(), + )); + } + + Ok(()) + } +} + +fn usize_as_u64(name: &str, value: usize) -> Result { + u64::try_from(value).map_err(|_| MoaError::ConfigError(format!("{name} is too large"))) +} + #[cfg(test)] mod tests { use super::{DEFAULT_MAX_IN_FLIGHT_TASKS, ExecutionConfig}; @@ -82,6 +333,22 @@ mod tests { planner_repair_attempts: 1, repeated_failure_limit: 3, max_in_flight_tasks: DEFAULT_MAX_IN_FLIGHT_TASKS, + maximum_horizon_seconds: 30 * 24 * 60 * 60, + maximum_activation_steps: 128, + dispatch_batch_size: DEFAULT_MAX_IN_FLIGHT_TASKS, + active_attempt_timeout_seconds: 10 * 60, + max_tenant_active_runs: 100, + max_fleet_active_runs: 1_000, + max_tenant_active_tasks: 256, + max_fleet_active_tasks: 4_096, + max_tenant_parked_runs: 10_000, + max_fleet_parked_runs: 100_000, + max_tenant_scheduled_triggers: 50_000, + max_fleet_scheduled_triggers: 500_000, + max_tenant_external_jobs: 1_000, + max_fleet_external_jobs: 10_000, + trigger_reconciliation_cadence_seconds: 60, + terminal_detail_retention_days: 30, max_tasks: 10_000, max_tokens: 10_000_000, max_tool_calls: 100_000, @@ -102,6 +369,7 @@ mod tests { let encoded = serde_json::to_value(ExecutionConfig::default()) .expect("serialize execution config defaults"); assert_eq!(encoded["max_in_flight_tasks"], DEFAULT_MAX_IN_FLIGHT_TASKS); + assert_eq!(encoded["dispatch_batch_size"], DEFAULT_MAX_IN_FLIGHT_TASKS); assert_eq!( serde_json::from_value::(encoded) .expect("deserialize execution config defaults") @@ -109,4 +377,74 @@ mod tests { DEFAULT_MAX_IN_FLIGHT_TASKS ); } + + #[test] + fn execution_config_rejects_inconsistent_activation_and_capacity_limits() { + // Pins: one activation cannot overrun its task window, timeout hierarchy, + // or a fleet ceiling through a larger tenant-local limit. + let mut batch = ExecutionConfig::default(); + batch.dispatch_batch_size = batch.max_in_flight_tasks + 1; + assert!(batch.validate().is_err()); + + let starving_batch = ExecutionConfig { + dispatch_batch_size: 2, + ..ExecutionConfig::default() + }; + let error = starving_batch + .validate() + .expect_err("a batch smaller than the three reconciliation lanes must fail"); + assert!( + error + .to_string() + .contains("dispatch_batch_size must be at least 3") + ); + + let mut timeout = ExecutionConfig::default(); + timeout.active_attempt_timeout_seconds = timeout.maximum_horizon_seconds + 1; + assert!(timeout.validate().is_err()); + + let mut reconciliation = ExecutionConfig::default(); + reconciliation.trigger_reconciliation_cadence_seconds = + reconciliation.active_attempt_timeout_seconds + 1; + assert!(reconciliation.validate().is_err()); + + let retention = ExecutionConfig { + terminal_detail_retention_days: 0, + ..ExecutionConfig::default() + }; + assert!(retention.validate().is_err()); + + let mut capacity = ExecutionConfig::default(); + capacity.max_tenant_active_tasks = capacity.max_fleet_active_tasks + 1; + assert!(capacity.validate().is_err()); + + let tenant_parking_entitlement = ExecutionConfig { + max_tenant_active_runs: 101, + max_tenant_parked_runs: 100, + ..ExecutionConfig::default() + }; + assert!( + tenant_parking_entitlement + .validate() + .expect_err("every active tenant run must retain parking entitlement") + .to_string() + .contains( + "max_tenant_active_runs must not exceed execution.max_tenant_parked_runs" + ) + ); + + let fleet_parking_entitlement = ExecutionConfig { + max_fleet_active_runs: 1_001, + max_fleet_parked_runs: 1_000, + max_tenant_parked_runs: 1_000, + ..ExecutionConfig::default() + }; + assert!( + fleet_parking_entitlement + .validate() + .expect_err("every active fleet run must retain parking entitlement") + .to_string() + .contains("max_fleet_active_runs must not exceed execution.max_fleet_parked_runs") + ); + } } diff --git a/crates/moa-config/src/lib.rs b/crates/moa-config/src/lib.rs index 9fec437af..d4fb39895 100644 --- a/crates/moa-config/src/lib.rs +++ b/crates/moa-config/src/lib.rs @@ -314,61 +314,7 @@ impl MoaConfig { u64::from(self.budgets.daily_tenant_cents), )?; - // Execution-run ceilings and the per-turn reservation estimates that - // consume them. A zero estimate reserves nothing, so it would let an - // unbounded number of turns run inside a bounded run budget. - let execution = &self.execution; - for (name, value) in [ - ( - "execution.repeated_failure_limit", - u64::from(execution.repeated_failure_limit), - ), - ( - "execution.max_in_flight_tasks", - u64::try_from(execution.max_in_flight_tasks).map_err(|_| { - MoaError::ConfigError("execution.max_in_flight_tasks is too large".to_string()) - })?, - ), - ("execution.max_tasks", execution.max_tasks), - ("execution.max_tokens", execution.max_tokens), - ("execution.max_tool_calls", execution.max_tool_calls), - ( - "execution.max_retrieved_bytes", - execution.max_retrieved_bytes, - ), - ("execution.max_cost_microusd", execution.max_cost_microusd), - ( - "execution.agent_turn_cost_microusd", - execution.agent_turn_cost_microusd, - ), - ("execution.agent_turn_tokens", execution.agent_turn_tokens), - ( - "execution.agent_turn_tool_calls", - execution.agent_turn_tool_calls, - ), - ( - "execution.agent_turn_retrieved_bytes", - execution.agent_turn_retrieved_bytes, - ), - ( - "execution.verifier_turn_cost_microusd", - execution.verifier_turn_cost_microusd, - ), - ( - "execution.verifier_turn_tokens", - execution.verifier_turn_tokens, - ), - ( - "execution.verifier_turn_tool_calls", - execution.verifier_turn_tool_calls, - ), - ( - "execution.verifier_turn_retrieved_bytes", - execution.verifier_turn_retrieved_bytes, - ), - ] { - require_positive_limit(name, value)?; - } + self.execution.validate()?; if self.database.uses_builtin_dev_url() { // Fails safe (the default targets localhost), so warn rather than reject: @@ -746,6 +692,52 @@ mod tests { ("execution.max_in_flight_tasks", |config| { config.execution.max_in_flight_tasks = 0 }), + ("execution.maximum_horizon_seconds", |config| { + config.execution.maximum_horizon_seconds = 0 + }), + ("execution.maximum_activation_steps", |config| { + config.execution.maximum_activation_steps = 0 + }), + ("execution.dispatch_batch_size", |config| { + config.execution.dispatch_batch_size = 0 + }), + ("execution.active_attempt_timeout_seconds", |config| { + config.execution.active_attempt_timeout_seconds = 0 + }), + ("execution.max_tenant_active_runs", |config| { + config.execution.max_tenant_active_runs = 0 + }), + ("execution.max_fleet_active_runs", |config| { + config.execution.max_fleet_active_runs = 0 + }), + ("execution.max_tenant_active_tasks", |config| { + config.execution.max_tenant_active_tasks = 0 + }), + ("execution.max_fleet_active_tasks", |config| { + config.execution.max_fleet_active_tasks = 0 + }), + ("execution.max_tenant_parked_runs", |config| { + config.execution.max_tenant_parked_runs = 0 + }), + ("execution.max_fleet_parked_runs", |config| { + config.execution.max_fleet_parked_runs = 0 + }), + ("execution.max_tenant_scheduled_triggers", |config| { + config.execution.max_tenant_scheduled_triggers = 0 + }), + ("execution.max_fleet_scheduled_triggers", |config| { + config.execution.max_fleet_scheduled_triggers = 0 + }), + ("execution.max_tenant_external_jobs", |config| { + config.execution.max_tenant_external_jobs = 0 + }), + ("execution.max_fleet_external_jobs", |config| { + config.execution.max_fleet_external_jobs = 0 + }), + ( + "execution.trigger_reconciliation_cadence_seconds", + |config| config.execution.trigger_reconciliation_cadence_seconds = 0, + ), ("execution.max_tasks", |config| { config.execution.max_tasks = 0 }), diff --git a/crates/moa-core/src/events.rs b/crates/moa-core/src/events.rs index 12fd5d712..50e420039 100644 --- a/crates/moa-core/src/events.rs +++ b/crates/moa-core/src/events.rs @@ -30,6 +30,66 @@ pub enum ExecutionTaskResultsRef { }, } +/// Public activity or parked-state distinction for detached execution progress. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ExecutionProgressPhase { + /// The controller or a bounded attempt is advancing work. + Running, + /// A task is parked for user input. + WaitingInput, + /// A governed action is parked for review. + WaitingReview, + /// A task is parked for an external signal. + WaitingSignal, + /// A task or run is parked until an absolute timer. + WaitingTimer, + /// A provider-owned asynchronous job is running outside MOA compute. + WaitingExternal, + /// An operator pause has been requested but attempts are still draining. + PauseRequested, + /// Active attempts are being fenced before the run becomes paused. + Pausing, + /// The run is fully paused and consumes no active execution capacity. + Paused, + /// A late callback or activation was fenced without advancing canonical state. + StaleWork, +} + +/// Audience expected to resolve the run's current public blocker. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ExecutionBlockerAudience { + /// The owning user must provide input. + User, + /// Another authorized agent must provide input. + Agent, + /// A tenant reviewer must decide. + TenantReviewer, + /// An external actor or callback must signal. + External, + /// Time or internal execution state is the only blocker. + System, +} + +/// Exact unconsumed and unreserved execution budget exposed with public progress. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ExecutionRemainingBudget { + /// Remaining billed cost in integer micro-US-dollars. + pub cost_microusd: Option, + /// Remaining model tokens. + pub tokens: Option, + /// Remaining logical tasks. + pub tasks: Option, + /// Remaining governed tool or capability calls. + pub tool_calls: Option, + /// Remaining bytes retrievable from external or memory sources. + pub retrieved_bytes: Option, + /// Absolute execution deadline; unlike counters, time is not consumed arithmetically. + pub deadline_at: Option>, +} + /// Compact aggregate progress for one detached execution run. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -42,6 +102,26 @@ pub struct ExecutionProgress { pub plan_revision: u64, /// Exhaustively mapped stable execution status. pub status: String, + /// Typed public distinction between active, parked, and pause states. + pub phase: ExecutionProgressPhase, + /// Time at which the current storage-only wait began. + pub waiting_since: Option>, + /// Earliest durable time at which the controller should be reactivated. + pub next_wake_at: Option>, + /// Latest durable scheduler progress time. + pub last_progress_at: DateTime, + /// Current provider job when progress is externally owned. + pub external_job_uid: Option, + /// Exact number of ready logical tasks. + pub ready_tasks: u64, + /// Exact number of active task attempts. + pub active_tasks: u64, + /// Exact number of logical tasks parked on durable waits. + pub parked_tasks: u64, + /// Audience expected to resolve the highest-priority current blocker. + pub blocker_audience: Option, + /// Budget remaining after cumulative consumption and live reservations. + pub remaining_budget: ExecutionRemainingBudget, /// Number of materialized logical tasks. pub total: u64, /// Number of successfully completed logical tasks. @@ -1324,6 +1404,7 @@ mod tests { run_uid: Uuid::from_u128(40), task_uid: Uuid::from_u128(41), generation: 2, + attempt_generation: 3, }, }; let event = Event::ActionReviewRequested { @@ -1816,6 +1897,23 @@ mod tests { originating_user_sequence_num: 9, plan_revision: 2, status: "running".to_string(), + phase: ExecutionProgressPhase::Running, + waiting_since: None, + next_wake_at: None, + last_progress_at: Utc::now(), + external_job_uid: None, + ready_tasks: 1, + active_tasks: 1, + parked_tasks: 0, + blocker_audience: None, + remaining_budget: ExecutionRemainingBudget { + cost_microusd: Some(80), + tokens: Some(800), + tasks: Some(2), + tool_calls: Some(4), + retrieved_bytes: Some(8_000), + deadline_at: None, + }, total: 4, completed: 2, failed: 1, diff --git a/crates/moa-core/src/traits/mod.rs b/crates/moa-core/src/traits/mod.rs index 854543b81..0638cc634 100644 --- a/crates/moa-core/src/traits/mod.rs +++ b/crates/moa-core/src/traits/mod.rs @@ -1009,6 +1009,7 @@ pub trait BuiltInTool: Send + Sync { schema: self.input_schema(), policy: self.policy_spec(), idempotency_class: self.idempotency_class(), + async_mode: crate::types::tools::ToolAsyncMode::SynchronousOnly, rollback: None, max_output_tokens: self.max_output_tokens(), } diff --git a/crates/moa-core/src/types/action_policy.rs b/crates/moa-core/src/types/action_policy.rs index 20385cee2..4f3259e41 100644 --- a/crates/moa-core/src/types/action_policy.rs +++ b/crates/moa-core/src/types/action_policy.rs @@ -299,8 +299,10 @@ pub struct ExecutionTaskOrigin { pub run_uid: Uuid, /// Owning persisted task identifier. pub task_uid: Uuid, - /// Task attempt generation fenced by the execution workflow. + /// Logical task generation fenced by the execution workflow. pub generation: u64, + /// Exact bounded attempt generation that owns provider dispatch and capacity. + pub attempt_generation: u64, } /// Durable compensation identity carried through policy, review, and dispatch. @@ -311,8 +313,10 @@ pub struct ExecutionCompensationOrigin { pub run_uid: Uuid, /// Stable compensation identifier within the execution run. pub compensation_id: Uuid, - /// Compensation generation fenced by the execution workflow. + /// Logical compensation generation fenced by the execution workflow. pub generation: u64, + /// Exact bounded compensation-attempt generation that owns provider dispatch and capacity. + pub attempt_generation: u64, } /// Exact owner that must be resumed when one action review resolves. @@ -812,6 +816,7 @@ mod tests { run_uid: Uuid::from_u128(10), task_uid: Uuid::from_u128(11), generation: 5, + attempt_generation: 8, }, }; assert_eq!(task.session_id(), session_id); @@ -830,6 +835,7 @@ mod tests { run_uid: Uuid::from_u128(12), compensation_id: Uuid::from_u128(13), generation: 6, + attempt_generation: 9, }, }; assert_eq!(compensation.session_id(), session_id); @@ -843,6 +849,7 @@ mod tests { run_uid: Uuid::from_u128(12), compensation_id: Uuid::from_u128(13), generation: 6, + attempt_generation: 9, }) ); assert_eq!(compensation.as_str(), "execution_compensation"); @@ -858,6 +865,7 @@ mod tests { run_uid: Uuid::from_u128(32), compensation_id: Uuid::from_u128(33), generation: 9, + attempt_generation: 12, }, }; let encoded = diff --git a/crates/moa-core/src/types/execution_planning.rs b/crates/moa-core/src/types/execution_planning.rs index c02f37b38..7bbc0795d 100644 --- a/crates/moa-core/src/types/execution_planning.rs +++ b/crates/moa-core/src/types/execution_planning.rs @@ -1,12 +1,13 @@ //! Cycle-free execution routing, planning-audit, provenance, and admission DTOs. -use chrono::{DateTime, Utc}; +use chrono::{DateTime, NaiveDateTime, Utc}; use schemars::JsonSchema; use serde::{Deserialize, Deserializer, Serialize}; use serde_json::Value; use uuid::Uuid; use crate::canonical_json::canonical_json_bytes; +use crate::traits::Identity; use crate::types::{ contact::ContactId, identifiers::{SessionId, TenantId}, @@ -726,6 +727,409 @@ pub struct ExecutionRunStarted { pub confirmation: Option, } +/// Lifecycle state of one tenant-owned recurring execution schedule. +#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ExecutionScheduleStatus { + /// New occurrences may be armed and admitted. + Active, + /// No occurrence is armed, while immutable schedule inputs are retained. + Paused, + /// The configured end or occurrence budget was exhausted. + Completed, + /// An operator permanently fenced future occurrences. + Cancelled, +} + +impl ExecutionScheduleStatus { + /// Returns the canonical database and wire label. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Active => "active", + Self::Paused => "paused", + Self::Completed => "completed", + Self::Cancelled => "cancelled", + } + } +} + +/// Policy applied when a scheduled occurrence became due while delivery was unavailable. +#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ExecutionScheduleMissedFirePolicy { + /// Discard missed occurrences and arm the first future wall-clock occurrence. + Skip, + /// Admit at most one catch-up occurrence, then return to the wall-clock series. + FireOnce, +} + +impl ExecutionScheduleMissedFirePolicy { + /// Returns the canonical database and wire label. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Skip => "skip", + Self::FireOnce => "fire_once", + } + } +} + +/// Policy applied when a preceding occurrence still owns a nonterminal run. +#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ExecutionScheduleOverlapPolicy { + /// Do not admit the overlapping occurrence. + Skip, + /// Retain at most one queued overlapping occurrence. + QueueOne, + /// Admit overlapping occurrences up to the configured concurrency bound. + Allow, +} + +impl ExecutionScheduleOverlapPolicy { + /// Returns the canonical database and wire label. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Skip => "skip", + Self::QueueOne => "queue_one", + Self::Allow => "allow", + } + } +} + +/// Resolution policy for ambiguous or nonexistent local wall-clock occurrences. +#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ExecutionScheduleDstPolicy { + /// Select the earlier UTC instant for an ambiguous local occurrence. + Earliest, + /// Select the later UTC instant for an ambiguous local occurrence. + Latest, + /// Omit ambiguous or nonexistent local occurrences. + Skip, +} + +impl ExecutionScheduleDstPolicy { + /// Returns the canonical database and wire label. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Earliest => "earliest", + Self::Latest => "latest", + Self::Skip => "skip", + } + } +} + +/// Trusted control-plane source that created an execution schedule. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum ExecutionScheduleOriginSource { + /// Direct authenticated tenant API request. + TenantApi, + /// Exact persisted session user event. + Session { + /// Owning session. + session_id: SessionId, + /// Exact persisted user event sequence. + originating_user_sequence_num: u64, + }, +} + +/// Immutable creation provenance for one tenant execution schedule. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ExecutionScheduleOrigin { + /// Replay-stable request identity. + pub request_uid: Uuid, + /// Exact authenticated creator admitted by the schedule service. + pub created_by: Identity, + /// Public entry point that originated the schedule. + pub source: ExecutionScheduleOriginSource, +} + +/// Immutable pinned template input copied into every schedule occurrence. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ExecutionScheduleTemplate { + /// Exact immutable artifact revision. + pub revision_uid: Uuid, + /// Canonical lowercase BLAKE3 hash of the template snapshot. + pub template_hash: String, + /// Complete bounded template snapshot needed to admit a fresh run. + pub snapshot: Value, +} + +/// Computes the canonical domain-separated hash stored with a schedule template snapshot. +pub fn execution_schedule_template_hash( + snapshot: &Value, +) -> Result { + let bytes = canonical_json_bytes(snapshot) + .map_err(|error| ExecutionPlanningContractError::Json(error.to_string()))?; + Ok(execution_planning_hash( + "moa.execution.schedule.template.v1", + &bytes, + )) +} + +/// Wall-clock and resource policy for one execution schedule. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ExecutionSchedulePolicy { + /// IANA timezone used to interpret the calendar expression. + pub timezone: String, + /// Five- or six-field cron expression evaluated in `timezone`. + pub calendar_expression: String, + /// Earliest UTC instant that may produce an occurrence. + pub start_at: DateTime, + /// Optional exclusive upper bound for occurrences. + pub end_at: Option>, + /// Missed occurrence behavior. + pub missed_fire_policy: ExecutionScheduleMissedFirePolicy, + /// Concurrent occurrence behavior. + pub overlap_policy: ExecutionScheduleOverlapPolicy, + /// Ambiguous/nonexistent local-time behavior. + pub dst_policy: ExecutionScheduleDstPolicy, + /// Maximum number of nonterminal occurrence runs admitted concurrently. + pub maximum_concurrent_runs: u64, + /// Exact approved execution budget copied into each fresh run. + pub occurrence_budget: Value, +} + +/// Authenticated request to create one tenant execution schedule. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ExecutionScheduleCreateRequest { + /// Tenant authorization and RLS boundary. + pub tenant_id: TenantId, + /// Caller-selected replay-stable schedule identity. + pub schedule_uid: Uuid, + /// Non-empty operator-facing schedule name. + pub name: String, + /// Immutable pinned template revision and snapshot. + pub template: ExecutionScheduleTemplate, + /// Exact identity under which every occurrence is admitted. + pub run_as_identity: Identity, + /// Immutable authenticated creation provenance. + pub origin: ExecutionScheduleOrigin, + /// Wall-clock, overlap, and occurrence resource policy. + pub policy: ExecutionSchedulePolicy, +} + +/// Tenant-scoped request targeting one schedule. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ExecutionScheduleRequest { + /// Tenant authorization and RLS boundary. + pub tenant_id: TenantId, + /// Target schedule. + pub schedule_uid: Uuid, +} + +/// Mutable policy replacement for one schedule incarnation. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ExecutionScheduleUpdateRequest { + /// Tenant authorization and RLS boundary. + pub tenant_id: TenantId, + /// Target schedule. + pub schedule_uid: Uuid, + /// Expected incarnation used as a compare-and-set fence. + pub expected_incarnation: u64, + /// Replacement operator-facing name. + pub name: String, + /// Replacement timing/resource policy; template, identity, and origin stay immutable. + pub policy: ExecutionSchedulePolicy, +} + +/// Bounded stable-page request for tenant schedules. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ExecutionScheduleListRequest { + /// Tenant authorization and RLS boundary. + pub tenant_id: TenantId, + /// Maximum rows to return, clamped by the repository. + pub limit: u32, + /// Exclusive schedule UID cursor from the preceding page. + pub cursor: Option, +} + +/// Persisted tenant schedule projection returned by control-plane handlers. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ExecutionScheduleRecord { + /// Stable schedule identity. + pub schedule_uid: Uuid, + /// Tenant authorization and RLS boundary. + pub tenant_id: TenantId, + /// Operator-facing schedule name. + pub name: String, + /// Immutable pinned template revision and snapshot. + pub template: ExecutionScheduleTemplate, + /// Exact identity copied into every occurrence run. + pub run_as_identity: Identity, + /// Immutable authenticated creation provenance. + pub origin: ExecutionScheduleOrigin, + /// Current timing/resource policy. + pub policy: ExecutionSchedulePolicy, + /// Current lifecycle state. + pub status: ExecutionScheduleStatus, + /// Monotonic fence for already-armed occurrence triggers. + pub schedule_incarnation: u64, + /// Last occurrence sequence considered in this incarnation. + pub last_occurrence_sequence: u64, + /// Exact UTC instant of the currently armed occurrence. + pub next_occurrence_at: Option>, + /// Local wall-clock value corresponding to `next_occurrence_at`. + pub next_occurrence_local: Option, + /// Time at which the schedule was paused. + pub paused_at: Option>, + /// Database-owned creation time. + pub created_at: DateTime, + /// Database-owned last mutation time. + pub updated_at: DateTime, +} + +/// Stable page of visible tenant schedules. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ExecutionSchedulePage { + /// Schedule rows in stable UUID order. + pub schedules: Vec, + /// Cursor for a subsequent page. + pub next_cursor: Option, +} + +/// Deterministic identities for one immutable schedule occurrence. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ExecutionScheduleOccurrenceIds { + /// Temporal occurrence trigger identity. + pub trigger_uid: Uuid, + /// Fresh execution-run identity for this occurrence. + pub run_uid: Uuid, + /// Initial run-activation outbox identity. + pub activation_dispatch_uid: Uuid, +} + +/// Derives all occurrence identities from the exact schedule generation tuple. +#[must_use] +pub fn execution_schedule_occurrence_ids( + schedule_uid: Uuid, + schedule_incarnation: u64, + occurrence_sequence: u64, +) -> ExecutionScheduleOccurrenceIds { + let name = format!("{schedule_uid}:{schedule_incarnation}:{occurrence_sequence}"); + let occurrence_namespace = Uuid::new_v5( + &Uuid::NAMESPACE_URL, + b"moa.execution.schedule.occurrence.v1", + ); + let occurrence_uid = Uuid::new_v5(&occurrence_namespace, name.as_bytes()); + ExecutionScheduleOccurrenceIds { + trigger_uid: Uuid::new_v5(&occurrence_uid, b"trigger"), + run_uid: Uuid::new_v5(&occurrence_uid, b"run"), + activation_dispatch_uid: Uuid::new_v5(&occurrence_uid, b"run-activation"), + } +} + +impl ExecutionScheduleCreateRequest { + /// Validates all tenant, immutable snapshot, time, and bounded scalar invariants. + pub fn validate(&self) -> Result<(), ExecutionPlanningContractError> { + if self.schedule_uid.is_nil() || self.template.revision_uid.is_nil() { + return Err(ExecutionPlanningContractError::InvalidField { + field: "schedule_uid/template.revision_uid".to_string(), + message: "identifiers must not be nil".to_string(), + }); + } + ensure_nonempty_bytes("name", &self.name, 256)?; + validate_hash("template.template_hash", &self.template.template_hash)?; + ensure_json_object("template.snapshot", &self.template.snapshot)?; + if self.template.template_hash != execution_schedule_template_hash(&self.template.snapshot)? + { + return Err(ExecutionPlanningContractError::InvalidField { + field: "template.template_hash".to_string(), + message: "must equal the canonical pinned template snapshot hash".to_string(), + }); + } + ensure_schedule_identity("run_as_identity", self.tenant_id, &self.run_as_identity)?; + ensure_schedule_identity("origin.created_by", self.tenant_id, &self.origin.created_by)?; + if self.origin.request_uid.is_nil() { + return Err(ExecutionPlanningContractError::InvalidField { + field: "origin.request_uid".to_string(), + message: "must not be nil".to_string(), + }); + } + validate_schedule_policy(&self.policy) + } +} + +impl ExecutionScheduleUpdateRequest { + /// Validates mutable policy bounds and the nonzero compare-and-set incarnation. + pub fn validate(&self) -> Result<(), ExecutionPlanningContractError> { + if self.schedule_uid.is_nil() || self.expected_incarnation == 0 { + return Err(ExecutionPlanningContractError::InvalidField { + field: "schedule_uid/expected_incarnation".to_string(), + message: "identifier and incarnation must be nonzero".to_string(), + }); + } + ensure_nonempty_bytes("name", &self.name, 256)?; + validate_schedule_policy(&self.policy) + } +} + +fn validate_schedule_policy( + policy: &ExecutionSchedulePolicy, +) -> Result<(), ExecutionPlanningContractError> { + ensure_nonempty_bytes("policy.timezone", &policy.timezone, 128)?; + ensure_nonempty_bytes( + "policy.calendar_expression", + &policy.calendar_expression, + 256, + )?; + if policy + .end_at + .is_some_and(|end_at| end_at <= policy.start_at) + { + return Err(ExecutionPlanningContractError::InvalidField { + field: "policy.end_at".to_string(), + message: "must be later than start_at".to_string(), + }); + } + if policy.maximum_concurrent_runs == 0 { + return Err(ExecutionPlanningContractError::InvalidField { + field: "policy.maximum_concurrent_runs".to_string(), + message: "must be greater than zero".to_string(), + }); + } + ensure_json_object("policy.occurrence_budget", &policy.occurrence_budget) +} + +fn ensure_schedule_identity( + field: &str, + tenant_id: TenantId, + identity: &Identity, +) -> Result<(), ExecutionPlanningContractError> { + if identity.id.is_nil() || identity.tenant_id != tenant_id { + return Err(ExecutionPlanningContractError::InvalidField { + field: field.to_string(), + message: "identity must be non-nil and belong to the schedule tenant".to_string(), + }); + } + Ok(()) +} + +fn ensure_json_object(field: &str, value: &Value) -> Result<(), ExecutionPlanningContractError> { + if !value.is_object() { + return Err(ExecutionPlanningContractError::InvalidField { + field: field.to_string(), + message: "must be a JSON object".to_string(), + }); + } + Ok(()) +} + impl ExecutionRunStarted { /// Validates the closed status/evidence matrix and plan-hash representation. pub fn validate(&self) -> Result<(), ExecutionPlanningContractError> { @@ -2105,6 +2509,18 @@ mod tests { )); } + #[test] + fn schedule_occurrence_ids_are_tuple_deterministic_and_generation_fenced() { + // Pins: retries derive the same trigger/run/outbox identities, while a new schedule + // incarnation cannot collide with an already-armed occurrence sequence. + let schedule_uid = Uuid::from_u128(91); + let first = execution_schedule_occurrence_ids(schedule_uid, 3, 7); + assert_eq!(first, execution_schedule_occurrence_ids(schedule_uid, 3, 7)); + assert_ne!(first, execution_schedule_occurrence_ids(schedule_uid, 4, 7)); + assert_ne!(first.trigger_uid, first.run_uid); + assert_ne!(first.run_uid, first.activation_dispatch_uid); + } + #[test] fn execution_source_provenance_rejects_cross_cohort_fields_and_hash_drift() { // Pins: generated and skill-template source cohorts stay closed and plan-hash bound. diff --git a/crates/moa-core/src/types/identifiers.rs b/crates/moa-core/src/types/identifiers.rs index 55020bcbf..bf9866558 100644 --- a/crates/moa-core/src/types/identifiers.rs +++ b/crates/moa-core/src/types/identifiers.rs @@ -104,6 +104,14 @@ uuid_id!( pub struct ExecutionTaskScopeId ); +uuid_id!( + /// Core boundary reference to one verified durable execution compensation. + /// + /// The owning `moa-execution` crate retains its `CompensationId` domain + /// type. Orchestration converts that verified UUID at the sandbox boundary. + pub struct ExecutionCompensationScopeId +); + uuid_id!( /// Identifier for one durable child-to-parent attention signal. pub struct AgentSignalId @@ -118,9 +126,9 @@ impl From for ToolCallId { #[cfg(test)] mod tests { use super::{ - ConnectorConnectionId, ExecutionRunScopeId, ExecutionTaskScopeId, ProviderAccountId, - SandboxWorkspaceId, StoragePartitionId, TenantId, WorkspaceCheckpointId, - WorkspaceOperationId, + ConnectorConnectionId, ExecutionCompensationScopeId, ExecutionRunScopeId, + ExecutionTaskScopeId, ProviderAccountId, SandboxWorkspaceId, StoragePartitionId, TenantId, + WorkspaceCheckpointId, WorkspaceOperationId, }; #[test] @@ -163,6 +171,7 @@ mod tests { ProviderAccountId(value), ExecutionRunScopeId(value), ExecutionTaskScopeId(value), + ExecutionCompensationScopeId(value), )) .expect("workspace identifiers should serialize"); let decoded: ( @@ -172,6 +181,7 @@ mod tests { ProviderAccountId, ExecutionRunScopeId, ExecutionTaskScopeId, + ExecutionCompensationScopeId, ) = serde_json::from_str(&encoded).expect("workspace identifiers should deserialize"); assert_eq!( @@ -183,6 +193,7 @@ mod tests { ProviderAccountId(value), ExecutionRunScopeId(value), ExecutionTaskScopeId(value), + ExecutionCompensationScopeId(value), ) ); } diff --git a/crates/moa-core/src/types/sandbox_workspace.rs b/crates/moa-core/src/types/sandbox_workspace.rs index bddf96c5c..10898bc7b 100644 --- a/crates/moa-core/src/types/sandbox_workspace.rs +++ b/crates/moa-core/src/types/sandbox_workspace.rs @@ -2,13 +2,15 @@ use std::path::PathBuf; +use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use crate::error::{MoaError, Result}; use crate::types::hands::HandHandle; use crate::types::identifiers::{ - ExecutionRunScopeId, ExecutionTaskScopeId, HandProvisioningOperationId, ProviderAccountId, - SandboxWorkspaceId, SessionId, TenantId, WorkspaceCheckpointId, WorkspaceOperationId, + ExecutionCompensationScopeId, ExecutionRunScopeId, ExecutionTaskScopeId, + HandProvisioningOperationId, ProviderAccountId, SandboxWorkspaceId, SessionId, TenantId, + WorkspaceCheckpointId, WorkspaceOperationId, }; use crate::types::worker::state::WorkerId; @@ -212,6 +214,8 @@ pub struct ProviderStorageRef { pub enum WorkspaceCapacityDimension { /// Logical workspaces. Workspaces, + /// Ephemeral sandbox compute instances with a live durable owner. + ActiveHands, /// Provider volumes. Volumes, /// Immutable checkpoint count. @@ -220,6 +224,64 @@ pub enum WorkspaceCapacityDimension { LogicalBytes, } +/// Exact durable execution owner whose bounded attempt released sandbox compute. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum ExecutionHandReleaseOwner { + /// Forward task owner with its exact logical generation. + Task { + /// Stable execution task. + task_id: ExecutionTaskScopeId, + /// Exact logical task generation. + logical_generation: u64, + }, + /// Rollback compensation owner with its exact logical generation. + Compensation { + /// Stable compensation registration. + compensation_id: ExecutionCompensationScopeId, + /// Exact logical compensation generation. + logical_generation: u64, + }, +} + +/// Durable proof that one exact execution attempt released its sandbox compute. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ExecutionHandReleaseReceipt { + /// Deterministic receipt identity. + pub receipt_id: uuid::Uuid, + /// Tenant owner. + pub tenant_id: TenantId, + /// Owning execution run. + pub run_id: ExecutionRunScopeId, + /// Exact task or compensation owner and logical generation. + pub owner: ExecutionHandReleaseOwner, + /// Exact bounded attempt generation. + pub attempt_generation: u64, + /// Released workspace, present for task-owned durable filesystems. + pub workspace_id: Option, + /// Exact writer generation checkpointed by the attempt. + pub writer_epoch: Option, + /// Exact compute instance generation destroyed by the attempt. + pub instance_generation: Option, + /// Provider-visible hand creation identity that was destroyed. + pub hand_provisioning_operation_id: Option, + /// Exact durable hand lease generation that was released. + pub hand_lease_generation: Option, + /// Verified portable checkpoint promoted as recovery authority. + pub checkpoint_id: Option, + /// Monotonic checkpoint generation. + pub checkpoint_generation: Option, + /// Verified canonical checkpoint manifest digest. + pub checkpoint_manifest_digest: Option, + /// Exact logical bytes charged to the checkpoint. + pub checkpoint_logical_bytes: Option, + /// Time the release operation was first requested. + pub requested_at: DateTime, + /// Time verified provider absence and durable release completed. + pub released_at: DateTime, +} + macro_rules! impl_persisted_workspace_labels { ($type:ty, $kind:literal, {$($variant:path => $label:literal),+ $(,)?}) => { impl $type { @@ -294,6 +356,7 @@ impl_persisted_workspace_labels!(ProviderStorageKind, "provider storage kind", { }); impl_persisted_workspace_labels!(WorkspaceCapacityDimension, "workspace capacity dimension", { WorkspaceCapacityDimension::Workspaces => "workspaces", + WorkspaceCapacityDimension::ActiveHands => "active_hands", WorkspaceCapacityDimension::Volumes => "volumes", WorkspaceCapacityDimension::Checkpoints => "checkpoints", WorkspaceCapacityDimension::LogicalBytes => "logical_bytes", @@ -424,6 +487,8 @@ pub struct WorkspaceCheckpointPublishRequest { pub hand: HandHandle, /// Parent committed revision being advanced, absent only at generation zero. pub parent_revision: Option, + /// Whether verified publication must destroy compute before reporting success. + pub release_compute: bool, } /// Request to restore one verified checkpoint into compute. diff --git a/crates/moa-core/src/types/tools.rs b/crates/moa-core/src/types/tools.rs index 4a429af0c..c2dddb6b0 100644 --- a/crates/moa-core/src/types/tools.rs +++ b/crates/moa-core/src/types/tools.rs @@ -2,6 +2,7 @@ use std::time::Duration; +use chrono::{DateTime, Utc}; use serde::ser::SerializeSeq; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use serde_json::Value; @@ -326,6 +327,123 @@ pub enum IdempotencyClass { NonIdempotent, } +/// Catalog-pinned provider completion mode for one governed tool contract. +/// +/// External-job capacity is reserved before provider dispatch only for tools +/// that explicitly opt into asynchronous completion. Returning an external job +/// from a synchronous-only contract is an invariant violation. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(tag = "mode", rename_all = "snake_case", deny_unknown_fields)] +pub enum ToolAsyncMode { + /// Every admitted invocation completes within the bounded provider call. + SynchronousOnly, + /// An admitted invocation may commit a provider-owned asynchronous job. + MayReturnExternalJob { + /// Registered adapter/provider key that owns start recovery and callbacks. + provider: String, + }, +} + +/// Exact pre-provider identity an asynchronous-capable tool must use for start. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ExternalJobStartContext { + /// MOA-owned job identity reserved before any provider network call. + pub external_job_uid: uuid::Uuid, + /// Catalog-pinned adapter/provider key. + pub provider: String, + /// Deterministic provider idempotency key used by start and recovery. + pub idempotency_key: String, +} + +/// Provider-owned asynchronous job returned after a capability has committed its start. +/// +/// MOA assigns its own durable external-job UID when this outcome is persisted. +/// These fields are the immutable provider identity and recovery contract needed +/// to authenticate callbacks, reconcile sparsely, and cancel without keeping a +/// workflow or sandbox active. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct AsyncToolJob { + /// Stable provider implementation name. + pub provider: String, + /// Provider-issued external job identity. + pub provider_job_id: String, + /// Stable provider idempotency key used for start, reconciliation, and cancel. + pub idempotency_key: String, + /// Vault or connection reference used to authenticate provider callbacks. + pub callback_auth_reference: String, + /// Latest bounded provider progress phase. + pub progress_phase: String, + /// Whether the provider exposes definitive cancellation. + pub cancel_supported: bool, + /// Earliest time at which sparse provider reconciliation may run. + pub next_reconcile_at: DateTime, +} + +/// Terminal provider outcome carried by an authenticated asynchronous callback. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "outcome", rename_all = "snake_case", deny_unknown_fields)] +pub enum AsyncToolJobTerminalOutcome { + /// Provider work completed with structured output. + Completed { + /// Provider result validated by the capability adapter. + output: Value, + }, + /// Provider work failed definitively. + Failed { + /// Structured provider failure evidence. + error: Value, + }, + /// Provider work was cancelled definitively. + Cancelled, + /// The provider effect may have committed but cannot be determined safely. + UnknownOutcome { + /// Structured ambiguity evidence for operator resolution. + error: Value, + }, +} + +/// Authenticated provider event accepted for one exact asynchronous-job generation. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "event", rename_all = "snake_case", deny_unknown_fields)] +pub enum AsyncToolJobCallbackOutcome { + /// The provider reports durable nonterminal progress. + Progress { + /// Latest bounded provider progress phase. + progress_phase: String, + /// Earliest time at which sparse provider reconciliation may run. + next_reconcile_at: DateTime, + }, + /// The provider reports a definitive or explicitly ambiguous terminal outcome. + Terminal { + /// Typed terminal provider outcome. + outcome: AsyncToolJobTerminalOutcome, + }, +} + +/// Result of requesting cancellation for one exact provider-job generation. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "outcome", rename_all = "snake_case", deny_unknown_fields)] +pub enum AsyncToolJobCancelOutcome { + /// Provider confirmed terminal cancellation. + Cancelled, + /// Provider accepted cancellation and requires later callback or reconciliation. + Accepted { + /// Earliest sparse reconciliation time. + next_reconcile_at: DateTime, + /// Latest provider progress phase. + progress_phase: String, + }, + /// Provider does not support cancellation for this job. + Unsupported, + /// Cancellation transport completed ambiguously. + UnknownOutcome { + /// Structured ambiguity evidence for operator resolution. + error: Value, + }, +} + /// Static action-policy metadata for a tool. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct ToolPolicySpec { @@ -992,6 +1110,8 @@ pub struct ToolDefinition { pub policy: ToolPolicySpec, /// Declared retry/idempotency semantics for the tool implementation. pub idempotency_class: IdempotencyClass, + /// Declared synchronous or asynchronous provider completion contract. + pub async_mode: ToolAsyncMode, /// Exact source-owned declaration of the governed tool that reverses this effect. pub rollback: Option, /// Approximate maximum output tokens persisted for one successful call. diff --git a/crates/moa-db/src/lib.rs b/crates/moa-db/src/lib.rs index 0b0e6f2b9..48aa5d13e 100644 --- a/crates/moa-db/src/lib.rs +++ b/crates/moa-db/src/lib.rs @@ -192,5 +192,152 @@ impl AsMut for ScopedConn<'_> { } pub(crate) fn map_sqlx_error(error: sqlx::Error) -> MoaError { - MoaError::StorageError(error.to_string()) + if is_retryable_sqlx_error(&error) { + MoaError::StorageUnavailable(error.to_string()) + } else { + MoaError::StorageError(error.to_string()) + } +} + +/// Returns whether replaying an idempotent operation may recover from this SQLx failure. +/// +/// The classification is intentionally narrow. Connection failures, pool availability, +/// transaction serialization/deadlock conflicts, and PostgreSQL shutdown or overload states +/// are transient. Query-shape, constraint, schema, and decode failures are permanent. +#[must_use] +pub fn is_retryable_sqlx_error(error: &sqlx::Error) -> bool { + match error { + sqlx::Error::Io(error) if is_transient_io(error.kind()) => true, + sqlx::Error::Tls(_) + | sqlx::Error::PoolTimedOut + | sqlx::Error::PoolClosed + | sqlx::Error::WorkerCrashed + | sqlx::Error::Protocol(_) + | sqlx::Error::BeginFailed => true, + sqlx::Error::Database(database_error) => database_error + .code() + .as_deref() + .is_some_and(is_retryable_postgres_sqlstate), + _ => false, + } +} + +fn is_transient_io(kind: std::io::ErrorKind) -> bool { + matches!( + kind, + std::io::ErrorKind::Interrupted + | std::io::ErrorKind::WouldBlock + | std::io::ErrorKind::TimedOut + | std::io::ErrorKind::ConnectionReset + | std::io::ErrorKind::ConnectionAborted + | std::io::ErrorKind::ConnectionRefused + | std::io::ErrorKind::NotConnected + | std::io::ErrorKind::NetworkDown + | std::io::ErrorKind::NetworkUnreachable + | std::io::ErrorKind::HostUnreachable + ) +} + +fn is_retryable_postgres_sqlstate(code: &str) -> bool { + code.starts_with("08") + || matches!( + code, + "40001" | "40P01" | "53300" | "53400" | "57P01" | "57P02" | "57P03" + ) +} + +#[cfg(test)] +mod tests { + use std::{borrow::Cow, error::Error as StdError, fmt}; + + use sqlx::error::{DatabaseError, ErrorKind}; + + use super::*; + + #[derive(Debug)] + struct TestDatabaseError { + code: &'static str, + kind: ErrorKind, + } + + impl fmt::Display for TestDatabaseError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(formatter, "test database error {}", self.code) + } + } + + impl StdError for TestDatabaseError {} + + impl DatabaseError for TestDatabaseError { + fn message(&self) -> &str { + "test database error" + } + + fn code(&self) -> Option> { + Some(Cow::Borrowed(self.code)) + } + + fn as_error(&self) -> &(dyn StdError + Send + Sync + 'static) { + self + } + + fn as_error_mut(&mut self) -> &mut (dyn StdError + Send + Sync + 'static) { + self + } + + fn into_error(self: Box) -> Box { + self + } + + fn kind(&self) -> ErrorKind { + match self.kind { + ErrorKind::UniqueViolation => ErrorKind::UniqueViolation, + _ => ErrorKind::Other, + } + } + } + + fn database_error(code: &'static str, kind: ErrorKind) -> sqlx::Error { + sqlx::Error::Database(Box::new(TestDatabaseError { code, kind })) + } + + #[test] + fn sqlx_storage_classification_separates_transient_from_permanent_failures() { + // Pins: retry-owning callers may replay pool and transaction-contention + // failures, but must not retry deterministic constraint violations. + for error in [ + sqlx::Error::PoolClosed, + sqlx::Error::PoolTimedOut, + database_error("40001", ErrorKind::Other), + database_error("40P01", ErrorKind::Other), + database_error("08006", ErrorKind::Other), + database_error("57P03", ErrorKind::Other), + ] { + assert!( + is_retryable_sqlx_error(&error), + "classified {error} as permanent" + ); + assert!( + matches!(map_sqlx_error(error), MoaError::StorageUnavailable(_)), + "transient SQLx failure lost its retry provenance" + ); + } + + let constraint = database_error("23505", ErrorKind::UniqueViolation); + assert!(!is_retryable_sqlx_error(&constraint)); + assert!(matches!( + map_sqlx_error(constraint), + MoaError::StorageError(_) + )); + + let invalid_data = sqlx::Error::Io(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "invalid database protocol payload", + )); + assert!(!is_retryable_sqlx_error(&invalid_data)); + assert!(matches!( + map_sqlx_error(invalid_data), + MoaError::StorageError(_) + )); + } } diff --git a/crates/moa-edge/README.md b/crates/moa-edge/README.md index 1cf3c78fd..983fec737 100644 --- a/crates/moa-edge/README.md +++ b/crates/moa-edge/README.md @@ -11,14 +11,16 @@ forwards requests to Restate ingress with trusted identity headers injected. - `connector_credential_proxy` — bounded exact-path forwarding to the private orchestrator credential ingress; it never targets Restate or accepts an upstream response body. +- `external_job_callback_proxy` — bounded raw provider-callback forwarding to + the same network-restricted internal orchestrator ingress. - `ingress` — Restate ingress path construction. - `headers` — the header contract between `moa-edge` and `moa-orchestrator`. - `mcp` — tenant-operations Model Context Protocol transport and tools (an inbound operator surface). - `tenant_accounts` — tenant-account application and persistence boundaries. -The edge binary requires `MOA_EDGE_CONNECTOR_CREDENTIAL_UPSTREAM` to be the -origin-only URL of the private orchestrator credential listener. That listener +The edge binary requires `MOA_EDGE_INTERNAL_INGRESS_UPSTREAM` to be the +origin-only URL of the private orchestrator credential-and-callback listener. That listener must be reachable only from edge workloads. `MOA_EDGE_CONNECTOR_MANAGEMENT_ENABLED` defaults to `false`. While false, the diff --git a/crates/moa-edge/src/external_job_callback_proxy.rs b/crates/moa-edge/src/external_job_callback_proxy.rs new file mode 100644 index 000000000..9797097aa --- /dev/null +++ b/crates/moa-edge/src/external_job_callback_proxy.rs @@ -0,0 +1,326 @@ +//! Fixed-origin proxy for authenticated asynchronous-provider callbacks. + +use std::time::Duration; + +use axum::body::Bytes; +use axum::http::{HeaderMap, StatusCode}; +use reqwest::{Client, Url}; +use thiserror::Error; +use uuid::Uuid; + +/// Public callback route registered by the edge. +pub const EXTERNAL_JOB_CALLBACK_PUBLIC_ROUTE: &str = "/v1/execution/external-jobs/{external_job_uid}/generations/{job_generation}/callbacks/{provider_event_id}"; + +const INTERNAL_CALLBACK_PREFIX: &str = "/internal/v1/execution/external-jobs"; + +/// Maximum raw callback body accepted before forwarding. +pub const MAX_EXTERNAL_JOB_CALLBACK_BODY_BYTES: usize = 256 * 1024; +/// Maximum number of callback headers accepted at either boundary. +pub const MAX_EXTERNAL_JOB_CALLBACK_HEADERS: usize = 64; +/// Maximum aggregate callback-header bytes accepted at either boundary. +pub const MAX_EXTERNAL_JOB_CALLBACK_HEADER_BYTES: usize = 32 * 1024; +/// Maximum provider event identity length. +pub const MAX_EXTERNAL_JOB_PROVIDER_EVENT_ID_BYTES: usize = 512; + +/// Immutable callback selector carried outside the untrusted body. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ExternalJobCallbackSelector { + /// Stable MOA external-job identity. + pub external_job_uid: Uuid, + /// Exact provider-job generation. + pub job_generation: u64, + /// Provider event identity used for durable deduplication. + pub provider_event_id: String, +} + +/// Failure to construct the fixed private callback proxy. +#[derive(Debug, Error)] +pub enum ExternalJobCallbackProxyBuildError { + /// The configured value was not an origin-only HTTP(S) URL. + #[error("external-job callback upstream must be an origin-only HTTP(S) URL")] + InvalidUpstream, + /// The hardened HTTP client could not be constructed. + #[error("build external-job callback HTTP client")] + Client(#[source] reqwest::Error), +} + +/// Sanitized callback proxy failure that never retains headers or body bytes. +#[derive(Debug, Error, PartialEq, Eq)] +pub enum ExternalJobCallbackProxyError { + /// Route selectors, headers, or body shape were invalid. + #[error("invalid external-job callback request")] + InvalidRequest, + /// Raw callback evidence exceeded a fixed pre-forwarding limit. + #[error("external-job callback request exceeds the size limit")] + RequestTooLarge, + /// The private listener could not be reached within its bounded timeout. + #[error("private external-job callback ingress unavailable")] + Transport, + /// The private listener returned a public-safe rejection status. + #[error("private external-job callback ingress rejected the request with status {status}")] + Rejected { + /// Sanitized status; no upstream response body is retained. + status: StatusCode, + }, + /// The private listener violated the empty-response contract. + #[error("private external-job callback ingress returned an invalid response contract")] + InvalidResponse, +} + +/// Exact-path client for the private non-Restate callback listener. +pub struct ExternalJobCallbackProxy { + http: Client, + origin: Url, +} + +impl ExternalJobCallbackProxy { + /// Builds a proxy whose upstream is an origin, never a caller-selected URL. + pub fn new( + upstream_origin: impl AsRef, + ) -> Result { + let origin = Url::parse(upstream_origin.as_ref()) + .map_err(|_| ExternalJobCallbackProxyBuildError::InvalidUpstream)?; + if !matches!(origin.scheme(), "http" | "https") + || origin.host_str().is_none() + || !origin.username().is_empty() + || origin.password().is_some() + || origin.query().is_some() + || origin.fragment().is_some() + || !matches!(origin.path(), "" | "/") + { + return Err(ExternalJobCallbackProxyBuildError::InvalidUpstream); + } + let http = Client::builder() + .timeout(Duration::from_secs(15)) + .connect_timeout(Duration::from_secs(5)) + .pool_max_idle_per_host(16) + .pool_idle_timeout(Duration::from_secs(60)) + .redirect(reqwest::redirect::Policy::none()) + .build() + .map_err(ExternalJobCallbackProxyBuildError::Client)?; + Ok(Self { http, origin }) + } + + /// Forwards bounded raw callback evidence to one fixed private path. + pub async fn forward( + &self, + selector: &ExternalJobCallbackSelector, + headers: &HeaderMap, + body: Bytes, + ) -> Result<(), ExternalJobCallbackProxyError> { + validate_selector(selector)?; + validate_headers(headers)?; + if body.is_empty() { + return Err(ExternalJobCallbackProxyError::InvalidRequest); + } + if body.len() > MAX_EXTERNAL_JOB_CALLBACK_BODY_BYTES { + return Err(ExternalJobCallbackProxyError::RequestTooLarge); + } + let url = callback_url(&self.origin, selector)?; + let mut request = self.http.post(url); + for (name, value) in headers { + if should_forward_header(name.as_str()) { + request = request.header(name.clone(), value.clone()); + } + } + request = moa_observability::propagation::with_reqwest_trace_headers(request).body(body); + let response = request + .send() + .await + .map_err(|_| ExternalJobCallbackProxyError::Transport)?; + let status = response.status(); + if status != StatusCode::NO_CONTENT { + if is_public_rejection_status(status) { + return Err(ExternalJobCallbackProxyError::Rejected { status }); + } + return Err(ExternalJobCallbackProxyError::InvalidResponse); + } + if response + .headers() + .get(reqwest::header::CONTENT_LENGTH) + .is_some_and(|length| length.as_bytes() != b"0") + { + return Err(ExternalJobCallbackProxyError::InvalidResponse); + } + let response_body = response + .bytes() + .await + .map_err(|_| ExternalJobCallbackProxyError::Transport)?; + if !response_body.is_empty() { + return Err(ExternalJobCallbackProxyError::InvalidResponse); + } + Ok(()) + } +} + +fn callback_url( + origin: &Url, + selector: &ExternalJobCallbackSelector, +) -> Result { + let mut url = origin.clone(); + url.path_segments_mut() + .map_err(|_| ExternalJobCallbackProxyError::InvalidRequest)? + .pop_if_empty() + .extend([ + "internal", + "v1", + "execution", + "external-jobs", + &selector.external_job_uid.to_string(), + "generations", + &selector.job_generation.to_string(), + "callbacks", + &selector.provider_event_id, + ]); + debug_assert!(url.path().starts_with(INTERNAL_CALLBACK_PREFIX)); + Ok(url) +} + +fn validate_selector( + selector: &ExternalJobCallbackSelector, +) -> Result<(), ExternalJobCallbackProxyError> { + if selector.external_job_uid.is_nil() + || selector.job_generation == 0 + || selector.provider_event_id.trim().is_empty() + || selector.provider_event_id.len() > MAX_EXTERNAL_JOB_PROVIDER_EVENT_ID_BYTES + || selector.provider_event_id.chars().any(char::is_control) + { + return Err(ExternalJobCallbackProxyError::InvalidRequest); + } + Ok(()) +} + +fn validate_headers(headers: &HeaderMap) -> Result<(), ExternalJobCallbackProxyError> { + if headers.len() > MAX_EXTERNAL_JOB_CALLBACK_HEADERS { + return Err(ExternalJobCallbackProxyError::RequestTooLarge); + } + let mut bytes = 0usize; + for name in headers.keys() { + let values = headers.get_all(name); + let mut values = values.iter(); + let value = values + .next() + .ok_or(ExternalJobCallbackProxyError::InvalidRequest)?; + if values.next().is_some() { + return Err(ExternalJobCallbackProxyError::InvalidRequest); + } + bytes = bytes + .checked_add(name.as_str().len()) + .and_then(|sum| sum.checked_add(value.as_bytes().len())) + .ok_or(ExternalJobCallbackProxyError::RequestTooLarge)?; + if bytes > MAX_EXTERNAL_JOB_CALLBACK_HEADER_BYTES { + return Err(ExternalJobCallbackProxyError::RequestTooLarge); + } + } + Ok(()) +} + +fn should_forward_header(name: &str) -> bool { + !name.eq_ignore_ascii_case("content-length") + && !crate::proxy::is_hop_by_hop_header(name) + && !crate::headers::is_moa_header(name) + && !crate::proxy::is_trace_context_header(name) +} + +fn is_public_rejection_status(status: StatusCode) -> bool { + matches!( + status, + StatusCode::BAD_REQUEST + | StatusCode::UNAUTHORIZED + | StatusCode::NOT_FOUND + | StatusCode::PAYLOAD_TOO_LARGE + | StatusCode::UNPROCESSABLE_ENTITY + | StatusCode::TOO_MANY_REQUESTS + | StatusCode::SERVICE_UNAVAILABLE + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::net::TcpListener; + use tokio::sync::oneshot; + + #[tokio::test] + async fn callback_proxy_preserves_signature_bytes_on_only_the_fixed_private_path() { + // Pins: provider authentication bytes are forwarded unchanged, while + // caller-selected MOA and hop-by-hop headers cannot cross the boundary. + let (origin, request_rx, server) = capture_one_request().await; + let proxy = ExternalJobCallbackProxy::new(origin).expect("build callback proxy"); + let selector = fixture_selector(); + let mut headers = HeaderMap::new(); + headers.insert( + "x-provider-signature", + "sha256=fixture".parse().expect("header"), + ); + headers.insert("authorization", "Bearer fixture".parse().expect("header")); + headers.insert("x-moa-tenant-id", "attacker".parse().expect("header")); + headers.insert("connection", "close".parse().expect("header")); + proxy + .forward(&selector, &headers, Bytes::from_static(b"{\"ok\":true}")) + .await + .expect("callback proxy should accept empty 204"); + + let request = request_rx.await.expect("captured request"); + assert!(request.starts_with(&format!( + "POST /internal/v1/execution/external-jobs/{}/generations/7/callbacks/event%2F11 HTTP/1.1\r\n", + selector.external_job_uid + ))); + let lower = request.to_ascii_lowercase(); + assert!(lower.contains("x-provider-signature: sha256=fixture")); + assert!(lower.contains("authorization: bearer fixture")); + assert!(!lower.contains("x-moa-tenant-id")); + server.await.expect("capture server"); + } + + #[tokio::test] + async fn callback_proxy_rejects_limits_before_transport_and_never_echoes_body() { + // Pins: oversized raw evidence is rejected without connecting, and its + // bytes never enter the stable proxy error. + let proxy = ExternalJobCallbackProxy::new("http://127.0.0.1:9") + .expect("build syntactic callback proxy"); + let secret = "provider-signature-secret"; + let error = proxy + .forward( + &fixture_selector(), + &HeaderMap::new(), + Bytes::from(vec![b'x'; MAX_EXTERNAL_JOB_CALLBACK_BODY_BYTES + 1]), + ) + .await + .expect_err("oversized callback must fail locally"); + assert_eq!(error, ExternalJobCallbackProxyError::RequestTooLarge); + assert!(!error.to_string().contains(secret)); + } + + fn fixture_selector() -> ExternalJobCallbackSelector { + ExternalJobCallbackSelector { + external_job_uid: Uuid::from_u128(1), + job_generation: 7, + provider_event_id: "event/11".to_string(), + } + } + + async fn capture_one_request() -> ( + String, + oneshot::Receiver, + tokio::task::JoinHandle<()>, + ) { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); + let addr = listener.local_addr().expect("address"); + let (request_tx, request_rx) = oneshot::channel(); + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.expect("accept"); + let mut bytes = vec![0; 16 * 1024]; + let count = stream.read(&mut bytes).await.expect("read"); + request_tx + .send(String::from_utf8_lossy(&bytes[..count]).into_owned()) + .ok(); + stream + .write_all(b"HTTP/1.1 204 No Content\r\nContent-Length: 0\r\n\r\n") + .await + .expect("write"); + }); + (format!("http://{addr}"), request_rx, server) + } +} diff --git a/crates/moa-edge/src/lib.rs b/crates/moa-edge/src/lib.rs index 096f93963..7cba09e50 100644 --- a/crates/moa-edge/src/lib.rs +++ b/crates/moa-edge/src/lib.rs @@ -1,6 +1,7 @@ //! Library surface for the MOA public HTTP edge. pub mod connector_credential_proxy; +pub mod external_job_callback_proxy; pub mod headers; mod ingress; pub mod mcp; diff --git a/crates/moa-edge/src/main.rs b/crates/moa-edge/src/main.rs index 598af3a72..9235cc056 100644 --- a/crates/moa-edge/src/main.rs +++ b/crates/moa-edge/src/main.rs @@ -12,6 +12,7 @@ use clap::Parser; use moa_authz::{FgaClient, FgaConfig}; use moa_config::{AuthzEngine, optional_config_secret}; use moa_edge::connector_credential_proxy::ConnectorCredentialProxy; +use moa_edge::external_job_callback_proxy::ExternalJobCallbackProxy; use moa_edge::mcp::{self, McpHttpConfig}; use moa_edge::proxy::OrchestratorProxy; use moa_edge::routes::{AppState, KnowledgeWebhookEdgeConfig}; @@ -27,9 +28,9 @@ struct Args { /// Internal Restate ingress base URL. #[arg(long, env = "MOA_EDGE_UPSTREAM")] upstream: Option, - /// Private orchestrator origin for connector credential ingress. - #[arg(long, env = "MOA_EDGE_CONNECTOR_CREDENTIAL_UPSTREAM")] - connector_credential_upstream: String, + /// Private orchestrator origin for credential and provider-callback ingresses. + #[arg(long, env = "MOA_EDGE_INTERNAL_INGRESS_UPSTREAM")] + internal_ingress_upstream: String, /// Exposes connector management and credential routes during staged rollout. #[arg( long, @@ -148,9 +149,13 @@ async fn main() -> anyhow::Result<()> { delivery, proxy: Arc::new(OrchestratorProxy::new(&upstream).context("build orchestrator proxy")?), connector_credentials: Arc::new( - ConnectorCredentialProxy::new(&args.connector_credential_upstream) + ConnectorCredentialProxy::new(&args.internal_ingress_upstream) .context("build private connector credential proxy")?, ), + external_job_callbacks: Arc::new( + ExternalJobCallbackProxy::new(&args.internal_ingress_upstream) + .context("build private external-job callback proxy")?, + ), clickhouse_lineage: moa_config .clickhouse .as_ref() diff --git a/crates/moa-edge/src/proxy.rs b/crates/moa-edge/src/proxy.rs index 0eec9a1a2..9019dd5aa 100644 --- a/crates/moa-edge/src/proxy.rs +++ b/crates/moa-edge/src/proxy.rs @@ -106,7 +106,7 @@ fn should_forward_header(name: &str, has_body: bool) -> bool { /// W3C trace-context headers are re-injected from the edge span, so the raw /// inbound values are dropped to avoid forwarding a conflicting second copy. -fn is_trace_context_header(name: &str) -> bool { +pub(crate) fn is_trace_context_header(name: &str) -> bool { name.eq_ignore_ascii_case(moa_observability::TRACEPARENT_HEADER) || name.eq_ignore_ascii_case(moa_observability::TRACESTATE_HEADER) } @@ -182,7 +182,8 @@ fn hex_value(byte: u8) -> Option { } } -fn is_hop_by_hop_header(name: &str) -> bool { +/// Returns whether an HTTP header is connection-specific and must not be proxied. +pub(crate) fn is_hop_by_hop_header(name: &str) -> bool { const HOP_BY_HOP: [&str; 9] = [ "host", "connection", diff --git a/crates/moa-edge/src/routes.rs b/crates/moa-edge/src/routes.rs index 2fd03d23d..fbb2c6145 100644 --- a/crates/moa-edge/src/routes.rs +++ b/crates/moa-edge/src/routes.rs @@ -2,6 +2,10 @@ #![allow(clippy::result_large_err)] use crate::connector_credential_proxy::ConnectorCredentialProxy; +use crate::external_job_callback_proxy::{ + EXTERNAL_JOB_CALLBACK_PUBLIC_ROUTE, ExternalJobCallbackProxy, + MAX_EXTERNAL_JOB_CALLBACK_BODY_BYTES, +}; use crate::ingress::call_path; use crate::{headers, proxy::OrchestratorProxy}; use axum::Router; @@ -44,6 +48,7 @@ pub(crate) mod auth_accounts; mod connectors; mod contact_messages; pub(crate) mod dashboard; +mod external_jobs; mod knowledge; pub(crate) mod lineage; mod memory; @@ -94,6 +99,8 @@ pub struct AppState { pub proxy: Arc, /// Exact-path proxy to private orchestrator credential ingress. pub connector_credentials: Arc, + /// Exact-path proxy to private asynchronous-provider callback ingress. + pub external_job_callbacks: Arc, /// ClickHouse lineage store when `[clickhouse]` is configured; lineage /// reads and offboarding deletes follow the write backend. pub clickhouse_lineage: Option>, @@ -180,6 +187,11 @@ pub(crate) fn base_router(state: AppState) -> Router { post(auth_accounts::set_user_password), ) .route("/v1/whoami", get(whoami::handle)) + .route( + EXTERNAL_JOB_CALLBACK_PUBLIC_ROUTE, + post(external_jobs::handle_callback) + .layer(DefaultBodyLimit::max(MAX_EXTERNAL_JOB_CALLBACK_BODY_BYTES)), + ) .route( "/.well-known/oauth-authorization-server", get(oauth::authorization_server_metadata), diff --git a/crates/moa-edge/src/routes/external_jobs.rs b/crates/moa-edge/src/routes/external_jobs.rs new file mode 100644 index 000000000..a982ada6a --- /dev/null +++ b/crates/moa-edge/src/routes/external_jobs.rs @@ -0,0 +1,78 @@ +//! Public asynchronous-provider callback route. + +use axum::body::Bytes; +use axum::extract::{Path, State}; +use axum::http::{HeaderMap, StatusCode}; +use axum::response::{IntoResponse, Response}; +use serde::Deserialize; +use uuid::Uuid; + +use crate::external_job_callback_proxy::{ + ExternalJobCallbackProxyError, ExternalJobCallbackSelector, +}; + +use super::AppState; + +/// Path selectors outside the provider-controlled callback body. +#[derive(Debug, Deserialize)] +pub(crate) struct ExternalJobCallbackPath { + external_job_uid: Uuid, + job_generation: u64, + provider_event_id: String, +} + +/// Forwards one bounded opaque provider callback to the private ingress. +// SAFETY: provider authentication is enforced on the raw callback before its bytes are parsed or persisted. +pub(crate) async fn handle_callback( + State(state): State, + Path(path): Path, + headers: HeaderMap, + body: Bytes, +) -> Response { + let selector = ExternalJobCallbackSelector { + external_job_uid: path.external_job_uid, + job_generation: path.job_generation, + provider_event_id: path.provider_event_id, + }; + match state + .external_job_callbacks + .forward(&selector, &headers, body) + .await + { + Ok(()) => StatusCode::NO_CONTENT.into_response(), + Err(error) => callback_error_response(error), + } +} + +fn callback_error_response(error: ExternalJobCallbackProxyError) -> Response { + let status = match error { + ExternalJobCallbackProxyError::InvalidRequest => StatusCode::BAD_REQUEST, + ExternalJobCallbackProxyError::RequestTooLarge => StatusCode::PAYLOAD_TOO_LARGE, + ExternalJobCallbackProxyError::Transport => StatusCode::SERVICE_UNAVAILABLE, + ExternalJobCallbackProxyError::Rejected { status } => status, + ExternalJobCallbackProxyError::InvalidResponse => StatusCode::BAD_GATEWAY, + }; + status.into_response() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn callback_errors_never_expose_upstream_or_provider_material() { + // Pins: public errors are status-only even when the private boundary + // rejects authentication or violates its response contract. + assert_eq!( + callback_error_response(ExternalJobCallbackProxyError::Rejected { + status: StatusCode::UNAUTHORIZED, + }) + .status(), + StatusCode::UNAUTHORIZED + ); + assert_eq!( + callback_error_response(ExternalJobCallbackProxyError::InvalidResponse).status(), + StatusCode::BAD_GATEWAY + ); + } +} diff --git a/crates/moa-edge/src/routes/session_stream.rs b/crates/moa-edge/src/routes/session_stream.rs index cad49050f..48a92f6dd 100644 --- a/crates/moa-edge/src/routes/session_stream.rs +++ b/crates/moa-edge/src/routes/session_stream.rs @@ -490,7 +490,18 @@ fn sse_event_name(event: &Event) -> &'static str { Event::WorkerHeartbeatStale { .. } => "worker_stale", Event::BrainResponse { .. } => "response", Event::ExecutionRunStarted(_) => "execution_started", - Event::ExecutionProgress(_) => "execution_progress", + Event::ExecutionProgress(progress) => match progress.phase { + moa_core::events::ExecutionProgressPhase::WaitingExternal => "execution_external_wait", + moa_core::events::ExecutionProgressPhase::PauseRequested + | moa_core::events::ExecutionProgressPhase::Pausing + | moa_core::events::ExecutionProgressPhase::Paused => "execution_pause", + moa_core::events::ExecutionProgressPhase::WaitingInput => "execution_input_wait", + moa_core::events::ExecutionProgressPhase::WaitingReview => "execution_review_wait", + moa_core::events::ExecutionProgressPhase::WaitingSignal => "execution_signal_wait", + moa_core::events::ExecutionProgressPhase::WaitingTimer => "execution_timer_wait", + moa_core::events::ExecutionProgressPhase::StaleWork => "execution_stale", + moa_core::events::ExecutionProgressPhase::Running => "execution_progress", + }, Event::ExecutionInputRequired(_) => "execution_input_request", Event::ExecutionCompleted(_) => "execution_completed", Event::ExecutionFailed { .. } | Event::ExecutionCancelled(_) => "execution_failed", @@ -727,13 +738,66 @@ mod tests { } #[test] - fn execution_delivery_events_use_only_the_five_stable_sse_names() { - // Pins: all durable run delivery frames use the public five-name contract; cancellation - // is a typed failed frame and synthesis retargets stream completion without a sixth name. + fn parked_execution_progress_uses_distinct_public_frames() { + // Pins: clients can distinguish timer/signal waits, drained pauses, and provider-owned + // asynchronous work without parsing a free-form status string. + let run_uid = Uuid::from_u128(80); + let progress = |phase| { + Event::ExecutionProgress(moa_core::events::ExecutionProgress { + run_uid, + originating_user_sequence_num: 1, + plan_revision: 1, + status: "deliberately_unparsed".to_string(), + phase, + waiting_since: Some(Utc::now()), + next_wake_at: None, + last_progress_at: Utc::now(), + external_job_uid: None, + ready_tasks: 0, + active_tasks: 0, + parked_tasks: 1, + blocker_audience: Some(moa_core::events::ExecutionBlockerAudience::System), + remaining_budget: moa_core::events::ExecutionRemainingBudget { + cost_microusd: Some(10), + tokens: Some(100), + tasks: Some(1), + tool_calls: Some(1), + retrieved_bytes: Some(1_000), + deadline_at: None, + }, + total: 1, + completed: 0, + failed: 0, + cancelled: 0, + }) + }; + + assert_eq!( + sse_event_name(&progress( + moa_core::events::ExecutionProgressPhase::WaitingTimer + )), + "execution_timer_wait" + ); + assert_eq!( + sse_event_name(&progress(moa_core::events::ExecutionProgressPhase::Paused)), + "execution_pause" + ); + assert_eq!( + sse_event_name(&progress( + moa_core::events::ExecutionProgressPhase::WaitingExternal + )), + "execution_external_wait" + ); + } + + #[test] + fn execution_terminal_and_active_delivery_events_use_typed_sse_names() { + // Pins: terminal delivery names stay stable while active progress is separately refined by + // parked-state phase; cancellation remains a typed failed frame. use moa_core::events::{ ExecutionFailureDisposition, ExecutionInputRequired, ExecutionProgress, - ExecutionRunEvidenceRef, ExecutionSynthesisRequested, ExecutionTaskResultsRef, - ExecutionTerminalSummary, + ExecutionProgressPhase, ExecutionRunEvidenceRef, ExecutionSynthesisRequested, + ExecutionTaskResultsRef, ExecutionTerminalSummary, }; let run_uid = Uuid::from_u128(81); @@ -761,6 +825,23 @@ mod tests { originating_user_sequence_num: 7, plan_revision: 2, status: "running".to_string(), + phase: ExecutionProgressPhase::Running, + waiting_since: None, + next_wake_at: None, + last_progress_at: Utc::now(), + external_job_uid: None, + ready_tasks: 3, + active_tasks: 1, + parked_tasks: 0, + blocker_audience: None, + remaining_budget: moa_core::events::ExecutionRemainingBudget { + cost_microusd: Some(50), + tokens: Some(500), + tasks: Some(5), + tool_calls: Some(10), + retrieved_bytes: Some(5_000), + deadline_at: None, + }, total: 9, completed: 4, failed: 1, @@ -988,6 +1069,7 @@ mod tests { run_uid: Uuid::from_u128(90), task_uid: Uuid::from_u128(91), generation: 1, + attempt_generation: 2, }, }), }; diff --git a/crates/moa-edge/tests/direct_read_routes_db.rs b/crates/moa-edge/tests/direct_read_routes_db.rs index 69ecdc36f..c2d3034a2 100644 --- a/crates/moa-edge/tests/direct_read_routes_db.rs +++ b/crates/moa-edge/tests/direct_read_routes_db.rs @@ -281,6 +281,10 @@ async fn start_edge_with_auth_upstream_and_connector_management( moa_edge::connector_credential_proxy::ConnectorCredentialProxy::new(upstream) .expect("credential proxy URL should be syntactically valid"), ), + external_job_callbacks: Arc::new( + moa_edge::external_job_callback_proxy::ExternalJobCallbackProxy::new(upstream) + .expect("callback proxy URL should be syntactically valid"), + ), clickhouse_lineage: None, clickhouse_analytics: None, }; diff --git a/crates/moa-edge/tests/direct_read_routes_db/graceful_shutdown_db.rs b/crates/moa-edge/tests/direct_read_routes_db/graceful_shutdown_db.rs index 39d68086b..2829901cf 100644 --- a/crates/moa-edge/tests/direct_read_routes_db/graceful_shutdown_db.rs +++ b/crates/moa-edge/tests/direct_read_routes_db/graceful_shutdown_db.rs @@ -106,10 +106,7 @@ impl SpawnedEdge { .env("MOA_DATABASE_URL", database_url) .env("MOA_EDGE_BIND", format!("127.0.0.1:{port}")) .env("MOA_EDGE_UPSTREAM", "http://127.0.0.1:1") - .env( - "MOA_EDGE_CONNECTOR_CREDENTIAL_UPSTREAM", - "http://127.0.0.1:1", - ) + .env("MOA_EDGE_INTERNAL_INGRESS_UPSTREAM", "http://127.0.0.1:1") .env("MOA_METRICS_EXPORTER", "disabled") // The shutdown arms log at info; the process default is warn, which // would leave the distinguishing line out of stdout entirely. diff --git a/crates/moa-edge/tests/session_message_attachments_docker.rs b/crates/moa-edge/tests/session_message_attachments_docker.rs index db20ad4ec..136845d5d 100644 --- a/crates/moa-edge/tests/session_message_attachments_docker.rs +++ b/crates/moa-edge/tests/session_message_attachments_docker.rs @@ -225,6 +225,10 @@ async fn start_edge( moa_edge::connector_credential_proxy::ConnectorCredentialProxy::new(upstream) .expect("credential proxy URL is valid"), ), + external_job_callbacks: Arc::new( + moa_edge::external_job_callback_proxy::ExternalJobCallbackProxy::new(upstream) + .expect("callback proxy URL is valid"), + ), clickhouse_lineage: None, clickhouse_analytics: None, }; diff --git a/crates/moa-eval/examples/generate_execution_corpus.rs b/crates/moa-eval/examples/generate_execution_corpus.rs index 22971ea7b..11b802ccf 100644 --- a/crates/moa-eval/examples/generate_execution_corpus.rs +++ b/crates/moa-eval/examples/generate_execution_corpus.rs @@ -473,6 +473,12 @@ fn contract_case(index: usize) -> ExecutionContractCase { }, plan: ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, + input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { + expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { + delay_seconds: 86_400, + }, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + }, input_schema: json!({ "type": "object", "properties": { diff --git a/crates/moa-eval/scenarios/execution/contract-recorded.jsonl b/crates/moa-eval/scenarios/execution/contract-recorded.jsonl index 7cbdcb5c8..87a7023bb 100644 --- a/crates/moa-eval/scenarios/execution/contract-recorded.jsonl +++ b/crates/moa-eval/scenarios/execution/contract-recorded.jsonl @@ -1,80 +1,80 @@ -{"schema_version":1,"case_id":"contract-000","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (000).","requirements":[{"id":"req-screen-000","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-000","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-000","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-000","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-000-a","issuer-000-b","issuer-000-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-000","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-000","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-000","description":"Require complete map coverage","requirement_ids":["req-screen-000"],"constraint_ids":["constraint-exclusions-000"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-000","description":"Require citations for every issuer","requirement_ids":["req-report-000"],"constraint_ids":["constraint-definition-000"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-000"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-000-a"},{"ticker":"issuer-000-b"},{"ticker":"issuer-000-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-000"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-000"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-000-a","issuer-000-b","issuer-000-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-001","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (001).","requirements":[{"id":"req-screen-001","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-001","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-001","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-001","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-001-a","issuer-001-b","issuer-001-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-001","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-001","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-001","description":"Require complete map coverage","requirement_ids":["req-screen-001"],"constraint_ids":["constraint-exclusions-001"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-001","description":"Require citations for every issuer","requirement_ids":["req-report-001"],"constraint_ids":["constraint-definition-001"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-001"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-001-a"},{"ticker":"issuer-001-b"},{"ticker":"issuer-001-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-001"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-001"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-001-a","issuer-001-b","issuer-001-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-002","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (002).","requirements":[{"id":"req-screen-002","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-002","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-002","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-002","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-002-a","issuer-002-b","issuer-002-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-002","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-002","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-002","description":"Require complete map coverage","requirement_ids":["req-screen-002"],"constraint_ids":["constraint-exclusions-002"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-002","description":"Require citations for every issuer","requirement_ids":["req-report-002"],"constraint_ids":["constraint-definition-002"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-002"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-002-a"},{"ticker":"issuer-002-b"},{"ticker":"issuer-002-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-002"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-002"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-002-a","issuer-002-b","issuer-002-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-003","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (003).","requirements":[{"id":"req-screen-003","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-003","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-003","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-003","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-003-a","issuer-003-b","issuer-003-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-003","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-003","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-003","description":"Require complete map coverage","requirement_ids":["req-screen-003"],"constraint_ids":["constraint-exclusions-003"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-003","description":"Require citations for every issuer","requirement_ids":["req-report-003"],"constraint_ids":["constraint-definition-003"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-003"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-003-a"},{"ticker":"issuer-003-b"},{"ticker":"issuer-003-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-003"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-003"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-003-a","issuer-003-b","issuer-003-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-004","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (004).","requirements":[{"id":"req-screen-004","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-004","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-004","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-004","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-004-a","issuer-004-b","issuer-004-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-004","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-004","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-004","description":"Require complete map coverage","requirement_ids":["req-screen-004"],"constraint_ids":["constraint-exclusions-004"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-004","description":"Require citations for every issuer","requirement_ids":["req-report-004"],"constraint_ids":["constraint-definition-004"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-004"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-004-a"},{"ticker":"issuer-004-b"},{"ticker":"issuer-004-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-004"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-004"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-004-a","issuer-004-b","issuer-004-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-005","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (005).","requirements":[{"id":"req-screen-005","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-005","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-005","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-005","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-005-a","issuer-005-b","issuer-005-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-005","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-005","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-005","description":"Require complete map coverage","requirement_ids":["req-screen-005"],"constraint_ids":["constraint-exclusions-005"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-005","description":"Require citations for every issuer","requirement_ids":["req-report-005"],"constraint_ids":["constraint-definition-005"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-005"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-005-a"},{"ticker":"issuer-005-b"},{"ticker":"issuer-005-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-005"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-005"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-005-a","issuer-005-b","issuer-005-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-006","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (006).","requirements":[{"id":"req-screen-006","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-006","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-006","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-006","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-006-a","issuer-006-b","issuer-006-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-006","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-006","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-006","description":"Require complete map coverage","requirement_ids":["req-screen-006"],"constraint_ids":["constraint-exclusions-006"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-006","description":"Require citations for every issuer","requirement_ids":["req-report-006"],"constraint_ids":["constraint-definition-006"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-006"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-006-a"},{"ticker":"issuer-006-b"},{"ticker":"issuer-006-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-006"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-006"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-006-a","issuer-006-b","issuer-006-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-007","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (007).","requirements":[{"id":"req-screen-007","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-007","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-007","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-007","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-007-a","issuer-007-b","issuer-007-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-007","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-007","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-007","description":"Require complete map coverage","requirement_ids":["req-screen-007"],"constraint_ids":["constraint-exclusions-007"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-007","description":"Require citations for every issuer","requirement_ids":["req-report-007"],"constraint_ids":["constraint-definition-007"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-007"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-007-a"},{"ticker":"issuer-007-b"},{"ticker":"issuer-007-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-007"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-007"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-007-a","issuer-007-b","issuer-007-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-008","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (008).","requirements":[{"id":"req-screen-008","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-008","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-008","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-008","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-008-a","issuer-008-b","issuer-008-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-008","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-008","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-008","description":"Require complete map coverage","requirement_ids":["req-screen-008"],"constraint_ids":["constraint-exclusions-008"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-008","description":"Require citations for every issuer","requirement_ids":["req-report-008"],"constraint_ids":["constraint-definition-008"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-008"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-008-a"},{"ticker":"issuer-008-b"},{"ticker":"issuer-008-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-008"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-008"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-008-a","issuer-008-b","issuer-008-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-009","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (009).","requirements":[{"id":"req-screen-009","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-009","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-009","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-009","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-009-a","issuer-009-b","issuer-009-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-009","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-009","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-009","description":"Require complete map coverage","requirement_ids":["req-screen-009"],"constraint_ids":["constraint-exclusions-009"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-009","description":"Require citations for every issuer","requirement_ids":["req-report-009"],"constraint_ids":["constraint-definition-009"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-009"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-009-a"},{"ticker":"issuer-009-b"},{"ticker":"issuer-009-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-009"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-009"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-009-a","issuer-009-b","issuer-009-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-010","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (010).","requirements":[{"id":"req-screen-010","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-010","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-010","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-010","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-010-a","issuer-010-b","issuer-010-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-010","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-010","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-010","description":"Require complete map coverage","requirement_ids":["req-screen-010"],"constraint_ids":["constraint-exclusions-010"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-010","description":"Require citations for every issuer","requirement_ids":["req-report-010"],"constraint_ids":["constraint-definition-010"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-010"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-010-a"},{"ticker":"issuer-010-b"},{"ticker":"issuer-010-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-010"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-010"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-010-a","issuer-010-b","issuer-010-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-011","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (011).","requirements":[{"id":"req-screen-011","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-011","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-011","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-011","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-011-a","issuer-011-b","issuer-011-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-011","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-011","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-011","description":"Require complete map coverage","requirement_ids":["req-screen-011"],"constraint_ids":["constraint-exclusions-011"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-011","description":"Require citations for every issuer","requirement_ids":["req-report-011"],"constraint_ids":["constraint-definition-011"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-011"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-011-a"},{"ticker":"issuer-011-b"},{"ticker":"issuer-011-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-011"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-011"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-011-a","issuer-011-b","issuer-011-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-012","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (012).","requirements":[{"id":"req-screen-012","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-012","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-012","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-012","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-012-a","issuer-012-b","issuer-012-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-012","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-012","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-012","description":"Require complete map coverage","requirement_ids":["req-screen-012"],"constraint_ids":["constraint-exclusions-012"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-012","description":"Require citations for every issuer","requirement_ids":["req-report-012"],"constraint_ids":["constraint-definition-012"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-012"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-012-a"},{"ticker":"issuer-012-b"},{"ticker":"issuer-012-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-012"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-012"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-012-a","issuer-012-b","issuer-012-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-013","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (013).","requirements":[{"id":"req-screen-013","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-013","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-013","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-013","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-013-a","issuer-013-b","issuer-013-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-013","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-013","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-013","description":"Require complete map coverage","requirement_ids":["req-screen-013"],"constraint_ids":["constraint-exclusions-013"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-013","description":"Require citations for every issuer","requirement_ids":["req-report-013"],"constraint_ids":["constraint-definition-013"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-013"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-013-a"},{"ticker":"issuer-013-b"},{"ticker":"issuer-013-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-013"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-013"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-013-a","issuer-013-b","issuer-013-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-014","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (014).","requirements":[{"id":"req-screen-014","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-014","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-014","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-014","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-014-a","issuer-014-b","issuer-014-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-014","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-014","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-014","description":"Require complete map coverage","requirement_ids":["req-screen-014"],"constraint_ids":["constraint-exclusions-014"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-014","description":"Require citations for every issuer","requirement_ids":["req-report-014"],"constraint_ids":["constraint-definition-014"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-014"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-014-a"},{"ticker":"issuer-014-b"},{"ticker":"issuer-014-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-014"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-014"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-014-a","issuer-014-b","issuer-014-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-015","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (015).","requirements":[{"id":"req-screen-015","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-015","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-015","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-015","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-015-a","issuer-015-b","issuer-015-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-015","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-015","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-015","description":"Require complete map coverage","requirement_ids":["req-screen-015"],"constraint_ids":["constraint-exclusions-015"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-015","description":"Require citations for every issuer","requirement_ids":["req-report-015"],"constraint_ids":["constraint-definition-015"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-015"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-015-a"},{"ticker":"issuer-015-b"},{"ticker":"issuer-015-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-015"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-015"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-015-a","issuer-015-b","issuer-015-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-016","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (016).","requirements":[{"id":"req-screen-016","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-016","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-016","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-016","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-016-a","issuer-016-b","issuer-016-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-016","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-016","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-016","description":"Require complete map coverage","requirement_ids":["req-screen-016"],"constraint_ids":["constraint-exclusions-016"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-016","description":"Require citations for every issuer","requirement_ids":["req-report-016"],"constraint_ids":["constraint-definition-016"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-016"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-016-a"},{"ticker":"issuer-016-b"},{"ticker":"issuer-016-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-016"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-016"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-016-a","issuer-016-b","issuer-016-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-017","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (017).","requirements":[{"id":"req-screen-017","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-017","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-017","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-017","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-017-a","issuer-017-b","issuer-017-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-017","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-017","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-017","description":"Require complete map coverage","requirement_ids":["req-screen-017"],"constraint_ids":["constraint-exclusions-017"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-017","description":"Require citations for every issuer","requirement_ids":["req-report-017"],"constraint_ids":["constraint-definition-017"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-017"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-017-a"},{"ticker":"issuer-017-b"},{"ticker":"issuer-017-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-017"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-017"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-017-a","issuer-017-b","issuer-017-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-018","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (018).","requirements":[{"id":"req-screen-018","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-018","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-018","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-018","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-018-a","issuer-018-b","issuer-018-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-018","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-018","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-018","description":"Require complete map coverage","requirement_ids":["req-screen-018"],"constraint_ids":["constraint-exclusions-018"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-018","description":"Require citations for every issuer","requirement_ids":["req-report-018"],"constraint_ids":["constraint-definition-018"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-018"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-018-a"},{"ticker":"issuer-018-b"},{"ticker":"issuer-018-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-018"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-018"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-018-a","issuer-018-b","issuer-018-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-019","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (019).","requirements":[{"id":"req-screen-019","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-019","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-019","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-019","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-019-a","issuer-019-b","issuer-019-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-019","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-019","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-019","description":"Require complete map coverage","requirement_ids":["req-screen-019"],"constraint_ids":["constraint-exclusions-019"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-019","description":"Require citations for every issuer","requirement_ids":["req-report-019"],"constraint_ids":["constraint-definition-019"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-019"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-019-a"},{"ticker":"issuer-019-b"},{"ticker":"issuer-019-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-019"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-019"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-019-a","issuer-019-b","issuer-019-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-020","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (020).","requirements":[{"id":"req-screen-020","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-020","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-020","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-020","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-020-a","issuer-020-b","issuer-020-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-020","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-020","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-020","description":"Require complete map coverage","requirement_ids":["req-screen-020"],"constraint_ids":["constraint-exclusions-020"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-020","description":"Require citations for every issuer","requirement_ids":["req-report-020"],"constraint_ids":["constraint-definition-020"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-020"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-020-a"},{"ticker":"issuer-020-b"},{"ticker":"issuer-020-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-020"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-020"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-020-a","issuer-020-b","issuer-020-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-021","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (021).","requirements":[{"id":"req-screen-021","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-021","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-021","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-021","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-021-a","issuer-021-b","issuer-021-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-021","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-021","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-021","description":"Require complete map coverage","requirement_ids":["req-screen-021"],"constraint_ids":["constraint-exclusions-021"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-021","description":"Require citations for every issuer","requirement_ids":["req-report-021"],"constraint_ids":["constraint-definition-021"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-021"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-021-a"},{"ticker":"issuer-021-b"},{"ticker":"issuer-021-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-021"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-021"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-021-a","issuer-021-b","issuer-021-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-022","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (022).","requirements":[{"id":"req-screen-022","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-022","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-022","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-022","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-022-a","issuer-022-b","issuer-022-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-022","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-022","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-022","description":"Require complete map coverage","requirement_ids":["req-screen-022"],"constraint_ids":["constraint-exclusions-022"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-022","description":"Require citations for every issuer","requirement_ids":["req-report-022"],"constraint_ids":["constraint-definition-022"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-022"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-022-a"},{"ticker":"issuer-022-b"},{"ticker":"issuer-022-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-022"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-022"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-022-a","issuer-022-b","issuer-022-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-023","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (023).","requirements":[{"id":"req-screen-023","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-023","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-023","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-023","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-023-a","issuer-023-b","issuer-023-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-023","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-023","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-023","description":"Require complete map coverage","requirement_ids":["req-screen-023"],"constraint_ids":["constraint-exclusions-023"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-023","description":"Require citations for every issuer","requirement_ids":["req-report-023"],"constraint_ids":["constraint-definition-023"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-023"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-023-a"},{"ticker":"issuer-023-b"},{"ticker":"issuer-023-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-023"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-023"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-023-a","issuer-023-b","issuer-023-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-024","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (024).","requirements":[{"id":"req-screen-024","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-024","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-024","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-024","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-024-a","issuer-024-b","issuer-024-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-024","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-024","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-024","description":"Require complete map coverage","requirement_ids":["req-screen-024"],"constraint_ids":["constraint-exclusions-024"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-024","description":"Require citations for every issuer","requirement_ids":["req-report-024"],"constraint_ids":["constraint-definition-024"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-024"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-024-a"},{"ticker":"issuer-024-b"},{"ticker":"issuer-024-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-024"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-024"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-024-a","issuer-024-b","issuer-024-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-025","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (025).","requirements":[{"id":"req-screen-025","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-025","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-025","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-025","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-025-a","issuer-025-b","issuer-025-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-025","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-025","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-025","description":"Require complete map coverage","requirement_ids":["req-screen-025"],"constraint_ids":["constraint-exclusions-025"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-025","description":"Require citations for every issuer","requirement_ids":["req-report-025"],"constraint_ids":["constraint-definition-025"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-025"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-025-a"},{"ticker":"issuer-025-b"},{"ticker":"issuer-025-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-025"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-025"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-025-a","issuer-025-b","issuer-025-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-026","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (026).","requirements":[{"id":"req-screen-026","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-026","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-026","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-026","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-026-a","issuer-026-b","issuer-026-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-026","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-026","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-026","description":"Require complete map coverage","requirement_ids":["req-screen-026"],"constraint_ids":["constraint-exclusions-026"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-026","description":"Require citations for every issuer","requirement_ids":["req-report-026"],"constraint_ids":["constraint-definition-026"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-026"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-026-a"},{"ticker":"issuer-026-b"},{"ticker":"issuer-026-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-026"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-026"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-026-a","issuer-026-b","issuer-026-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-027","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (027).","requirements":[{"id":"req-screen-027","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-027","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-027","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-027","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-027-a","issuer-027-b","issuer-027-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-027","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-027","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-027","description":"Require complete map coverage","requirement_ids":["req-screen-027"],"constraint_ids":["constraint-exclusions-027"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-027","description":"Require citations for every issuer","requirement_ids":["req-report-027"],"constraint_ids":["constraint-definition-027"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-027"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-027-a"},{"ticker":"issuer-027-b"},{"ticker":"issuer-027-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-027"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-027"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-027-a","issuer-027-b","issuer-027-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-028","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (028).","requirements":[{"id":"req-screen-028","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-028","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-028","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-028","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-028-a","issuer-028-b","issuer-028-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-028","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-028","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-028","description":"Require complete map coverage","requirement_ids":["req-screen-028"],"constraint_ids":["constraint-exclusions-028"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-028","description":"Require citations for every issuer","requirement_ids":["req-report-028"],"constraint_ids":["constraint-definition-028"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-028"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-028-a"},{"ticker":"issuer-028-b"},{"ticker":"issuer-028-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-028"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-028"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-028-a","issuer-028-b","issuer-028-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-029","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (029).","requirements":[{"id":"req-screen-029","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-029","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-029","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-029","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-029-a","issuer-029-b","issuer-029-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-029","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-029","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-029","description":"Require complete map coverage","requirement_ids":["req-screen-029"],"constraint_ids":["constraint-exclusions-029"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-029","description":"Require citations for every issuer","requirement_ids":["req-report-029"],"constraint_ids":["constraint-definition-029"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-029"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-029-a"},{"ticker":"issuer-029-b"},{"ticker":"issuer-029-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-029"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-029"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-029-a","issuer-029-b","issuer-029-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-030","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (030).","requirements":[{"id":"req-screen-030","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-030","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-030","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-030","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-030-a","issuer-030-b","issuer-030-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-030","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-030","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-030","description":"Require complete map coverage","requirement_ids":["req-screen-030"],"constraint_ids":["constraint-exclusions-030"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-030","description":"Require citations for every issuer","requirement_ids":["req-report-030"],"constraint_ids":["constraint-definition-030"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-030"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-030-a"},{"ticker":"issuer-030-b"},{"ticker":"issuer-030-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-030"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-030"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-030-a","issuer-030-b","issuer-030-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-031","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (031).","requirements":[{"id":"req-screen-031","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-031","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-031","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-031","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-031-a","issuer-031-b","issuer-031-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-031","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-031","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-031","description":"Require complete map coverage","requirement_ids":["req-screen-031"],"constraint_ids":["constraint-exclusions-031"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-031","description":"Require citations for every issuer","requirement_ids":["req-report-031"],"constraint_ids":["constraint-definition-031"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-031"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-031-a"},{"ticker":"issuer-031-b"},{"ticker":"issuer-031-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-031"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-031"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-031-a","issuer-031-b","issuer-031-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-032","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (032).","requirements":[{"id":"req-screen-032","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-032","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-032","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-032","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-032-a","issuer-032-b","issuer-032-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-032","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-032","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-032","description":"Require complete map coverage","requirement_ids":["req-screen-032"],"constraint_ids":["constraint-exclusions-032"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-032","description":"Require citations for every issuer","requirement_ids":["req-report-032"],"constraint_ids":["constraint-definition-032"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-032"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-032-a"},{"ticker":"issuer-032-b"},{"ticker":"issuer-032-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-032"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-032"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-032-a","issuer-032-b","issuer-032-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-033","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (033).","requirements":[{"id":"req-screen-033","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-033","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-033","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-033","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-033-a","issuer-033-b","issuer-033-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-033","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-033","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-033","description":"Require complete map coverage","requirement_ids":["req-screen-033"],"constraint_ids":["constraint-exclusions-033"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-033","description":"Require citations for every issuer","requirement_ids":["req-report-033"],"constraint_ids":["constraint-definition-033"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-033"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-033-a"},{"ticker":"issuer-033-b"},{"ticker":"issuer-033-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-033"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-033"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-033-a","issuer-033-b","issuer-033-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-034","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (034).","requirements":[{"id":"req-screen-034","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-034","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-034","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-034","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-034-a","issuer-034-b","issuer-034-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-034","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-034","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-034","description":"Require complete map coverage","requirement_ids":["req-screen-034"],"constraint_ids":["constraint-exclusions-034"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-034","description":"Require citations for every issuer","requirement_ids":["req-report-034"],"constraint_ids":["constraint-definition-034"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-034"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-034-a"},{"ticker":"issuer-034-b"},{"ticker":"issuer-034-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-034"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-034"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-034-a","issuer-034-b","issuer-034-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-035","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (035).","requirements":[{"id":"req-screen-035","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-035","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-035","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-035","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-035-a","issuer-035-b","issuer-035-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-035","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-035","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-035","description":"Require complete map coverage","requirement_ids":["req-screen-035"],"constraint_ids":["constraint-exclusions-035"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-035","description":"Require citations for every issuer","requirement_ids":["req-report-035"],"constraint_ids":["constraint-definition-035"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-035"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-035-a"},{"ticker":"issuer-035-b"},{"ticker":"issuer-035-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-035"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-035"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-035-a","issuer-035-b","issuer-035-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-036","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (036).","requirements":[{"id":"req-screen-036","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-036","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-036","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-036","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-036-a","issuer-036-b","issuer-036-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-036","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-036","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-036","description":"Require complete map coverage","requirement_ids":["req-screen-036"],"constraint_ids":["constraint-exclusions-036"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-036","description":"Require citations for every issuer","requirement_ids":["req-report-036"],"constraint_ids":["constraint-definition-036"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-036"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-036-a"},{"ticker":"issuer-036-b"},{"ticker":"issuer-036-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-036"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-036"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-036-a","issuer-036-b","issuer-036-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-037","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (037).","requirements":[{"id":"req-screen-037","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-037","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-037","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-037","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-037-a","issuer-037-b","issuer-037-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-037","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-037","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-037","description":"Require complete map coverage","requirement_ids":["req-screen-037"],"constraint_ids":["constraint-exclusions-037"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-037","description":"Require citations for every issuer","requirement_ids":["req-report-037"],"constraint_ids":["constraint-definition-037"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-037"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-037-a"},{"ticker":"issuer-037-b"},{"ticker":"issuer-037-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-037"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-037"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-037-a","issuer-037-b","issuer-037-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-038","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (038).","requirements":[{"id":"req-screen-038","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-038","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-038","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-038","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-038-a","issuer-038-b","issuer-038-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-038","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-038","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-038","description":"Require complete map coverage","requirement_ids":["req-screen-038"],"constraint_ids":["constraint-exclusions-038"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-038","description":"Require citations for every issuer","requirement_ids":["req-report-038"],"constraint_ids":["constraint-definition-038"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-038"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-038-a"},{"ticker":"issuer-038-b"},{"ticker":"issuer-038-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-038"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-038"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-038-a","issuer-038-b","issuer-038-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-039","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (039).","requirements":[{"id":"req-screen-039","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-039","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-039","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-039","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-039-a","issuer-039-b","issuer-039-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-039","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-039","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-039","description":"Require complete map coverage","requirement_ids":["req-screen-039"],"constraint_ids":["constraint-exclusions-039"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-039","description":"Require citations for every issuer","requirement_ids":["req-report-039"],"constraint_ids":["constraint-definition-039"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-039"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-039-a"},{"ticker":"issuer-039-b"},{"ticker":"issuer-039-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-039"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-039"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-039-a","issuer-039-b","issuer-039-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-040","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (040).","requirements":[{"id":"req-screen-040","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-040","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-040","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-040","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-040-a","issuer-040-b","issuer-040-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-040","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-040","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-040","description":"Require complete map coverage","requirement_ids":["req-screen-040"],"constraint_ids":["constraint-exclusions-040"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-040","description":"Require citations for every issuer","requirement_ids":["req-report-040"],"constraint_ids":["constraint-definition-040"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-040"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-040-a"},{"ticker":"issuer-040-b"},{"ticker":"issuer-040-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-040"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-040"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-040-a","issuer-040-b","issuer-040-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-041","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (041).","requirements":[{"id":"req-screen-041","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-041","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-041","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-041","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-041-a","issuer-041-b","issuer-041-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-041","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-041","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-041","description":"Require complete map coverage","requirement_ids":["req-screen-041"],"constraint_ids":["constraint-exclusions-041"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-041","description":"Require citations for every issuer","requirement_ids":["req-report-041"],"constraint_ids":["constraint-definition-041"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-041"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-041-a"},{"ticker":"issuer-041-b"},{"ticker":"issuer-041-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-041"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-041"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-041-a","issuer-041-b","issuer-041-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-042","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (042).","requirements":[{"id":"req-screen-042","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-042","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-042","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-042","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-042-a","issuer-042-b","issuer-042-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-042","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-042","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-042","description":"Require complete map coverage","requirement_ids":["req-screen-042"],"constraint_ids":["constraint-exclusions-042"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-042","description":"Require citations for every issuer","requirement_ids":["req-report-042"],"constraint_ids":["constraint-definition-042"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-042"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-042-a"},{"ticker":"issuer-042-b"},{"ticker":"issuer-042-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-042"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-042"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-042-a","issuer-042-b","issuer-042-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-043","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (043).","requirements":[{"id":"req-screen-043","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-043","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-043","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-043","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-043-a","issuer-043-b","issuer-043-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-043","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-043","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-043","description":"Require complete map coverage","requirement_ids":["req-screen-043"],"constraint_ids":["constraint-exclusions-043"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-043","description":"Require citations for every issuer","requirement_ids":["req-report-043"],"constraint_ids":["constraint-definition-043"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-043"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-043-a"},{"ticker":"issuer-043-b"},{"ticker":"issuer-043-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-043"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-043"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-043-a","issuer-043-b","issuer-043-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-044","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (044).","requirements":[{"id":"req-screen-044","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-044","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-044","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-044","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-044-a","issuer-044-b","issuer-044-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-044","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-044","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-044","description":"Require complete map coverage","requirement_ids":["req-screen-044"],"constraint_ids":["constraint-exclusions-044"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-044","description":"Require citations for every issuer","requirement_ids":["req-report-044"],"constraint_ids":["constraint-definition-044"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-044"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-044-a"},{"ticker":"issuer-044-b"},{"ticker":"issuer-044-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-044"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-044"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-044-a","issuer-044-b","issuer-044-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-045","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (045).","requirements":[{"id":"req-screen-045","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-045","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-045","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-045","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-045-a","issuer-045-b","issuer-045-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-045","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-045","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-045","description":"Require complete map coverage","requirement_ids":["req-screen-045"],"constraint_ids":["constraint-exclusions-045"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-045","description":"Require citations for every issuer","requirement_ids":["req-report-045"],"constraint_ids":["constraint-definition-045"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-045"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-045-a"},{"ticker":"issuer-045-b"},{"ticker":"issuer-045-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-045"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-045"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-045-a","issuer-045-b","issuer-045-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-046","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (046).","requirements":[{"id":"req-screen-046","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-046","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-046","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-046","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-046-a","issuer-046-b","issuer-046-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-046","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-046","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-046","description":"Require complete map coverage","requirement_ids":["req-screen-046"],"constraint_ids":["constraint-exclusions-046"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-046","description":"Require citations for every issuer","requirement_ids":["req-report-046"],"constraint_ids":["constraint-definition-046"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-046"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-046-a"},{"ticker":"issuer-046-b"},{"ticker":"issuer-046-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-046"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-046"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-046-a","issuer-046-b","issuer-046-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-047","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (047).","requirements":[{"id":"req-screen-047","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-047","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-047","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-047","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-047-a","issuer-047-b","issuer-047-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-047","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-047","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-047","description":"Require complete map coverage","requirement_ids":["req-screen-047"],"constraint_ids":["constraint-exclusions-047"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-047","description":"Require citations for every issuer","requirement_ids":["req-report-047"],"constraint_ids":["constraint-definition-047"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-047"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-047-a"},{"ticker":"issuer-047-b"},{"ticker":"issuer-047-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-047"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-047"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-047-a","issuer-047-b","issuer-047-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-048","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (048).","requirements":[{"id":"req-screen-048","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-048","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-048","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-048","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-048-a","issuer-048-b","issuer-048-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-048","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-048","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-048","description":"Require complete map coverage","requirement_ids":["req-screen-048"],"constraint_ids":["constraint-exclusions-048"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-048","description":"Require citations for every issuer","requirement_ids":["req-report-048"],"constraint_ids":["constraint-definition-048"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-048"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-048-a"},{"ticker":"issuer-048-b"},{"ticker":"issuer-048-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-048"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-048"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-048-a","issuer-048-b","issuer-048-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-049","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (049).","requirements":[{"id":"req-screen-049","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-049","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-049","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-049","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-049-a","issuer-049-b","issuer-049-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-049","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-049","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-049","description":"Require complete map coverage","requirement_ids":["req-screen-049"],"constraint_ids":["constraint-exclusions-049"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-049","description":"Require citations for every issuer","requirement_ids":["req-report-049"],"constraint_ids":["constraint-definition-049"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-049"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-049-a"},{"ticker":"issuer-049-b"},{"ticker":"issuer-049-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-049"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-049"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-049-a","issuer-049-b","issuer-049-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-050","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (050).","requirements":[{"id":"req-screen-050","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-050","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-050","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-050","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-050-a","issuer-050-b","issuer-050-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-050","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-050","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-050","description":"Require complete map coverage","requirement_ids":["req-screen-050"],"constraint_ids":["constraint-exclusions-050"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-050","description":"Require citations for every issuer","requirement_ids":["req-report-050"],"constraint_ids":["constraint-definition-050"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-050"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-050-a"},{"ticker":"issuer-050-b"},{"ticker":"issuer-050-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-050"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-050"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-050-a","issuer-050-b","issuer-050-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-051","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (051).","requirements":[{"id":"req-screen-051","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-051","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-051","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-051","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-051-a","issuer-051-b","issuer-051-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-051","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-051","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-051","description":"Require complete map coverage","requirement_ids":["req-screen-051"],"constraint_ids":["constraint-exclusions-051"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-051","description":"Require citations for every issuer","requirement_ids":["req-report-051"],"constraint_ids":["constraint-definition-051"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-051"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-051-a"},{"ticker":"issuer-051-b"},{"ticker":"issuer-051-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-051"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-051"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-051-a","issuer-051-b","issuer-051-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-052","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (052).","requirements":[{"id":"req-screen-052","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-052","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-052","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-052","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-052-a","issuer-052-b","issuer-052-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-052","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-052","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-052","description":"Require complete map coverage","requirement_ids":["req-screen-052"],"constraint_ids":["constraint-exclusions-052"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-052","description":"Require citations for every issuer","requirement_ids":["req-report-052"],"constraint_ids":["constraint-definition-052"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-052"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-052-a"},{"ticker":"issuer-052-b"},{"ticker":"issuer-052-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-052"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-052"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-052-a","issuer-052-b","issuer-052-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-053","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (053).","requirements":[{"id":"req-screen-053","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-053","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-053","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-053","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-053-a","issuer-053-b","issuer-053-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-053","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-053","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-053","description":"Require complete map coverage","requirement_ids":["req-screen-053"],"constraint_ids":["constraint-exclusions-053"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-053","description":"Require citations for every issuer","requirement_ids":["req-report-053"],"constraint_ids":["constraint-definition-053"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-053"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-053-a"},{"ticker":"issuer-053-b"},{"ticker":"issuer-053-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-053"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-053"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-053-a","issuer-053-b","issuer-053-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-054","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (054).","requirements":[{"id":"req-screen-054","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-054","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-054","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-054","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-054-a","issuer-054-b","issuer-054-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-054","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-054","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-054","description":"Require complete map coverage","requirement_ids":["req-screen-054"],"constraint_ids":["constraint-exclusions-054"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-054","description":"Require citations for every issuer","requirement_ids":["req-report-054"],"constraint_ids":["constraint-definition-054"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-054"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-054-a"},{"ticker":"issuer-054-b"},{"ticker":"issuer-054-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-054"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-054"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-054-a","issuer-054-b","issuer-054-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-055","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (055).","requirements":[{"id":"req-screen-055","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-055","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-055","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-055","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-055-a","issuer-055-b","issuer-055-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-055","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-055","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-055","description":"Require complete map coverage","requirement_ids":["req-screen-055"],"constraint_ids":["constraint-exclusions-055"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-055","description":"Require citations for every issuer","requirement_ids":["req-report-055"],"constraint_ids":["constraint-definition-055"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-055"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-055-a"},{"ticker":"issuer-055-b"},{"ticker":"issuer-055-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-055"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-055"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-055-a","issuer-055-b","issuer-055-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-056","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (056).","requirements":[{"id":"req-screen-056","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-056","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-056","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-056","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-056-a","issuer-056-b","issuer-056-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-056","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-056","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-056","description":"Require complete map coverage","requirement_ids":["req-screen-056"],"constraint_ids":["constraint-exclusions-056"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-056","description":"Require citations for every issuer","requirement_ids":["req-report-056"],"constraint_ids":["constraint-definition-056"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-056"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-056-a"},{"ticker":"issuer-056-b"},{"ticker":"issuer-056-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-056"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-056"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-056-a","issuer-056-b","issuer-056-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-057","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (057).","requirements":[{"id":"req-screen-057","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-057","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-057","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-057","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-057-a","issuer-057-b","issuer-057-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-057","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-057","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-057","description":"Require complete map coverage","requirement_ids":["req-screen-057"],"constraint_ids":["constraint-exclusions-057"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-057","description":"Require citations for every issuer","requirement_ids":["req-report-057"],"constraint_ids":["constraint-definition-057"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-057"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-057-a"},{"ticker":"issuer-057-b"},{"ticker":"issuer-057-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-057"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-057"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-057-a","issuer-057-b","issuer-057-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-058","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (058).","requirements":[{"id":"req-screen-058","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-058","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-058","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-058","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-058-a","issuer-058-b","issuer-058-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-058","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-058","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-058","description":"Require complete map coverage","requirement_ids":["req-screen-058"],"constraint_ids":["constraint-exclusions-058"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-058","description":"Require citations for every issuer","requirement_ids":["req-report-058"],"constraint_ids":["constraint-definition-058"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-058"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-058-a"},{"ticker":"issuer-058-b"},{"ticker":"issuer-058-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-058"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-058"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-058-a","issuer-058-b","issuer-058-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-059","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (059).","requirements":[{"id":"req-screen-059","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-059","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-059","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-059","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-059-a","issuer-059-b","issuer-059-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-059","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-059","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-059","description":"Require complete map coverage","requirement_ids":["req-screen-059"],"constraint_ids":["constraint-exclusions-059"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-059","description":"Require citations for every issuer","requirement_ids":["req-report-059"],"constraint_ids":["constraint-definition-059"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-059"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-059-a"},{"ticker":"issuer-059-b"},{"ticker":"issuer-059-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-059"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-059"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-059-a","issuer-059-b","issuer-059-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-060","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (060).","requirements":[{"id":"req-screen-060","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-060","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-060","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-060","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-060-a","issuer-060-b","issuer-060-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-060","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-060","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-060","description":"Require complete map coverage","requirement_ids":["req-screen-060"],"constraint_ids":["constraint-exclusions-060"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-060","description":"Require citations for every issuer","requirement_ids":["req-report-060"],"constraint_ids":["constraint-definition-060"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-060"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-060-a"},{"ticker":"issuer-060-b"},{"ticker":"issuer-060-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-060"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-060"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-060-a","issuer-060-b","issuer-060-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-061","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (061).","requirements":[{"id":"req-screen-061","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-061","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-061","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-061","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-061-a","issuer-061-b","issuer-061-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-061","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-061","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-061","description":"Require complete map coverage","requirement_ids":["req-screen-061"],"constraint_ids":["constraint-exclusions-061"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-061","description":"Require citations for every issuer","requirement_ids":["req-report-061"],"constraint_ids":["constraint-definition-061"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-061"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-061-a"},{"ticker":"issuer-061-b"},{"ticker":"issuer-061-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-061"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-061"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-061-a","issuer-061-b","issuer-061-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-062","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (062).","requirements":[{"id":"req-screen-062","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-062","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-062","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-062","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-062-a","issuer-062-b","issuer-062-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-062","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-062","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-062","description":"Require complete map coverage","requirement_ids":["req-screen-062"],"constraint_ids":["constraint-exclusions-062"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-062","description":"Require citations for every issuer","requirement_ids":["req-report-062"],"constraint_ids":["constraint-definition-062"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-062"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-062-a"},{"ticker":"issuer-062-b"},{"ticker":"issuer-062-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-062"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-062"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-062-a","issuer-062-b","issuer-062-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-063","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (063).","requirements":[{"id":"req-screen-063","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-063","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-063","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-063","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-063-a","issuer-063-b","issuer-063-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-063","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-063","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-063","description":"Require complete map coverage","requirement_ids":["req-screen-063"],"constraint_ids":["constraint-exclusions-063"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-063","description":"Require citations for every issuer","requirement_ids":["req-report-063"],"constraint_ids":["constraint-definition-063"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-063"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-063-a"},{"ticker":"issuer-063-b"},{"ticker":"issuer-063-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-063"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-063"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-063-a","issuer-063-b","issuer-063-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-064","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (064).","requirements":[{"id":"req-screen-064","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-064","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-064","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-064","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-064-a","issuer-064-b","issuer-064-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-064","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-064","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-064","description":"Require complete map coverage","requirement_ids":["req-screen-064"],"constraint_ids":["constraint-exclusions-064"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-064","description":"Require citations for every issuer","requirement_ids":["req-report-064"],"constraint_ids":["constraint-definition-064"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-064"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-064-a"},{"ticker":"issuer-064-b"},{"ticker":"issuer-064-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-064"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-064"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-064-a","issuer-064-b","issuer-064-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-065","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (065).","requirements":[{"id":"req-screen-065","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-065","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-065","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-065","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-065-a","issuer-065-b","issuer-065-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-065","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-065","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-065","description":"Require complete map coverage","requirement_ids":["req-screen-065"],"constraint_ids":["constraint-exclusions-065"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-065","description":"Require citations for every issuer","requirement_ids":["req-report-065"],"constraint_ids":["constraint-definition-065"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-065"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-065-a"},{"ticker":"issuer-065-b"},{"ticker":"issuer-065-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-065"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-065"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-065-a","issuer-065-b","issuer-065-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-066","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (066).","requirements":[{"id":"req-screen-066","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-066","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-066","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-066","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-066-a","issuer-066-b","issuer-066-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-066","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-066","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-066","description":"Require complete map coverage","requirement_ids":["req-screen-066"],"constraint_ids":["constraint-exclusions-066"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-066","description":"Require citations for every issuer","requirement_ids":["req-report-066"],"constraint_ids":["constraint-definition-066"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-066"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-066-a"},{"ticker":"issuer-066-b"},{"ticker":"issuer-066-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-066"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-066"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-066-a","issuer-066-b","issuer-066-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-067","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (067).","requirements":[{"id":"req-screen-067","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-067","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-067","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-067","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-067-a","issuer-067-b","issuer-067-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-067","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-067","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-067","description":"Require complete map coverage","requirement_ids":["req-screen-067"],"constraint_ids":["constraint-exclusions-067"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-067","description":"Require citations for every issuer","requirement_ids":["req-report-067"],"constraint_ids":["constraint-definition-067"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-067"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-067-a"},{"ticker":"issuer-067-b"},{"ticker":"issuer-067-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-067"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-067"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-067-a","issuer-067-b","issuer-067-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-068","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (068).","requirements":[{"id":"req-screen-068","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-068","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-068","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-068","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-068-a","issuer-068-b","issuer-068-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-068","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-068","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-068","description":"Require complete map coverage","requirement_ids":["req-screen-068"],"constraint_ids":["constraint-exclusions-068"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-068","description":"Require citations for every issuer","requirement_ids":["req-report-068"],"constraint_ids":["constraint-definition-068"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-068"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-068-a"},{"ticker":"issuer-068-b"},{"ticker":"issuer-068-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-068"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-068"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-068-a","issuer-068-b","issuer-068-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-069","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (069).","requirements":[{"id":"req-screen-069","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-069","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-069","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-069","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-069-a","issuer-069-b","issuer-069-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-069","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-069","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-069","description":"Require complete map coverage","requirement_ids":["req-screen-069"],"constraint_ids":["constraint-exclusions-069"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-069","description":"Require citations for every issuer","requirement_ids":["req-report-069"],"constraint_ids":["constraint-definition-069"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-069"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-069-a"},{"ticker":"issuer-069-b"},{"ticker":"issuer-069-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-069"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-069"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-069-a","issuer-069-b","issuer-069-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-070","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (070).","requirements":[{"id":"req-screen-070","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-070","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-070","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-070","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-070-a","issuer-070-b","issuer-070-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-070","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-070","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-070","description":"Require complete map coverage","requirement_ids":["req-screen-070"],"constraint_ids":["constraint-exclusions-070"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-070","description":"Require citations for every issuer","requirement_ids":["req-report-070"],"constraint_ids":["constraint-definition-070"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-070"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-070-a"},{"ticker":"issuer-070-b"},{"ticker":"issuer-070-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-070"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-070"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-070-a","issuer-070-b","issuer-070-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-071","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (071).","requirements":[{"id":"req-screen-071","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-071","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-071","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-071","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-071-a","issuer-071-b","issuer-071-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-071","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-071","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-071","description":"Require complete map coverage","requirement_ids":["req-screen-071"],"constraint_ids":["constraint-exclusions-071"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-071","description":"Require citations for every issuer","requirement_ids":["req-report-071"],"constraint_ids":["constraint-definition-071"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-071"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-071-a"},{"ticker":"issuer-071-b"},{"ticker":"issuer-071-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-071"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-071"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-071-a","issuer-071-b","issuer-071-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-072","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (072).","requirements":[{"id":"req-screen-072","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-072","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-072","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-072","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-072-a","issuer-072-b","issuer-072-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-072","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-072","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-072","description":"Require complete map coverage","requirement_ids":["req-screen-072"],"constraint_ids":["constraint-exclusions-072"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-072","description":"Require citations for every issuer","requirement_ids":["req-report-072"],"constraint_ids":["constraint-definition-072"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-072"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-072-a"},{"ticker":"issuer-072-b"},{"ticker":"issuer-072-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-072"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-072"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-072-a","issuer-072-b","issuer-072-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-073","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (073).","requirements":[{"id":"req-screen-073","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-073","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-073","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-073","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-073-a","issuer-073-b","issuer-073-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-073","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-073","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-073","description":"Require complete map coverage","requirement_ids":["req-screen-073"],"constraint_ids":["constraint-exclusions-073"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-073","description":"Require citations for every issuer","requirement_ids":["req-report-073"],"constraint_ids":["constraint-definition-073"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-073"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-073-a"},{"ticker":"issuer-073-b"},{"ticker":"issuer-073-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-073"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-073"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-073-a","issuer-073-b","issuer-073-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-074","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (074).","requirements":[{"id":"req-screen-074","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-074","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-074","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-074","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-074-a","issuer-074-b","issuer-074-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-074","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-074","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-074","description":"Require complete map coverage","requirement_ids":["req-screen-074"],"constraint_ids":["constraint-exclusions-074"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-074","description":"Require citations for every issuer","requirement_ids":["req-report-074"],"constraint_ids":["constraint-definition-074"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-074"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-074-a"},{"ticker":"issuer-074-b"},{"ticker":"issuer-074-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-074"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-074"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-074-a","issuer-074-b","issuer-074-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-075","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (075).","requirements":[{"id":"req-screen-075","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-075","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-075","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-075","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-075-a","issuer-075-b","issuer-075-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-075","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-075","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-075","description":"Require complete map coverage","requirement_ids":["req-screen-075"],"constraint_ids":["constraint-exclusions-075"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-075","description":"Require citations for every issuer","requirement_ids":["req-report-075"],"constraint_ids":["constraint-definition-075"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-075"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-075-a"},{"ticker":"issuer-075-b"},{"ticker":"issuer-075-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-075"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-075"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-075-a","issuer-075-b","issuer-075-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-076","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (076).","requirements":[{"id":"req-screen-076","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-076","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-076","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-076","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-076-a","issuer-076-b","issuer-076-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-076","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-076","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-076","description":"Require complete map coverage","requirement_ids":["req-screen-076"],"constraint_ids":["constraint-exclusions-076"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-076","description":"Require citations for every issuer","requirement_ids":["req-report-076"],"constraint_ids":["constraint-definition-076"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-076"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-076-a"},{"ticker":"issuer-076-b"},{"ticker":"issuer-076-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-076"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-076"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-076-a","issuer-076-b","issuer-076-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-077","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (077).","requirements":[{"id":"req-screen-077","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-077","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-077","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-077","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-077-a","issuer-077-b","issuer-077-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-077","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-077","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-077","description":"Require complete map coverage","requirement_ids":["req-screen-077"],"constraint_ids":["constraint-exclusions-077"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-077","description":"Require citations for every issuer","requirement_ids":["req-report-077"],"constraint_ids":["constraint-definition-077"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-077"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-077-a"},{"ticker":"issuer-077-b"},{"ticker":"issuer-077-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-077"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-077"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-077-a","issuer-077-b","issuer-077-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-078","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (078).","requirements":[{"id":"req-screen-078","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-078","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-078","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-078","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-078-a","issuer-078-b","issuer-078-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-078","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-078","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-078","description":"Require complete map coverage","requirement_ids":["req-screen-078"],"constraint_ids":["constraint-exclusions-078"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-078","description":"Require citations for every issuer","requirement_ids":["req-report-078"],"constraint_ids":["constraint-definition-078"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-078"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-078-a"},{"ticker":"issuer-078-b"},{"ticker":"issuer-078-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-078"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-078"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-078-a","issuer-078-b","issuer-078-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-079","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (079).","requirements":[{"id":"req-screen-079","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-079","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-079","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-079","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-079-a","issuer-079-b","issuer-079-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-079","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-079","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-079","description":"Require complete map coverage","requirement_ids":["req-screen-079"],"constraint_ids":["constraint-exclusions-079"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-079","description":"Require citations for every issuer","requirement_ids":["req-report-079"],"constraint_ids":["constraint-definition-079"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-079"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-079-a"},{"ticker":"issuer-079-b"},{"ticker":"issuer-079-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-079"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-079"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-079-a","issuer-079-b","issuer-079-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-000","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (000).","requirements":[{"id":"req-screen-000","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-000","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-000","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-000","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-000-a","issuer-000-b","issuer-000-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-000","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-000","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-000","description":"Require complete map coverage","requirement_ids":["req-screen-000"],"constraint_ids":["constraint-exclusions-000"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-000","description":"Require citations for every issuer","requirement_ids":["req-report-000"],"constraint_ids":["constraint-definition-000"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-000"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-000-a"},{"ticker":"issuer-000-b"},{"ticker":"issuer-000-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-000"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-000"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-000-a","issuer-000-b","issuer-000-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-001","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (001).","requirements":[{"id":"req-screen-001","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-001","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-001","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-001","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-001-a","issuer-001-b","issuer-001-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-001","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-001","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-001","description":"Require complete map coverage","requirement_ids":["req-screen-001"],"constraint_ids":["constraint-exclusions-001"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-001","description":"Require citations for every issuer","requirement_ids":["req-report-001"],"constraint_ids":["constraint-definition-001"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-001"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-001-a"},{"ticker":"issuer-001-b"},{"ticker":"issuer-001-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-001"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-001"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-001-a","issuer-001-b","issuer-001-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-002","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (002).","requirements":[{"id":"req-screen-002","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-002","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-002","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-002","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-002-a","issuer-002-b","issuer-002-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-002","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-002","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-002","description":"Require complete map coverage","requirement_ids":["req-screen-002"],"constraint_ids":["constraint-exclusions-002"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-002","description":"Require citations for every issuer","requirement_ids":["req-report-002"],"constraint_ids":["constraint-definition-002"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-002"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-002-a"},{"ticker":"issuer-002-b"},{"ticker":"issuer-002-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-002"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-002"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-002-a","issuer-002-b","issuer-002-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-003","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (003).","requirements":[{"id":"req-screen-003","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-003","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-003","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-003","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-003-a","issuer-003-b","issuer-003-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-003","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-003","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-003","description":"Require complete map coverage","requirement_ids":["req-screen-003"],"constraint_ids":["constraint-exclusions-003"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-003","description":"Require citations for every issuer","requirement_ids":["req-report-003"],"constraint_ids":["constraint-definition-003"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-003"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-003-a"},{"ticker":"issuer-003-b"},{"ticker":"issuer-003-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-003"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-003"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-003-a","issuer-003-b","issuer-003-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-004","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (004).","requirements":[{"id":"req-screen-004","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-004","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-004","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-004","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-004-a","issuer-004-b","issuer-004-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-004","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-004","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-004","description":"Require complete map coverage","requirement_ids":["req-screen-004"],"constraint_ids":["constraint-exclusions-004"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-004","description":"Require citations for every issuer","requirement_ids":["req-report-004"],"constraint_ids":["constraint-definition-004"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-004"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-004-a"},{"ticker":"issuer-004-b"},{"ticker":"issuer-004-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-004"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-004"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-004-a","issuer-004-b","issuer-004-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-005","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (005).","requirements":[{"id":"req-screen-005","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-005","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-005","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-005","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-005-a","issuer-005-b","issuer-005-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-005","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-005","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-005","description":"Require complete map coverage","requirement_ids":["req-screen-005"],"constraint_ids":["constraint-exclusions-005"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-005","description":"Require citations for every issuer","requirement_ids":["req-report-005"],"constraint_ids":["constraint-definition-005"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-005"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-005-a"},{"ticker":"issuer-005-b"},{"ticker":"issuer-005-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-005"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-005"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-005-a","issuer-005-b","issuer-005-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-006","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (006).","requirements":[{"id":"req-screen-006","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-006","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-006","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-006","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-006-a","issuer-006-b","issuer-006-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-006","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-006","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-006","description":"Require complete map coverage","requirement_ids":["req-screen-006"],"constraint_ids":["constraint-exclusions-006"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-006","description":"Require citations for every issuer","requirement_ids":["req-report-006"],"constraint_ids":["constraint-definition-006"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-006"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-006-a"},{"ticker":"issuer-006-b"},{"ticker":"issuer-006-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-006"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-006"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-006-a","issuer-006-b","issuer-006-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-007","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (007).","requirements":[{"id":"req-screen-007","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-007","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-007","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-007","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-007-a","issuer-007-b","issuer-007-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-007","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-007","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-007","description":"Require complete map coverage","requirement_ids":["req-screen-007"],"constraint_ids":["constraint-exclusions-007"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-007","description":"Require citations for every issuer","requirement_ids":["req-report-007"],"constraint_ids":["constraint-definition-007"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-007"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-007-a"},{"ticker":"issuer-007-b"},{"ticker":"issuer-007-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-007"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-007"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-007-a","issuer-007-b","issuer-007-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-008","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (008).","requirements":[{"id":"req-screen-008","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-008","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-008","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-008","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-008-a","issuer-008-b","issuer-008-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-008","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-008","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-008","description":"Require complete map coverage","requirement_ids":["req-screen-008"],"constraint_ids":["constraint-exclusions-008"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-008","description":"Require citations for every issuer","requirement_ids":["req-report-008"],"constraint_ids":["constraint-definition-008"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-008"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-008-a"},{"ticker":"issuer-008-b"},{"ticker":"issuer-008-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-008"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-008"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-008-a","issuer-008-b","issuer-008-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-009","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (009).","requirements":[{"id":"req-screen-009","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-009","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-009","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-009","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-009-a","issuer-009-b","issuer-009-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-009","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-009","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-009","description":"Require complete map coverage","requirement_ids":["req-screen-009"],"constraint_ids":["constraint-exclusions-009"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-009","description":"Require citations for every issuer","requirement_ids":["req-report-009"],"constraint_ids":["constraint-definition-009"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-009"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-009-a"},{"ticker":"issuer-009-b"},{"ticker":"issuer-009-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-009"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-009"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-009-a","issuer-009-b","issuer-009-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-010","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (010).","requirements":[{"id":"req-screen-010","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-010","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-010","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-010","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-010-a","issuer-010-b","issuer-010-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-010","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-010","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-010","description":"Require complete map coverage","requirement_ids":["req-screen-010"],"constraint_ids":["constraint-exclusions-010"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-010","description":"Require citations for every issuer","requirement_ids":["req-report-010"],"constraint_ids":["constraint-definition-010"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-010"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-010-a"},{"ticker":"issuer-010-b"},{"ticker":"issuer-010-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-010"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-010"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-010-a","issuer-010-b","issuer-010-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-011","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (011).","requirements":[{"id":"req-screen-011","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-011","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-011","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-011","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-011-a","issuer-011-b","issuer-011-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-011","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-011","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-011","description":"Require complete map coverage","requirement_ids":["req-screen-011"],"constraint_ids":["constraint-exclusions-011"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-011","description":"Require citations for every issuer","requirement_ids":["req-report-011"],"constraint_ids":["constraint-definition-011"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-011"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-011-a"},{"ticker":"issuer-011-b"},{"ticker":"issuer-011-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-011"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-011"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-011-a","issuer-011-b","issuer-011-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-012","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (012).","requirements":[{"id":"req-screen-012","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-012","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-012","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-012","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-012-a","issuer-012-b","issuer-012-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-012","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-012","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-012","description":"Require complete map coverage","requirement_ids":["req-screen-012"],"constraint_ids":["constraint-exclusions-012"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-012","description":"Require citations for every issuer","requirement_ids":["req-report-012"],"constraint_ids":["constraint-definition-012"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-012"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-012-a"},{"ticker":"issuer-012-b"},{"ticker":"issuer-012-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-012"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-012"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-012-a","issuer-012-b","issuer-012-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-013","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (013).","requirements":[{"id":"req-screen-013","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-013","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-013","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-013","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-013-a","issuer-013-b","issuer-013-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-013","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-013","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-013","description":"Require complete map coverage","requirement_ids":["req-screen-013"],"constraint_ids":["constraint-exclusions-013"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-013","description":"Require citations for every issuer","requirement_ids":["req-report-013"],"constraint_ids":["constraint-definition-013"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-013"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-013-a"},{"ticker":"issuer-013-b"},{"ticker":"issuer-013-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-013"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-013"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-013-a","issuer-013-b","issuer-013-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-014","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (014).","requirements":[{"id":"req-screen-014","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-014","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-014","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-014","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-014-a","issuer-014-b","issuer-014-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-014","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-014","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-014","description":"Require complete map coverage","requirement_ids":["req-screen-014"],"constraint_ids":["constraint-exclusions-014"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-014","description":"Require citations for every issuer","requirement_ids":["req-report-014"],"constraint_ids":["constraint-definition-014"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-014"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-014-a"},{"ticker":"issuer-014-b"},{"ticker":"issuer-014-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-014"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-014"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-014-a","issuer-014-b","issuer-014-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-015","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (015).","requirements":[{"id":"req-screen-015","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-015","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-015","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-015","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-015-a","issuer-015-b","issuer-015-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-015","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-015","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-015","description":"Require complete map coverage","requirement_ids":["req-screen-015"],"constraint_ids":["constraint-exclusions-015"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-015","description":"Require citations for every issuer","requirement_ids":["req-report-015"],"constraint_ids":["constraint-definition-015"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-015"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-015-a"},{"ticker":"issuer-015-b"},{"ticker":"issuer-015-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-015"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-015"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-015-a","issuer-015-b","issuer-015-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-016","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (016).","requirements":[{"id":"req-screen-016","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-016","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-016","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-016","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-016-a","issuer-016-b","issuer-016-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-016","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-016","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-016","description":"Require complete map coverage","requirement_ids":["req-screen-016"],"constraint_ids":["constraint-exclusions-016"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-016","description":"Require citations for every issuer","requirement_ids":["req-report-016"],"constraint_ids":["constraint-definition-016"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-016"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-016-a"},{"ticker":"issuer-016-b"},{"ticker":"issuer-016-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-016"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-016"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-016-a","issuer-016-b","issuer-016-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-017","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (017).","requirements":[{"id":"req-screen-017","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-017","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-017","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-017","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-017-a","issuer-017-b","issuer-017-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-017","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-017","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-017","description":"Require complete map coverage","requirement_ids":["req-screen-017"],"constraint_ids":["constraint-exclusions-017"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-017","description":"Require citations for every issuer","requirement_ids":["req-report-017"],"constraint_ids":["constraint-definition-017"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-017"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-017-a"},{"ticker":"issuer-017-b"},{"ticker":"issuer-017-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-017"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-017"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-017-a","issuer-017-b","issuer-017-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-018","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (018).","requirements":[{"id":"req-screen-018","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-018","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-018","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-018","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-018-a","issuer-018-b","issuer-018-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-018","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-018","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-018","description":"Require complete map coverage","requirement_ids":["req-screen-018"],"constraint_ids":["constraint-exclusions-018"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-018","description":"Require citations for every issuer","requirement_ids":["req-report-018"],"constraint_ids":["constraint-definition-018"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-018"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-018-a"},{"ticker":"issuer-018-b"},{"ticker":"issuer-018-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-018"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-018"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-018-a","issuer-018-b","issuer-018-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-019","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (019).","requirements":[{"id":"req-screen-019","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-019","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-019","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-019","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-019-a","issuer-019-b","issuer-019-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-019","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-019","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-019","description":"Require complete map coverage","requirement_ids":["req-screen-019"],"constraint_ids":["constraint-exclusions-019"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-019","description":"Require citations for every issuer","requirement_ids":["req-report-019"],"constraint_ids":["constraint-definition-019"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-019"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-019-a"},{"ticker":"issuer-019-b"},{"ticker":"issuer-019-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-019"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-019"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-019-a","issuer-019-b","issuer-019-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-020","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (020).","requirements":[{"id":"req-screen-020","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-020","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-020","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-020","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-020-a","issuer-020-b","issuer-020-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-020","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-020","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-020","description":"Require complete map coverage","requirement_ids":["req-screen-020"],"constraint_ids":["constraint-exclusions-020"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-020","description":"Require citations for every issuer","requirement_ids":["req-report-020"],"constraint_ids":["constraint-definition-020"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-020"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-020-a"},{"ticker":"issuer-020-b"},{"ticker":"issuer-020-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-020"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-020"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-020-a","issuer-020-b","issuer-020-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-021","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (021).","requirements":[{"id":"req-screen-021","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-021","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-021","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-021","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-021-a","issuer-021-b","issuer-021-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-021","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-021","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-021","description":"Require complete map coverage","requirement_ids":["req-screen-021"],"constraint_ids":["constraint-exclusions-021"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-021","description":"Require citations for every issuer","requirement_ids":["req-report-021"],"constraint_ids":["constraint-definition-021"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-021"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-021-a"},{"ticker":"issuer-021-b"},{"ticker":"issuer-021-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-021"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-021"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-021-a","issuer-021-b","issuer-021-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-022","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (022).","requirements":[{"id":"req-screen-022","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-022","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-022","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-022","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-022-a","issuer-022-b","issuer-022-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-022","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-022","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-022","description":"Require complete map coverage","requirement_ids":["req-screen-022"],"constraint_ids":["constraint-exclusions-022"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-022","description":"Require citations for every issuer","requirement_ids":["req-report-022"],"constraint_ids":["constraint-definition-022"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-022"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-022-a"},{"ticker":"issuer-022-b"},{"ticker":"issuer-022-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-022"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-022"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-022-a","issuer-022-b","issuer-022-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-023","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (023).","requirements":[{"id":"req-screen-023","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-023","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-023","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-023","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-023-a","issuer-023-b","issuer-023-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-023","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-023","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-023","description":"Require complete map coverage","requirement_ids":["req-screen-023"],"constraint_ids":["constraint-exclusions-023"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-023","description":"Require citations for every issuer","requirement_ids":["req-report-023"],"constraint_ids":["constraint-definition-023"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-023"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-023-a"},{"ticker":"issuer-023-b"},{"ticker":"issuer-023-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-023"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-023"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-023-a","issuer-023-b","issuer-023-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-024","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (024).","requirements":[{"id":"req-screen-024","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-024","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-024","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-024","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-024-a","issuer-024-b","issuer-024-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-024","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-024","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-024","description":"Require complete map coverage","requirement_ids":["req-screen-024"],"constraint_ids":["constraint-exclusions-024"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-024","description":"Require citations for every issuer","requirement_ids":["req-report-024"],"constraint_ids":["constraint-definition-024"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-024"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-024-a"},{"ticker":"issuer-024-b"},{"ticker":"issuer-024-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-024"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-024"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-024-a","issuer-024-b","issuer-024-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-025","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (025).","requirements":[{"id":"req-screen-025","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-025","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-025","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-025","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-025-a","issuer-025-b","issuer-025-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-025","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-025","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-025","description":"Require complete map coverage","requirement_ids":["req-screen-025"],"constraint_ids":["constraint-exclusions-025"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-025","description":"Require citations for every issuer","requirement_ids":["req-report-025"],"constraint_ids":["constraint-definition-025"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-025"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-025-a"},{"ticker":"issuer-025-b"},{"ticker":"issuer-025-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-025"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-025"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-025-a","issuer-025-b","issuer-025-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-026","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (026).","requirements":[{"id":"req-screen-026","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-026","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-026","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-026","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-026-a","issuer-026-b","issuer-026-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-026","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-026","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-026","description":"Require complete map coverage","requirement_ids":["req-screen-026"],"constraint_ids":["constraint-exclusions-026"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-026","description":"Require citations for every issuer","requirement_ids":["req-report-026"],"constraint_ids":["constraint-definition-026"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-026"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-026-a"},{"ticker":"issuer-026-b"},{"ticker":"issuer-026-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-026"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-026"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-026-a","issuer-026-b","issuer-026-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-027","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (027).","requirements":[{"id":"req-screen-027","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-027","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-027","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-027","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-027-a","issuer-027-b","issuer-027-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-027","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-027","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-027","description":"Require complete map coverage","requirement_ids":["req-screen-027"],"constraint_ids":["constraint-exclusions-027"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-027","description":"Require citations for every issuer","requirement_ids":["req-report-027"],"constraint_ids":["constraint-definition-027"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-027"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-027-a"},{"ticker":"issuer-027-b"},{"ticker":"issuer-027-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-027"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-027"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-027-a","issuer-027-b","issuer-027-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-028","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (028).","requirements":[{"id":"req-screen-028","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-028","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-028","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-028","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-028-a","issuer-028-b","issuer-028-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-028","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-028","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-028","description":"Require complete map coverage","requirement_ids":["req-screen-028"],"constraint_ids":["constraint-exclusions-028"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-028","description":"Require citations for every issuer","requirement_ids":["req-report-028"],"constraint_ids":["constraint-definition-028"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-028"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-028-a"},{"ticker":"issuer-028-b"},{"ticker":"issuer-028-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-028"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-028"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-028-a","issuer-028-b","issuer-028-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-029","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (029).","requirements":[{"id":"req-screen-029","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-029","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-029","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-029","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-029-a","issuer-029-b","issuer-029-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-029","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-029","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-029","description":"Require complete map coverage","requirement_ids":["req-screen-029"],"constraint_ids":["constraint-exclusions-029"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-029","description":"Require citations for every issuer","requirement_ids":["req-report-029"],"constraint_ids":["constraint-definition-029"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-029"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-029-a"},{"ticker":"issuer-029-b"},{"ticker":"issuer-029-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-029"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-029"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-029-a","issuer-029-b","issuer-029-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-030","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (030).","requirements":[{"id":"req-screen-030","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-030","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-030","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-030","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-030-a","issuer-030-b","issuer-030-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-030","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-030","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-030","description":"Require complete map coverage","requirement_ids":["req-screen-030"],"constraint_ids":["constraint-exclusions-030"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-030","description":"Require citations for every issuer","requirement_ids":["req-report-030"],"constraint_ids":["constraint-definition-030"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-030"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-030-a"},{"ticker":"issuer-030-b"},{"ticker":"issuer-030-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-030"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-030"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-030-a","issuer-030-b","issuer-030-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-031","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (031).","requirements":[{"id":"req-screen-031","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-031","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-031","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-031","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-031-a","issuer-031-b","issuer-031-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-031","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-031","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-031","description":"Require complete map coverage","requirement_ids":["req-screen-031"],"constraint_ids":["constraint-exclusions-031"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-031","description":"Require citations for every issuer","requirement_ids":["req-report-031"],"constraint_ids":["constraint-definition-031"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-031"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-031-a"},{"ticker":"issuer-031-b"},{"ticker":"issuer-031-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-031"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-031"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-031-a","issuer-031-b","issuer-031-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-032","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (032).","requirements":[{"id":"req-screen-032","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-032","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-032","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-032","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-032-a","issuer-032-b","issuer-032-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-032","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-032","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-032","description":"Require complete map coverage","requirement_ids":["req-screen-032"],"constraint_ids":["constraint-exclusions-032"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-032","description":"Require citations for every issuer","requirement_ids":["req-report-032"],"constraint_ids":["constraint-definition-032"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-032"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-032-a"},{"ticker":"issuer-032-b"},{"ticker":"issuer-032-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-032"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-032"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-032-a","issuer-032-b","issuer-032-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-033","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (033).","requirements":[{"id":"req-screen-033","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-033","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-033","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-033","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-033-a","issuer-033-b","issuer-033-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-033","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-033","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-033","description":"Require complete map coverage","requirement_ids":["req-screen-033"],"constraint_ids":["constraint-exclusions-033"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-033","description":"Require citations for every issuer","requirement_ids":["req-report-033"],"constraint_ids":["constraint-definition-033"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-033"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-033-a"},{"ticker":"issuer-033-b"},{"ticker":"issuer-033-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-033"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-033"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-033-a","issuer-033-b","issuer-033-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-034","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (034).","requirements":[{"id":"req-screen-034","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-034","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-034","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-034","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-034-a","issuer-034-b","issuer-034-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-034","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-034","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-034","description":"Require complete map coverage","requirement_ids":["req-screen-034"],"constraint_ids":["constraint-exclusions-034"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-034","description":"Require citations for every issuer","requirement_ids":["req-report-034"],"constraint_ids":["constraint-definition-034"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-034"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-034-a"},{"ticker":"issuer-034-b"},{"ticker":"issuer-034-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-034"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-034"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-034-a","issuer-034-b","issuer-034-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-035","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (035).","requirements":[{"id":"req-screen-035","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-035","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-035","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-035","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-035-a","issuer-035-b","issuer-035-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-035","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-035","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-035","description":"Require complete map coverage","requirement_ids":["req-screen-035"],"constraint_ids":["constraint-exclusions-035"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-035","description":"Require citations for every issuer","requirement_ids":["req-report-035"],"constraint_ids":["constraint-definition-035"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-035"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-035-a"},{"ticker":"issuer-035-b"},{"ticker":"issuer-035-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-035"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-035"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-035-a","issuer-035-b","issuer-035-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-036","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (036).","requirements":[{"id":"req-screen-036","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-036","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-036","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-036","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-036-a","issuer-036-b","issuer-036-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-036","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-036","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-036","description":"Require complete map coverage","requirement_ids":["req-screen-036"],"constraint_ids":["constraint-exclusions-036"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-036","description":"Require citations for every issuer","requirement_ids":["req-report-036"],"constraint_ids":["constraint-definition-036"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-036"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-036-a"},{"ticker":"issuer-036-b"},{"ticker":"issuer-036-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-036"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-036"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-036-a","issuer-036-b","issuer-036-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-037","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (037).","requirements":[{"id":"req-screen-037","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-037","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-037","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-037","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-037-a","issuer-037-b","issuer-037-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-037","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-037","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-037","description":"Require complete map coverage","requirement_ids":["req-screen-037"],"constraint_ids":["constraint-exclusions-037"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-037","description":"Require citations for every issuer","requirement_ids":["req-report-037"],"constraint_ids":["constraint-definition-037"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-037"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-037-a"},{"ticker":"issuer-037-b"},{"ticker":"issuer-037-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-037"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-037"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-037-a","issuer-037-b","issuer-037-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-038","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (038).","requirements":[{"id":"req-screen-038","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-038","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-038","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-038","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-038-a","issuer-038-b","issuer-038-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-038","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-038","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-038","description":"Require complete map coverage","requirement_ids":["req-screen-038"],"constraint_ids":["constraint-exclusions-038"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-038","description":"Require citations for every issuer","requirement_ids":["req-report-038"],"constraint_ids":["constraint-definition-038"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-038"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-038-a"},{"ticker":"issuer-038-b"},{"ticker":"issuer-038-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-038"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-038"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-038-a","issuer-038-b","issuer-038-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-039","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (039).","requirements":[{"id":"req-screen-039","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-039","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-039","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-039","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-039-a","issuer-039-b","issuer-039-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-039","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-039","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-039","description":"Require complete map coverage","requirement_ids":["req-screen-039"],"constraint_ids":["constraint-exclusions-039"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-039","description":"Require citations for every issuer","requirement_ids":["req-report-039"],"constraint_ids":["constraint-definition-039"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-039"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-039-a"},{"ticker":"issuer-039-b"},{"ticker":"issuer-039-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-039"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-039"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-039-a","issuer-039-b","issuer-039-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-040","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (040).","requirements":[{"id":"req-screen-040","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-040","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-040","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-040","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-040-a","issuer-040-b","issuer-040-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-040","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-040","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-040","description":"Require complete map coverage","requirement_ids":["req-screen-040"],"constraint_ids":["constraint-exclusions-040"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-040","description":"Require citations for every issuer","requirement_ids":["req-report-040"],"constraint_ids":["constraint-definition-040"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-040"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-040-a"},{"ticker":"issuer-040-b"},{"ticker":"issuer-040-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-040"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-040"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-040-a","issuer-040-b","issuer-040-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-041","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (041).","requirements":[{"id":"req-screen-041","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-041","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-041","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-041","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-041-a","issuer-041-b","issuer-041-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-041","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-041","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-041","description":"Require complete map coverage","requirement_ids":["req-screen-041"],"constraint_ids":["constraint-exclusions-041"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-041","description":"Require citations for every issuer","requirement_ids":["req-report-041"],"constraint_ids":["constraint-definition-041"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-041"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-041-a"},{"ticker":"issuer-041-b"},{"ticker":"issuer-041-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-041"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-041"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-041-a","issuer-041-b","issuer-041-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-042","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (042).","requirements":[{"id":"req-screen-042","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-042","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-042","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-042","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-042-a","issuer-042-b","issuer-042-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-042","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-042","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-042","description":"Require complete map coverage","requirement_ids":["req-screen-042"],"constraint_ids":["constraint-exclusions-042"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-042","description":"Require citations for every issuer","requirement_ids":["req-report-042"],"constraint_ids":["constraint-definition-042"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-042"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-042-a"},{"ticker":"issuer-042-b"},{"ticker":"issuer-042-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-042"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-042"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-042-a","issuer-042-b","issuer-042-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-043","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (043).","requirements":[{"id":"req-screen-043","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-043","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-043","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-043","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-043-a","issuer-043-b","issuer-043-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-043","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-043","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-043","description":"Require complete map coverage","requirement_ids":["req-screen-043"],"constraint_ids":["constraint-exclusions-043"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-043","description":"Require citations for every issuer","requirement_ids":["req-report-043"],"constraint_ids":["constraint-definition-043"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-043"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-043-a"},{"ticker":"issuer-043-b"},{"ticker":"issuer-043-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-043"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-043"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-043-a","issuer-043-b","issuer-043-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-044","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (044).","requirements":[{"id":"req-screen-044","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-044","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-044","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-044","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-044-a","issuer-044-b","issuer-044-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-044","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-044","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-044","description":"Require complete map coverage","requirement_ids":["req-screen-044"],"constraint_ids":["constraint-exclusions-044"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-044","description":"Require citations for every issuer","requirement_ids":["req-report-044"],"constraint_ids":["constraint-definition-044"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-044"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-044-a"},{"ticker":"issuer-044-b"},{"ticker":"issuer-044-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-044"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-044"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-044-a","issuer-044-b","issuer-044-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-045","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (045).","requirements":[{"id":"req-screen-045","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-045","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-045","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-045","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-045-a","issuer-045-b","issuer-045-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-045","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-045","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-045","description":"Require complete map coverage","requirement_ids":["req-screen-045"],"constraint_ids":["constraint-exclusions-045"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-045","description":"Require citations for every issuer","requirement_ids":["req-report-045"],"constraint_ids":["constraint-definition-045"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-045"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-045-a"},{"ticker":"issuer-045-b"},{"ticker":"issuer-045-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-045"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-045"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-045-a","issuer-045-b","issuer-045-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-046","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (046).","requirements":[{"id":"req-screen-046","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-046","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-046","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-046","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-046-a","issuer-046-b","issuer-046-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-046","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-046","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-046","description":"Require complete map coverage","requirement_ids":["req-screen-046"],"constraint_ids":["constraint-exclusions-046"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-046","description":"Require citations for every issuer","requirement_ids":["req-report-046"],"constraint_ids":["constraint-definition-046"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-046"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-046-a"},{"ticker":"issuer-046-b"},{"ticker":"issuer-046-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-046"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-046"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-046-a","issuer-046-b","issuer-046-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-047","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (047).","requirements":[{"id":"req-screen-047","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-047","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-047","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-047","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-047-a","issuer-047-b","issuer-047-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-047","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-047","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-047","description":"Require complete map coverage","requirement_ids":["req-screen-047"],"constraint_ids":["constraint-exclusions-047"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-047","description":"Require citations for every issuer","requirement_ids":["req-report-047"],"constraint_ids":["constraint-definition-047"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-047"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-047-a"},{"ticker":"issuer-047-b"},{"ticker":"issuer-047-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-047"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-047"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-047-a","issuer-047-b","issuer-047-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-048","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (048).","requirements":[{"id":"req-screen-048","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-048","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-048","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-048","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-048-a","issuer-048-b","issuer-048-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-048","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-048","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-048","description":"Require complete map coverage","requirement_ids":["req-screen-048"],"constraint_ids":["constraint-exclusions-048"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-048","description":"Require citations for every issuer","requirement_ids":["req-report-048"],"constraint_ids":["constraint-definition-048"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-048"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-048-a"},{"ticker":"issuer-048-b"},{"ticker":"issuer-048-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-048"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-048"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-048-a","issuer-048-b","issuer-048-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-049","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (049).","requirements":[{"id":"req-screen-049","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-049","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-049","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-049","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-049-a","issuer-049-b","issuer-049-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-049","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-049","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-049","description":"Require complete map coverage","requirement_ids":["req-screen-049"],"constraint_ids":["constraint-exclusions-049"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-049","description":"Require citations for every issuer","requirement_ids":["req-report-049"],"constraint_ids":["constraint-definition-049"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-049"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-049-a"},{"ticker":"issuer-049-b"},{"ticker":"issuer-049-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-049"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-049"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-049-a","issuer-049-b","issuer-049-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-050","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (050).","requirements":[{"id":"req-screen-050","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-050","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-050","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-050","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-050-a","issuer-050-b","issuer-050-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-050","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-050","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-050","description":"Require complete map coverage","requirement_ids":["req-screen-050"],"constraint_ids":["constraint-exclusions-050"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-050","description":"Require citations for every issuer","requirement_ids":["req-report-050"],"constraint_ids":["constraint-definition-050"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-050"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-050-a"},{"ticker":"issuer-050-b"},{"ticker":"issuer-050-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-050"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-050"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-050-a","issuer-050-b","issuer-050-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-051","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (051).","requirements":[{"id":"req-screen-051","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-051","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-051","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-051","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-051-a","issuer-051-b","issuer-051-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-051","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-051","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-051","description":"Require complete map coverage","requirement_ids":["req-screen-051"],"constraint_ids":["constraint-exclusions-051"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-051","description":"Require citations for every issuer","requirement_ids":["req-report-051"],"constraint_ids":["constraint-definition-051"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-051"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-051-a"},{"ticker":"issuer-051-b"},{"ticker":"issuer-051-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-051"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-051"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-051-a","issuer-051-b","issuer-051-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-052","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (052).","requirements":[{"id":"req-screen-052","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-052","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-052","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-052","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-052-a","issuer-052-b","issuer-052-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-052","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-052","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-052","description":"Require complete map coverage","requirement_ids":["req-screen-052"],"constraint_ids":["constraint-exclusions-052"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-052","description":"Require citations for every issuer","requirement_ids":["req-report-052"],"constraint_ids":["constraint-definition-052"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-052"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-052-a"},{"ticker":"issuer-052-b"},{"ticker":"issuer-052-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-052"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-052"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-052-a","issuer-052-b","issuer-052-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-053","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (053).","requirements":[{"id":"req-screen-053","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-053","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-053","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-053","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-053-a","issuer-053-b","issuer-053-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-053","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-053","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-053","description":"Require complete map coverage","requirement_ids":["req-screen-053"],"constraint_ids":["constraint-exclusions-053"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-053","description":"Require citations for every issuer","requirement_ids":["req-report-053"],"constraint_ids":["constraint-definition-053"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-053"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-053-a"},{"ticker":"issuer-053-b"},{"ticker":"issuer-053-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-053"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-053"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-053-a","issuer-053-b","issuer-053-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-054","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (054).","requirements":[{"id":"req-screen-054","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-054","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-054","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-054","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-054-a","issuer-054-b","issuer-054-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-054","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-054","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-054","description":"Require complete map coverage","requirement_ids":["req-screen-054"],"constraint_ids":["constraint-exclusions-054"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-054","description":"Require citations for every issuer","requirement_ids":["req-report-054"],"constraint_ids":["constraint-definition-054"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-054"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-054-a"},{"ticker":"issuer-054-b"},{"ticker":"issuer-054-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-054"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-054"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-054-a","issuer-054-b","issuer-054-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-055","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (055).","requirements":[{"id":"req-screen-055","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-055","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-055","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-055","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-055-a","issuer-055-b","issuer-055-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-055","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-055","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-055","description":"Require complete map coverage","requirement_ids":["req-screen-055"],"constraint_ids":["constraint-exclusions-055"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-055","description":"Require citations for every issuer","requirement_ids":["req-report-055"],"constraint_ids":["constraint-definition-055"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-055"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-055-a"},{"ticker":"issuer-055-b"},{"ticker":"issuer-055-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-055"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-055"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-055-a","issuer-055-b","issuer-055-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-056","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (056).","requirements":[{"id":"req-screen-056","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-056","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-056","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-056","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-056-a","issuer-056-b","issuer-056-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-056","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-056","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-056","description":"Require complete map coverage","requirement_ids":["req-screen-056"],"constraint_ids":["constraint-exclusions-056"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-056","description":"Require citations for every issuer","requirement_ids":["req-report-056"],"constraint_ids":["constraint-definition-056"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-056"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-056-a"},{"ticker":"issuer-056-b"},{"ticker":"issuer-056-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-056"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-056"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-056-a","issuer-056-b","issuer-056-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-057","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (057).","requirements":[{"id":"req-screen-057","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-057","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-057","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-057","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-057-a","issuer-057-b","issuer-057-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-057","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-057","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-057","description":"Require complete map coverage","requirement_ids":["req-screen-057"],"constraint_ids":["constraint-exclusions-057"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-057","description":"Require citations for every issuer","requirement_ids":["req-report-057"],"constraint_ids":["constraint-definition-057"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-057"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-057-a"},{"ticker":"issuer-057-b"},{"ticker":"issuer-057-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-057"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-057"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-057-a","issuer-057-b","issuer-057-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-058","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (058).","requirements":[{"id":"req-screen-058","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-058","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-058","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-058","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-058-a","issuer-058-b","issuer-058-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-058","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-058","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-058","description":"Require complete map coverage","requirement_ids":["req-screen-058"],"constraint_ids":["constraint-exclusions-058"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-058","description":"Require citations for every issuer","requirement_ids":["req-report-058"],"constraint_ids":["constraint-definition-058"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-058"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-058-a"},{"ticker":"issuer-058-b"},{"ticker":"issuer-058-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-058"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-058"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-058-a","issuer-058-b","issuer-058-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-059","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (059).","requirements":[{"id":"req-screen-059","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-059","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-059","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-059","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-059-a","issuer-059-b","issuer-059-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-059","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-059","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-059","description":"Require complete map coverage","requirement_ids":["req-screen-059"],"constraint_ids":["constraint-exclusions-059"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-059","description":"Require citations for every issuer","requirement_ids":["req-report-059"],"constraint_ids":["constraint-definition-059"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-059"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-059-a"},{"ticker":"issuer-059-b"},{"ticker":"issuer-059-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-059"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-059"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-059-a","issuer-059-b","issuer-059-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-060","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (060).","requirements":[{"id":"req-screen-060","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-060","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-060","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-060","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-060-a","issuer-060-b","issuer-060-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-060","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-060","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-060","description":"Require complete map coverage","requirement_ids":["req-screen-060"],"constraint_ids":["constraint-exclusions-060"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-060","description":"Require citations for every issuer","requirement_ids":["req-report-060"],"constraint_ids":["constraint-definition-060"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-060"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-060-a"},{"ticker":"issuer-060-b"},{"ticker":"issuer-060-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-060"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-060"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-060-a","issuer-060-b","issuer-060-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-061","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (061).","requirements":[{"id":"req-screen-061","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-061","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-061","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-061","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-061-a","issuer-061-b","issuer-061-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-061","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-061","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-061","description":"Require complete map coverage","requirement_ids":["req-screen-061"],"constraint_ids":["constraint-exclusions-061"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-061","description":"Require citations for every issuer","requirement_ids":["req-report-061"],"constraint_ids":["constraint-definition-061"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-061"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-061-a"},{"ticker":"issuer-061-b"},{"ticker":"issuer-061-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-061"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-061"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-061-a","issuer-061-b","issuer-061-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-062","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (062).","requirements":[{"id":"req-screen-062","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-062","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-062","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-062","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-062-a","issuer-062-b","issuer-062-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-062","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-062","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-062","description":"Require complete map coverage","requirement_ids":["req-screen-062"],"constraint_ids":["constraint-exclusions-062"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-062","description":"Require citations for every issuer","requirement_ids":["req-report-062"],"constraint_ids":["constraint-definition-062"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-062"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-062-a"},{"ticker":"issuer-062-b"},{"ticker":"issuer-062-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-062"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-062"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-062-a","issuer-062-b","issuer-062-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-063","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (063).","requirements":[{"id":"req-screen-063","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-063","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-063","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-063","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-063-a","issuer-063-b","issuer-063-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-063","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-063","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-063","description":"Require complete map coverage","requirement_ids":["req-screen-063"],"constraint_ids":["constraint-exclusions-063"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-063","description":"Require citations for every issuer","requirement_ids":["req-report-063"],"constraint_ids":["constraint-definition-063"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-063"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-063-a"},{"ticker":"issuer-063-b"},{"ticker":"issuer-063-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-063"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-063"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-063-a","issuer-063-b","issuer-063-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-064","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (064).","requirements":[{"id":"req-screen-064","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-064","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-064","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-064","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-064-a","issuer-064-b","issuer-064-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-064","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-064","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-064","description":"Require complete map coverage","requirement_ids":["req-screen-064"],"constraint_ids":["constraint-exclusions-064"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-064","description":"Require citations for every issuer","requirement_ids":["req-report-064"],"constraint_ids":["constraint-definition-064"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-064"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-064-a"},{"ticker":"issuer-064-b"},{"ticker":"issuer-064-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-064"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-064"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-064-a","issuer-064-b","issuer-064-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-065","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (065).","requirements":[{"id":"req-screen-065","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-065","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-065","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-065","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-065-a","issuer-065-b","issuer-065-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-065","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-065","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-065","description":"Require complete map coverage","requirement_ids":["req-screen-065"],"constraint_ids":["constraint-exclusions-065"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-065","description":"Require citations for every issuer","requirement_ids":["req-report-065"],"constraint_ids":["constraint-definition-065"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-065"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-065-a"},{"ticker":"issuer-065-b"},{"ticker":"issuer-065-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-065"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-065"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-065-a","issuer-065-b","issuer-065-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-066","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (066).","requirements":[{"id":"req-screen-066","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-066","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-066","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-066","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-066-a","issuer-066-b","issuer-066-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-066","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-066","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-066","description":"Require complete map coverage","requirement_ids":["req-screen-066"],"constraint_ids":["constraint-exclusions-066"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-066","description":"Require citations for every issuer","requirement_ids":["req-report-066"],"constraint_ids":["constraint-definition-066"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-066"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-066-a"},{"ticker":"issuer-066-b"},{"ticker":"issuer-066-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-066"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-066"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-066-a","issuer-066-b","issuer-066-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-067","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (067).","requirements":[{"id":"req-screen-067","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-067","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-067","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-067","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-067-a","issuer-067-b","issuer-067-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-067","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-067","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-067","description":"Require complete map coverage","requirement_ids":["req-screen-067"],"constraint_ids":["constraint-exclusions-067"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-067","description":"Require citations for every issuer","requirement_ids":["req-report-067"],"constraint_ids":["constraint-definition-067"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-067"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-067-a"},{"ticker":"issuer-067-b"},{"ticker":"issuer-067-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-067"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-067"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-067-a","issuer-067-b","issuer-067-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-068","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (068).","requirements":[{"id":"req-screen-068","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-068","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-068","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-068","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-068-a","issuer-068-b","issuer-068-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-068","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-068","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-068","description":"Require complete map coverage","requirement_ids":["req-screen-068"],"constraint_ids":["constraint-exclusions-068"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-068","description":"Require citations for every issuer","requirement_ids":["req-report-068"],"constraint_ids":["constraint-definition-068"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-068"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-068-a"},{"ticker":"issuer-068-b"},{"ticker":"issuer-068-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-068"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-068"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-068-a","issuer-068-b","issuer-068-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-069","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (069).","requirements":[{"id":"req-screen-069","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-069","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-069","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-069","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-069-a","issuer-069-b","issuer-069-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-069","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-069","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-069","description":"Require complete map coverage","requirement_ids":["req-screen-069"],"constraint_ids":["constraint-exclusions-069"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-069","description":"Require citations for every issuer","requirement_ids":["req-report-069"],"constraint_ids":["constraint-definition-069"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-069"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-069-a"},{"ticker":"issuer-069-b"},{"ticker":"issuer-069-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-069"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-069"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-069-a","issuer-069-b","issuer-069-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-070","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (070).","requirements":[{"id":"req-screen-070","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-070","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-070","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-070","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-070-a","issuer-070-b","issuer-070-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-070","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-070","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-070","description":"Require complete map coverage","requirement_ids":["req-screen-070"],"constraint_ids":["constraint-exclusions-070"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-070","description":"Require citations for every issuer","requirement_ids":["req-report-070"],"constraint_ids":["constraint-definition-070"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-070"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-070-a"},{"ticker":"issuer-070-b"},{"ticker":"issuer-070-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-070"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-070"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-070-a","issuer-070-b","issuer-070-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-071","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (071).","requirements":[{"id":"req-screen-071","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-071","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-071","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-071","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-071-a","issuer-071-b","issuer-071-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-071","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-071","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-071","description":"Require complete map coverage","requirement_ids":["req-screen-071"],"constraint_ids":["constraint-exclusions-071"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-071","description":"Require citations for every issuer","requirement_ids":["req-report-071"],"constraint_ids":["constraint-definition-071"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-071"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-071-a"},{"ticker":"issuer-071-b"},{"ticker":"issuer-071-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-071"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-071"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-071-a","issuer-071-b","issuer-071-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-072","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (072).","requirements":[{"id":"req-screen-072","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-072","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-072","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-072","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-072-a","issuer-072-b","issuer-072-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-072","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-072","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-072","description":"Require complete map coverage","requirement_ids":["req-screen-072"],"constraint_ids":["constraint-exclusions-072"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-072","description":"Require citations for every issuer","requirement_ids":["req-report-072"],"constraint_ids":["constraint-definition-072"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-072"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-072-a"},{"ticker":"issuer-072-b"},{"ticker":"issuer-072-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-072"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-072"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-072-a","issuer-072-b","issuer-072-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-073","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (073).","requirements":[{"id":"req-screen-073","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-073","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-073","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-073","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-073-a","issuer-073-b","issuer-073-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-073","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-073","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-073","description":"Require complete map coverage","requirement_ids":["req-screen-073"],"constraint_ids":["constraint-exclusions-073"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-073","description":"Require citations for every issuer","requirement_ids":["req-report-073"],"constraint_ids":["constraint-definition-073"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-073"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-073-a"},{"ticker":"issuer-073-b"},{"ticker":"issuer-073-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-073"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-073"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-073-a","issuer-073-b","issuer-073-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-074","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (074).","requirements":[{"id":"req-screen-074","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-074","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-074","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-074","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-074-a","issuer-074-b","issuer-074-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-074","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-074","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-074","description":"Require complete map coverage","requirement_ids":["req-screen-074"],"constraint_ids":["constraint-exclusions-074"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-074","description":"Require citations for every issuer","requirement_ids":["req-report-074"],"constraint_ids":["constraint-definition-074"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-074"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-074-a"},{"ticker":"issuer-074-b"},{"ticker":"issuer-074-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-074"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-074"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-074-a","issuer-074-b","issuer-074-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-075","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (075).","requirements":[{"id":"req-screen-075","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-075","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-075","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-075","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-075-a","issuer-075-b","issuer-075-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-075","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-075","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-075","description":"Require complete map coverage","requirement_ids":["req-screen-075"],"constraint_ids":["constraint-exclusions-075"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-075","description":"Require citations for every issuer","requirement_ids":["req-report-075"],"constraint_ids":["constraint-definition-075"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-075"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-075-a"},{"ticker":"issuer-075-b"},{"ticker":"issuer-075-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-075"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-075"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-075-a","issuer-075-b","issuer-075-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-076","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (076).","requirements":[{"id":"req-screen-076","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-076","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-076","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-076","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-076-a","issuer-076-b","issuer-076-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-076","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-076","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-076","description":"Require complete map coverage","requirement_ids":["req-screen-076"],"constraint_ids":["constraint-exclusions-076"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-076","description":"Require citations for every issuer","requirement_ids":["req-report-076"],"constraint_ids":["constraint-definition-076"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-076"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-076-a"},{"ticker":"issuer-076-b"},{"ticker":"issuer-076-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-076"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-076"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-076-a","issuer-076-b","issuer-076-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-077","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (077).","requirements":[{"id":"req-screen-077","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-077","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-077","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-077","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-077-a","issuer-077-b","issuer-077-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-077","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-077","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-077","description":"Require complete map coverage","requirement_ids":["req-screen-077"],"constraint_ids":["constraint-exclusions-077"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-077","description":"Require citations for every issuer","requirement_ids":["req-report-077"],"constraint_ids":["constraint-definition-077"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-077"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-077-a"},{"ticker":"issuer-077-b"},{"ticker":"issuer-077-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-077"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-077"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-077-a","issuer-077-b","issuer-077-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-078","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (078).","requirements":[{"id":"req-screen-078","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-078","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-078","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-078","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-078-a","issuer-078-b","issuer-078-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-078","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-078","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-078","description":"Require complete map coverage","requirement_ids":["req-screen-078"],"constraint_ids":["constraint-exclusions-078"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-078","description":"Require citations for every issuer","requirement_ids":["req-report-078"],"constraint_ids":["constraint-definition-078"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-078"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-078-a"},{"ticker":"issuer-078-b"},{"ticker":"issuer-078-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-078"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-078"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-078-a","issuer-078-b","issuer-078-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-079","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (079).","requirements":[{"id":"req-screen-079","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-079","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-079","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-079","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-079-a","issuer-079-b","issuer-079-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-079","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-079","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-079","description":"Require complete map coverage","requirement_ids":["req-screen-079"],"constraint_ids":["constraint-exclusions-079"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-079","description":"Require citations for every issuer","requirement_ids":["req-report-079"],"constraint_ids":["constraint-definition-079"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-079"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-079-a"},{"ticker":"issuer-079-b"},{"ticker":"issuer-079-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-079"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-079"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-079-a","issuer-079-b","issuer-079-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} diff --git a/crates/moa-eval/scenarios/execution/manifest.toml b/crates/moa-eval/scenarios/execution/manifest.toml index 20cd75a48..4d87de89b 100644 --- a/crates/moa-eval/scenarios/execution/manifest.toml +++ b/crates/moa-eval/scenarios/execution/manifest.toml @@ -7,7 +7,7 @@ count = 328 [contract] path = "contract-recorded.jsonl" -sha256 = "8703c2c3f99dfb583265a51967d1abd49c5de029d20254d90a5853285a1ad575" +sha256 = "a7d2f680db94d508f8265ac8551fcfd46cdc137605b1efd18aa4e45c000e110a" count = 80 [task_quality] diff --git a/crates/moa-eval/src/execution/snapshot.rs b/crates/moa-eval/src/execution/snapshot.rs index b8833034b..abd5bda40 100644 --- a/crates/moa-eval/src/execution/snapshot.rs +++ b/crates/moa-eval/src/execution/snapshot.rs @@ -129,6 +129,8 @@ pub enum ExecutionTaskKindSummary { Review, /// Named external signal wait. WaitSignal, + /// Storage-only timer wait. + WaitUntil, /// Terminal output task. Output, /// Bounded semantic completion verifier. @@ -457,6 +459,7 @@ fn redact_task(record: &ExecutionTaskRecord) -> Result { } => (ExecutionTaskKindSummary::Agent, capability_refs.clone()), LogicalTaskKind::Review { .. } => (ExecutionTaskKindSummary::Review, Vec::new()), LogicalTaskKind::WaitSignal { .. } => (ExecutionTaskKindSummary::WaitSignal, Vec::new()), + LogicalTaskKind::WaitUntil { .. } => (ExecutionTaskKindSummary::WaitUntil, Vec::new()), LogicalTaskKind::Output { .. } => (ExecutionTaskKindSummary::Output, Vec::new()), LogicalTaskKind::CompletionVerifier { .. } => { (ExecutionTaskKindSummary::CompletionVerifier, Vec::new()) diff --git a/crates/moa-eval/tests/eval_offline/execution_snapshot.rs b/crates/moa-eval/tests/eval_offline/execution_snapshot.rs index 89c65f72a..4a6349aa5 100644 --- a/crates/moa-eval/tests/eval_offline/execution_snapshot.rs +++ b/crates/moa-eval/tests/eval_offline/execution_snapshot.rs @@ -9,12 +9,15 @@ use moa_artifacts::execution_plan::{ ExecutionPlanDefinition, ExecutionRequirement, ExecutionTaskOutcome, ExecutionTaskResult, ExecutionUsage, RetryPolicy, }; -use moa_core::types::{ - execution_planning::{ - ExecutionPlannerCallKind, ExecutionPlannerOutcome, ExecutionPlanningAuditEnvelope, - ExecutionPlanningAuditPayload, ExecutionSourceProvenance, +use moa_core::{ + traits::{Identity, IdentityType}, + types::{ + execution_planning::{ + ExecutionPlannerCallKind, ExecutionPlannerOutcome, ExecutionPlanningAuditEnvelope, + ExecutionPlanningAuditPayload, ExecutionSourceProvenance, + }, + identifiers::{SessionId, TenantId, UserId}, }, - identifiers::{SessionId, TenantId, UserId}, }; use moa_eval::execution::{ ExecutionEvalSnapshot, ExecutionHarnessEvidence, ExecutionSessionEventSummary, @@ -25,7 +28,10 @@ use moa_execution::{ ExecutionValidationReport, budget::BudgetLedger, compiler::CanonicalExecutionPlan, - repository::{ExecutionRunRecord, ExecutionSchedulingSnapshot, ExecutionTaskRecord}, + repository::{ + ExecutionActivationState, ExecutionAttemptState, ExecutionRunRecord, + ExecutionSchedulingSnapshot, ExecutionTaskRecord, + }, state::{ ExecutionNodeStatus, ExecutionProjection, ExecutionRunStatus, ExecutionSourceKind, ExecutionTaskId, ExecutionTaskProjection, ExecutionTaskStatus, ExecutionTerminalCause, @@ -193,6 +199,13 @@ fn runtime_parts( planning_context_uid: Uuid::from_u128(0x48f8_f1f3_6a67_c90a_7f8f_2f2f_57f5_c444), planning_context_hash: ExecutionHash::from_bytes([4; 32]), owner_user_id: UserId::new("execution-eval-user"), + admitted_identity: Identity { + identity_type: IdentityType::Operator, + id: Uuid::from_u128(0x38f8_f1f3_6a67_c90a_7f8f_2f2f_57f5_c333), + tenant_id, + api_key_id: None, + acting_on_behalf_of: None, + }, goal: goal(), initial_plan: plan.clone(), active_plan: plan.clone(), @@ -227,6 +240,29 @@ fn runtime_parts( terminal_evidence: terminal.2, terminal_reason: terminal.3, status: run_status, + controller_generation: 1, + activation_state: if run_status.is_terminal() { + ExecutionActivationState::Terminal + } else { + ExecutionActivationState::Advancing + }, + next_wake_at: None, + waiting_since: None, + last_progress_at: now, + pause_requested_at: None, + paused_at: None, + ready_task_count: 0, + active_task_count: 0, + waiting_task_count: 0, + waiting_input_task_count: 0, + waiting_review_task_count: 0, + waiting_signal_task_count: 0, + waiting_timer_task_count: 0, + waiting_external_task_count: 0, + waiting_replan_task_count: 0, + waiting_input_user_task_count: 0, + waiting_input_tenant_admin_task_count: 0, + waiting_input_external_task_count: 0, approved_budget: approved_budget.clone(), reserved: ExecutionEstimate::default(), consumed: estimate, @@ -236,6 +272,7 @@ fn runtime_parts( progress_failed_tasks: failed_tasks, progress_cancelled_tasks: cancelled_tasks, waiting_reasons: Vec::new(), + waiting_reasons_truncated: false, wake_epoch: 1, processed_wake_epoch: 1, next_compensation_sequence: 1, @@ -359,6 +396,20 @@ fn task_record(run_uid: Uuid, item_key: &str, status: ExecutionTaskStatus) -> Ex status, attempt: 1, generation: 1, + attempt_generation: 1, + attempt_state: if terminal { + ExecutionAttemptState::Terminal + } else { + ExecutionAttemptState::Running + }, + attempt_started_at: Some(now), + last_progress_at: now, + attempt_deadline_at: None, + waiting_since: None, + ready_at: None, + external_job_uid: None, + active_dispatch_uid: None, + dispatch_sequence: 0, input: json!({ "issuer": item_key, "secret": RAW_TASK_SECRET }), resume_input_history: Vec::new(), kind: LogicalTaskKind::Capability { @@ -427,6 +478,12 @@ fn canonical_plan(catalog_hash: ExecutionHash) -> CanonicalExecutionPlan { CanonicalExecutionPlan { definition: ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, + input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { + expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::At { + at: fixed_time() + chrono::TimeDelta::hours(1), + }, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + }, input_schema: json!({ "type": "object" }), output_schema: json!({ "type": "object" }), nodes: vec![moa_artifacts::execution_plan::ExecutionNode { @@ -474,7 +531,7 @@ fn budget() -> ExecutionBudgetLimit { max_tasks: Some(10), max_tool_calls: Some(10), max_retrieved_bytes: Some(10), - deadline_at: None, + deadline_at: Some(fixed_time() + chrono::TimeDelta::days(1)), } } diff --git a/crates/moa-execution/src/capability.rs b/crates/moa-execution/src/capability.rs index d7a3cce35..4ccddab63 100644 --- a/crates/moa-execution/src/capability.rs +++ b/crates/moa-execution/src/capability.rs @@ -15,7 +15,7 @@ use moa_core::{ types::{ action_policy::{ActionClass, ActionPolicyEffect, RiskLevel}, identifiers::{ConnectorConnectionId, TenantId}, - tools::IdempotencyClass, + tools::{IdempotencyClass, ToolAsyncMode}, }, }; use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error as _}; @@ -35,6 +35,8 @@ pub const AMENDMENT_OPERATIONS_HASH_DOMAIN: &str = "moa.execution.amendment-oper pub const FAILURE_HASH_DOMAIN: &str = "moa.execution.failure"; /// Domain separator for structured task-output hashes. pub const TASK_OUTPUT_HASH_DOMAIN: &str = "moa.execution.task-output"; +/// Domain separator for persisted deterministic node-aggregate output hashes. +pub const NODE_OUTPUT_HASH_DOMAIN: &str = "moa.execution.node-output"; /// A 32-byte BLAKE3 digest serialized as 64 lowercase hexadecimal characters. #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] @@ -227,8 +229,12 @@ pub struct ExecutionCapability { pub default_effect: ActionPolicyEffect, /// Replay and retry safety classification. pub idempotency_class: IdempotencyClass, + /// Provider completion mode pinned into the catalog hash. + pub async_mode: ToolAsyncMode, /// Resource-execution class. pub execution_class: ExecutionClass, + /// Whether invocation requires a live sandbox workspace. + pub requires_sandbox: bool, /// Source provenance for this catalog entry. pub source: CapabilitySource, /// Canonical policy floor and artifact identity carried to durable dispatch. @@ -260,6 +266,28 @@ impl CapabilityRollbackContract { impl ExecutionCapability { pub(crate) fn validate_policy_context(&self) -> Result<()> { + if matches!( + self.source, + CapabilitySource::HandTool { .. } | CapabilitySource::SkillCode { .. } + ) && !self.requires_sandbox + { + return Err(Error::InvalidProjection { + message: format!( + "capability {} source requires sandbox execution metadata", + self.reference.name + ), + }); + } + if let ToolAsyncMode::MayReturnExternalJob { provider } = &self.async_mode + && provider.trim().is_empty() + { + return Err(Error::InvalidProjection { + message: format!( + "capability {} declares an empty external-job provider key", + self.reference.name + ), + }); + } if self.policy_context.source != self.source { return Err(Error::InvalidProjection { message: format!( @@ -727,6 +755,11 @@ pub fn task_output_hash(output: &Value) -> Result { hash_serializable(TASK_OUTPUT_HASH_DOMAIN, output) } +/// Computes a domain-separated hash of one deterministic node aggregate output. +pub fn node_output_hash(output: &Value) -> Result { + hash_serializable(NODE_OUTPUT_HASH_DOMAIN, output) +} + /// Computes a deterministic version string from owned capability metadata. pub fn capability_version(domain: &str, metadata: &Value) -> Result { Ok(hash_serializable(domain, metadata)?.to_string()) @@ -786,7 +819,7 @@ mod tests { use moa_artifacts::reference::ArtifactRef; use moa_core::types::{ action_policy::{ActionClass, ActionPolicyEffect, RiskLevel}, - tools::IdempotencyClass, + tools::{IdempotencyClass, ToolAsyncMode}, }; use serde_json::json; use uuid::Uuid; @@ -811,7 +844,12 @@ mod tests { risk_level: RiskLevel::Low, default_effect: ActionPolicyEffect::Allow, idempotency_class: IdempotencyClass::Idempotent, + async_mode: ToolAsyncMode::SynchronousOnly, execution_class: ExecutionClass::Data, + requires_sandbox: matches!( + &source, + CapabilitySource::HandTool { .. } | CapabilitySource::SkillCode { .. } + ), source, policy_context, estimate: ExecutionEstimate { @@ -858,6 +896,32 @@ mod tests { ); } + #[test] + fn capability_catalog_hash_pins_async_provider_mode() { + // Pins: changing whether a provider may return an external job changes + // the immutable dispatch contract and therefore must invalidate the + // catalog hash used by compiled plans and durable retries. + let synchronous = capability( + "render", + CapabilitySource::McpTool { + server: "renderer".to_string(), + tool_name: "mcp__renderer__render".to_string(), + remote_name: "render".to_string(), + }, + ); + let mut asynchronous = synchronous.clone(); + asynchronous.async_mode = ToolAsyncMode::MayReturnExternalJob { + provider: "fixture-renderer".to_string(), + }; + + let synchronous = ExecutionCapabilityCatalog::build(vec![synchronous]) + .expect("synchronous catalog must build"); + let asynchronous = ExecutionCapabilityCatalog::build(vec![asynchronous]) + .expect("asynchronous catalog must build"); + + assert_ne!(synchronous.catalog_hash, asynchronous.catalog_hash); + } + #[test] fn capability_catalog_build_rejects_a_reference_claimed_twice() { // Pins: a reference identifies exactly one capability. Two entries under diff --git a/crates/moa-execution/src/compiler/amendment.rs b/crates/moa-execution/src/compiler/amendment.rs index 085a596e1..09b08e1ef 100644 --- a/crates/moa-execution/src/compiler/amendment.rs +++ b/crates/moa-execution/src/compiler/amendment.rs @@ -4,13 +4,13 @@ use super::*; pub(super) fn apply_amendment( amendment: &PlanAmendment, - projection: &ExecutionProjection, + projection: &ExecutionAmendmentProjection, active: &ExecutionPlanDefinition, definition: &mut ExecutionPlanDefinition, report: &mut ExecutionValidationReport, ) { let waiting_replan_nodes = projection - .tasks + .replan_tasks .iter() .filter(|task| task.status == ExecutionTaskStatus::WaitingReplan) .map(|task| task.node_id.as_str()) @@ -205,15 +205,12 @@ pub(super) fn apply_amendment( } } -pub(super) fn node_has_started(node_id: &str, projection: &ExecutionProjection) -> bool { +pub(super) fn node_has_started(node_id: &str, projection: &ExecutionAmendmentProjection) -> bool { projection .node_statuses .get(node_id) .is_some_and(|status| *status != ExecutionNodeStatus::Pending) - || projection - .tasks - .iter() - .any(|task| task.node_id == node_id && task.status != ExecutionTaskStatus::Pending) + || projection.started_node_ids.contains(node_id) } pub(super) fn validate_budget_narrowing( @@ -358,7 +355,7 @@ pub(super) fn map_items_are_equal_or_narrower( pub(super) fn node_is_replaceable( node_id: &str, - projection: &ExecutionProjection, + projection: &ExecutionAmendmentProjection, allow_waiting_replan: bool, ) -> bool { let status = projection @@ -366,22 +363,18 @@ pub(super) fn node_is_replaceable( .get(node_id) .copied() .unwrap_or(ExecutionNodeStatus::Pending); - let task_evidence_is_replaceable = projection - .tasks - .iter() - .filter(|task| task.node_id == node_id) - .all(|task| { - task.status == ExecutionTaskStatus::Pending - || (allow_waiting_replan && task.status == ExecutionTaskStatus::WaitingReplan) - }); - task_evidence_is_replaceable - && (status == ExecutionNodeStatus::Pending - || (allow_waiting_replan && status == ExecutionNodeStatus::Waiting)) + let started = projection.started_node_ids.contains(node_id); + (!started && status == ExecutionNodeStatus::Pending) + || (allow_waiting_replan + && status == ExecutionNodeStatus::Waiting + && projection.replan_tasks.iter().any(|task| { + task.node_id == node_id && task.status == ExecutionTaskStatus::WaitingReplan + })) } pub(super) fn is_downstream_of_completed( node: &ExecutionNode, - projection: &ExecutionProjection, + projection: &ExecutionAmendmentProjection, active: &ExecutionPlanDefinition, ) -> bool { if node.depends_on.iter().any(|dependency| { diff --git a/crates/moa-execution/src/compiler/estimate.rs b/crates/moa-execution/src/compiler/estimate.rs index 313af8a7f..bb1d9a49c 100644 --- a/crates/moa-execution/src/compiler/estimate.rs +++ b/crates/moa-execution/src/compiler/estimate.rs @@ -58,7 +58,7 @@ pub(super) fn estimate_plan( pub(super) fn estimate_remaining_plan( goal: &ExecutionGoalContract, plan: &ExecutionPlanDefinition, - projection: &ExecutionProjection, + projection: &ExecutionAmendmentProjection, catalog: &ExecutionCapabilityCatalog, config: &ExecutionConfig, report: &mut ExecutionValidationReport, @@ -98,18 +98,6 @@ pub(super) fn estimate_remaining_plan( let CompletionCheckKind::AgentVerifier { max_turns, .. } = check.kind else { continue; }; - let node_id = format!("@check/{}", check.id); - if projection.tasks.iter().any(|task| { - task.node_id == node_id - && matches!( - task.status, - ExecutionTaskStatus::Completed - | ExecutionTaskStatus::Failed - | ExecutionTaskStatus::Cancelled - ) - }) { - continue; - } match verifier_estimate(config, max_turns) .and_then(|estimate| total.checked_add(estimate, "remaining verifier estimate")) { @@ -189,6 +177,7 @@ pub(super) fn estimate_node( }), ExecutionOperation::Review { .. } | ExecutionOperation::WaitSignal { .. } + | ExecutionOperation::WaitUntil { .. } | ExecutionOperation::Output { .. } => Ok(ExecutionEstimate { tasks: 1, ..ExecutionEstimate::default() diff --git a/crates/moa-execution/src/compiler/mod.rs b/crates/moa-execution/src/compiler/mod.rs index 5e6ba585c..f8deb1416 100644 --- a/crates/moa-execution/src/compiler/mod.rs +++ b/crates/moa-execution/src/compiler/mod.rs @@ -6,6 +6,9 @@ mod validation; use amendment::*; use estimate::*; +use validation::activation_bounds::{ + validate_completion_activation_bounds, validate_plan_activation_bound, +}; use validation::schema_references::{validate_declared_reference_paths, validate_schemas}; use validation::{ append_artifact_reports, append_error, validate_amendment_reference_narrowing, @@ -18,8 +21,8 @@ use chrono::{DateTime, Utc}; use moa_artifacts::{ execution_plan::{ CompletionCheckKind, ExecutionBudgetLimit, ExecutionGoalContract, ExecutionNode, - ExecutionOperation, ExecutionPlanDefinition, ExecutionReducer, MapTask, PlanAmendment, - PlanAmendmentOperation, + ExecutionOperation, ExecutionPlanDefinition, ExecutionReducer, ExecutionTemporalTarget, + ExecutionWaitExpiryAction, MapTask, PlanAmendment, PlanAmendmentOperation, }, reference::ArtifactRef, validation::{validate_execution_goal_contract, validate_execution_plan_definition}, @@ -38,7 +41,7 @@ use crate::{ ExecutionEstimate, ExecutionHash, canonical_sort_key, catalog_hash, plan_hash, }, schema::validate_instance, - state::{ExecutionNodeStatus, ExecutionProjection, ExecutionTaskStatus}, + state::{ExecutionAmendmentProjection, ExecutionNodeStatus, ExecutionTaskStatus}, }; /// Complete input to deterministic initial execution compilation. @@ -74,7 +77,7 @@ pub struct ValidateAmendmentRequest { /// Restricted pending/downstream patch. pub amendment: PlanAmendment, /// Current durable run projection. - pub projection: ExecutionProjection, + pub projection: ExecutionAmendmentProjection, /// Current immutable capability catalog. pub catalog: ExecutionCapabilityCatalog, /// Original immutable authorization envelope. @@ -197,6 +200,8 @@ pub fn compile(request: CompileExecutionRequest) -> CompileExecutionOutcome { let mut report = ExecutionValidationReport::default(); append_artifact_reports(&request.goal, &request.plan, &mut report); validate_goal_plan_links(&request.goal, &request.plan, &mut report); + validate_plan_activation_bound(&request.plan, &request.config, &mut report); + validate_completion_activation_bounds(&request.goal, &request.config, &mut report); validate_catalog(&request.catalog, &mut report); validate_authorization(&request.authorization, &mut report); validate_schemas(&request.goal, &request.plan, &mut report); @@ -214,18 +219,15 @@ pub fn compile(request: CompileExecutionRequest) -> CompileExecutionOutcome { &request.authorization, &mut report, ); - - if request - .approved_budget - .deadline_at - .is_some_and(|deadline| request.now > deadline) - { - report.error( - "deadline_exceeded", - "approved_budget.deadline_at", - "approved execution deadline has already elapsed", - ); - } + append_execution_config_validation(&request.config, &mut report); + validate_temporal_contract( + &request.plan, + request.approved_budget.deadline_at, + request.now, + &request.config, + "approved_budget.deadline_at", + &mut report, + ); let estimate = estimate_plan( &request.goal, @@ -344,6 +346,8 @@ pub fn validate_amendment(request: ValidateAmendmentRequest) -> AmendmentValidat append_artifact_reports(&request.goal, &definition, &mut report); validate_goal_plan_links(&request.goal, &definition, &mut report); + validate_plan_activation_bound(&definition, &request.config, &mut report); + validate_completion_activation_bounds(&request.goal, &request.config, &mut report); validate_schemas(&request.goal, &definition, &mut report); validate_declared_reference_paths(&request.goal, &definition, &mut report); validate_plan_references( @@ -353,17 +357,15 @@ pub fn validate_amendment(request: ValidateAmendmentRequest) -> AmendmentValidat &mut report, ); - if request - .remaining_budget - .deadline_at - .is_some_and(|deadline| request.now > deadline) - { - report.error( - "deadline_exceeded", - "remaining_budget.deadline_at", - "execution deadline has already elapsed", - ); - } + append_execution_config_validation(&request.config, &mut report); + validate_temporal_contract( + &definition, + request.remaining_budget.deadline_at, + request.now, + &request.config, + "remaining_budget.deadline_at", + &mut report, + ); let full_estimate = estimate_plan( &request.goal, @@ -433,5 +435,189 @@ pub fn validate_amendment(request: ValidateAmendmentRequest) -> AmendmentValidat } } +fn append_execution_config_validation( + config: &ExecutionConfig, + report: &mut ExecutionValidationReport, +) { + if let Err(error) = config.validate() { + report.error("invalid_execution_config", "config", error.to_string()); + } +} + +fn validate_temporal_contract( + plan: &ExecutionPlanDefinition, + deadline_at: Option>, + now: DateTime, + config: &ExecutionConfig, + deadline_path: &str, + report: &mut ExecutionValidationReport, +) { + let Some(deadline_at) = deadline_at else { + report.error( + "missing_deadline", + deadline_path, + "durable execution requires an absolute deadline", + ); + return; + }; + if deadline_at <= now { + report.error( + "deadline_exceeded", + deadline_path, + "execution deadline must be later than the validation time", + ); + return; + } + let horizon_seconds = deadline_at + .signed_duration_since(now) + .to_std() + .map(|duration| duration.as_secs()); + if !matches!( + horizon_seconds, + Ok(seconds) if seconds <= config.maximum_horizon_seconds + ) { + report.error( + "deadline_out_of_horizon", + deadline_path, + format!( + "execution deadline exceeds the configured maximum horizon of {} seconds", + config.maximum_horizon_seconds + ), + ); + } + + validate_wait_policy( + &plan.input_wait_policy, + "plan.input_wait_policy", + now, + deadline_at, + report, + ); + for (index, node) in plan.nodes.iter().enumerate() { + let path = format!("plan.nodes[{index}].operation"); + match &node.operation { + ExecutionOperation::Review { wait_policy, .. } + | ExecutionOperation::WaitSignal { wait_policy, .. } => { + validate_wait_policy( + wait_policy, + &format!("{path}.wait_policy"), + now, + deadline_at, + report, + ); + if let ExecutionWaitExpiryAction::ContinueWith { output } = &wait_policy.on_expiry + && let Err(error) = validate_instance( + &node.output_schema, + output, + "wait_policy.on_expiry.output", + ) + { + append_error( + report, + "invalid_wait_expiry_output", + format!("{path}.wait_policy.on_expiry.output"), + error, + ); + } + } + ExecutionOperation::WaitUntil { wake, result } => { + validate_temporal_target(wake, &format!("{path}.wake"), now, deadline_at, report); + if !value_contains_binding(result) + && let Err(error) = + validate_instance(&node.output_schema, result, "wait_until.result") + { + append_error( + report, + "invalid_wait_until_result", + format!("{path}.result"), + error, + ); + } + } + ExecutionOperation::Capability { .. } + | ExecutionOperation::Agent { .. } + | ExecutionOperation::Map { .. } + | ExecutionOperation::Reduce { .. } + | ExecutionOperation::Output { .. } => {} + } + } +} + +fn validate_wait_policy( + policy: &moa_artifacts::execution_plan::ExecutionWaitPolicy, + path: &str, + now: DateTime, + deadline_at: DateTime, + report: &mut ExecutionValidationReport, +) { + validate_temporal_target( + &policy.expiry, + &format!("{path}.expiry"), + now, + deadline_at, + report, + ); +} + +fn validate_temporal_target( + target: &ExecutionTemporalTarget, + path: &str, + now: DateTime, + deadline_at: DateTime, + report: &mut ExecutionValidationReport, +) { + match target { + ExecutionTemporalTarget::At { at } => { + if *at <= now { + report.error( + "temporal_target_elapsed", + path, + "absolute temporal target must be later than the validation time", + ); + } + if *at >= deadline_at { + report.error( + "temporal_target_after_deadline", + path, + "temporal target must be earlier than the run deadline", + ); + } + } + ExecutionTemporalTarget::After { delay_seconds } => { + let remaining_seconds = deadline_at + .signed_duration_since(now) + .to_std() + .map(|duration| duration.as_secs()) + .unwrap_or_default(); + if *delay_seconds == 0 { + report.error( + "temporal_delay_zero", + path, + "relative temporal delay must be greater than zero", + ); + } else if *delay_seconds >= remaining_seconds { + report.error( + "temporal_target_after_deadline", + path, + "relative temporal delay must fit strictly inside the remaining run horizon", + ); + } + } + } +} + +fn value_contains_binding(value: &Value) -> bool { + match value { + Value::Object(object) => { + object.contains_key("$ref") + || object.contains_key("$item") + || object.contains_key("$item_key") + || object.values().any(value_contains_binding) + } + Value::Array(values) => values.iter().any(value_contains_binding), + Value::Null | Value::Bool(_) | Value::Number(_) | Value::String(_) => false, + } +} + #[cfg(test)] mod tests; diff --git a/crates/moa-execution/src/compiler/tests.rs b/crates/moa-execution/src/compiler/tests.rs index 0dcf35e74..2d2e7412a 100644 --- a/crates/moa-execution/src/compiler/tests.rs +++ b/crates/moa-execution/src/compiler/tests.rs @@ -1,12 +1,13 @@ //! Unit tests for deterministic execution compilation. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use chrono::Utc; use moa_artifacts::execution_plan::{ CompletionCheck, CompletionCheckKind, ExecutionBudgetLimit, ExecutionCancelPolicy, ExecutionGoalContract, ExecutionNode, ExecutionOperation, ExecutionPlanDefinition, - ExecutionRequirement, PlanAmendment, PlanAmendmentOperation, RetryPolicy, + ExecutionRequirement, ExecutionTemporalTarget, ExecutionWaitExpiryAction, ExecutionWaitPolicy, + PlanAmendment, PlanAmendmentOperation, RetryPolicy, }; use moa_config::ExecutionConfig; use serde_json::json; @@ -54,13 +55,14 @@ fn execution_planning_amendment_cannot_remove_sole_goal_serving_output() { node_id: "output".to_string(), }], }, - projection: ExecutionProjection { + projection: ExecutionAmendmentProjection { plan_revision: 1, node_statuses: BTreeMap::from([( "output".to_string(), crate::state::ExecutionNodeStatus::Pending, )]), - tasks: Vec::new(), + started_node_ids: BTreeSet::new(), + replan_tasks: Vec::new(), }, catalog: request.catalog, authorization: request.authorization, @@ -77,6 +79,148 @@ fn execution_planning_amendment_cannot_remove_sole_goal_serving_output() { ); } +#[test] +fn execution_planning_compiler_rejects_completion_metadata_over_activation_bound() { + // Pins: bounded terminal evaluation never inherits an unbounded goal/check collection from + // an otherwise valid plan; completion metadata has its own ceiling within the activation. + let mut request = output_only_compile_request(); + request.config.maximum_activation_steps = 1; + + let outcome = compile(request); + + assert!(outcome.compiled.is_none()); + assert!( + outcome + .report + .issues + .iter() + .any(|issue| { issue.code == "completion_metadata_exceeds_activation_bound" }) + ); +} + +#[test] +fn execution_planning_compiler_rejects_plan_nodes_over_activation_bound() { + // Pins: run admission can seed every node aggregate atomically because canonical plans cannot + // encode high cardinality as nodes; large work must remain inside pageable map/reduce tasks. + let mut request = output_only_compile_request(); + request.config.maximum_activation_steps = 2; + let template = request.plan.nodes[0].clone(); + request.plan.nodes = (0..2) + .map(|index| ExecutionNode { + id: format!("output_{index}"), + ..template.clone() + }) + .collect(); + let exact_bound = compile(request.clone()); + assert!( + exact_bound + .report + .issues + .iter() + .all(|issue| issue.code != "plan_nodes_exceed_activation_bound") + ); + + request.plan.nodes.push(ExecutionNode { + id: "output_2".to_string(), + ..template + }); + + let outcome = compile(request); + + assert!(outcome.compiled.is_none()); + assert!( + outcome + .report + .issues + .iter() + .any(|issue| issue.code == "plan_nodes_exceed_activation_bound") + ); +} + +#[test] +fn execution_planning_amendment_rejects_result_over_activation_bound() { + // Pins: an amendment cannot bypass the initial node bound by appending many small nodes. + let request = output_only_compile_request(); + let compiled = compile(request.clone()) + .compiled + .expect("output-only fixture should compile"); + let template = compiled.plan.definition.nodes[0].clone(); + let outcome = validate_amendment(ValidateAmendmentRequest { + goal: compiled.goal, + active_plan: compiled.plan, + amendment: PlanAmendment { + base_plan_revision: 1, + reason: "Attempt to bypass the node bound".to_string(), + evidence: json!({}), + operations: (0..2) + .map(|index| PlanAmendmentOperation::AddNode { + node: ExecutionNode { + id: format!("extra_{index}"), + ..template.clone() + }, + }) + .collect(), + }, + projection: ExecutionAmendmentProjection { + plan_revision: 1, + node_statuses: BTreeMap::from([( + "output".to_string(), + crate::state::ExecutionNodeStatus::Completed, + )]), + started_node_ids: BTreeSet::from(["output".to_string()]), + replan_tasks: Vec::new(), + }, + catalog: request.catalog, + authorization: request.authorization, + remaining_budget: generous_budget(), + config: ExecutionConfig { + maximum_activation_steps: 2, + ..ExecutionConfig::default() + }, + now: Utc::now(), + }); + + assert!(outcome.plan.is_none()); + assert!( + outcome + .report + .issues + .iter() + .any(|issue| issue.code == "plan_nodes_exceed_activation_bound") + ); +} + +#[test] +fn execution_planning_compiler_rejects_verifiers_over_dispatch_batch() { + // Pins: verifier materialization cannot commit more compute-ready tasks than one hard + // dispatcher batch even when the general activation-step limit is larger. + let mut request = output_only_compile_request(); + request.config.dispatch_batch_size = 1; + for index in 0..2 { + request.goal.completion_checks.push(CompletionCheck { + id: format!("verifier_{index}"), + description: format!("Verify completion pass {index}."), + requirement_ids: vec!["req_report".to_string()], + constraint_ids: Vec::new(), + kind: CompletionCheckKind::AgentVerifier { + instructions: "Return a persisted pass/fail verdict.".to_string(), + max_turns: 1, + }, + }); + } + + let outcome = compile(request); + + assert!(outcome.compiled.is_none()); + assert!( + outcome + .report + .issues + .iter() + .any(|issue| { issue.code == "completion_verifiers_exceed_dispatch_bound" }) + ); +} + fn output_only_compile_request() -> CompileExecutionRequest { let catalog = ExecutionCapabilityCatalog::build(Vec::new()) .expect("empty capability catalog should be valid"); @@ -100,6 +244,10 @@ fn output_only_compile_request() -> CompileExecutionRequest { }, plan: ExecutionPlanDefinition { cancel_policy: ExecutionCancelPolicy::RetainEffects, + input_wait_policy: ExecutionWaitPolicy { + expiry: ExecutionTemporalTarget::After { delay_seconds: 60 }, + on_expiry: ExecutionWaitExpiryAction::FailTask, + }, input_schema: json!({ "type": "object" }), output_schema: json!({ "type": "object" }), nodes: vec![ExecutionNode { @@ -140,6 +288,6 @@ fn generous_budget() -> ExecutionBudgetLimit { max_tasks: Some(100), max_tool_calls: Some(100), max_retrieved_bytes: Some(1_000_000), - deadline_at: None, + deadline_at: Some(Utc::now() + chrono::Duration::days(1)), } } diff --git a/crates/moa-execution/src/compiler/validation.rs b/crates/moa-execution/src/compiler/validation.rs index 1c6b816b2..114d42fbd 100644 --- a/crates/moa-execution/src/compiler/validation.rs +++ b/crates/moa-execution/src/compiler/validation.rs @@ -1,5 +1,6 @@ //! Structural, schema, reference, catalog, and authorization validation. +pub(super) mod activation_bounds; pub(super) mod schema_references; use self::schema_references::validate_one_schema; @@ -375,6 +376,7 @@ pub(super) fn collect_plan_references(plan: &ExecutionPlanDefinition) -> PlanRef }, ExecutionOperation::Review { .. } | ExecutionOperation::WaitSignal { .. } + | ExecutionOperation::WaitUntil { .. } | ExecutionOperation::Output { .. } => {} } if let Some(compensation) = &node.compensation { @@ -554,6 +556,7 @@ pub(super) fn validate_cancel_compensation_coverage( }, ExecutionOperation::Review { .. } | ExecutionOperation::WaitSignal { .. } + | ExecutionOperation::WaitUntil { .. } | ExecutionOperation::Output { .. } => {} } } @@ -639,6 +642,13 @@ pub(super) fn validate_compensations( "compensator must be idempotent for durable retry", ); } + if compensator.requires_sandbox { + report.error( + "sandbox_compensator_unsupported", + format!("{path}.compensator"), + "compensator requires a sandbox workspace, which durable compensation does not support", + ); + } let mut decoded_targets = Vec::>::new(); for (binding_index, binding) in compensation.input_mapping.bindings.iter().enumerate() { @@ -893,6 +903,7 @@ pub(super) fn validate_agent_tool_name_ambiguity( | ExecutionOperation::Reduce { .. } | ExecutionOperation::Review { .. } | ExecutionOperation::WaitSignal { .. } + | ExecutionOperation::WaitUntil { .. } | ExecutionOperation::Output { .. } => {} } } diff --git a/crates/moa-execution/src/compiler/validation/activation_bounds.rs b/crates/moa-execution/src/compiler/validation/activation_bounds.rs new file mode 100644 index 000000000..7ee9f54cb --- /dev/null +++ b/crates/moa-execution/src/compiler/validation/activation_bounds.rs @@ -0,0 +1,87 @@ +//! Activation and dispatch bounds for durable long-horizon execution plans. + +use moa_artifacts::execution_plan::{ + CompletionCheckKind, ExecutionGoalContract, ExecutionPlanDefinition, +}; +use moa_config::ExecutionConfig; + +use crate::compiler::ExecutionValidationReport; + +/// Rejects completion metadata that cannot be evaluated within bounded activations. +pub(in crate::compiler) fn validate_completion_activation_bounds( + goal: &ExecutionGoalContract, + config: &ExecutionConfig, + report: &mut ExecutionValidationReport, +) { + let metadata_count = goal + .requirements + .len() + .saturating_add(goal.constraints.len()) + .saturating_add(goal.deliverables.len()) + .saturating_add(goal.coverage.len()) + .saturating_add(goal.completion_checks.len()); + if metadata_count > config.maximum_activation_steps { + report.error( + "completion_metadata_exceeds_activation_bound", + "goal", + format!( + "completion metadata count {metadata_count} exceeds one activation bound {}", + config.maximum_activation_steps + ), + ); + } + let referenced_node_count = goal + .completion_checks + .iter() + .map(|check| match &check.kind { + CompletionCheckKind::RequiredNodes { node_ids } + | CompletionCheckKind::Citations { node_ids, .. } => node_ids.len(), + CompletionCheckKind::MapCoverage { .. } => 1, + CompletionCheckKind::OutputSchema | CompletionCheckKind::AgentVerifier { .. } => 0, + }) + .fold(0_usize, usize::saturating_add); + if referenced_node_count > config.maximum_activation_steps { + report.error( + "completion_node_references_exceed_activation_bound", + "goal.completion_checks", + format!( + "completion node-reference count {referenced_node_count} exceeds one activation bound {}", + config.maximum_activation_steps + ), + ); + } + let verifier_count = goal + .completion_checks + .iter() + .filter(|check| matches!(check.kind, CompletionCheckKind::AgentVerifier { .. })) + .count(); + if verifier_count > config.dispatch_batch_size { + report.error( + "completion_verifiers_exceed_dispatch_bound", + "goal.completion_checks", + format!( + "completion verifier count {verifier_count} exceeds one dispatch batch {}", + config.dispatch_batch_size + ), + ); + } +} + +/// Rejects plans whose node-state seed or aggregate projection cannot fit one activation bound. +pub(in crate::compiler) fn validate_plan_activation_bound( + plan: &ExecutionPlanDefinition, + config: &ExecutionConfig, + report: &mut ExecutionValidationReport, +) { + if plan.nodes.len() > config.maximum_activation_steps { + report.error( + "plan_nodes_exceed_activation_bound", + "plan.nodes", + format!( + "plan node count {} exceeds one activation bound {}; high-cardinality work must use paged map or reduce tasks", + plan.nodes.len(), + config.maximum_activation_steps + ), + ); + } +} diff --git a/crates/moa-execution/src/compiler/validation/schema_references.rs b/crates/moa-execution/src/compiler/validation/schema_references.rs index 383f839fe..61042fbf2 100644 --- a/crates/moa-execution/src/compiler/validation/schema_references.rs +++ b/crates/moa-execution/src/compiler/validation/schema_references.rs @@ -114,6 +114,13 @@ pub(in crate::compiler) fn validate_declared_reference_paths( &output_schemas, report, ), + ExecutionOperation::WaitUntil { result, .. } => validate_dynamic_reference_paths( + &format!("{root}.operation.result"), + result, + plan, + &output_schemas, + report, + ), ExecutionOperation::Capability { .. } | ExecutionOperation::Agent { .. } | ExecutionOperation::Review { .. } diff --git a/crates/moa-execution/src/completion.rs b/crates/moa-execution/src/completion.rs index 3bcf3dcb1..b503b1b7d 100644 --- a/crates/moa-execution/src/completion.rs +++ b/crates/moa-execution/src/completion.rs @@ -440,11 +440,6 @@ pub fn cancellation_terminal_evidence( plan: &CanonicalExecutionPlan, projection: &ExecutionProjection, ) -> Result { - let declared_requirement_ids = goal - .requirements - .iter() - .map(|requirement| requirement.id.as_str()) - .collect::>(); let completed_node_ids = projection .tasks .iter() @@ -456,11 +451,32 @@ pub fn cancellation_terminal_evidence( }) .map(|task| task.node_id.as_str()) .collect::>(); + cancellation_terminal_evidence_from_completed_nodes(goal, plan, &completed_node_ids) +} + +/// Builds cancellation evidence from a bounded set of completed plan-node identities. +pub fn cancellation_terminal_evidence_from_completed_nodes( + goal: &ExecutionGoalContract, + plan: &CanonicalExecutionPlan, + completed_node_ids: &BTreeSet, +) -> Result +where + S: AsRef + Ord, +{ + let declared_requirement_ids = goal + .requirements + .iter() + .map(|requirement| requirement.id.as_str()) + .collect::>(); let evidenced_requirement_ids = plan .definition .nodes .iter() - .filter(|node| completed_node_ids.contains(node.id.as_str())) + .filter(|node| { + completed_node_ids + .iter() + .any(|completed| completed.as_ref() == node.id.as_str()) + }) .flat_map(|node| node.requirement_ids.iter().map(String::as_str)) .filter(|requirement_id| declared_requirement_ids.contains(requirement_id)) .collect::>(); @@ -849,13 +865,21 @@ fn evaluate_coverage( ExecutionTaskStatus::Completed => { completed_keys.insert(task.item_key.clone()); } - ExecutionTaskStatus::Failed | ExecutionTaskStatus::Cancelled => { + ExecutionTaskStatus::Failed + | ExecutionTaskStatus::UnknownOutcome + | ExecutionTaskStatus::Cancelled => { failed_keys.insert(task.item_key.clone()); } ExecutionTaskStatus::Pending + | ExecutionTaskStatus::Ready | ExecutionTaskStatus::Reserved + | ExecutionTaskStatus::Dispatching | ExecutionTaskStatus::Running | ExecutionTaskStatus::WaitingInput + | ExecutionTaskStatus::WaitingReview + | ExecutionTaskStatus::WaitingSignal + | ExecutionTaskStatus::WaitingTimer + | ExecutionTaskStatus::WaitingExternal | ExecutionTaskStatus::WaitingReplan | ExecutionTaskStatus::Skipped => {} } @@ -938,16 +962,21 @@ fn evaluate_requirements( fn is_blocked(request: &CompletionEvaluationRequest) -> bool { request.projection.tasks.iter().any(|task| { - task.status == ExecutionTaskStatus::WaitingInput - || task.outcome.as_ref().is_some_and(|outcome| { - matches!( - outcome.result, - ExecutionTaskResult::Failed { - class: ExecutionFailureClass::AuthorizationDenied, - .. - } - ) - }) + matches!( + task.status, + ExecutionTaskStatus::WaitingInput + | ExecutionTaskStatus::WaitingReview + | ExecutionTaskStatus::WaitingSignal + | ExecutionTaskStatus::WaitingExternal + ) || task.outcome.as_ref().is_some_and(|outcome| { + matches!( + outcome.result, + ExecutionTaskResult::Failed { + class: ExecutionFailureClass::AuthorizationDenied, + .. + } + ) + }) }) || request.plan.definition.nodes.iter().any(|node| { request.projection.node_statuses.get(&node.id) == Some(&ExecutionNodeStatus::Waiting) && matches!( @@ -1152,6 +1181,7 @@ fn map_capability( | ExecutionOperation::Reduce { .. } | ExecutionOperation::Review { .. } | ExecutionOperation::WaitSignal { .. } + | ExecutionOperation::WaitUntil { .. } | ExecutionOperation::Output { .. } => None, } } diff --git a/crates/moa-execution/src/error.rs b/crates/moa-execution/src/error.rs index caccffc0a..0bc045995 100644 --- a/crates/moa-execution/src/error.rs +++ b/crates/moa-execution/src/error.rs @@ -54,6 +54,12 @@ pub enum Error { /// Resource dimension that overran. dimension: &'static str, }, + /// Fleet or tenant admission has no room for one durable execution resource. + #[error("execution capacity saturated for {dimension}")] + CapacitySaturated { + /// Closed execution-capacity dimension; never includes owner labels. + dimension: &'static str, + }, /// A ledger transition supplied an invalid reservation or usage counter. #[error("invalid budget ledger transition: {message}")] InvalidBudgetLedger { @@ -90,4 +96,73 @@ pub enum Error { /// Database failure with operation context. message: String, }, + /// A database operation failed with its SQLx provenance intact. + #[error("execution repository database error: {source}")] + Database { + /// Original SQLx failure used to determine whether replay is safe. + #[source] + source: sqlx::Error, + }, + /// A shared database helper reported transient storage unavailability. + #[error("execution repository storage unavailable: {message}")] + StorageUnavailable { + /// Human-readable failure context retained by the shared database boundary. + message: String, + }, +} + +impl Error { + /// Returns whether a retry-owning boundary may safely replay this storage failure. + #[must_use] + pub fn is_retryable_storage(&self) -> bool { + match self { + Self::Database { source } => moa_db::is_retryable_sqlx_error(source), + Self::StorageUnavailable { .. } => true, + _ => false, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn repository_error_keeps_sqlx_retry_provenance() { + // Pins: the execution repository preserves a concrete transient SQLx + // failure until the Restate boundary makes the retry decision. + let error = Error::Database { + source: sqlx::Error::PoolClosed, + }; + + assert!(error.is_retryable_storage()); + assert!(matches!( + error, + Error::Database { + source: sqlx::Error::PoolClosed + } + )); + } + + #[test] + fn repository_error_does_not_retry_terminal_storage_or_decode_failures() { + // Pins: repository invariants and corrupt row projections remain terminal + // even though they originate at the database boundary. + for error in [ + Error::Storage { + message: "missing required row".to_string(), + }, + Error::InvalidRepositoryData { + message: "invalid persisted status".to_string(), + }, + Error::Database { + source: sqlx::Error::RowNotFound, + }, + ] { + assert!( + !error.is_retryable_storage(), + "classified {error} as retryable" + ); + } + } } diff --git a/crates/moa-execution/src/interpreter/aggregate.rs b/crates/moa-execution/src/interpreter/aggregate.rs index c44c05616..9895b0c67 100644 --- a/crates/moa-execution/src/interpreter/aggregate.rs +++ b/crates/moa-execution/src/interpreter/aggregate.rs @@ -98,6 +98,7 @@ pub(super) fn derive_aggregate_nodes( | ExecutionOperation::Agent { .. } | ExecutionOperation::Review { .. } | ExecutionOperation::WaitSignal { .. } + | ExecutionOperation::WaitUntil { .. } | ExecutionOperation::Output { .. } => AggregateState::Pending, }; @@ -326,7 +327,9 @@ pub(super) fn derive_reduce_rounds( )?; completed.push(output); } - ExecutionTaskStatus::Failed => return Ok(AggregateState::Failed), + ExecutionTaskStatus::Failed | ExecutionTaskStatus::UnknownOutcome => { + return Ok(AggregateState::Failed); + } ExecutionTaskStatus::Cancelled => return Ok(AggregateState::Cancelled), ExecutionTaskStatus::Skipped => { return Err(Error::InvalidProjection { @@ -334,9 +337,15 @@ pub(super) fn derive_reduce_rounds( }); } ExecutionTaskStatus::Pending + | ExecutionTaskStatus::Ready | ExecutionTaskStatus::Reserved + | ExecutionTaskStatus::Dispatching | ExecutionTaskStatus::Running | ExecutionTaskStatus::WaitingInput + | ExecutionTaskStatus::WaitingReview + | ExecutionTaskStatus::WaitingSignal + | ExecutionTaskStatus::WaitingTimer + | ExecutionTaskStatus::WaitingExternal | ExecutionTaskStatus::WaitingReplan => return Ok(AggregateState::Pending), } } diff --git a/crates/moa-execution/src/interpreter/materialize.rs b/crates/moa-execution/src/interpreter/materialize.rs index d2ed5bfbd..fb12c8386 100644 --- a/crates/moa-execution/src/interpreter/materialize.rs +++ b/crates/moa-execution/src/interpreter/materialize.rs @@ -2,6 +2,385 @@ use super::*; +pub(super) fn materialize_node_page( + request: &ScheduleRequest, + node: &ExecutionNode, + outputs: &BTreeMap, + cursor: u64, + limit: u32, + reduce: Option<&ReduceMaterializationPageInput>, +) -> Result { + if let ExecutionOperation::Map { + items, + item_key, + max_items, + task, + .. + } = &node.operation + { + return materialize_map_page( + request, node, outputs, items, item_key, *max_items, task, cursor, limit, + ); + } + if let ExecutionOperation::Reduce { + items, + max_items, + reducer, + batch_size, + } = &node.operation + { + let reduce = reduce.ok_or_else(|| Error::InvalidProjection { + message: format!("reduce node {} requires a persisted round cursor", node.id), + })?; + return materialize_reduce_page( + request, + node, + outputs, + items, + *max_items, + reducer, + *batch_size, + limit, + reduce, + ); + } + if reduce.is_some() { + return Err(Error::InvalidProjection { + message: format!("non-reduce node {} received a reduce cursor", node.id), + }); + } + if cursor > 0 { + return Ok(NodeMaterializationPage { + tasks: Vec::new(), + next_cursor: cursor, + source_exhausted: true, + reduce_cursor: None, + terminal_output: None, + }); + } + let mut tasks = materialize_node(request, node, outputs)?; + let limit = usize::try_from(limit).map_err(|_| Error::ArithmeticOverflow { + context: format!("node {} materialization page limit", node.id), + })?; + let source_exhausted = tasks.len() <= limit; + tasks.truncate(limit); + let next_cursor = u64::try_from(tasks.len()).map_err(|_| Error::ArithmeticOverflow { + context: format!("node {} materialization page cursor", node.id), + })?; + Ok(NodeMaterializationPage { + tasks, + next_cursor, + source_exhausted, + reduce_cursor: None, + terminal_output: None, + }) +} + +#[allow(clippy::too_many_arguments)] +fn materialize_reduce_page( + request: &ScheduleRequest, + node: &ExecutionNode, + outputs: &BTreeMap, + items: &Value, + max_items: u64, + reducer: &ExecutionReducer, + batch_size: u32, + limit: u32, + cursor: &ReduceMaterializationPageInput, +) -> Result { + if cursor.round == 0 { + return Err(Error::InvalidProjection { + message: format!("reduce node {} round must be one-based", node.id), + }); + } + let batch_size = usize::try_from(batch_size).map_err(|_| Error::ArithmeticOverflow { + context: format!("reduce {} batch size", node.id), + })?; + let limit = usize::try_from(limit).map_err(|_| Error::ArithmeticOverflow { + context: format!("reduce {} page limit", node.id), + })?; + let owned_round_one; + let (page_inputs, round_input_count) = if cursor.round == 1 { + if !cursor.page_inputs.is_empty() { + return Err(Error::InvalidProjection { + message: format!( + "reduce node {} round one reads immutable plan items", + node.id + ), + }); + } + let dependencies = node.depends_on.iter().cloned().collect::>(); + let resolved = resolve_bindings( + items, + &BindingContext { + run_input: &request.run_input, + node_outputs: outputs, + dependencies: &dependencies, + item: None, + item_key: None, + }, + )?; + owned_round_one = resolved.as_array().cloned().ok_or_else(|| Error::Binding { + path: format!("node.{}.operation.items", node.id), + message: "reduce items must resolve to an array".to_string(), + })?; + let actual_count = + u64::try_from(owned_round_one.len()).map_err(|_| Error::ArithmeticOverflow { + context: format!("reduce {} round-one input count", node.id), + })?; + if actual_count == 0 { + return Err(Error::InvalidProjection { + message: format!("reduce {} requires at least one item", node.id), + }); + } + if cursor + .round_input_count + .is_some_and(|persisted| persisted != actual_count) + || actual_count > max_items + { + return Err(Error::InvalidProjection { + message: format!("reduce node {} round-one input count changed", node.id), + }); + } + if actual_count == 1 { + let output = owned_round_one[0].clone(); + validate_instance( + &node.output_schema, + &output, + &format!("node.{}.output", node.id), + )?; + return Ok(NodeMaterializationPage { + tasks: Vec::new(), + next_cursor: cursor.batch_cursor, + source_exhausted: true, + reduce_cursor: Some(ReduceMaterializationCursor { + round: cursor.round, + batch_cursor: cursor.batch_cursor, + round_input_count: actual_count, + }), + terminal_output: Some(output), + }); + } + let start = usize::try_from(cursor.batch_cursor) + .ok() + .and_then(|batch| batch.checked_mul(batch_size)) + .ok_or_else(|| Error::ArithmeticOverflow { + context: format!("reduce {} round-one page start", node.id), + })?; + let requested_items = + limit + .checked_mul(batch_size) + .ok_or_else(|| Error::ArithmeticOverflow { + context: format!("reduce {} round-one page length", node.id), + })?; + let end = start + .saturating_add(requested_items) + .min(owned_round_one.len()); + ( + owned_round_one + .get(start..end) + .ok_or_else(|| Error::InvalidProjection { + message: format!("reduce node {} batch cursor exceeds round input", node.id), + })?, + actual_count, + ) + } else { + let round_input_count = + cursor + .round_input_count + .ok_or_else(|| Error::InvalidProjection { + message: format!( + "reduce node {} later round requires its persisted input count", + node.id + ), + })?; + let start = cursor + .batch_cursor + .checked_mul( + u64::try_from(batch_size).map_err(|_| Error::ArithmeticOverflow { + context: format!("reduce {} batch size", node.id), + })?, + ) + .ok_or_else(|| Error::ArithmeticOverflow { + context: format!("reduce {} page input offset", node.id), + })?; + let remaining = round_input_count.saturating_sub(start); + let expected = remaining.min( + u64::try_from(limit) + .map_err(|_| Error::ArithmeticOverflow { + context: format!("reduce {} page limit", node.id), + })? + .checked_mul( + u64::try_from(batch_size).map_err(|_| Error::ArithmeticOverflow { + context: format!("reduce {} batch size", node.id), + })?, + ) + .ok_or_else(|| Error::ArithmeticOverflow { + context: format!("reduce {} page input length", node.id), + })?, + ); + if u64::try_from(cursor.page_inputs.len()).map_err(|_| Error::ArithmeticOverflow { + context: format!("reduce {} page input count", node.id), + })? != expected + { + return Err(Error::InvalidProjection { + message: format!("reduce node {} page input slice is not contiguous", node.id), + }); + } + (cursor.page_inputs.as_slice(), round_input_count) + }; + + let mut tasks = Vec::with_capacity(page_inputs.len().div_ceil(batch_size)); + for (page_batch, batch) in page_inputs.chunks(batch_size).enumerate() { + let batch_index = cursor + .batch_cursor + .checked_add( + u64::try_from(page_batch).map_err(|_| Error::ArithmeticOverflow { + context: format!("reduce {} page batch index", node.id), + })?, + ) + .ok_or_else(|| Error::ArithmeticOverflow { + context: format!("reduce {} batch index", node.id), + })?; + let item_key = format!("r{}:b{batch_index}", cursor.round); + let input = json!({ + "round": cursor.round, + "batch_index": batch_index, + "items": batch, + }); + validate_reducer_capability_input(request, reducer, &input)?; + let reservation = reducer_reservation(request, reducer, node.retry.max_attempts)?; + tasks.push(logical_task( + request, + node, + item_key, + input, + reducer_kind(reducer), + reservation, + )?); + } + let materialized = u64::try_from(tasks.len()).map_err(|_| Error::ArithmeticOverflow { + context: format!("reduce {} materialized batch count", node.id), + })?; + let next_cursor = cursor + .batch_cursor + .checked_add(materialized) + .ok_or_else(|| Error::ArithmeticOverflow { + context: format!("reduce {} next batch cursor", node.id), + })?; + let total_batches = round_input_count.div_ceil(u64::try_from(batch_size).map_err(|_| { + Error::ArithmeticOverflow { + context: format!("reduce {} batch size", node.id), + } + })?); + Ok(NodeMaterializationPage { + tasks, + next_cursor, + source_exhausted: next_cursor == total_batches, + reduce_cursor: Some(ReduceMaterializationCursor { + round: cursor.round, + batch_cursor: cursor.batch_cursor, + round_input_count, + }), + terminal_output: None, + }) +} + +#[allow(clippy::too_many_arguments)] +fn materialize_map_page( + request: &ScheduleRequest, + node: &ExecutionNode, + outputs: &BTreeMap, + items: &Value, + item_key_pointer: &str, + max_items: u64, + task: &MapTask, + cursor: u64, + limit: u32, +) -> Result { + let dependencies = node.depends_on.iter().cloned().collect::>(); + let base = BindingContext { + run_input: &request.run_input, + node_outputs: outputs, + dependencies: &dependencies, + item: None, + item_key: None, + }; + let resolved = resolve_bindings(items, &base)?; + let values = resolved.as_array().ok_or_else(|| Error::Binding { + path: format!("node.{}.operation.items", node.id), + message: "map items must resolve to an array".to_string(), + })?; + let count = u64::try_from(values.len()).map_err(|_| Error::ArithmeticOverflow { + context: format!("map {} item count", node.id), + })?; + if count > max_items { + return Err(Error::InvalidProjection { + message: format!("map {} exceeds max_items", node.id), + }); + } + if cursor > count { + return Err(Error::InvalidProjection { + message: format!("map {} materialization cursor exceeds item count", node.id), + }); + } + if values.is_empty() { + let output = json!({ "items": [] }); + validate_instance( + &node.output_schema, + &output, + &format!("node.{}.output", node.id), + )?; + return Ok(NodeMaterializationPage { + tasks: Vec::new(), + next_cursor: cursor, + source_exhausted: true, + reduce_cursor: None, + terminal_output: Some(output), + }); + } + let start = usize::try_from(cursor).map_err(|_| Error::ArithmeticOverflow { + context: format!("map {} materialization cursor", node.id), + })?; + let end_u64 = cursor.saturating_add(u64::from(limit)).min(count); + let end = usize::try_from(end_u64).map_err(|_| Error::ArithmeticOverflow { + context: format!("map {} materialization page end", node.id), + })?; + let mut page = Vec::with_capacity(end.saturating_sub(start)); + let mut page_keys = BTreeSet::new(); + for item in &values[start..end] { + let item_key = extract_map_key(item, item_key_pointer)?; + if !page_keys.insert(item_key.clone()) { + return Err(Error::InvalidProjection { + message: format!("map {} produced duplicate item key {item_key}", node.id), + }); + } + let context = BindingContext { + item: Some(item), + item_key: Some(&item_key), + ..base + }; + let input = resolve_bindings(&node.input, &context)?; + validate_map_capability_input(request, task, &input)?; + let reservation = map_task_reservation(request, task, node.retry.max_attempts)?; + page.push(logical_task( + request, + node, + item_key, + input, + map_kind(task), + reservation, + )?); + } + Ok(NodeMaterializationPage { + tasks: page, + next_cursor: end_u64, + source_exhausted: end_u64 == count, + reduce_cursor: None, + terminal_output: None, + }) +} + pub(super) fn materialize_node( request: &ScheduleRequest, node: &ExecutionNode, @@ -41,6 +420,7 @@ pub(super) fn materialize_node( | ExecutionOperation::Agent { .. } | ExecutionOperation::Review { .. } | ExecutionOperation::WaitSignal { .. } + | ExecutionOperation::WaitUntil { .. } | ExecutionOperation::Output { .. } => { if request .projection @@ -321,11 +701,23 @@ pub(super) fn logical_kind( capability_refs: capability_refs.clone(), max_turns: *max_turns, }), - ExecutionOperation::Review { prompt } => Ok(LogicalTaskKind::Review { + ExecutionOperation::Review { + prompt, + wait_policy, + } => Ok(LogicalTaskKind::Review { prompt: prompt.clone(), + wait_policy: wait_policy.clone(), }), - ExecutionOperation::WaitSignal { signal_name } => Ok(LogicalTaskKind::WaitSignal { + ExecutionOperation::WaitSignal { + signal_name, + wait_policy, + } => Ok(LogicalTaskKind::WaitSignal { signal_name: signal_name.clone(), + wait_policy: wait_policy.clone(), + }), + ExecutionOperation::WaitUntil { wake, result } => Ok(LogicalTaskKind::WaitUntil { + wake: wake.clone(), + result: resolve_bindings(result, context)?, }), ExecutionOperation::Output { value } => Ok(LogicalTaskKind::Output { value: resolve_bindings(value, context)?, diff --git a/crates/moa-execution/src/interpreter/mod.rs b/crates/moa-execution/src/interpreter/mod.rs index 629860aa0..16a750824 100644 --- a/crates/moa-execution/src/interpreter/mod.rs +++ b/crates/moa-execution/src/interpreter/mod.rs @@ -5,6 +5,7 @@ mod compensation; mod materialize; mod projection; mod reservation; +mod temporal_wait; mod terminal; use aggregate::*; @@ -12,15 +13,24 @@ pub use compensation::resolve_compensation_input; use materialize::*; use projection::*; use reservation::*; +use temporal_wait::*; use terminal::*; +/// Derives the bounded reservation for one persisted completion verifier. +pub(crate) fn verifier_turn_reservation( + config: &ExecutionConfig, + max_turns: u32, +) -> Result { + turn_reservation(config, max_turns, 1, true) +} + use std::collections::{BTreeMap, BTreeSet}; use chrono::{DateTime, Utc}; use moa_artifacts::execution_plan::{ CapabilityReference, CompletionCheckKind, ExecutionFailureClass, ExecutionGoalContract, ExecutionNode, ExecutionOperation, ExecutionReducer, ExecutionTaskOutcome, ExecutionTaskResult, - MapTask, RetryPolicy, + ExecutionTemporalTarget, MapTask, RetryPolicy, }; use moa_config::ExecutionConfig; @@ -106,6 +116,7 @@ fn task_output_schema(node: &ExecutionNode) -> &Value { | ExecutionOperation::Reduce { .. } | ExecutionOperation::Review { .. } | ExecutionOperation::WaitSignal { .. } + | ExecutionOperation::WaitUntil { .. } | ExecutionOperation::Output { .. } => &node.output_schema, } } @@ -144,6 +155,120 @@ pub struct ScheduleOutcome { pub effective_projection: ExecutionProjection, } +/// One bounded deterministic logical-task page for a single eligible node. +#[derive(Clone, Debug, PartialEq)] +pub struct NodeMaterializationPage { + /// Stable logical tasks for this source cursor page. + pub tasks: Vec, + /// Cursor immediately after this page. + pub next_cursor: u64, + /// Whether the deterministic materialization source is exhausted. + pub source_exhausted: bool, + /// Exact reduce-round source fence used for this page, when this is a reduce node. + pub reduce_cursor: Option, + /// Aggregate output for a source that completes without creating a logical task. + pub terminal_output: Option, +} + +/// Exact reduce-round source position used to derive one materialization page. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ReduceMaterializationCursor { + /// One-based reduction round. + pub round: u64, + /// Number of batches committed before this page. + pub batch_cursor: u64, + /// Total input values consumed by this round. + pub round_input_count: u64, +} + +/// Bounded source slice and persisted cursor for one reduce round. +#[derive(Clone, Debug, PartialEq)] +pub struct ReduceMaterializationPageInput { + /// One-based reduction round. + pub round: u64, + /// Number of batches already materialized in this round. + pub batch_cursor: u64, + /// Persisted input count, or `None` only while round one resolves immutable plan items. + pub round_input_count: Option, + /// Exact contiguous input values for this page; empty in round one, whose source is the plan. + pub page_inputs: Vec, +} + +/// Materializes only one bounded eligible-node page without a full task projection. +pub fn materialize_node_page( + request: &ScheduleRequest, + node_id: &str, + referenced_outputs: &BTreeMap, + cursor: u64, + limit: u32, + reduce: Option<&ReduceMaterializationPageInput>, +) -> Result { + if limit == 0 || limit > 1_000 { + return Err(Error::InvalidProjection { + message: "node materialization page limit must be 1..=1000".to_string(), + }); + } + validate_scheduler_catalog(&request.catalog)?; + let node = request + .plan + .definition + .nodes + .iter() + .find(|node| node.id == node_id) + .ok_or_else(|| Error::InvalidProjection { + message: format!("active plan has no node `{node_id}`"), + })?; + if node + .depends_on + .iter() + .any(|dependency| !referenced_outputs.contains_key(dependency)) + { + return Err(Error::InvalidProjection { + message: format!("node `{node_id}` is missing a direct dependency output"), + }); + } + materialize::materialize_node_page(request, node, referenced_outputs, cursor, limit, reduce) +} + +/// Resolves an exact or wait-entry-relative temporal target and fences it by the run deadline. +pub fn resolve_temporal_target( + target: &ExecutionTemporalTarget, + wait_entered_at: DateTime, + run_deadline_at: DateTime, +) -> Result> { + let due_at = match target { + ExecutionTemporalTarget::At { at } => *at, + ExecutionTemporalTarget::After { delay_seconds } => { + if *delay_seconds == 0 { + return Err(Error::InvalidProjection { + message: "relative temporal delay must be greater than zero".to_string(), + }); + } + let seconds = i64::try_from(*delay_seconds).map_err(|_| Error::InvalidProjection { + message: "relative temporal delay exceeds supported timestamp range".to_string(), + })?; + let delay = chrono::TimeDelta::try_seconds(seconds).ok_or_else(|| { + Error::InvalidProjection { + message: "relative temporal delay exceeds supported timestamp range" + .to_string(), + } + })?; + wait_entered_at + .checked_add_signed(delay) + .ok_or_else(|| Error::InvalidProjection { + message: "relative temporal target overflows supported timestamp range" + .to_string(), + })? + } + }; + if due_at >= run_deadline_at { + return Err(Error::InvalidProjection { + message: "temporal target must be earlier than the run deadline".to_string(), + }); + } + Ok(due_at) +} + /// Returns map nodes whose first deterministic materialization contains zero items. /// /// The repository uses these node IDs to persist a zero-fan-out marker even though @@ -232,7 +357,7 @@ pub fn schedule(mut request: ScheduleRequest) -> Result { .budget_ledger .limit .deadline_at - .is_some_and(|deadline| request.now > deadline) + .is_some_and(|deadline| request.now >= deadline) { let decision = completion_terminal( &request, @@ -244,6 +369,13 @@ pub fn schedule(mut request: ScheduleRequest) -> Result { }); } + if let Some(settlement) = ready_wait_settlement(&request, &outputs)? { + return Ok(ScheduleOutcome { + decision: ScheduleDecision::SettleWait(settlement), + effective_projection: request.projection, + }); + } + let mut ready = Vec::new(); let mut dependency_waits = BTreeSet::new(); for node in &request.plan.definition.nodes { diff --git a/crates/moa-execution/src/interpreter/projection.rs b/crates/moa-execution/src/interpreter/projection.rs index 8afbe0c67..02a875c3b 100644 --- a/crates/moa-execution/src/interpreter/projection.rs +++ b/crates/moa-execution/src/interpreter/projection.rs @@ -49,6 +49,7 @@ pub(super) fn validate_projection(request: &ScheduleRequest) -> Result<()> { task.status, ExecutionTaskStatus::Completed | ExecutionTaskStatus::Failed + | ExecutionTaskStatus::UnknownOutcome | ExecutionTaskStatus::Cancelled | ExecutionTaskStatus::WaitingInput | ExecutionTaskStatus::WaitingReplan diff --git a/crates/moa-execution/src/interpreter/reservation.rs b/crates/moa-execution/src/interpreter/reservation.rs index ecfcbb3e6..20cace7c6 100644 --- a/crates/moa-execution/src/interpreter/reservation.rs +++ b/crates/moa-execution/src/interpreter/reservation.rs @@ -16,6 +16,7 @@ pub(super) fn operation_reservation( } ExecutionOperation::Review { .. } | ExecutionOperation::WaitSignal { .. } + | ExecutionOperation::WaitUntil { .. } | ExecutionOperation::Output { .. } => Ok(ExecutionEstimate { tasks: 1, ..ExecutionEstimate::default() diff --git a/crates/moa-execution/src/interpreter/temporal_wait.rs b/crates/moa-execution/src/interpreter/temporal_wait.rs new file mode 100644 index 000000000..04cc94300 --- /dev/null +++ b/crates/moa-execution/src/interpreter/temporal_wait.rs @@ -0,0 +1,196 @@ +//! Pure projection and due-settlement decisions for storage-only execution waits. + +use std::collections::{BTreeMap, BTreeSet}; + +use chrono::{DateTime, Utc}; +use moa_artifacts::execution_plan::{ + ExecutionOperation, ExecutionTaskResult, ExecutionTemporalTarget, +}; +use serde_json::Value; + +use super::ScheduleRequest; +use crate::{ + Result, + bindings::{BindingContext, resolve_bindings}, + state::{ExecutionNodeStatus, ExecutionTaskStatus, WaitSettlement, WaitingReason}, +}; + +pub(super) fn waiting_reasons( + request: &ScheduleRequest, + dependency_waits: BTreeSet, +) -> Vec { + let by_id = request + .plan + .definition + .nodes + .iter() + .map(|node| (node.id.as_str(), node)) + .collect::>(); + let mut waiting = Vec::new(); + if request.projection.tasks.iter().any(|task| { + matches!( + task.status, + ExecutionTaskStatus::Pending + | ExecutionTaskStatus::Ready + | ExecutionTaskStatus::Reserved + | ExecutionTaskStatus::Dispatching + | ExecutionTaskStatus::Running + | ExecutionTaskStatus::WaitingReplan + ) + }) { + waiting.push(WaitingReason::RunningTasks); + } + for task in &request.projection.tasks { + if task.status == ExecutionTaskStatus::WaitingExternal { + waiting.push(WaitingReason::External { + task_id: task.task_id, + }); + } + if task.status == ExecutionTaskStatus::WaitingInput + && let Some(outcome) = &task.outcome + && let ExecutionTaskResult::NeedsInput { question, audience } = &outcome.result + { + waiting.push(WaitingReason::Input { + task_id: task.task_id, + audience: audience.clone(), + question: question.clone(), + wait_policy: request.plan.definition.input_wait_policy.clone(), + }); + } + if request.projection.node_statuses.get(&task.node_id) + == Some(&ExecutionNodeStatus::Waiting) + && let Some(node) = by_id.get(task.node_id.as_str()) + { + match &node.operation { + ExecutionOperation::Review { + prompt, + wait_policy, + } => waiting.push(WaitingReason::Review { + task_id: task.task_id, + prompt: prompt.clone(), + wait_policy: wait_policy.clone(), + }), + ExecutionOperation::WaitSignal { + signal_name, + wait_policy, + } => { + waiting.push(WaitingReason::Signal { + task_id: task.task_id, + signal_name: signal_name.clone(), + wait_policy: wait_policy.clone(), + }); + } + ExecutionOperation::WaitUntil { wake, .. } => { + waiting.push(WaitingReason::Timer { + task_id: task.task_id, + wake: wake.clone(), + }); + } + ExecutionOperation::Capability { .. } + | ExecutionOperation::Agent { .. } + | ExecutionOperation::Map { .. } + | ExecutionOperation::Reduce { .. } + | ExecutionOperation::Output { .. } => {} + } + } + } + if !dependency_waits.is_empty() { + waiting.push(WaitingReason::Dependencies { + node_ids: dependency_waits.into_iter().collect(), + }); + } + waiting +} + +pub(super) fn ready_wait_settlement( + request: &ScheduleRequest, + outputs: &BTreeMap, +) -> Result> { + let by_id = request + .plan + .definition + .nodes + .iter() + .map(|node| (node.id.as_str(), node)) + .collect::>(); + let mut settlements = Vec::new(); + for task in &request.projection.tasks { + if task.status == ExecutionTaskStatus::WaitingInput + && temporal_target_is_due( + &request.plan.definition.input_wait_policy.expiry, + request.now, + ) + { + settlements.push(( + task.task_id, + WaitSettlement::WaitExpired { + task_id: task.task_id, + action: request.plan.definition.input_wait_policy.on_expiry.clone(), + }, + )); + continue; + } + let Some(node) = by_id.get(task.node_id.as_str()) else { + continue; + }; + let settlement = match &node.operation { + ExecutionOperation::Review { wait_policy, .. } + if matches!( + task.status, + ExecutionTaskStatus::Running | ExecutionTaskStatus::WaitingReview + ) && temporal_target_is_due(&wait_policy.expiry, request.now) => + { + Some(WaitSettlement::WaitExpired { + task_id: task.task_id, + action: wait_policy.on_expiry.clone(), + }) + } + ExecutionOperation::WaitSignal { wait_policy, .. } + if matches!( + task.status, + ExecutionTaskStatus::Running | ExecutionTaskStatus::WaitingSignal + ) && temporal_target_is_due(&wait_policy.expiry, request.now) => + { + Some(WaitSettlement::WaitExpired { + task_id: task.task_id, + action: wait_policy.on_expiry.clone(), + }) + } + ExecutionOperation::WaitUntil { wake, result } + if matches!( + task.status, + ExecutionTaskStatus::Running | ExecutionTaskStatus::WaitingTimer + ) && temporal_target_is_due(wake, request.now) => + { + let dependencies = node.depends_on.iter().cloned().collect::>(); + let output = resolve_bindings( + result, + &BindingContext { + run_input: &request.run_input, + node_outputs: outputs, + dependencies: &dependencies, + item: None, + item_key: None, + }, + )?; + Some(WaitSettlement::TimerElapsed { + task_id: task.task_id, + output, + }) + } + _ => None, + }; + if let Some(settlement) = settlement { + settlements.push((task.task_id, settlement)); + } + } + settlements.sort_by_key(|(task_id, _)| *task_id); + Ok(settlements + .into_iter() + .next() + .map(|(_, settlement)| settlement)) +} + +fn temporal_target_is_due(target: &ExecutionTemporalTarget, now: DateTime) -> bool { + matches!(target, ExecutionTemporalTarget::At { at } if now >= *at) +} diff --git a/crates/moa-execution/src/interpreter/terminal.rs b/crates/moa-execution/src/interpreter/terminal.rs index 9eea56fe8..9c192ca98 100644 --- a/crates/moa-execution/src/interpreter/terminal.rs +++ b/crates/moa-execution/src/interpreter/terminal.rs @@ -1,72 +1,8 @@ -//! Waiting and terminal scheduler decisions. +//! Terminal completion and verifier scheduler decisions. +use super::temporal_wait::waiting_reasons; use super::*; -pub(super) fn waiting_reasons( - request: &ScheduleRequest, - dependency_waits: BTreeSet, -) -> Vec { - let by_id = request - .plan - .definition - .nodes - .iter() - .map(|node| (node.id.as_str(), node)) - .collect::>(); - let mut waiting = Vec::new(); - if request.projection.tasks.iter().any(|task| { - matches!( - task.status, - ExecutionTaskStatus::Pending - | ExecutionTaskStatus::Reserved - | ExecutionTaskStatus::Running - | ExecutionTaskStatus::WaitingReplan - ) - }) { - waiting.push(WaitingReason::RunningTasks); - } - for task in &request.projection.tasks { - if task.status == ExecutionTaskStatus::WaitingInput - && let Some(outcome) = &task.outcome - && let ExecutionTaskResult::NeedsInput { question, audience } = &outcome.result - { - waiting.push(WaitingReason::Input { - task_id: task.task_id, - audience: audience.clone(), - question: question.clone(), - }); - } - if request.projection.node_statuses.get(&task.node_id) - == Some(&ExecutionNodeStatus::Waiting) - && let Some(node) = by_id.get(task.node_id.as_str()) - { - match &node.operation { - ExecutionOperation::Review { prompt } => waiting.push(WaitingReason::Review { - task_id: task.task_id, - prompt: prompt.clone(), - }), - ExecutionOperation::WaitSignal { signal_name } => { - waiting.push(WaitingReason::Signal { - task_id: task.task_id, - signal_name: signal_name.clone(), - }); - } - ExecutionOperation::Capability { .. } - | ExecutionOperation::Agent { .. } - | ExecutionOperation::Map { .. } - | ExecutionOperation::Reduce { .. } - | ExecutionOperation::Output { .. } => {} - } - } - } - if !dependency_waits.is_empty() { - waiting.push(WaitingReason::Dependencies { - node_ids: dependency_waits.into_iter().collect(), - }); - } - waiting -} - pub(super) fn schedule_verifiers_or_complete(request: ScheduleRequest) -> Result { let terminal = terminal_output(&request.plan, &request.projection); let preliminary = evaluate_completion(CompletionEvaluationRequest { @@ -143,6 +79,7 @@ pub(super) fn schedule_verifiers_or_complete(request: ScheduleRequest) -> Result task.status, ExecutionTaskStatus::Completed | ExecutionTaskStatus::Failed + | ExecutionTaskStatus::UnknownOutcome | ExecutionTaskStatus::Cancelled ) }) { @@ -369,6 +306,7 @@ pub(super) fn operation_capability(operation: &ExecutionOperation) -> Option None, } } @@ -401,6 +339,7 @@ pub(super) const fn is_terminal_task_status(status: ExecutionTaskStatus) -> bool ExecutionTaskStatus::Completed | ExecutionTaskStatus::Skipped | ExecutionTaskStatus::Failed + | ExecutionTaskStatus::UnknownOutcome | ExecutionTaskStatus::Cancelled ) } diff --git a/crates/moa-execution/src/interpreter/tests.rs b/crates/moa-execution/src/interpreter/tests.rs index d8b38e54c..e75bf6c97 100644 --- a/crates/moa-execution/src/interpreter/tests.rs +++ b/crates/moa-execution/src/interpreter/tests.rs @@ -4,7 +4,8 @@ use chrono::{Duration, Utc}; use moa_artifacts::execution_plan::{ CapabilityReference, CompensationInputMapping, ExecutionBudgetLimit, ExecutionCancelPolicy, ExecutionCompensation, ExecutionGoalContract, ExecutionNode, ExecutionOperation, - ExecutionPlanDefinition, MapTask, RetryPolicy, + ExecutionPlanDefinition, ExecutionTemporalTarget, ExecutionWaitExpiryAction, + ExecutionWaitPolicy, MapTask, RetryPolicy, }; use super::*; @@ -22,6 +23,7 @@ fn map_execution_task_validates_the_item_output_schema() { let plan = CanonicalExecutionPlan { definition: ExecutionPlanDefinition { cancel_policy: ExecutionCancelPolicy::RetainEffects, + input_wait_policy: test_input_wait_policy(), input_schema: serde_json::json!({}), output_schema: serde_json::json!({}), nodes: vec![ExecutionNode { @@ -136,6 +138,7 @@ fn empty_map_is_reported_as_first_materialization_without_a_logical_task() { plan: CanonicalExecutionPlan { definition: ExecutionPlanDefinition { cancel_policy: ExecutionCancelPolicy::RetainEffects, + input_wait_policy: test_input_wait_policy(), input_schema: serde_json::json!({}), output_schema: serde_json::json!({}), nodes: vec![map_node], @@ -230,6 +233,7 @@ fn only_direct_capability_task_materializes_compensation_contract() { plan: CanonicalExecutionPlan { definition: ExecutionPlanDefinition { cancel_policy: ExecutionCancelPolicy::CompensateCommitted, + input_wait_policy: test_input_wait_policy(), input_schema: serde_json::json!({}), output_schema: serde_json::json!({}), nodes: vec![direct_node.clone()], @@ -309,3 +313,10 @@ fn only_direct_capability_task_materializes_compensation_contract() { .expect("aggregate capability task should materialize"); assert_eq!(aggregate.compensation, None); } + +fn test_input_wait_policy() -> ExecutionWaitPolicy { + ExecutionWaitPolicy { + expiry: ExecutionTemporalTarget::After { delay_seconds: 60 }, + on_expiry: ExecutionWaitExpiryAction::FailTask, + } +} diff --git a/crates/moa-execution/src/lib.rs b/crates/moa-execution/src/lib.rs index 7d81186ea..8026653d4 100644 --- a/crates/moa-execution/src/lib.rs +++ b/crates/moa-execution/src/lib.rs @@ -42,7 +42,10 @@ pub use completion::{ evaluate_completion, execution_terminal_reason, }; pub use error::Error; -pub use interpreter::{ScheduleRequest, ready_empty_map_nodes, schedule}; +pub use interpreter::{ + NodeMaterializationPage, ReduceMaterializationCursor, ReduceMaterializationPageInput, + ScheduleRequest, materialize_node_page, ready_empty_map_nodes, schedule, +}; pub use replan::{ReplanDecision, ReplanEvaluationRequest, ReplanStopReason, evaluate_replan_stop}; pub use state::{ExecutionSourceKind, ExecutionTerminalReason}; diff --git a/crates/moa-execution/src/repository/admission.rs b/crates/moa-execution/src/repository/admission.rs index 42d3c4af0..dfae3ab7b 100644 --- a/crates/moa-execution/src/repository/admission.rs +++ b/crates/moa-execution/src/repository/admission.rs @@ -1,7 +1,16 @@ //! Row-locked external-effect admission for forward tasks and compensations. +use moa_core::types::action_policy::{ + ActionReviewOwner, ExecutionCompensationOrigin, ExecutionTaskOrigin, +}; + use super::*; -use super::{rows::*, sql::*}; +use super::{ + compensation::CompensationAttemptState, + rows::*, + sql::*, + task::{checkpoint_review_uid, task_checkpoint_from_row}, +}; impl ExecutionRepository { /// Linearizes one external-effect admission against the run's terminal fence. @@ -23,21 +32,20 @@ impl ExecutionRepository { .map_err(sqlx_error)? else { conn.commit().await.map_err(storage_error)?; - return Ok(ExecutionEffectAdmissionOutcome::Rejected( - ExecutionToolDispatchRejection::OriginNotFound, - )); + return Ok(rejected(ExecutionToolDispatchRejection::OriginNotFound)); }; let run = run_from_row(&run_row)?; if run.session_id != session_id { conn.commit().await.map_err(storage_error)?; - return Ok(ExecutionEffectAdmissionOutcome::Rejected( - ExecutionToolDispatchRejection::OriginNotFound, - )); + return Ok(rejected(ExecutionToolDispatchRejection::OriginNotFound)); } + let rejection = match owner { ExecutionEffectOwner::Task { task_id, generation, + attempt_generation, + phase, } => { let Some(task_row) = sqlx::query(LOAD_TASK_FOR_UPDATE_SQL) .bind(run_uid) @@ -47,9 +55,7 @@ impl ExecutionRepository { .map_err(sqlx_error)? else { conn.commit().await.map_err(storage_error)?; - return Ok(ExecutionEffectAdmissionOutcome::Rejected( - ExecutionToolDispatchRejection::OriginNotFound, - )); + return Ok(rejected(ExecutionToolDispatchRejection::OriginNotFound)); }; let task = task_from_row(&task_row)?; if run.status.is_terminal() @@ -58,9 +64,13 @@ impl ExecutionRepository { || run.manual_repair_required { Some(ExecutionToolDispatchRejection::RunNotDispatchable) - } else if task.generation != generation { + } else if task.generation != generation + || task.attempt_generation != attempt_generation + { Some(ExecutionToolDispatchRejection::StaleGeneration) - } else if task.status != ExecutionTaskStatus::Running { + } else if !task_effect_phase_is_current(&mut conn, &run, &task, phase, session_id) + .await? + { Some(ExecutionToolDispatchRejection::OperationNotRunning) } else { None @@ -69,28 +79,45 @@ impl ExecutionRepository { ExecutionEffectOwner::Compensation { compensation_id, generation, + attempt_generation, + phase, } => { - let Some(compensation_row) = sqlx::query(LOAD_COMPENSATION_FOR_UPDATE_SQL) - .bind(run_uid) - .bind(compensation_id.as_uuid()) - .fetch_optional(conn.as_mut()) - .await - .map_err(sqlx_error)? + let Some(compensation_row) = sqlx::query( + "SELECT * FROM moa.execution_compensation \ + WHERE run_uid = $1 AND compensation_id = $2 FOR UPDATE", + ) + .bind(run_uid) + .bind(compensation_id.as_uuid()) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? else { conn.commit().await.map_err(storage_error)?; - return Ok(ExecutionEffectAdmissionOutcome::Rejected( - ExecutionToolDispatchRejection::OriginNotFound, - )); + return Ok(rejected(ExecutionToolDispatchRejection::OriginNotFound)); }; let compensation = compensation_from_row(&compensation_row)?; + let current_attempt_generation = + required_u64(&compensation_row, "attempt_generation")?; if run.status != ExecutionRunStatus::Compensating || run.pending_terminal.is_none() || run.manual_repair_required { Some(ExecutionToolDispatchRejection::RunNotDispatchable) - } else if compensation.generation != generation { + } else if compensation.generation != generation + || current_attempt_generation != attempt_generation + { Some(ExecutionToolDispatchRejection::StaleGeneration) - } else if compensation.status != CompensationStatus::Running { + } else if !compensation_effect_phase_is_current( + &mut conn, + &run, + &compensation, + &compensation_row, + attempt_generation, + phase, + session_id, + ) + .await? + { Some(ExecutionToolDispatchRejection::OperationNotRunning) } else { None @@ -98,10 +125,160 @@ impl ExecutionRepository { } }; conn.commit().await.map_err(storage_error)?; - Ok( - rejection.map_or(ExecutionEffectAdmissionOutcome::Admitted, |reason| { - ExecutionEffectAdmissionOutcome::Rejected(reason) - }), - ) + Ok(rejection.map_or(ExecutionEffectAdmissionOutcome::Admitted, rejected)) + } +} + +const fn rejected(reason: ExecutionToolDispatchRejection) -> ExecutionEffectAdmissionOutcome { + ExecutionEffectAdmissionOutcome::Rejected(reason) +} + +async fn task_effect_phase_is_current( + conn: &mut ScopedConn<'_>, + run: &ExecutionRunRecord, + task: &ExecutionTaskRecord, + phase: ExecutionEffectPhase, + session_id: SessionId, +) -> Result { + match phase { + ExecutionEffectPhase::Direct => Ok(task.status == ExecutionTaskStatus::Running + && task.attempt_state == ExecutionAttemptState::Running), + ExecutionEffectPhase::Reviewed { review_uid } => { + if review_uid.is_nil() + || task.status != ExecutionTaskStatus::WaitingReview + || task.attempt_state != ExecutionAttemptState::Waiting + { + return Ok(false); + } + let checkpoint = sqlx::query( + "SELECT * FROM moa.execution_task_checkpoint \ + WHERE tenant_id=$1 AND run_uid=$2 AND task_id=$3 \ + AND superseded_at IS NULL FOR UPDATE", + ) + .bind(run.tenant_id.0) + .bind(run.run_uid) + .bind(task.task_id.as_uuid()) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(checkpoint) = checkpoint else { + return Err(Error::InvalidRepositoryData { + message: "waiting-review task has no current attempt checkpoint".to_string(), + }); + }; + let checkpoint = task_checkpoint_from_row(&checkpoint)?; + if checkpoint.controller_generation != run.controller_generation + || checkpoint.task_generation != task.generation + || checkpoint.attempt_generation != task.attempt_generation + || checkpoint_review_uid(&checkpoint.payload) != Some(review_uid) + { + return Ok(false); + } + let expected_owner = ActionReviewOwner::ExecutionTask { + session_id, + origin: ExecutionTaskOrigin { + run_uid: run.run_uid, + task_uid: task.task_id.as_uuid(), + generation: task.generation, + attempt_generation: task.attempt_generation, + }, + }; + current_claimed_review_matches( + conn, + run.tenant_id, + session_id, + review_uid, + &expected_owner, + ) + .await + } } } + +async fn compensation_effect_phase_is_current( + conn: &mut ScopedConn<'_>, + run: &ExecutionRunRecord, + compensation: &CompensationRegistrationProjection, + row: &PgRow, + attempt_generation: u64, + phase: ExecutionEffectPhase, + session_id: SessionId, +) -> Result { + let attempt_state = CompensationAttemptState::from_str( + &row.try_get::("attempt_state") + .map_err(row_error)?, + )?; + match phase { + ExecutionEffectPhase::Direct => Ok(compensation.status == CompensationStatus::Running + && attempt_state == CompensationAttemptState::Running), + ExecutionEffectPhase::Reviewed { review_uid } => { + if review_uid.is_nil() + || compensation.status != CompensationStatus::Running + || attempt_state != CompensationAttemptState::WaitingReview + { + return Ok(false); + } + let persisted = row + .try_get::, _>("outcome") + .map_err(row_error)? + .map(serde_json::from_value::) + .transpose()? + .ok_or_else(|| Error::InvalidRepositoryData { + message: "waiting-review compensation has no persisted review audit" + .to_string(), + })?; + if !persisted.review_audit.iter().any(|entry| { + entry.review_uid == review_uid + && entry.generation == compensation.generation + && !entry.accepted + }) { + return Ok(false); + } + let expected_owner = ActionReviewOwner::ExecutionCompensation { + session_id, + origin: ExecutionCompensationOrigin { + run_uid: run.run_uid, + compensation_id: compensation.compensation_id.as_uuid(), + generation: compensation.generation, + attempt_generation, + }, + }; + current_claimed_review_matches( + conn, + run.tenant_id, + session_id, + review_uid, + &expected_owner, + ) + .await + } + } +} + +async fn current_claimed_review_matches( + conn: &mut ScopedConn<'_>, + tenant_id: TenantId, + session_id: SessionId, + review_uid: Uuid, + expected_owner: &ActionReviewOwner, +) -> Result { + let expected_owner = serde_json::to_value(expected_owner)?; + let row = sqlx::query_scalar::<_, bool>( + "SELECT TRUE FROM public.tenant_action_reviews \ + WHERE tenant_id=$1 AND session_id=$2 AND id=$3 \ + AND status='pending' AND owner_registered_at IS NOT NULL \ + AND execution_requested_at IS NOT NULL AND execution_tool_call_id IS NOT NULL \ + AND owner_release_delivered_at IS NULL \ + AND envelope ->> 'review_id' = $3::TEXT \ + AND envelope -> 'owner' = $4 \ + FOR UPDATE", + ) + .bind(tenant_id.0) + .bind(session_id.0) + .bind(review_uid) + .bind(expected_owner) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + Ok(row.is_some()) +} diff --git a/crates/moa-execution/src/repository/amendment.rs b/crates/moa-execution/src/repository/amendment.rs new file mode 100644 index 000000000..7c5306ab2 --- /dev/null +++ b/crates/moa-execution/src/repository/amendment.rs @@ -0,0 +1,207 @@ +//! Bounded persisted evidence for restricted execution-plan amendments. + +use std::collections::BTreeMap; + +use moa_config::ExecutionConfig; + +use super::*; +use super::{outcome_support::task_failure_fingerprint_input, projection::budget_ledger, rows::*}; +use crate::{replan::failure_fingerprint, state::ExecutionAmendmentProjection}; + +/// One session- and revision-fenced amendment evidence request. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct AmendmentProjectionRequest { + /// Run being amended. + pub run_uid: Uuid, + /// Parent session that owns the mutation boundary. + pub session_id: SessionId, + /// Exact active plan revision accepted by the amendment. + pub expected_plan_revision: u64, +} + +/// Compact compiler and loop-stop evidence loaded in one strictly bounded transaction. +#[derive(Clone, Debug, PartialEq)] +pub struct ExecutionAmendmentSnapshot { + /// Canonical run, immutable pins, and exact generation counters. + pub run: ExecutionRunRecord, + /// Current persisted budget ledger. + pub budget_ledger: BudgetLedger, + /// Compiler-bounded aggregate node state and exact replan origin. + pub projection: ExecutionAmendmentProjection, + /// Prior persisted occurrences of the current normalized failure fingerprint. + pub prior_failure_fingerprint_counts: BTreeMap, +} + +/// Result of the one-call bounded amendment projection load. +#[derive(Clone, Debug, PartialEq)] +pub enum AmendmentProjectionOutcome { + /// Every bounded evidence source is ready for pure amendment validation. + Ready(Box), + /// No run exists under the supplied tenant/contact/session scope. + NotFound, + /// The requested plan revision or replan origin is no longer current. + Conflict, +} + +impl ExecutionRepository { + /// Loads compiler-bounded node aggregates, one replan task, and one indexed scalar count. + pub async fn load_amendment_projection_for_session( + &self, + scope: ExecutionScope, + config: &ExecutionConfig, + request: AmendmentProjectionRequest, + ) -> Result { + let mut conn = scope.begin(&self.pool).await?; + let Some(run_row) = sqlx::query( + "SELECT * FROM moa.execution_run WHERE run_uid=$1 AND session_id=$2 FOR UPDATE", + ) + .bind(request.run_uid) + .bind(request.session_id.0) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + conn.commit().await.map_err(storage_error)?; + return Ok(AmendmentProjectionOutcome::NotFound); + }; + let run = run_from_row(&run_row)?; + if run.plan_revision != request.expected_plan_revision + || run.status != ExecutionRunStatus::WaitingReplan + { + conn.commit().await.map_err(storage_error)?; + return Ok(AmendmentProjectionOutcome::Conflict); + } + if run.active_plan.definition.nodes.len() > config.maximum_activation_steps { + return Err(Error::InvalidRepositoryData { + message: "persisted plan exceeds its compiler-validated node activation bound" + .to_string(), + }); + } + + let replan_rows = sqlx::query( + "SELECT *, failure_fingerprint AS persisted_failure_fingerprint \ + FROM moa.execution_task WHERE run_uid=$1 AND status='waiting_replan' \ + ORDER BY task_id LIMIT 2", + ) + .bind(run.run_uid) + .fetch_all(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let [replan_row] = replan_rows.as_slice() else { + conn.commit().await.map_err(storage_error)?; + return Ok(AmendmentProjectionOutcome::Conflict); + }; + let replan_task = task_from_row(replan_row)?; + let failure = task_failure_fingerprint_input(&replan_task).ok_or_else(|| { + Error::InvalidRepositoryData { + message: "waiting-replan task has no fingerprintable persisted outcome".to_string(), + } + })?; + let fingerprint = failure_fingerprint(&failure)?; + let fingerprint_text = fingerprint.to_string(); + let persisted_fingerprint: Option = replan_row + .try_get("persisted_failure_fingerprint") + .map_err(row_error)?; + if persisted_fingerprint.as_deref() != Some(fingerprint_text.as_str()) { + return Err(Error::InvalidRepositoryData { + message: "waiting-replan task failure fingerprint does not match its outcome" + .to_string(), + }); + } + let prior_count = sqlx::query_scalar::<_, i64>( + "SELECT COUNT(*) FROM moa.execution_task \ + WHERE run_uid=$1 AND task_id<>$2 AND failure_fingerprint=$3", + ) + .bind(run.run_uid) + .bind(replan_task.task_id.as_uuid()) + .bind(&fingerprint_text) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let prior_count = u32::try_from(prior_count).map_err(|_| Error::ArithmeticOverflow { + context: "prior amendment failure fingerprint count".to_string(), + })?; + + let node_limit = i64::try_from(config.maximum_activation_steps) + .map_err(|_| Error::ArithmeticOverflow { + context: "amendment node activation bound".to_string(), + })? + .checked_add(1) + .ok_or_else(|| Error::ArithmeticOverflow { + context: "amendment node activation bound".to_string(), + })?; + let node_rows = sqlx::query( + "SELECT node_id,node_status,total_task_count FROM moa.execution_node_state \ + WHERE run_uid=$1 AND node_id NOT LIKE '@check/%' \ + ORDER BY node_order,node_state_uid LIMIT $2", + ) + .bind(run.run_uid) + .bind(node_limit) + .fetch_all(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if node_rows.len() != run.active_plan.definition.nodes.len() + || node_rows.len() > config.maximum_activation_steps + { + return Err(Error::InvalidRepositoryData { + message: "amendment node aggregates do not exactly cover the bounded active plan" + .to_string(), + }); + } + let mut node_statuses = BTreeMap::new(); + let mut started_node_ids = std::collections::BTreeSet::new(); + for row in node_rows { + let node_id: String = row.try_get("node_id").map_err(row_error)?; + let status = amendment_node_status( + &row.try_get::("node_status").map_err(row_error)?, + )?; + if status != ExecutionNodeStatus::Pending || required_u64(&row, "total_task_count")? > 0 + { + started_node_ids.insert(node_id.clone()); + } + node_statuses.insert(node_id, status); + } + let projection = ExecutionAmendmentProjection { + plan_revision: run.plan_revision, + node_statuses, + started_node_ids, + replan_tasks: vec![task_projection(&replan_task)], + }; + let snapshot = ExecutionAmendmentSnapshot { + budget_ledger: budget_ledger(&run), + run, + projection, + prior_failure_fingerprint_counts: BTreeMap::from([(fingerprint, prior_count)]), + }; + conn.commit().await.map_err(storage_error)?; + Ok(AmendmentProjectionOutcome::Ready(Box::new(snapshot))) + } +} + +fn task_projection(task: &ExecutionTaskRecord) -> ExecutionTaskProjection { + ExecutionTaskProjection { + task_id: task.task_id, + node_id: task.node_id.clone(), + item_key: task.item_key.clone(), + status: task.status, + attempt: task.attempt, + generation: task.generation, + input: task.input.clone(), + outcome: task.current_outcome.clone(), + } +} + +fn amendment_node_status(value: &str) -> Result { + match value { + "pending" => Ok(ExecutionNodeStatus::Pending), + "ready" | "running" => Ok(ExecutionNodeStatus::Running), + "waiting" => Ok(ExecutionNodeStatus::Waiting), + "completed" => Ok(ExecutionNodeStatus::Completed), + "skipped" => Ok(ExecutionNodeStatus::Skipped), + "failed" => Ok(ExecutionNodeStatus::Failed), + "cancelled" => Ok(ExecutionNodeStatus::Cancelled), + other => Err(Error::InvalidRepositoryData { + message: format!("unknown amendment node status `{other}`"), + }), + } +} diff --git a/crates/moa-execution/src/repository/audit.rs b/crates/moa-execution/src/repository/audit.rs index 1126bb31f..fc8e2d949 100644 --- a/crates/moa-execution/src/repository/audit.rs +++ b/crates/moa-execution/src/repository/audit.rs @@ -1,6 +1,135 @@ //! Execution-template admission and normalized planning audit persistence. use super::*; + +/// Input used to insert one immutable origin-bound planning-context snapshot. +#[derive(Clone, Debug)] +pub struct NewExecutionPlanningContext { + /// Exact immutable snapshot whose canonical bytes are hashed. + pub snapshot: ExecutionPlanningContextSnapshot, + /// Domain-separated hash of the canonical snapshot bytes. + pub planning_context_hash: ExecutionHash, +} + +/// Persisted immutable planning-context projection. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct ExecutionPlanningContextRecord { + /// Durable planning-context identifier. + pub planning_context_uid: Uuid, + /// Exact immutable snapshot. + pub snapshot: ExecutionPlanningContextSnapshot, + /// Domain-separated hash of the canonical snapshot bytes. + pub planning_context_hash: ExecutionHash, + /// Database-owned creation timestamp. + pub created_at: DateTime, +} + +/// Result of inserting or replaying one unique origin-bound planning context. +#[derive(Clone, Debug, PartialEq)] +pub enum PlanningContextWriteOutcome { + /// The immutable snapshot was inserted. + Created(ExecutionPlanningContextRecord), + /// The exact immutable snapshot already existed for the origin. + Replayed(ExecutionPlanningContextRecord), + /// The unique origin already exists with different immutable bytes or scope. + Conflict, +} + +/// Persisted low-cardinality evidence for one route-audit insertion. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct RouteAuditEvidence { + /// Deterministic UUIDv5 audit identifier. + pub audit_uid: Uuid, + /// Respond, Execute, or NeedsInput decision. + pub decision: ExecutionRouteKind, + /// Selected strategy, present exactly for Execute. + pub strategy: Option, + /// Redacted trusted-bypass or classifier provenance. + pub provenance: ExecutionRouteProvenance, + /// First durable acceptance timestamp. + pub accepted_at: DateTime, +} + +/// Durable result of inserting one normalized route audit. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub enum RouteAuditWriteOutcome { + /// This transaction inserted the first route row. + Applied(RouteAuditEvidence), + /// The exact semantic route row already existed. + Replayed(RouteAuditEvidence), + /// The logical key already carries different route semantics. + Conflict { + /// Deterministic audit identifier for the conflicting logical key. + audit_uid: Uuid, + }, +} + +/// Persisted low-cardinality evidence for one planner-call audit insertion. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct PlannerCallAuditEvidence { + /// Deterministic UUIDv5 audit identifier. + pub audit_uid: Uuid, + /// Exact closed planner call kind. + pub call: ExecutionPlannerCallKind, + /// Exact closed planner outcome. + pub outcome: ExecutionPlannerOutcome, + /// First persisted measured duration. + pub duration_micros: u64, + /// Candidate hash when required by the outcome. + pub candidate_hash: Option, +} + +/// Durable result of inserting one normalized planner-call audit. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub enum PlannerCallAuditWriteOutcome { + /// This transaction inserted the first planner-call row. + Applied(PlannerCallAuditEvidence), + /// The exact semantic planner-call row already existed. + Replayed(PlannerCallAuditEvidence), + /// The logical key already carries different planner-call semantics. + Conflict { + /// Deterministic audit identifier for the conflicting logical key. + audit_uid: Uuid, + }, +} + +/// Persisted low-cardinality evidence for one compiler-audit insertion. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct CompileAuditEvidence { + /// Deterministic UUIDv5 audit identifier. + pub audit_uid: Uuid, + /// Exact closed compiler source. + pub source: ExecutionCompileSource, + /// Exact closed compiler outcome. + pub outcome: ExecutionCompileOutcome, + /// First persisted measured duration. + pub duration_micros: u64, + /// Hash of the strict compile candidate. + pub candidate_hash: String, + /// Accepted final plan hash, when compilation succeeded. + pub final_plan_hash: Option, +} + +/// Durable result of inserting one normalized compiler audit. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub enum CompileAuditWriteOutcome { + /// This transaction inserted the first compiler row. + Applied(CompileAuditEvidence), + /// The exact semantic compiler row already existed. + Replayed(CompileAuditEvidence), + /// The logical key already carries different compiler semantics. + Conflict { + /// Deterministic audit identifier for the conflicting logical key. + audit_uid: Uuid, + }, +} + +const LOAD_PLANNING_CONTEXT_FOR_SESSION_SQL: &str = r#" + SELECT planning_context_uid, snapshot, planning_context_hash, created_at + FROM moa.execution_planning_context + WHERE planning_context_uid = $1 + AND session_id = $2 +"#; use super::{audit_codec::*, rows::*, sql::*}; impl ExecutionRepository { @@ -177,6 +306,24 @@ impl ExecutionRepository { row.as_ref().map(planning_context_from_row).transpose() } + /// Loads one immutable planning context only when it belongs to the expected session. + pub async fn load_planning_context_for_session( + &self, + scope: ExecutionScope, + planning_context_uid: Uuid, + expected_session_id: SessionId, + ) -> Result> { + let mut conn = scope.begin(&self.pool).await?; + let row = sqlx::query(LOAD_PLANNING_CONTEXT_FOR_SESSION_SQL) + .bind(planning_context_uid) + .bind(expected_session_id.0) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + conn.commit().await.map_err(storage_error)?; + row.as_ref().map(planning_context_from_row).transpose() + } + /// Inserts or exactly replays one normalized route-audit row. pub async fn write_route_audit( &self, diff --git a/crates/moa-execution/src/repository/audit_codec.rs b/crates/moa-execution/src/repository/audit_codec.rs index 624959bcc..6f6366076 100644 --- a/crates/moa-execution/src/repository/audit_codec.rs +++ b/crates/moa-execution/src/repository/audit_codec.rs @@ -1,5 +1,6 @@ //! Normalized planning-audit identities, labels, and row codecs. +use super::audit::{CompileAuditEvidence, PlannerCallAuditEvidence, RouteAuditEvidence}; use super::rows::{optional_u64, required_u64}; use super::*; diff --git a/crates/moa-execution/src/repository/capacity.rs b/crates/moa-execution/src/repository/capacity.rs new file mode 100644 index 000000000..2cbf4cc42 --- /dev/null +++ b/crates/moa-execution/src/repository/capacity.rs @@ -0,0 +1,1563 @@ +//! PostgreSQL-owned fleet and tenant admission for bounded execution attempts. + +use chrono::{DateTime, Duration, Utc}; +use moa_config::ExecutionConfig; + +use crate::wire::ExecutionTaskAttemptRequest; + +use super::*; +use super::{ + materialize::DbEstimate, + outbox::{ExecutionDispatchKind, NewExecutionDispatch, enqueue_dispatch_in_conn}, + ready::transition_node_counters_in_tx, + rows::*, + run::active_run_capacity_request, + sql::*, + trigger::{ExecutionTriggerKind, NewExecutionTrigger, create_trigger_with_dispatch_in_conn}, +}; + +const MAX_ADMISSION_BATCH: u32 = 1_000; +const FAIRNESS_QUANTUM: i64 = 1_000_000; +const CAPACITY_RESERVATION_NAMESPACE: Uuid = + Uuid::from_u128(0x5b72_581c_d6f1_5a0b_9097_a267_eb1c_18d4); + +/// Closed execution resource dimensions enforced by fleet and tenant buckets. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ExecutionCapacityDimension { + /// One admitted nonterminal execution run. + ActiveRuns, + /// One active forward or compensation attempt. + ActiveTasks, + /// One run parked entirely in durable storage. + ParkedRuns, + /// One pending durable trigger. + ScheduledTriggers, + /// One nonterminal provider-owned external job. + ExternalJobs, +} + +impl ExecutionCapacityDimension { + /// Returns the stable PostgreSQL resource-dimension discriminator. + pub const fn as_str(self) -> &'static str { + match self { + Self::ActiveRuns => "active_runs", + Self::ActiveTasks => "active_tasks", + Self::ParkedRuns => "parked_runs", + Self::ScheduledTriggers => "scheduled_triggers", + Self::ExternalJobs => "external_jobs", + } + } + + const fn limits(self, config: &ExecutionConfig) -> (u32, u32) { + match self { + Self::ActiveRuns => (config.max_fleet_active_runs, config.max_tenant_active_runs), + Self::ActiveTasks => ( + config.max_fleet_active_tasks, + config.max_tenant_active_tasks, + ), + Self::ParkedRuns => (config.max_fleet_parked_runs, config.max_tenant_parked_runs), + Self::ScheduledTriggers => ( + config.max_fleet_scheduled_triggers, + config.max_tenant_scheduled_triggers, + ), + Self::ExternalJobs => ( + config.max_fleet_external_jobs, + config.max_tenant_external_jobs, + ), + } + } + + const fn lock_order(self) -> u8 { + match self { + Self::ActiveRuns => 0, + Self::ActiveTasks => 1, + Self::ParkedRuns => 2, + Self::ScheduledTriggers => 3, + Self::ExternalJobs => 4, + } + } +} + +/// Exact owner columns for one execution capacity reservation. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ExecutionCapacityOwner { + /// Run lifetime or parked-generation capacity. + Run, + /// Pending durable trigger capacity. + Trigger { trigger_uid: Uuid }, + /// Provider-owned external job capacity. + ExternalJob { external_job_uid: Uuid }, +} + +/// Generic capacity request shared by run, trigger, and external-job transactions. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ExecutionCapacityRequest { + /// Deterministic idempotency identity for this exact owner fence. + pub reservation_uid: Uuid, + /// Owning tenant. + pub tenant_id: TenantId, + /// Owning execution run, absent only for schedule-owned triggers. + pub run_uid: Option, + /// Run generation that owns the reservation, absent only with `run_uid`. + pub controller_generation: Option, + /// Closed resource dimension. + pub dimension: ExecutionCapacityDimension, + /// Exact owner-column shape. + pub owner: ExecutionCapacityOwner, + /// Optional expiry used by repair scans. + pub expires_at: Option>, +} + +/// Idempotent generic capacity admission outcome. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum CapacityReserveOutcome { + /// Both counters and the exact receipt were committed. + Reserved, + /// The exact active receipt already exists. + Replayed, + /// Fleet or tenant capacity is exhausted; no counter changed. + Saturated, +} + +/// Admission outcome for ActiveRuns plus its mandatory parked-capacity entitlement. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum ActiveRunCapacityReserveOutcome { + /// Both counters and the exact ActiveRuns receipt were committed. + Reserved, + /// The exact ActiveRuns receipt already exists. + Replayed, + /// The named fleet or tenant ceiling rejected admission without changing counters. + Saturated(ExecutionCapacityDimension), +} + +/// Derives a stable reservation UID from one immutable resource owner fence. +#[must_use] +pub fn execution_capacity_reservation_uid( + dimension: ExecutionCapacityDimension, + owner_uid: Uuid, + controller_generation: Option, +) -> Uuid { + let name = controller_generation.map_or_else( + || format!("{}:{owner_uid}", dimension.as_str()), + |generation| format!("{}:{owner_uid}:{generation}", dimension.as_str()), + ); + Uuid::new_v5(&CAPACITY_RESERVATION_NAMESPACE, name.as_bytes()) +} + +/// Builds the exact receipt requested when one controller wake parks a run in storage. +#[must_use] +pub(super) fn parked_run_capacity_request( + run: &ExecutionRunRecord, + wake_epoch: u64, +) -> ExecutionCapacityRequest { + let owner_name = format!( + "parked_runs:{}:{}:{wake_epoch}", + run.run_uid, run.controller_generation + ); + ExecutionCapacityRequest { + reservation_uid: Uuid::new_v5(&CAPACITY_RESERVATION_NAMESPACE, owner_name.as_bytes()), + tenant_id: run.tenant_id, + run_uid: Some(run.run_uid), + controller_generation: Some(run.controller_generation), + dimension: ExecutionCapacityDimension::ParkedRuns, + owner: ExecutionCapacityOwner::Run, + expires_at: None, + } +} + +/// One task attempt admitted atomically with capacity, outbox, and watchdog state. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ExecutionAdmissionItem { + /// Immutable dispatch identity and bounded workflow key. + pub dispatch_uid: Uuid, + /// Exact active-capacity reservation released by attempt settlement. + pub capacity_reservation_uid: Uuid, + /// Exact watchdog trigger for the admitted attempt generation. + pub watchdog_trigger_uid: Uuid, + /// Durable delayed-delivery dispatch for the exact watchdog trigger. + pub watchdog_dispatch_uid: Uuid, + /// Tenant that owns the attempt. + pub tenant_id: TenantId, + /// Run that owns the logical task. + pub run_uid: Uuid, + /// Stable logical task identifier. + pub task_id: ExecutionTaskId, + /// Current run-controller generation. + pub controller_generation: u64, + /// Current bounded task-attempt generation. + pub attempt_generation: u64, + /// Absolute deadline enforced by the watchdog. + pub attempt_deadline_at: DateTime, +} + +/// Result of one bounded fleet admission pass. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ExecutionAdmissionBatch { + /// Attempts committed for durable dispatch in weighted-fair order. + pub admitted: Vec, + /// Earliest useful retry time when capacity or ready work prevented a full batch. + pub retry_after: Option>, + /// Oldest ready-queue timestamp observed by the locked admission snapshot. + pub oldest_ready_at: Option>, +} + +/// Result of releasing one exact active task-capacity reservation. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum CapacityReleaseOutcome { + /// The reservation and both authoritative counters were released. + Released, + /// The same reservation had already been released. + AlreadyReleased, + /// No matching reservation exists. + NotFound, + /// The reservation belongs to another attempt generation or dispatch. + Stale, +} + +impl ExecutionRepository { + /// Admits a bounded weighted-fair batch of globally ready task attempts. + /// + /// PostgreSQL is the sole correctness owner. Each item commits the Ready-to-Dispatching + /// transition, run-budget reservation, fleet and tenant capacity, task-attempt outbox, and + /// watchdog delivery before it is returned to the dispatcher. + pub async fn admit_ready_attempts( + &self, + config: &ExecutionConfig, + requested_limit: u32, + now: DateTime, + ) -> Result { + let limit = requested_limit.min(MAX_ADMISSION_BATCH); + if limit == 0 { + return Ok(ExecutionAdmissionBatch { + admitted: Vec::new(), + retry_after: None, + oldest_ready_at: None, + }); + } + let deadline = now + .checked_add_signed(Duration::seconds( + i64::try_from(config.active_attempt_timeout_seconds).map_err(|_| { + Error::InvalidRepositoryInput { + message: "active attempt timeout exceeds chrono duration".to_string(), + } + })?, + )) + .ok_or_else(|| Error::InvalidRepositoryInput { + message: "active attempt deadline is not representable".to_string(), + })?; + let retry_after = now + .checked_add_signed(Duration::seconds( + i64::try_from(config.trigger_reconciliation_cadence_seconds).map_err(|_| { + Error::InvalidRepositoryInput { + message: "execution reconciliation cadence exceeds chrono duration" + .to_string(), + } + })?, + )) + .ok_or_else(|| Error::InvalidRepositoryInput { + message: "execution retry time is not representable".to_string(), + })?; + + let mut conn = ExecutionScope::ControlPlane.begin(&self.pool).await?; + let oldest_ready_at = sqlx::query_scalar::<_, Option>>( + "SELECT MIN(ready_at) FROM moa.execution_task WHERE status='ready'", + ) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + ensure_fleet_bucket( + conn.as_mut(), + "active_tasks", + i64::from(config.max_fleet_active_tasks), + ) + .await?; + let fleet_available = lock_capacity_bucket( + conn.as_mut(), + "fleet", + None, + "active_tasks", + i64::from(config.max_fleet_active_tasks), + ) + .await?; + if fleet_available == 0 { + conn.commit().await.map_err(storage_error)?; + return Ok(ExecutionAdmissionBatch { + admitted: Vec::new(), + retry_after: Some(retry_after), + oldest_ready_at, + }); + } + + let bounded_limit = + usize::try_from(u64::from(limit).min(fleet_available)).map_err(|_| { + Error::InvalidRepositoryInput { + message: "admission batch does not fit in memory".to_string(), + } + })?; + let mut admitted = Vec::with_capacity(bounded_limit); + let mut saturated_tenants = Vec::::new(); + let mut exhausted_runs = Vec::::new(); + while admitted.len() < bounded_limit { + let Some(tenant_id) = select_fair_ready_tenant( + &mut conn, + config.max_in_flight_tasks, + &saturated_tenants, + &exhausted_runs, + now, + ) + .await? + else { + break; + }; + ensure_tenant_bucket( + conn.as_mut(), + TenantId::from(tenant_id), + "active_tasks", + i64::from(config.max_tenant_active_tasks), + ) + .await?; + let tenant_available = lock_capacity_bucket( + conn.as_mut(), + "tenant", + Some(tenant_id), + "active_tasks", + i64::from(config.max_tenant_active_tasks), + ) + .await?; + if tenant_available == 0 { + saturated_tenants.push(tenant_id); + continue; + } + + let Some((run, task)) = lock_oldest_ready_task( + &mut conn, + TenantId::from(tenant_id), + config.max_in_flight_tasks, + &exhausted_runs, + now, + ) + .await? + else { + saturated_tenants.push(tenant_id); + continue; + }; + let estimate = DbEstimate::try_from(task.estimate)?; + let budget = sqlx::query(RESERVE_RUN_BUDGET_SQL) + .bind(run.run_uid) + .bind(estimate.cost_microusd) + .bind(estimate.tokens) + .bind(estimate.tasks) + .bind(estimate.tool_calls) + .bind(estimate.retrieved_bytes) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if budget.rows_affected() != 1 { + exhausted_runs.push(run.run_uid); + continue; + } + + let dispatch_uid = Uuid::now_v7(); + let capacity_reservation_uid = Uuid::now_v7(); + let watchdog_trigger_uid = Uuid::now_v7(); + let attempt_generation = task.attempt_generation; + let watchdog = create_trigger_with_dispatch_in_conn( + conn.as_mut(), + config, + &NewExecutionTrigger { + trigger_uid: watchdog_trigger_uid, + tenant_id: TenantId::from(tenant_id), + run_uid: Some(run.run_uid), + task_id: Some(task.task_id.as_uuid()), + compensation_id: None, + schedule_uid: None, + kind: ExecutionTriggerKind::TaskWatchdog, + controller_generation: Some(run.controller_generation), + attempt_generation: Some(attempt_generation), + compensation_generation: None, + compensation_attempt_generation: None, + schedule_incarnation: None, + occurrence_sequence: None, + due_at: deadline, + payload: json!({}), + }, + ) + .await?; + let dispatch_payload = serde_json::to_value(ExecutionTaskAttemptRequest { + dispatch_uid, + capacity_reservation_uid, + watchdog_trigger_uid, + watchdog_dispatch_uid: watchdog.dispatch.dispatch_uid, + run_uid: run.run_uid, + task_id: task.task_id, + controller_generation: run.controller_generation, + attempt_generation, + attempt_deadline_at: deadline, + tenant_id: run.tenant_id, + })?; + enqueue_dispatch_in_conn( + conn.as_mut(), + &NewExecutionDispatch { + dispatch_uid, + tenant_id: TenantId::from(tenant_id), + run_uid: Some(run.run_uid), + task_id: Some(task.task_id.as_uuid()), + compensation_id: None, + trigger_uid: None, + external_job_uid: None, + kind: ExecutionDispatchKind::TaskAttempt, + controller_generation: Some(run.controller_generation), + wake_epoch: None, + attempt_generation: Some(attempt_generation), + compensation_generation: None, + compensation_attempt_generation: None, + not_before_at: now, + payload: dispatch_payload, + }, + ) + .await?; + let task_row = sqlx::query( + "UPDATE moa.execution_task \ + SET status = 'dispatching', attempt_state = 'dispatching', \ + attempt_started_at = $5, attempt_deadline_at = $6, waiting_since = NULL, \ + ready_at = NULL, active_dispatch_uid = $7, \ + dispatch_sequence = dispatch_sequence + 1, \ + reserved_cost_microusd = $8, reserved_tokens = $9, \ + reserved_tasks = $10, reserved_tool_calls = $11, \ + reserved_retrieved_bytes = $12, reserved_at = NOW(), \ + last_progress_at = NOW(), updated_at = NOW() \ + WHERE run_uid = $1 AND task_id = $2 AND status = 'ready' \ + AND ready_at IS NOT NULL AND ready_at <= $5 \ + AND EXISTS (SELECT 1 FROM moa.execution_run AS current_run \ + WHERE current_run.run_uid = $1 \ + AND current_run.controller_generation = $3) \ + AND attempt_generation = $4 \ + RETURNING *", + ) + .bind(run.run_uid) + .bind(task.task_id.as_uuid()) + .bind(to_i64(run.controller_generation, "controller generation")?) + .bind(to_i64(attempt_generation, "attempt generation")?) + .bind(now) + .bind(deadline) + .bind(dispatch_uid) + .bind(estimate.cost_microusd) + .bind(estimate.tokens) + .bind(estimate.tasks) + .bind(estimate.tool_calls) + .bind(estimate.retrieved_bytes) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(task_row) = task_row else { + return Err(Error::InvalidRepositoryData { + message: "locked ready task lost its dispatch fence".to_string(), + }); + }; + let _ = task_from_row(&task_row)?; + + sqlx::query( + "INSERT INTO moa.execution_capacity_reservation (\ + reservation_uid, tenant_id, run_uid, task_id, controller_generation, \ + attempt_generation, resource_dimension, quantity, expires_at\ + ) VALUES ($1, $2, $3, $4, $5, $6, 'active_tasks', 1, $7)", + ) + .bind(capacity_reservation_uid) + .bind(tenant_id) + .bind(run.run_uid) + .bind(task.task_id.as_uuid()) + .bind(to_i64(run.controller_generation, "controller generation")?) + .bind(to_i64(attempt_generation, "attempt generation")?) + .bind(deadline) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + increment_capacity(conn.as_mut(), "fleet", None, "active_tasks", 1).await?; + increment_capacity(conn.as_mut(), "tenant", Some(tenant_id), "active_tasks", 1).await?; + transition_node_counters_in_tx( + &mut conn, + run.run_uid, + &task.node_id, + &task.item_key, + ExecutionTaskStatus::Ready, + ExecutionTaskStatus::Dispatching, + ) + .await?; + advance_tenant_fairness(&mut conn, tenant_id, now).await?; + admitted.push(ExecutionAdmissionItem { + dispatch_uid, + capacity_reservation_uid, + watchdog_trigger_uid, + watchdog_dispatch_uid: watchdog.dispatch.dispatch_uid, + tenant_id: TenantId::from(tenant_id), + run_uid: run.run_uid, + task_id: task.task_id, + controller_generation: run.controller_generation, + attempt_generation, + attempt_deadline_at: deadline, + }); + } + conn.commit().await.map_err(storage_error)?; + Ok(ExecutionAdmissionBatch { + retry_after: (admitted.len() + < usize::try_from(limit).map_err(|_| Error::InvalidRepositoryInput { + message: "admission request does not fit in memory".to_string(), + })?) + .then_some(retry_after), + admitted, + oldest_ready_at, + }) + } + + /// Releases one exact active-task capacity receipt after generation settlement. + pub async fn release_task_attempt_capacity( + &self, + reservation_uid: Uuid, + run_uid: Uuid, + task_id: ExecutionTaskId, + attempt_generation: u64, + ) -> Result { + let mut conn = ExecutionScope::ControlPlane.begin(&self.pool).await?; + let outcome = release_task_capacity_in_tx( + &mut conn, + reservation_uid, + run_uid, + task_id, + attempt_generation, + ) + .await?; + conn.commit().await.map_err(storage_error)?; + Ok(outcome) + } +} + +/// Reserves one non-task execution resource inside its owner's creation transaction. +pub(super) async fn reserve_capacity_in_tx( + conn: &mut PgConnection, + config: &ExecutionConfig, + request: ExecutionCapacityRequest, +) -> Result { + validate_generic_capacity_request(&request)?; + let dimension = request.dimension.as_str(); + let (fleet_limit, tenant_limit) = request.dimension.limits(config); + ensure_fleet_bucket(conn, dimension, i64::from(fleet_limit)).await?; + ensure_tenant_bucket(conn, request.tenant_id, dimension, i64::from(tenant_limit)).await?; + let fleet_available = + lock_capacity_bucket(conn, "fleet", None, dimension, i64::from(fleet_limit)).await?; + let tenant_available = lock_capacity_bucket( + conn, + "tenant", + Some(request.tenant_id.0), + dimension, + i64::from(tenant_limit), + ) + .await?; + let existing = sqlx::query( + "SELECT tenant_id, run_uid, controller_generation, resource_dimension, state, \ + trigger_uid, external_job_uid \ + FROM moa.execution_capacity_reservation WHERE reservation_uid = $1 FOR UPDATE", + ) + .bind(request.reservation_uid) + .fetch_optional(&mut *conn) + .await + .map_err(sqlx_error)?; + if let Some(existing) = existing { + let state: String = existing.try_get("state").map_err(row_error)?; + let matches = existing + .try_get::("tenant_id") + .map_err(row_error)? + == request.tenant_id.0 + && existing + .try_get::, _>("run_uid") + .map_err(row_error)? + == request.run_uid + && optional_u64(&existing, "controller_generation")? == request.controller_generation + && existing + .try_get::("resource_dimension") + .map_err(row_error)? + == dimension + && existing + .try_get::, _>("trigger_uid") + .map_err(row_error)? + == owner_trigger_uid(request.owner) + && existing + .try_get::, _>("external_job_uid") + .map_err(row_error)? + == owner_external_job_uid(request.owner); + if !matches { + return Err(Error::InvalidRepositoryData { + message: "capacity reservation UID is bound to different immutable coordinates" + .to_string(), + }); + } + return match state.as_str() { + "reserved" | "reconciling" => Ok(CapacityReserveOutcome::Replayed), + "released" => Err(Error::InvalidRepositoryInput { + message: "released capacity owner fence cannot be reacquired".to_string(), + }), + other => Err(Error::InvalidRepositoryData { + message: format!("unknown capacity reservation state `{other}`"), + }), + }; + } + if fleet_available == 0 || tenant_available == 0 { + return Ok(CapacityReserveOutcome::Saturated); + } + sqlx::query( + "INSERT INTO moa.execution_capacity_reservation (\ + reservation_uid, tenant_id, run_uid, trigger_uid, external_job_uid, \ + controller_generation, resource_dimension, quantity, expires_at\ + ) VALUES ($1, $2, $3, $4, $5, $6, $7, 1, $8)", + ) + .bind(request.reservation_uid) + .bind(request.tenant_id.0) + .bind(request.run_uid) + .bind(owner_trigger_uid(request.owner)) + .bind(owner_external_job_uid(request.owner)) + .bind( + request + .controller_generation + .map(|generation| to_i64(generation, "capacity controller generation")) + .transpose()?, + ) + .bind(dimension) + .bind(request.expires_at) + .execute(&mut *conn) + .await + .map_err(sqlx_error)?; + increment_capacity(conn, "fleet", None, dimension, 1).await?; + increment_capacity(conn, "tenant", Some(request.tenant_id.0), dimension, 1).await?; + Ok(CapacityReserveOutcome::Reserved) +} + +/// Prelocks multiple capacity dimensions in the one global deadlock-free order. +/// +/// Multi-resource transactions must call this before their first reserve or release. Each +/// dimension locks its fleet bucket before its tenant bucket, and dimensions always sort by the +/// closed order `active_runs`, `active_tasks`, `parked_runs`, `scheduled_triggers`, `external_jobs`. +pub(super) async fn prelock_capacity_dimensions_in_tx( + conn: &mut PgConnection, + config: &ExecutionConfig, + tenant_id: TenantId, + dimensions: &[ExecutionCapacityDimension], +) -> Result<()> { + let mut dimensions = dimensions.to_vec(); + dimensions.sort_by_key(|dimension| dimension.lock_order()); + dimensions.dedup(); + for dimension in dimensions { + let label = dimension.as_str(); + let (fleet_limit, tenant_limit) = dimension.limits(config); + ensure_fleet_bucket(conn, label, i64::from(fleet_limit)).await?; + ensure_tenant_bucket(conn, tenant_id, label, i64::from(tenant_limit)).await?; + lock_capacity_bucket(conn, "fleet", None, label, i64::from(fleet_limit)).await?; + lock_capacity_bucket( + conn, + "tenant", + Some(tenant_id.0), + label, + i64::from(tenant_limit), + ) + .await?; + } + Ok(()) +} + +/// Prelocks already-created capacity buckets in the canonical multi-dimension order. +/// +/// Terminal settlement uses this variant because exact committed receipts prove the bucket rows +/// exist and terminal code must not require mutable runtime configuration merely to release them. +pub(super) async fn prelock_existing_capacity_dimensions_in_tx( + conn: &mut PgConnection, + tenant_id: TenantId, + dimensions: &[ExecutionCapacityDimension], +) -> Result<()> { + let mut dimensions = dimensions.to_vec(); + dimensions.sort_by_key(|dimension| dimension.lock_order()); + dimensions.dedup(); + if dimensions.is_empty() { + return Ok(()); + } + let labels = dimensions + .iter() + .map(|dimension| dimension.as_str()) + .collect::>(); + let locked = sqlx::query_as::<_, (String, Option, String)>( + "SELECT bucket.scope_kind, bucket.tenant_id, bucket.resource_dimension \ + FROM moa.execution_capacity_bucket AS bucket \ + WHERE bucket.resource_dimension = ANY($2::TEXT[]) \ + AND ((bucket.scope_kind = 'fleet' AND bucket.tenant_id IS NULL) \ + OR (bucket.scope_kind = 'tenant' AND bucket.tenant_id = $1)) \ + ORDER BY CASE bucket.resource_dimension \ + WHEN 'active_runs' THEN 0 WHEN 'active_tasks' THEN 1 \ + WHEN 'parked_runs' THEN 2 WHEN 'scheduled_triggers' THEN 3 \ + WHEN 'external_jobs' THEN 4 ELSE 5 END, \ + CASE bucket.scope_kind WHEN 'fleet' THEN 0 ELSE 1 END \ + FOR UPDATE", + ) + .bind(tenant_id.0) + .bind(&labels) + .fetch_all(&mut *conn) + .await + .map_err(sqlx_error)?; + let expected_len = dimensions.len().saturating_mul(2); + if locked.len() != expected_len { + return Err(Error::InvalidRepositoryData { + message: "missing canonical capacity buckets during existing-row prelock".to_string(), + }); + } + for (index, (scope_kind, owner, dimension)) in locked.iter().enumerate() { + let expected_dimension = dimensions[index / 2].as_str(); + let expected_scope = if index % 2 == 0 { "fleet" } else { "tenant" }; + let expected_owner = (index % 2 == 1).then_some(tenant_id.0); + if scope_kind != expected_scope + || *owner != expected_owner + || dimension != expected_dimension + { + return Err(Error::InvalidRepositoryData { + message: "capacity buckets violated canonical existing-row lock order".to_string(), + }); + } + } + Ok(()) +} + +/// Reserves ActiveRuns only when the admitted population retains a parking entitlement. +/// +/// The caller must prelock both `ActiveRuns` and `ParkedRuns`. Counting the two mutually +/// exclusive ownership classes under those locks makes `ParkedRuns` the resident-run ceiling, +/// so an admitted active run can always transfer into storage-only parking without deadlock or +/// a capacity race. +pub(super) async fn reserve_active_run_capacity_in_tx( + conn: &mut PgConnection, + config: &ExecutionConfig, + request: ExecutionCapacityRequest, +) -> Result { + if request.dimension != ExecutionCapacityDimension::ActiveRuns { + return Err(Error::InvalidRepositoryInput { + message: "active-run admission requires the active_runs capacity dimension".to_string(), + }); + } + let receipt_exists = sqlx::query_scalar::<_, bool>( + "SELECT EXISTS (SELECT 1 FROM moa.execution_capacity_reservation \ + WHERE reservation_uid=$1)", + ) + .bind(request.reservation_uid) + .fetch_one(&mut *conn) + .await + .map_err(sqlx_error)?; + if receipt_exists { + return match reserve_capacity_in_tx(conn, config, request).await? { + CapacityReserveOutcome::Reserved => Ok(ActiveRunCapacityReserveOutcome::Reserved), + CapacityReserveOutcome::Replayed => Ok(ActiveRunCapacityReserveOutcome::Replayed), + CapacityReserveOutcome::Saturated => Ok(ActiveRunCapacityReserveOutcome::Saturated( + ExecutionCapacityDimension::ActiveRuns, + )), + }; + } + let fleet_has_entitlement = resident_run_capacity_has_room(conn, "fleet", None).await?; + let tenant_has_entitlement = + resident_run_capacity_has_room(conn, "tenant", Some(request.tenant_id.0)).await?; + if !fleet_has_entitlement || !tenant_has_entitlement { + return Ok(ActiveRunCapacityReserveOutcome::Saturated( + ExecutionCapacityDimension::ParkedRuns, + )); + } + match reserve_capacity_in_tx(conn, config, request).await? { + CapacityReserveOutcome::Reserved => Ok(ActiveRunCapacityReserveOutcome::Reserved), + CapacityReserveOutcome::Replayed => Ok(ActiveRunCapacityReserveOutcome::Replayed), + CapacityReserveOutcome::Saturated => Ok(ActiveRunCapacityReserveOutcome::Saturated( + ExecutionCapacityDimension::ActiveRuns, + )), + } +} + +async fn resident_run_capacity_has_room( + conn: &mut PgConnection, + scope_kind: &str, + tenant_id: Option, +) -> Result { + let row = sqlx::query_as::<_, (i64, i64, i64)>( + "SELECT active.reserved_quantity,parked.reserved_quantity,parked.limit_value \ + FROM moa.execution_capacity_bucket AS active \ + JOIN moa.execution_capacity_bucket AS parked \ + ON parked.scope_kind=active.scope_kind \ + AND parked.tenant_id IS NOT DISTINCT FROM active.tenant_id \ + WHERE active.scope_kind=$1 AND active.tenant_id IS NOT DISTINCT FROM $2 \ + AND active.resource_dimension='active_runs' \ + AND parked.resource_dimension='parked_runs'", + ) + .bind(scope_kind) + .bind(tenant_id) + .fetch_optional(&mut *conn) + .await + .map_err(sqlx_error)? + .ok_or_else(|| Error::InvalidRepositoryData { + message: format!("missing {scope_kind} resident-run capacity buckets"), + })?; + let resident_count = row.0.checked_add(row.1).ok_or(Error::ArithmeticOverflow { + context: "resident-run capacity count".to_string(), + })?; + Ok(resident_count < row.2) +} + +/// Releases one exact non-task execution capacity receipt in the owner's settlement transaction. +pub(super) async fn release_capacity_in_tx( + conn: &mut PgConnection, + request: ExecutionCapacityRequest, +) -> Result { + validate_generic_capacity_request(&request)?; + let dimension = request.dimension.as_str(); + lock_existing_capacity_bucket(conn, "fleet", None, dimension).await?; + lock_existing_capacity_bucket(conn, "tenant", Some(request.tenant_id.0), dimension).await?; + let row = sqlx::query( + "SELECT tenant_id, run_uid, controller_generation, resource_dimension, state, \ + trigger_uid, external_job_uid \ + FROM moa.execution_capacity_reservation WHERE reservation_uid = $1 FOR UPDATE", + ) + .bind(request.reservation_uid) + .fetch_optional(&mut *conn) + .await + .map_err(sqlx_error)?; + let Some(row) = row else { + return Ok(CapacityReleaseOutcome::NotFound); + }; + let matches = row.try_get::("tenant_id").map_err(row_error)? == request.tenant_id.0 + && row + .try_get::, _>("run_uid") + .map_err(row_error)? + == request.run_uid + && optional_u64(&row, "controller_generation")? == request.controller_generation + && row + .try_get::("resource_dimension") + .map_err(row_error)? + == dimension + && row + .try_get::, _>("trigger_uid") + .map_err(row_error)? + == owner_trigger_uid(request.owner) + && row + .try_get::, _>("external_job_uid") + .map_err(row_error)? + == owner_external_job_uid(request.owner); + if !matches { + return Ok(CapacityReleaseOutcome::Stale); + } + let state: String = row.try_get("state").map_err(row_error)?; + if state == "released" { + return Ok(CapacityReleaseOutcome::AlreadyReleased); + } + if state != "reserved" && state != "reconciling" { + return Err(Error::InvalidRepositoryData { + message: format!("unknown capacity reservation state `{state}`"), + }); + } + for (scope_kind, tenant_id) in [("fleet", None), ("tenant", Some(request.tenant_id.0))] { + let updated = sqlx::query( + "UPDATE moa.execution_capacity_bucket \ + SET reserved_quantity = reserved_quantity - 1, version = version + 1, \ + updated_at = NOW() \ + WHERE scope_kind = $1 AND tenant_id IS NOT DISTINCT FROM $2 \ + AND resource_dimension = $3 AND reserved_quantity >= 1", + ) + .bind(scope_kind) + .bind(tenant_id) + .bind(dimension) + .execute(&mut *conn) + .await + .map_err(sqlx_error)?; + if updated.rows_affected() != 1 { + return Err(Error::InvalidRepositoryData { + message: format!("{scope_kind} {dimension} capacity underflow"), + }); + } + } + sqlx::query( + "UPDATE moa.execution_capacity_reservation \ + SET state = 'released', released_at = NOW(), updated_at = NOW() \ + WHERE reservation_uid = $1", + ) + .bind(request.reservation_uid) + .execute(&mut *conn) + .await + .map_err(sqlx_error)?; + Ok(CapacityReleaseOutcome::Released) +} + +/// Releases the one active parked-run receipt before reactivation or terminal settlement. +pub(super) async fn release_parked_run_capacity_in_tx( + conn: &mut PgConnection, + tenant_id: TenantId, + run_uid: Uuid, + controller_generation: u64, +) -> Result { + let reservation_uid = sqlx::query_scalar::<_, Uuid>( + "SELECT reservation_uid FROM moa.execution_capacity_reservation \ + WHERE tenant_id = $1 AND run_uid = $2 AND controller_generation = $3 \ + AND resource_dimension = 'parked_runs' \ + AND state IN ('reserved', 'reconciling')", + ) + .bind(tenant_id.0) + .bind(run_uid) + .bind(to_i64( + controller_generation, + "parked-run controller generation", + )?) + .fetch_optional(&mut *conn) + .await + .map_err(sqlx_error)?; + let Some(reservation_uid) = reservation_uid else { + return Ok(CapacityReleaseOutcome::NotFound); + }; + release_capacity_in_tx( + conn, + ExecutionCapacityRequest { + reservation_uid, + tenant_id, + run_uid: Some(run_uid), + controller_generation: Some(controller_generation), + dimension: ExecutionCapacityDimension::ParkedRuns, + owner: ExecutionCapacityOwner::Run, + expires_at: None, + }, + ) + .await +} + +/// Atomically transfers one active run into a storage-only parked receipt. +pub(super) async fn transfer_active_run_to_parked_in_tx( + conn: &mut PgConnection, + config: &ExecutionConfig, + run: &ExecutionRunRecord, + wake_epoch: u64, +) -> Result { + let parked = + reserve_capacity_in_tx(conn, config, parked_run_capacity_request(run, wake_epoch)).await?; + if parked == CapacityReserveOutcome::Saturated { + return Ok(parked); + } + match release_capacity_in_tx( + conn, + active_run_capacity_request(run.tenant_id, run.run_uid), + ) + .await? + { + CapacityReleaseOutcome::Released | CapacityReleaseOutcome::AlreadyReleased => Ok(parked), + CapacityReleaseOutcome::NotFound | CapacityReleaseOutcome::Stale => { + Err(Error::InvalidRepositoryData { + message: "parked run is missing its exact active-runs capacity receipt".to_string(), + }) + } + } +} + +/// Atomically restores ActiveRuns ownership before one parked run is reactivated. +pub(super) async fn transfer_parked_run_to_active_in_tx( + conn: &mut PgConnection, + tenant_id: TenantId, + run_uid: Uuid, + controller_generation: u64, +) -> Result { + let parked_exists = sqlx::query_scalar::<_, bool>( + "SELECT EXISTS (SELECT 1 FROM moa.execution_capacity_reservation \ + WHERE tenant_id=$1 AND run_uid=$2 AND controller_generation=$3 \ + AND resource_dimension='parked_runs' AND state IN ('reserved','reconciling'))", + ) + .bind(tenant_id.0) + .bind(run_uid) + .bind(to_i64( + controller_generation, + "parked-run controller generation", + )?) + .fetch_one(&mut *conn) + .await + .map_err(sqlx_error)?; + let active = reactivate_active_run_capacity_in_tx(conn, tenant_id, run_uid).await?; + if active == CapacityReserveOutcome::Saturated { + return Ok(active); + } + if !parked_exists { + if active == CapacityReserveOutcome::Reserved { + return Err(Error::InvalidRepositoryData { + message: "released active-runs receipt has no parked-runs transfer owner" + .to_string(), + }); + } + return Ok(active); + } + match release_parked_run_capacity_in_tx(conn, tenant_id, run_uid, controller_generation).await? + { + CapacityReleaseOutcome::Released | CapacityReleaseOutcome::AlreadyReleased => Ok(active), + CapacityReleaseOutcome::NotFound | CapacityReleaseOutcome::Stale => { + Err(Error::InvalidRepositoryData { + message: "reactivated run lost its exact parked-runs capacity receipt".to_string(), + }) + } + } +} + +async fn reactivate_active_run_capacity_in_tx( + conn: &mut PgConnection, + tenant_id: TenantId, + run_uid: Uuid, +) -> Result { + let request = active_run_capacity_request(tenant_id, run_uid); + let row = sqlx::query( + "SELECT tenant_id,run_uid,controller_generation,resource_dimension,state \ + FROM moa.execution_capacity_reservation WHERE reservation_uid=$1 FOR UPDATE", + ) + .bind(request.reservation_uid) + .fetch_optional(&mut *conn) + .await + .map_err(sqlx_error)? + .ok_or_else(|| Error::InvalidRepositoryData { + message: "execution run is missing its lifetime active-runs capacity receipt".to_string(), + })?; + if row.try_get::("tenant_id").map_err(row_error)? != tenant_id.0 + || row + .try_get::, _>("run_uid") + .map_err(row_error)? + != Some(run_uid) + || optional_u64(&row, "controller_generation")? != request.controller_generation + || row + .try_get::("resource_dimension") + .map_err(row_error)? + != ExecutionCapacityDimension::ActiveRuns.as_str() + { + return Err(Error::InvalidRepositoryData { + message: "active-runs capacity receipt has mismatched immutable coordinates" + .to_string(), + }); + } + let state: String = row.try_get("state").map_err(row_error)?; + match state.as_str() { + "reserved" | "reconciling" => return Ok(CapacityReserveOutcome::Replayed), + "released" => {} + other => { + return Err(Error::InvalidRepositoryData { + message: format!("unknown capacity reservation state `{other}`"), + }); + } + } + let dimension = ExecutionCapacityDimension::ActiveRuns.as_str(); + let fleet_available = capacity_bucket_has_room(conn, "fleet", None, dimension).await?; + let tenant_available = + capacity_bucket_has_room(conn, "tenant", Some(tenant_id.0), dimension).await?; + if !fleet_available || !tenant_available { + return Ok(CapacityReserveOutcome::Saturated); + } + sqlx::query( + "UPDATE moa.execution_capacity_reservation \ + SET state='reserved',released_at=NULL,updated_at=NOW() WHERE reservation_uid=$1", + ) + .bind(request.reservation_uid) + .execute(&mut *conn) + .await + .map_err(sqlx_error)?; + increment_capacity(conn, "fleet", None, dimension, 1).await?; + increment_capacity(conn, "tenant", Some(tenant_id.0), dimension, 1).await?; + Ok(CapacityReserveOutcome::Reserved) +} + +async fn capacity_bucket_has_room( + conn: &mut PgConnection, + scope_kind: &str, + tenant_id: Option, + dimension: &str, +) -> Result { + sqlx::query_scalar( + "SELECT reserved_quantity < limit_value FROM moa.execution_capacity_bucket \ + WHERE scope_kind=$1 AND tenant_id IS NOT DISTINCT FROM $2 \ + AND resource_dimension=$3 FOR UPDATE", + ) + .bind(scope_kind) + .bind(tenant_id) + .bind(dimension) + .fetch_optional(&mut *conn) + .await + .map_err(sqlx_error)? + .ok_or_else(|| Error::InvalidRepositoryData { + message: format!("missing {scope_kind} {dimension} capacity bucket"), + }) +} + +/// Releases the single ActiveRuns or ParkedRuns receipt owned by a nonterminal run. +pub(super) async fn release_owned_run_capacity_in_tx( + conn: &mut PgConnection, + tenant_id: TenantId, + run_uid: Uuid, + controller_generation: u64, +) -> Result<()> { + let active_owned = sqlx::query_scalar::<_, bool>( + "SELECT EXISTS (SELECT 1 FROM moa.execution_capacity_reservation \ + WHERE tenant_id=$1 AND run_uid=$2 AND resource_dimension='active_runs' \ + AND state IN ('reserved','reconciling'))", + ) + .bind(tenant_id.0) + .bind(run_uid) + .fetch_one(&mut *conn) + .await + .map_err(sqlx_error)?; + let parked_owned = sqlx::query_scalar::<_, bool>( + "SELECT EXISTS (SELECT 1 FROM moa.execution_capacity_reservation \ + WHERE tenant_id=$1 AND run_uid=$2 AND controller_generation=$3 \ + AND resource_dimension='parked_runs' AND state IN ('reserved','reconciling'))", + ) + .bind(tenant_id.0) + .bind(run_uid) + .bind(to_i64( + controller_generation, + "parked-run controller generation", + )?) + .fetch_one(&mut *conn) + .await + .map_err(sqlx_error)?; + match (active_owned, parked_owned) { + (true, false) => { + match release_capacity_in_tx(conn, active_run_capacity_request(tenant_id, run_uid)) + .await? + { + CapacityReleaseOutcome::Released | CapacityReleaseOutcome::AlreadyReleased => { + Ok(()) + } + CapacityReleaseOutcome::NotFound | CapacityReleaseOutcome::Stale => { + Err(Error::InvalidRepositoryData { + message: "terminal run lost its exact active-runs capacity receipt" + .to_string(), + }) + } + } + } + (false, true) => { + match release_parked_run_capacity_in_tx(conn, tenant_id, run_uid, controller_generation) + .await? + { + CapacityReleaseOutcome::Released | CapacityReleaseOutcome::AlreadyReleased => { + Ok(()) + } + CapacityReleaseOutcome::NotFound | CapacityReleaseOutcome::Stale => { + Err(Error::InvalidRepositoryData { + message: "terminal run lost its exact parked-runs capacity receipt" + .to_string(), + }) + } + } + } + (true, true) => Err(Error::InvalidRepositoryData { + message: "execution run simultaneously owns active-runs and parked-runs capacity" + .to_string(), + }), + (false, false) => Err(Error::InvalidRepositoryData { + message: "nonterminal execution run owns no active-runs or parked-runs capacity" + .to_string(), + }), + } +} + +fn validate_generic_capacity_request(request: &ExecutionCapacityRequest) -> Result<()> { + let valid = matches!( + (request.dimension, request.owner), + ( + ExecutionCapacityDimension::ActiveRuns | ExecutionCapacityDimension::ParkedRuns, + ExecutionCapacityOwner::Run + ) | ( + ExecutionCapacityDimension::ScheduledTriggers, + ExecutionCapacityOwner::Trigger { .. } + ) | ( + ExecutionCapacityDimension::ExternalJobs, + ExecutionCapacityOwner::ExternalJob { .. } + ) + ); + let run_fence_valid = match ( + request.dimension, + request.run_uid, + request.controller_generation, + ) { + (ExecutionCapacityDimension::ScheduledTriggers, None, None) => true, + (_, Some(run_uid), Some(generation)) => !run_uid.is_nil() && generation > 0, + _ => false, + }; + let owner_identity_valid = match request.owner { + ExecutionCapacityOwner::Run => true, + ExecutionCapacityOwner::Trigger { trigger_uid } => !trigger_uid.is_nil(), + ExecutionCapacityOwner::ExternalJob { external_job_uid } => !external_job_uid.is_nil(), + }; + if !valid || !run_fence_valid || !owner_identity_valid || request.reservation_uid.is_nil() { + return Err(Error::InvalidRepositoryInput { + message: "generic capacity request has an invalid dimension/owner shape".to_string(), + }); + } + Ok(()) +} + +const fn owner_trigger_uid(owner: ExecutionCapacityOwner) -> Option { + match owner { + ExecutionCapacityOwner::Trigger { trigger_uid } => Some(trigger_uid), + ExecutionCapacityOwner::Run | ExecutionCapacityOwner::ExternalJob { .. } => None, + } +} + +const fn owner_external_job_uid(owner: ExecutionCapacityOwner) -> Option { + match owner { + ExecutionCapacityOwner::ExternalJob { external_job_uid } => Some(external_job_uid), + ExecutionCapacityOwner::Run | ExecutionCapacityOwner::Trigger { .. } => None, + } +} + +async fn ensure_fleet_bucket(conn: &mut PgConnection, dimension: &str, limit: i64) -> Result<()> { + sqlx::query( + "INSERT INTO moa.execution_capacity_bucket (\ + capacity_bucket_uid, scope_kind, tenant_id, resource_dimension, limit_value\ + ) VALUES ($1, 'fleet', NULL, $2, $3) ON CONFLICT DO NOTHING", + ) + .bind(Uuid::now_v7()) + .bind(dimension) + .bind(limit) + .execute(&mut *conn) + .await + .map_err(sqlx_error)?; + Ok(()) +} + +async fn ensure_tenant_bucket( + conn: &mut PgConnection, + tenant_id: TenantId, + dimension: &str, + limit: i64, +) -> Result<()> { + sqlx::query( + "INSERT INTO moa.execution_capacity_bucket (\ + capacity_bucket_uid, scope_kind, tenant_id, resource_dimension, limit_value\ + ) VALUES ($1, 'tenant', $2, $3, $4) ON CONFLICT DO NOTHING", + ) + .bind(Uuid::now_v7()) + .bind(tenant_id.0) + .bind(dimension) + .bind(limit) + .execute(&mut *conn) + .await + .map_err(sqlx_error)?; + Ok(()) +} + +async fn lock_capacity_bucket( + conn: &mut PgConnection, + scope_kind: &str, + tenant_id: Option, + dimension: &str, + configured_limit: i64, +) -> Result { + let row = sqlx::query( + "SELECT limit_value, reserved_quantity \ + FROM moa.execution_capacity_bucket \ + WHERE scope_kind = $1 AND tenant_id IS NOT DISTINCT FROM $2 \ + AND resource_dimension = $3 FOR UPDATE", + ) + .bind(scope_kind) + .bind(tenant_id) + .bind(dimension) + .fetch_one(&mut *conn) + .await + .map_err(sqlx_error)?; + let persisted_limit: i64 = row.try_get("limit_value").map_err(row_error)?; + let reserved: i64 = row.try_get("reserved_quantity").map_err(row_error)?; + if persisted_limit != configured_limit { + sqlx::query( + "UPDATE moa.execution_capacity_bucket \ + SET limit_value = $4, version = version + 1, updated_at = NOW() \ + WHERE scope_kind = $1 AND tenant_id IS NOT DISTINCT FROM $2 \ + AND resource_dimension = $3", + ) + .bind(scope_kind) + .bind(tenant_id) + .bind(dimension) + .bind(configured_limit) + .execute(&mut *conn) + .await + .map_err(sqlx_error)?; + } + u64::try_from(configured_limit.saturating_sub(reserved)).map_err(|_| { + Error::InvalidRepositoryData { + message: "execution capacity availability is negative".to_string(), + } + }) +} + +async fn select_fair_ready_tenant( + conn: &mut ScopedConn<'_>, + per_run_limit: usize, + saturated_tenants: &[Uuid], + exhausted_runs: &[Uuid], + observed_at: DateTime, +) -> Result> { + let per_run_limit = + i64::try_from(per_run_limit).map_err(|_| Error::InvalidRepositoryInput { + message: "per-run active task limit exceeds PostgreSQL BIGINT".to_string(), + })?; + sqlx::query_scalar( + "SELECT dispatch.tenant_id \ + FROM moa.execution_tenant_dispatch_state AS dispatch \ + WHERE NOT (dispatch.tenant_id = ANY($1::UUID[])) \ + AND EXISTS (\ + SELECT 1 FROM moa.execution_task AS task \ + JOIN moa.execution_run AS run ON run.run_uid = task.run_uid \ + WHERE task.tenant_id = dispatch.tenant_id AND task.status = 'ready' \ + AND task.ready_at IS NOT NULL AND task.ready_at <= $4 \ + AND NOT (task.run_uid = ANY($3::UUID[])) \ + AND run.status IN ('queued', 'running') \ + AND run.activation_state <> 'paused' \ + AND run.pending_terminal_status IS NULL \ + AND run.active_task_count < $2\ + ) \ + ORDER BY dispatch.virtual_finish, dispatch.last_dispatched_at NULLS FIRST, \ + dispatch.tenant_id \ + LIMIT 1 FOR UPDATE", + ) + .bind(saturated_tenants) + .bind(per_run_limit) + .bind(exhausted_runs) + .bind(observed_at) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error) +} + +async fn lock_oldest_ready_task( + conn: &mut ScopedConn<'_>, + tenant_id: TenantId, + per_run_limit: usize, + exhausted_runs: &[Uuid], + observed_at: DateTime, +) -> Result> { + let per_run_limit = + i64::try_from(per_run_limit).map_err(|_| Error::InvalidRepositoryInput { + message: "per-run active task limit exceeds PostgreSQL BIGINT".to_string(), + })?; + let candidate: Option<(Uuid, Uuid)> = sqlx::query_as( + "SELECT task.run_uid, task.task_id \ + FROM moa.execution_task AS task \ + JOIN moa.execution_run AS run ON run.run_uid = task.run_uid \ + JOIN moa.execution_node_state AS node \ + ON node.run_uid = task.run_uid AND node.node_id = task.node_id \ + WHERE task.tenant_id = $1 AND task.status = 'ready' \ + AND task.ready_at IS NOT NULL AND task.ready_at <= $4 \ + AND NOT (task.run_uid = ANY($3::UUID[])) \ + AND run.status IN ('queued', 'running') \ + AND run.activation_state <> 'paused' \ + AND run.pending_terminal_status IS NULL \ + AND run.active_task_count < $2 \ + ORDER BY task.ready_at, node.node_order, task.item_key, task.task_id \ + LIMIT 1", + ) + .bind(tenant_id.0) + .bind(per_run_limit) + .bind(exhausted_runs) + .bind(observed_at) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some((run_uid, task_id)) = candidate else { + return Ok(None); + }; + let run_row = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) + .bind(run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let run = run_from_row(&run_row)?; + let task_row = sqlx::query(LOAD_TASK_FOR_UPDATE_SQL) + .bind(run_uid) + .bind(task_id) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let task = task_from_row(&task_row)?; + if task.status != ExecutionTaskStatus::Ready + || !matches!( + run.status, + ExecutionRunStatus::Queued | ExecutionRunStatus::Running + ) + || run.activation_state == ExecutionActivationState::Paused + || run.pending_terminal.is_some() + || run.active_task_count + >= u64::try_from(per_run_limit).map_err(|_| Error::InvalidRepositoryData { + message: "persisted per-run active task limit is negative".to_string(), + })? + { + return Ok(None); + } + Ok(Some((run, task))) +} + +async fn increment_capacity( + conn: &mut PgConnection, + scope_kind: &str, + tenant_id: Option, + dimension: &str, + quantity: i64, +) -> Result<()> { + let updated = sqlx::query( + "UPDATE moa.execution_capacity_bucket \ + SET reserved_quantity = reserved_quantity + $4, version = version + 1, \ + updated_at = NOW() \ + WHERE scope_kind = $1 AND tenant_id IS NOT DISTINCT FROM $2 \ + AND resource_dimension = $3 \ + AND reserved_quantity <= limit_value - $4", + ) + .bind(scope_kind) + .bind(tenant_id) + .bind(dimension) + .bind(quantity) + .execute(&mut *conn) + .await + .map_err(sqlx_error)?; + if updated.rows_affected() != 1 { + return Err(Error::InvalidRepositoryData { + message: format!("locked {scope_kind} {dimension} capacity was over-admitted"), + }); + } + Ok(()) +} + +async fn advance_tenant_fairness( + conn: &mut ScopedConn<'_>, + tenant_id: Uuid, + now: DateTime, +) -> Result<()> { + sqlx::query( + "WITH active_floor AS (\ + SELECT COALESCE(MIN(state.virtual_finish), 0) AS value \ + FROM moa.execution_tenant_dispatch_state AS state \ + WHERE EXISTS (\ + SELECT 1 FROM moa.execution_task AS task \ + WHERE task.tenant_id = state.tenant_id AND task.status = 'ready'\ + )\ + ) \ + UPDATE moa.execution_tenant_dispatch_state AS state \ + SET virtual_finish = GREATEST(state.virtual_finish, active_floor.value) \ + + ($2::NUMERIC / state.weight), \ + deficit = state.deficit + state.weight - 1, \ + last_dispatched_at = $3, version = state.version + 1, updated_at = NOW() \ + FROM active_floor WHERE state.tenant_id = $1", + ) + .bind(tenant_id) + .bind(FAIRNESS_QUANTUM) + .bind(now) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + Ok(()) +} + +/// Releases one task capacity receipt inside the caller's state-settlement transaction. +pub(super) async fn release_task_capacity_in_tx( + conn: &mut ScopedConn<'_>, + reservation_uid: Uuid, + run_uid: Uuid, + task_id: ExecutionTaskId, + attempt_generation: u64, +) -> Result { + let tenant_id = sqlx::query_scalar::<_, Uuid>( + "SELECT tenant_id FROM moa.execution_capacity_reservation \ + WHERE reservation_uid = $1 AND run_uid = $2 AND task_id = $3", + ) + .bind(reservation_uid) + .bind(run_uid) + .bind(task_id.as_uuid()) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(tenant_id) = tenant_id else { + return Ok(CapacityReleaseOutcome::NotFound); + }; + lock_existing_capacity_bucket(conn.as_mut(), "fleet", None, "active_tasks").await?; + lock_existing_capacity_bucket(conn.as_mut(), "tenant", Some(tenant_id), "active_tasks").await?; + + let row = sqlx::query( + "SELECT tenant_id, state, attempt_generation \ + FROM moa.execution_capacity_reservation \ + WHERE reservation_uid = $1 AND run_uid = $2 AND task_id = $3 \ + FOR UPDATE", + ) + .bind(reservation_uid) + .bind(run_uid) + .bind(task_id.as_uuid()) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(row) = row else { + return Ok(CapacityReleaseOutcome::NotFound); + }; + let locked_tenant_id: Uuid = row.try_get("tenant_id").map_err(row_error)?; + if locked_tenant_id != tenant_id { + return Err(Error::InvalidRepositoryData { + message: "capacity reservation tenant changed while locking".to_string(), + }); + } + let state: String = row.try_get("state").map_err(row_error)?; + let persisted_generation = required_u64(&row, "attempt_generation")?; + if persisted_generation != attempt_generation { + return Ok(CapacityReleaseOutcome::Stale); + } + if state == "released" { + return Ok(CapacityReleaseOutcome::AlreadyReleased); + } + if state != "reserved" && state != "reconciling" { + return Err(Error::InvalidRepositoryData { + message: format!("unknown active-task capacity reservation state `{state}`"), + }); + } + for (scope_kind, owner) in [("fleet", None), ("tenant", Some(tenant_id))] { + let updated = sqlx::query( + "UPDATE moa.execution_capacity_bucket \ + SET reserved_quantity = reserved_quantity - 1, version = version + 1, \ + updated_at = NOW() \ + WHERE scope_kind = $1 AND tenant_id IS NOT DISTINCT FROM $2 \ + AND resource_dimension = 'active_tasks' AND reserved_quantity >= 1", + ) + .bind(scope_kind) + .bind(owner) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if updated.rows_affected() != 1 { + return Err(Error::InvalidRepositoryData { + message: format!("{scope_kind} active-task capacity underflow"), + }); + } + } + sqlx::query( + "UPDATE moa.execution_capacity_reservation \ + SET state = 'released', released_at = NOW(), updated_at = NOW() \ + WHERE reservation_uid = $1", + ) + .bind(reservation_uid) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + Ok(CapacityReleaseOutcome::Released) +} + +async fn lock_existing_capacity_bucket( + conn: &mut PgConnection, + scope_kind: &str, + tenant_id: Option, + dimension: &str, +) -> Result<()> { + let exists = sqlx::query_scalar::<_, Uuid>( + "SELECT capacity_bucket_uid FROM moa.execution_capacity_bucket \ + WHERE scope_kind = $1 AND tenant_id IS NOT DISTINCT FROM $2 \ + AND resource_dimension = $3 FOR UPDATE", + ) + .bind(scope_kind) + .bind(tenant_id) + .bind(dimension) + .fetch_optional(&mut *conn) + .await + .map_err(sqlx_error)?; + if exists.is_none() { + return Err(Error::InvalidRepositoryData { + message: format!("missing {scope_kind} {dimension} capacity bucket"), + }); + } + Ok(()) +} diff --git a/crates/moa-execution/src/repository/compensation.rs b/crates/moa-execution/src/repository/compensation.rs index c95c49225..7e67c0090 100644 --- a/crates/moa-execution/src/repository/compensation.rs +++ b/crates/moa-execution/src/repository/compensation.rs @@ -1,8 +1,333 @@ //! Compensation registration, fencing, reverse-order claims, and finalization. use super::*; -use super::{projection::budget_ledger, rows::*, sql::*}; -use crate::interpreter::resolve_compensation_input; +use super::{ + capacity::{ + ExecutionCapacityDimension, prelock_capacity_dimensions_in_tx, + release_owned_run_capacity_in_tx, + }, + external_job::{ + ExecutionExternalJobCancellationRequestOutcome, ExecutionExternalJobIntentReleaseOutcome, + ExecutionExternalJobOwner, ExecutionExternalJobRecord, ExecutionExternalJobState, + NewExecutionExternalJobIntent, load_external_job_for_update_in_conn, + release_external_job_intent_in_conn, request_external_job_cancellation_in_conn, + }, + outbox::{ + ExecutionDispatchKind, ExecutionDispatchRecord, NewExecutionDispatch, + enqueue_dispatch_in_conn, + }, + outcome::record_task_outcome_in_conn, + projection::budget_ledger, + ready::transition_node_counters_in_tx, + rows::*, + run::enqueue_run_activation_in_conn, + sql::*, + task::settle_external_job_terminal_in_conn as settle_task_external_job_terminal_in_conn, + terminal::{ + PendingTerminalAdvanceCommit, PendingTerminalAdvanceOutcome, PendingTerminalAdvanceStage, + ReplanStopReceipt, drain_run_triggers_page_in_conn, + }, + trigger::{ + ExecutionTriggerKind, ExecutionTriggerWrite, NewExecutionTrigger, + create_trigger_with_dispatch_in_conn, + }, +}; +use crate::{ + interpreter::resolve_compensation_input, + state::{ + ExecutionLimitStop, ExecutionTerminalCause, ExecutionTerminalEvidence, + cancelled_task_outcome, + }, + wire::{ + ExecutionAttemptCancelReason, ExecutionCompensationAttemptCancelRequest, + ExecutionCompensationAttemptRequest, ExecutionCompensationReleaseIntent, + ExecutionTaskAttemptCancelRequest, + }, +}; +use chrono::Duration; +use moa_artifacts::execution_plan::ExecutionCancelPolicy; +use moa_config::ExecutionConfig; +use moa_core::types::sandbox_workspace::{ExecutionHandReleaseOwner, ExecutionHandReleaseReceipt}; + +const PENDING_TERMINAL_CANCEL_NAMESPACE: Uuid = + Uuid::from_u128(0xd3d4_9744_5c24_58cc_8be8_4806_faba_1837); +const MAX_PENDING_TERMINAL_PAGE_SIZE: u32 = 1_000; + +/// Durable lifecycle of one bounded compensation-attempt slice. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub enum CompensationAttemptState { + /// No slice currently owns capacity or a dispatch. + Idle, + /// The immutable slice dispatch is pending delivery. + Dispatching, + /// The slice is actively executing. + Running, + /// Provider teardown was requested while capacity remains owned. + Cancelling, + /// The slice is parked on an exact action-policy review. + WaitingReview, + /// The slice is parked on asynchronous provider-owned work. + WaitingExternal, + /// The logical compensation settled definitively. + Terminal, + /// The compensating effect may have committed without an authoritative result. + UnknownOutcome, +} + +impl CompensationAttemptState { + /// Returns the canonical database label. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Idle => "idle", + Self::Dispatching => "dispatching", + Self::Running => "running", + Self::Cancelling => "cancelling", + Self::WaitingReview => "waiting_review", + Self::WaitingExternal => "waiting_external", + Self::Terminal => "terminal", + Self::UnknownOutcome => "unknown_outcome", + } + } +} + +impl FromStr for CompensationAttemptState { + type Err = Error; + + fn from_str(value: &str) -> Result { + match value { + "idle" => Ok(Self::Idle), + "dispatching" => Ok(Self::Dispatching), + "running" => Ok(Self::Running), + "cancelling" => Ok(Self::Cancelling), + "waiting_review" => Ok(Self::WaitingReview), + "waiting_external" => Ok(Self::WaitingExternal), + "terminal" => Ok(Self::Terminal), + "unknown_outcome" => Ok(Self::UnknownOutcome), + _ => Err(Error::InvalidRepositoryData { + message: format!("unknown compensation attempt state `{value}`"), + }), + } + } +} + +/// Current durable compensation slice plus its admitted principal. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct CompensationAttemptRecord { + /// Immutable logical compensation registration. + pub registration: CompensationRegistrationProjection, + /// Authoritative locked run, including admitted identity, session, catalog, and scope. + pub run: ExecutionRunRecord, + /// Run-controller generation fencing this slice. + pub controller_generation: u64, + /// Bounded slice generation, separate from logical effect generation. + pub attempt_generation: u64, + /// Current bounded slice state. + pub attempt_state: CompensationAttemptState, + /// First active timestamp for the current slice. + pub attempt_started_at: Option>, + /// Latest monotonic progress timestamp. + pub last_progress_at: DateTime, + /// Absolute watchdog deadline for an active slice. + pub attempt_deadline_at: Option>, + /// Time at which an external review wait began. + pub waiting_since: Option>, + /// Immutable dispatch identity for the current slice. + pub active_dispatch_uid: Option, + /// Exact asynchronous provider job owned by the parked compensation. + pub external_job_uid: Option, + /// Truthful ownership-transfer intent while the slice is cancelling. + pub release_intent: Option, + /// Monotonic count of dispatches created for this registration. + pub dispatch_sequence: u64, +} + +/// Exact identity fence carried by a bounded compensation workflow. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct CompensationAttemptFence { + /// Owning execution run. + pub run_uid: Uuid, + /// Stable logical compensation registration. + pub compensation_id: CompensationId, + /// Exact run-controller generation. + pub controller_generation: u64, + /// Exact logical effect generation. + pub compensation_generation: u64, + /// Exact bounded slice generation. + pub attempt_generation: u64, + /// Immutable dispatch identity of the bounded slice. + pub dispatch_uid: Uuid, +} + +/// One admitted compensation slice and all durable delivery receipts. +#[derive(Clone, Debug, PartialEq)] +pub struct CompensationAttemptAdmission { + /// Current compensation attempt projection. + pub attempt: CompensationAttemptRecord, + /// Exact shared-capacity reservation released by settlement. + pub capacity_reservation_uid: Uuid, + /// Immutable compensation-attempt dispatch. + pub dispatch: ExecutionDispatchRecord, + /// Immutable watchdog trigger. + pub watchdog: ExecutionTriggerWrite, +} + +/// Result of atomically selecting and admitting the next reverse-order compensation. +#[derive(Clone, Debug, PartialEq)] +pub enum CompensationAttemptAdmissionOutcome { + /// The highest unsettled registration entered Dispatching with capacity. + Admitted(Box), + /// The exact active slice was already admitted. + Replayed(Box), + /// Shared fleet or tenant capacity is currently exhausted. + CapacityUnavailable { + /// Earliest useful sparse admission retry. + retry_at: DateTime, + }, + /// Every registered compensation is settled. + Complete, + /// No visible run exists. + NotFound, + /// The run, reverse-order registration, or state is not dispatchable. + Conflict, +} + +/// Result shared by exact compensation-attempt transitions. +#[derive(Clone, Debug, PartialEq)] +pub enum CompensationAttemptWriteOutcome { + /// The transition changed canonical state. + Applied(CompensationAttemptRecord), + /// The exact transition had already been applied. + Replayed(CompensationAttemptRecord), + /// No visible run or registration exists. + NotFound, + /// A generation, dispatch, review, deadline, or state fence rejected the write. + Conflict, +} + +/// Result of claiming exact compensation teardown ownership before provider I/O. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub enum CompensationAttemptReleaseClaimOutcome { + /// The active slice entered the non-dispatchable cancelling phase. + Applied(CompensationAttemptRecord), + /// The exact cancellation request already owns the cancelling phase. + Replayed(CompensationAttemptRecord), + /// No exact run or compensation registration exists. + NotFound, + /// An immutable controller, generation, dispatch, capacity, or watchdog fence differed. + Stale, + /// The compensation slice is not currently eligible to relinquish ownership. + InvalidState, +} + +/// Result of fencing a compensation after provider recovery proved no job started. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub enum CompensationExternalNotStartedReleaseClaimOutcome { + /// The active slice entered Cancelling with a truthful retry release intent. + Applied { + /// Exact request carried through verified sandbox teardown and finalization. + request: ExecutionCompensationAttemptCancelRequest, + /// Current compensation projection after the phase-one fence. + attempt: CompensationAttemptRecord, + }, + /// The exact phase-one fence was already persisted. + Replayed { + /// Exact request carried through verified sandbox teardown and finalization. + request: ExecutionCompensationAttemptCancelRequest, + /// Current compensation projection after the phase-one fence. + attempt: CompensationAttemptRecord, + }, + /// A prior exact finalizer already returned the slice to Idle. + AlreadySettled, + /// No exact run or compensation registration exists. + NotFound, + /// An immutable owner, generation, capacity, or watchdog coordinate differed. + Stale, + /// The compensation cannot enter recovery teardown from its current state. + InvalidState, +} + +/// Result of adopting a recovered, already-started provider job into compensation teardown. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub enum CompensationRecoveredExternalReleaseClaimOutcome { + /// The exact provider job and compensation teardown fence were attached atomically. + Applied { + /// Exact request carried through verified sandbox teardown and finalization. + request: ExecutionCompensationAttemptCancelRequest, + /// Current compensation projection after the phase-one fence. + attempt: CompensationAttemptRecord, + }, + /// The same exact recovered job was already attached to the cancelling slice. + Replayed { + /// Exact request carried through verified sandbox teardown and finalization. + request: ExecutionCompensationAttemptCancelRequest, + /// Current compensation projection after the phase-one fence. + attempt: CompensationAttemptRecord, + }, + /// The recovered job is already owned by a storage-only or terminal compensation state. + AlreadySettled, + /// No exact run, compensation, or provider job exists. + NotFound, + /// An immutable owner, generation, capacity, or watchdog coordinate differed. + Stale, + /// The compensation cannot enter recovery teardown from its current state. + InvalidState, +} + +/// Result of consuming one storage-only compensation action-review resolution. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub enum CompensationReviewResolutionOutcome { + /// The exact current review resolution settled or requeued the compensation. + Applied(CompensationAttemptRecord), + /// The same semantic resolution had already been consumed. + Replayed(CompensationAttemptRecord), + /// The decision arrived before the active attempt completed its durable park. + NotReady, + /// The run or compensation registration does not exist. + NotFound, + /// The logical generation or review identity is obsolete. + Stale, +} + +/// Result of parking a released compensation attempt on asynchronous provider work. +#[derive(Clone, Debug, PartialEq)] +pub enum CompensationAttemptExternalOutcome { + /// The exact provider job and storage-only wait committed atomically. + Applied { + /// Waiting or immediately settled compensation projection. + attempt: CompensationAttemptRecord, + /// Durable provider job. + external_job: ExecutionExternalJobRecord, + }, + /// The same exact provider job had already been attached. + Replayed { + /// Current compensation projection. + attempt: CompensationAttemptRecord, + /// Existing durable provider job. + external_job: ExecutionExternalJobRecord, + }, + /// No exact run, compensation, or capacity receipt exists. + NotFound, + /// An immutable attempt or external-job coordinate differed. + Stale, + /// The compensation cannot yield from its current state. + InvalidState, +} + +/// Result of consuming one terminal provider job into its exact compensation wait. +#[derive(Clone, Debug, PartialEq)] +pub enum CompensationExternalJobSettlementOutcome { + /// The terminal provider result settled or requeued the compensation. + Applied(CompensationAttemptRecord), + /// The same terminal provider result had already been consumed. + Replayed(CompensationAttemptRecord), + /// The callback is durable, but active sandbox ownership must be released first. + DeferredRelease(CompensationAttemptRecord), + /// The provider job no longer owns the current compensation attempt. + Stale, + /// No owning compensation exists. + NotFound, +} async fn load_replan_stop_task( conn: &mut ScopedConn<'_>, @@ -19,7 +344,7 @@ async fn load_replan_stop_task( .transpose() } -fn replan_stop_receipt_audit(receipt: &ReplanStopReceipt) -> Value { +fn replan_stop_receipt_audit(receipt: &ReplanStopReceipt, recorded_at: DateTime) -> Value { json!({ "kind": "replan_stop_fenced", "accepted": true, @@ -27,384 +352,397 @@ fn replan_stop_receipt_audit(receipt: &ReplanStopReceipt) -> Value { "task_generation": receipt.task_generation, "base_plan_revision": receipt.base_plan_revision, "amendment_hash": receipt.amendment_hash, - "recorded_at": Utc::now(), + "recorded_at": recorded_at, }) } -fn task_has_replan_stop_receipt(task: &ExecutionTaskRecord, receipt: &ReplanStopReceipt) -> bool { - let task_id = receipt.task_id.to_string(); - let amendment_hash = receipt.amendment_hash.to_string(); - task.task_id == receipt.task_id - && task.outcome_audit.iter().any(|entry| { - entry.get("kind").and_then(Value::as_str) == Some("replan_stop_fenced") - && entry.get("accepted").and_then(Value::as_bool) == Some(true) - && entry.get("task_id").and_then(Value::as_str) == Some(task_id.as_str()) - && entry.get("task_generation").and_then(Value::as_u64) - == Some(receipt.task_generation) - && entry.get("base_plan_revision").and_then(Value::as_u64) - == Some(receipt.base_plan_revision) - && entry.get("amendment_hash").and_then(Value::as_str) - == Some(amendment_hash.as_str()) - }) -} - impl ExecutionRepository { - /// Persists a terminal intent and fences all new forward admission before task settlement. - pub async fn fence_run_for_terminal( - &self, - scope: ExecutionScope, - run_uid: Uuid, - expected_revision: u64, - expected_wake_epoch: u64, - pending_terminal: PendingExecutionTerminal, - ) -> Result { - self.fence_run_for_terminal_inner( - scope, - run_uid, - expected_revision, - expected_wake_epoch, - pending_terminal, - None, - ) - .await - } - - /// Persists an amendment-driven replan-stop fence and its exact replay receipt atomically. - pub async fn fence_replan_stop( + /// Fences one due approved deadline and advances one bounded terminal-drain page. + #[allow(clippy::too_many_arguments)] + pub async fn fence_deadline_and_enqueue_settlement( &self, + config: &ExecutionConfig, scope: ExecutionScope, run_uid: Uuid, - expected_revision: u64, + controller_generation: u64, expected_wake_epoch: u64, - pending_terminal: PendingExecutionTerminal, - receipt: ReplanStopReceipt, - ) -> Result { - if receipt.base_plan_revision != expected_revision - || !matches!( - pending_terminal.terminal_evidence.cause, - ExecutionTerminalCause::ReplanStop { .. } - ) + now: DateTime, + page_limit: u32, + ) -> Result { + validate_pending_terminal_page_limit(page_limit)?; + let mut conn = scope.begin(&self.pool).await?; + let Some(run) = load_and_lock_pending_terminal_run(&mut conn, config, run_uid).await? + else { + conn.commit().await.map_err(storage_error)?; + return Ok(PendingTerminalAdvanceOutcome::NotFound); + }; + if run.controller_generation != controller_generation + || run.wake_epoch != expected_wake_epoch { - return Err(Error::InvalidRepositoryInput { - message: "replan-stop receipt must match the fenced revision and terminal cause" - .to_string(), + conn.commit().await.map_err(storage_error)?; + return Ok(PendingTerminalAdvanceOutcome::Conflict); + } + if expected_wake_epoch <= run.processed_wake_epoch { + let commit = replayed_pending_terminal_commit(&mut conn, config, run).await?; + conn.commit().await.map_err(storage_error)?; + return Ok(PendingTerminalAdvanceOutcome::Replayed(Box::new(commit))); + } + let Some(deadline_at) = run.approved_budget.deadline_at else { + return Err(Error::InvalidRepositoryData { + message: "durable execution run is missing its approved deadline".to_string(), }); + }; + if deadline_at > now || run.status.is_terminal() { + conn.commit().await.map_err(storage_error)?; + return Ok(PendingTerminalAdvanceOutcome::Conflict); } - self.fence_run_for_terminal_inner( - scope, - run_uid, - expected_revision, + let requirement_count = u64::try_from(run.goal.requirements.len()).map_err(|_| { + Error::InvalidRepositoryData { + message: "execution requirement count exceeds u64".to_string(), + } + })?; + let pending = PendingExecutionTerminal { + status: ExecutionRunStatus::Failed, + reason: ExecutionTerminalReason::DeadlineExceeded, + terminal_evidence: ExecutionTerminalEvidence { + cause: ExecutionTerminalCause::LimitStop { + reason: ExecutionLimitStop::DeadlineExceeded, + }, + satisfied_requirement_count: 0, + requirement_count, + }, + completion_check_results: Vec::new(), + terminal_gaps: vec!["approved execution deadline elapsed".to_string()], + output: run.output.clone(), + cancellation_reason: None, + }; + pending.validate()?; + let new_pending = run.pending_terminal.is_none().then_some(pending); + advance_pending_terminal_page_in_conn( + conn, + config, + run, + controller_generation, expected_wake_epoch, - pending_terminal, - Some(receipt), + new_pending, + now, + page_limit, ) .await } - async fn fence_run_for_terminal_inner( + /// Persists one completion-derived terminal intent and advances its first bounded drain page. + #[allow(clippy::too_many_arguments)] + pub async fn fence_completion_terminal_and_enqueue_settlement( &self, + config: &ExecutionConfig, scope: ExecutionScope, run_uid: Uuid, - expected_revision: u64, + controller_generation: u64, expected_wake_epoch: u64, - pending_terminal: PendingExecutionTerminal, - replan_stop_receipt: Option, - ) -> Result { - pending_terminal.validate()?; + pending: PendingExecutionTerminal, + now: DateTime, + page_limit: u32, + ) -> Result { + validate_pending_terminal_page_limit(page_limit)?; + pending.validate()?; let mut conn = scope.begin(&self.pool).await?; - let Some(run_row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) - .bind(run_uid) - .fetch_optional(conn.as_mut()) - .await - .map_err(sqlx_error)? + let Some(run) = load_and_lock_pending_terminal_run(&mut conn, config, run_uid).await? else { conn.commit().await.map_err(storage_error)?; - return Ok(TerminalFenceOutcome::NotFound); + return Ok(PendingTerminalAdvanceOutcome::NotFound); }; - let current = run_from_row(&run_row)?; - if current.pending_terminal.as_ref() == Some(&pending_terminal) { - if let Some(receipt) = replan_stop_receipt { - let Some(task) = load_replan_stop_task(&mut conn, run_uid, receipt.task_id).await? - else { - conn.commit().await.map_err(storage_error)?; - return Ok(TerminalFenceOutcome::NotFound); - }; - if !task_has_replan_stop_receipt(&task, &receipt) { - conn.commit().await.map_err(storage_error)?; - return Ok(TerminalFenceOutcome::Conflict); - } - } - let tasks_to_settle = load_nonterminal_tasks(&mut conn, run_uid).await?; + if run.controller_generation != controller_generation + || run.wake_epoch != expected_wake_epoch + { conn.commit().await.map_err(storage_error)?; - return Ok(TerminalFenceOutcome::Replayed(Box::new( - TerminalFenceCommit { - run: current, - tasks_to_settle, - }, - ))); + return Ok(PendingTerminalAdvanceOutcome::Conflict); + } + if expected_wake_epoch <= run.processed_wake_epoch { + let commit = replayed_pending_terminal_commit(&mut conn, config, run).await?; + conn.commit().await.map_err(storage_error)?; + return Ok(PendingTerminalAdvanceOutcome::Replayed(Box::new(commit))); } - if current.plan_revision != expected_revision - || current.wake_epoch != expected_wake_epoch - || current.status.is_terminal() - || current.status == ExecutionRunStatus::Compensating - || current.pending_terminal.is_some() + if run.status.is_terminal() + || run + .pending_terminal + .as_ref() + .is_some_and(|current| current != &pending) { conn.commit().await.map_err(storage_error)?; - return Ok(TerminalFenceOutcome::Conflict); + return Ok(PendingTerminalAdvanceOutcome::Conflict); } - let replan_stop_task = if let Some(receipt) = replan_stop_receipt { - let Some(task) = load_replan_stop_task(&mut conn, run_uid, receipt.task_id).await? - else { - conn.commit().await.map_err(storage_error)?; - return Ok(TerminalFenceOutcome::NotFound); - }; - if task.plan_revision != receipt.base_plan_revision - || task.generation != receipt.task_generation - || task.status != ExecutionTaskStatus::WaitingReplan - || !matches!( - task.current_outcome.as_ref().map(|outcome| &outcome.result), - Some(ExecutionTaskResult::NeedsReplan { .. }) - ) - { - conn.commit().await.map_err(storage_error)?; - return Ok(TerminalFenceOutcome::Conflict); - } - Some((task, receipt)) - } else { - None - }; - let row = sqlx::query(FENCE_RUN_FOR_COMPENSATION_SQL) - .bind(run_uid) - .bind(to_i64(expected_revision, "expected plan revision")?) - .bind(to_i64(expected_wake_epoch, "expected wake epoch")?) - .bind(pending_terminal.status.as_str()) - .bind(pending_terminal.reason.as_str()) - .bind(serde_json::to_value(PendingTerminalEvidencePayload { - terminal_evidence: pending_terminal.terminal_evidence.clone(), - completion_check_results: pending_terminal.completion_check_results.clone(), - terminal_gaps: pending_terminal.terminal_gaps.clone(), - })?) - .bind(&pending_terminal.output) - .bind(&pending_terminal.cancellation_reason) - .fetch_optional(conn.as_mut()) - .await - .map_err(sqlx_error)?; - let Some(row) = row else { - conn.rollback().await.map_err(storage_error)?; - return Ok(TerminalFenceOutcome::Conflict); - }; - let run = run_from_row(&row)?; - if let Some((task, receipt)) = replan_stop_task { - sqlx::query(APPEND_TASK_OUTCOME_AUDIT_SQL) - .bind(run_uid) - .bind(task.task_id.as_uuid()) - .bind(replan_stop_receipt_audit(&receipt)) - .fetch_one(conn.as_mut()) - .await - .map_err(sqlx_error)?; - } - let tasks_to_settle = load_nonterminal_tasks(&mut conn, run_uid).await?; - conn.commit().await.map_err(storage_error)?; - Ok(TerminalFenceOutcome::Applied(Box::new( - TerminalFenceCommit { - run, - tasks_to_settle, - }, - ))) + advance_pending_terminal_page_in_conn( + conn, + config, + run, + controller_generation, + expected_wake_epoch, + Some(pending), + now, + page_limit, + ) + .await } - /// Enters `compensating` after every fenced forward task has durably settled. - pub async fn begin_compensation( + /// Persists an exact replan-stop receipt and advances its first bounded terminal-drain page. + #[allow(clippy::too_many_arguments)] + pub async fn fence_replan_stop_and_enqueue_settlement( &self, + config: &ExecutionConfig, scope: ExecutionScope, run_uid: Uuid, + controller_generation: u64, expected_revision: u64, expected_wake_epoch: u64, - ) -> Result { + pending: PendingExecutionTerminal, + receipt: ReplanStopReceipt, + now: DateTime, + page_limit: u32, + ) -> Result { + validate_pending_terminal_page_limit(page_limit)?; + pending.validate()?; + if receipt.base_plan_revision != expected_revision + || !matches!( + pending.terminal_evidence.cause, + ExecutionTerminalCause::ReplanStop { .. } + ) + { + return Err(Error::InvalidRepositoryInput { + message: "replan-stop receipt must match the fenced revision and terminal cause" + .to_string(), + }); + } let mut conn = scope.begin(&self.pool).await?; - let Some(run_row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) - .bind(run_uid) - .fetch_optional(conn.as_mut()) - .await - .map_err(sqlx_error)? + let Some(run) = load_and_lock_pending_terminal_run(&mut conn, config, run_uid).await? else { conn.commit().await.map_err(storage_error)?; - return Ok(BeginCompensationOutcome::NotFound); + return Ok(PendingTerminalAdvanceOutcome::NotFound); }; - let current = run_from_row(&run_row)?; - let pending_tasks = load_nonterminal_tasks(&mut conn, run_uid).await?; - if !pending_tasks.is_empty() { + if run.controller_generation != controller_generation + || run.plan_revision != expected_revision + || run.wake_epoch != expected_wake_epoch + || run.status.is_terminal() + || run.status == ExecutionRunStatus::Compensating + || run + .pending_terminal + .as_ref() + .is_some_and(|current| current != &pending) + { conn.commit().await.map_err(storage_error)?; - return Ok(BeginCompensationOutcome::ForwardTasksPending(pending_tasks)); + return Ok(PendingTerminalAdvanceOutcome::Conflict); } - let registrations = load_compensations(&mut conn, run_uid).await?; - let all_tasks = load_nonterminal_or_terminal_tasks(&mut conn, run_uid).await?; - for task in all_tasks.iter().filter(|task| { - task.compensation_contract.is_some() && task.status == ExecutionTaskStatus::Completed - }) { - if !registrations - .iter() - .any(|registration| registration.forward_task_id == task.task_id) - { - return Err(Error::InvalidRepositoryData { - message: format!( - "completed forward task {} is missing its atomic compensation registration", - task.task_id - ), - }); + let Some(task) = load_replan_stop_task(&mut conn, run_uid, receipt.task_id).await? else { + conn.commit().await.map_err(storage_error)?; + return Ok(PendingTerminalAdvanceOutcome::NotFound); + }; + let receipt_exists: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM moa.execution_amendment_receipt \ + WHERE tenant_id=$1 AND run_uid=$2 AND base_plan_revision=$3 \ + AND amendment_hash=$4 AND receipt_kind='replan_stop' \ + AND superseded_task_id=$5 AND task_generation=$6 \ + AND cardinality(task_ids_to_release)=0)", + ) + .bind(run.tenant_id.0) + .bind(run.run_uid) + .bind(to_i64( + receipt.base_plan_revision, + "replan-stop plan revision", + )?) + .bind(receipt.amendment_hash.to_string()) + .bind(receipt.task_id.as_uuid()) + .bind(to_i64( + receipt.task_generation, + "replan-stop task generation", + )?) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let intent = sqlx::query( + "SELECT tenant_id,controller_generation,wake_epoch,origin_task_id,task_generation, \ + base_plan_revision,stop_reason,amendment_hash \ + FROM moa.execution_replan_stop_intent WHERE run_uid=$1 FOR UPDATE", + ) + .bind(run.run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if run.pending_terminal.is_some() { + if !receipt_exists || intent.is_some() { + conn.commit().await.map_err(storage_error)?; + return Ok(PendingTerminalAdvanceOutcome::Conflict); } + if expected_wake_epoch <= run.processed_wake_epoch { + let commit = replayed_pending_terminal_commit(&mut conn, config, run).await?; + conn.commit().await.map_err(storage_error)?; + return Ok(PendingTerminalAdvanceOutcome::Replayed(Box::new(commit))); + } + return advance_pending_terminal_page_in_conn( + conn, + config, + run, + controller_generation, + expected_wake_epoch, + None, + now, + page_limit, + ) + .await; } - if current.status == ExecutionRunStatus::Compensating - && current.plan_revision == expected_revision - && current.pending_terminal.is_some() - { + let Some(intent) = intent else { conn.commit().await.map_err(storage_error)?; - return Ok(BeginCompensationOutcome::Replayed(Box::new( - BeginCompensationCommit { - run: current, - registrations, - }, - ))); - } - if current.plan_revision != expected_revision - || current.wake_epoch != expected_wake_epoch - || current.pending_terminal.is_none() - || current.status.is_terminal() + return Ok(PendingTerminalAdvanceOutcome::Conflict); + }; + let expected_stop_reason = match &pending.terminal_evidence.cause { + ExecutionTerminalCause::ReplanStop { reason } => reason.as_str(), + _ => unreachable!("validated replan-stop terminal cause"), + }; + let intent_exact = intent.try_get::("tenant_id").map_err(row_error)? + == run.tenant_id.0 + && required_u64(&intent, "controller_generation")? == controller_generation + && required_u64(&intent, "wake_epoch")? == expected_wake_epoch + && intent + .try_get::("origin_task_id") + .map_err(row_error)? + == receipt.task_id.as_uuid() + && required_u64(&intent, "task_generation")? == receipt.task_generation + && required_u64(&intent, "base_plan_revision")? == receipt.base_plan_revision + && intent + .try_get::("stop_reason") + .map_err(row_error)? + == expected_stop_reason + && intent + .try_get::("amendment_hash") + .map_err(row_error)? + == receipt.amendment_hash.to_string(); + if receipt_exists + || !intent_exact + || task.plan_revision != receipt.base_plan_revision + || task.generation != receipt.task_generation + || task.status != ExecutionTaskStatus::WaitingReplan + || !matches!( + task.current_outcome.as_ref().map(|outcome| &outcome.result), + Some(ExecutionTaskResult::NeedsReplan { .. }) + ) { conn.commit().await.map_err(storage_error)?; - return Ok(BeginCompensationOutcome::Conflict); + return Ok(PendingTerminalAdvanceOutcome::Conflict); } - if registrations.is_empty() && !current.manual_repair_required { - conn.commit().await.map_err(storage_error)?; - return Ok(BeginCompensationOutcome::NoCompensations(Box::new(current))); + sqlx::query( + "INSERT INTO moa.execution_amendment_receipt \ + (tenant_id,run_uid,base_plan_revision,amendment_hash,receipt_kind, \ + superseded_task_id,task_generation,task_ids_to_release,created_at) \ + VALUES ($1,$2,$3,$4,'replan_stop',$5,$6,'{}'::UUID[],$7)", + ) + .bind(run.tenant_id.0) + .bind(run.run_uid) + .bind(to_i64( + receipt.base_plan_revision, + "replan-stop plan revision", + )?) + .bind(receipt.amendment_hash.to_string()) + .bind(receipt.task_id.as_uuid()) + .bind(to_i64( + receipt.task_generation, + "replan-stop task generation", + )?) + .bind(now) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let deleted = sqlx::query( + "DELETE FROM moa.execution_replan_stop_intent WHERE tenant_id=$1 AND run_uid=$2 \ + AND controller_generation=$3 AND wake_epoch=$4", + ) + .bind(run.tenant_id.0) + .bind(run.run_uid) + .bind(to_i64(controller_generation, "controller generation")?) + .bind(to_i64(expected_wake_epoch, "expected wake epoch")?) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if deleted.rows_affected() != 1 { + return Err(Error::InvalidRepositoryData { + message: "replan-stop fence lost its exact durable intent".to_string(), + }); } - let row = sqlx::query(BEGIN_COMPENSATION_SQL) + sqlx::query(APPEND_TASK_OUTCOME_AUDIT_SQL) .bind(run_uid) - .bind(to_i64(expected_revision, "expected plan revision")?) - .bind(to_i64(expected_wake_epoch, "expected wake epoch")?) - .fetch_optional(conn.as_mut()) + .bind(task.task_id.as_uuid()) + .bind(replan_stop_receipt_audit(&receipt, now)) + .fetch_one(conn.as_mut()) .await .map_err(sqlx_error)?; - let Some(row) = row else { - conn.rollback().await.map_err(storage_error)?; - return Ok(BeginCompensationOutcome::Conflict); - }; - let run = run_from_row(&row)?; - conn.commit().await.map_err(storage_error)?; - Ok(BeginCompensationOutcome::Applied(Box::new( - BeginCompensationCommit { run, registrations }, - ))) + advance_pending_terminal_page_in_conn( + conn, + config, + run, + controller_generation, + expected_wake_epoch, + Some(pending), + now, + page_limit, + ) + .await } - /// Installs a held terminal intent after every fenced forward task has settled. - pub async fn finalize_fenced_terminal( + /// Advances one bounded page of an already-fenced pending-terminal drain. + #[allow(clippy::too_many_arguments)] + pub async fn advance_pending_terminal_settlement( &self, + config: &ExecutionConfig, scope: ExecutionScope, run_uid: Uuid, - expected_revision: u64, + controller_generation: u64, expected_wake_epoch: u64, - ) -> Result { + now: DateTime, + page_limit: u32, + ) -> Result { + validate_pending_terminal_page_limit(page_limit)?; let mut conn = scope.begin(&self.pool).await?; - let Some(run_row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) - .bind(run_uid) - .fetch_optional(conn.as_mut()) - .await - .map_err(sqlx_error)? + let Some(run) = load_and_lock_pending_terminal_run(&mut conn, config, run_uid).await? else { conn.commit().await.map_err(storage_error)?; - return Ok(FencedTerminalFinalizationOutcome::NotFound); + return Ok(PendingTerminalAdvanceOutcome::NotFound); }; - let run = run_from_row(&run_row)?; - if run.status.is_terminal() && run.pending_terminal.is_none() { - let manual = run.manual_repair_required - && run.status == ExecutionRunStatus::Failed - && run.terminal_reason == Some(ExecutionTerminalReason::CompensationFailed); - conn.commit().await.map_err(storage_error)?; - return Ok(if manual { - FencedTerminalFinalizationOutcome::ManualRepairRequired(run) - } else { - FencedTerminalFinalizationOutcome::Replayed(run) - }); - } - let pending_tasks = load_nonterminal_tasks(&mut conn, run_uid).await?; - if !pending_tasks.is_empty() { - conn.commit().await.map_err(storage_error)?; - return Ok(FencedTerminalFinalizationOutcome::ForwardTasksPending( - pending_tasks, - )); - } - if run.plan_revision != expected_revision + if run.controller_generation != controller_generation || run.wake_epoch != expected_wake_epoch - || run.pending_terminal.is_none() - || run.status.is_terminal() - || run.status == ExecutionRunStatus::Compensating { conn.commit().await.map_err(storage_error)?; - return Ok(FencedTerminalFinalizationOutcome::Conflict); + return Ok(PendingTerminalAdvanceOutcome::Conflict); } - let pending = run - .pending_terminal - .ok_or_else(|| Error::InvalidRepositoryData { - message: "fenced run lost pending terminal intent".to_string(), - })?; - if run.manual_repair_required { - let registrations = load_compensations(&mut conn, run_uid).await?; - let (compensation_id, outcome) = - compensation_failure_evidence(&mut conn, run_uid, ®istrations).await?; - let evidence = ExecutionTerminalEvidence { - cause: ExecutionTerminalCause::CompensationFailure { - original_status: pending.status, - original_reason: pending.reason, - original_cause: Box::new(pending.terminal_evidence.cause.clone()), - compensation_id, - outcome, - }, - satisfied_requirement_count: pending.terminal_evidence.satisfied_requirement_count, - requirement_count: pending.terminal_evidence.requirement_count, - }; - let row = finalize_compensation_run( - &mut conn, - run_uid, - CompensationTerminalWrite { - status: ExecutionRunStatus::Failed, - reason: ExecutionTerminalReason::CompensationFailed, - evidence: &evidence, - completion_check_results: &pending.completion_check_results, - terminal_gaps: &pending.terminal_gaps, - output: pending.output, - manual_repair_required: true, - }, - ) - .await?; - let run = run_from_row(&row)?; + if expected_wake_epoch <= run.processed_wake_epoch { + let commit = replayed_pending_terminal_commit(&mut conn, config, run).await?; conn.commit().await.map_err(storage_error)?; - return Ok(FencedTerminalFinalizationOutcome::ManualRepairRequired(run)); + return Ok(PendingTerminalAdvanceOutcome::Replayed(Box::new(commit))); } - let row = finalize_compensation_run( - &mut conn, - run_uid, - CompensationTerminalWrite { - status: pending.status, - reason: pending.reason, - evidence: &pending.terminal_evidence, - completion_check_results: &pending.completion_check_results, - terminal_gaps: &pending.terminal_gaps, - output: pending.output, - manual_repair_required: false, - }, + if run.pending_terminal.is_none() || run.status.is_terminal() { + conn.commit().await.map_err(storage_error)?; + return Ok(PendingTerminalAdvanceOutcome::Conflict); + } + advance_pending_terminal_page_in_conn( + conn, + config, + run, + controller_generation, + expected_wake_epoch, + None, + now, + page_limit, ) - .await?; - let run = run_from_row(&row)?; - conn.commit().await.map_err(storage_error)?; - Ok(FencedTerminalFinalizationOutcome::Finalized(run)) + .await } - /// Loads one complete compensation driver snapshot in strict reverse sequence order. - pub async fn load_compensation_snapshot( + /// Admits the highest unsettled compensation into one bounded durable slice. + pub async fn admit_next_compensation_attempt( &self, scope: ExecutionScope, + config: &ExecutionConfig, run_uid: Uuid, - ) -> Result> { + now: DateTime, + ) -> Result { + let deadline = checked_attempt_deadline(config, now)?; + let retry_at = checked_retry_at(config, now)?; let mut conn = scope.begin(&self.pool).await?; let Some(run_row) = sqlx::query(LOAD_RUN_SQL) .bind(run_uid) @@ -413,30 +751,16 @@ impl ExecutionRepository { .map_err(sqlx_error)? else { conn.commit().await.map_err(storage_error)?; - return Ok(None); + return Ok(CompensationAttemptAdmissionOutcome::NotFound); }; - let run = run_from_row(&run_row)?; - let registrations = load_compensations(&mut conn, run_uid).await?; - let nonterminal_forward_tasks = load_nonterminal_tasks(&mut conn, run_uid).await?; - let manual_repair_required = run.manual_repair_required; - conn.commit().await.map_err(storage_error)?; - Ok(Some(ExecutionCompensationSnapshot { - run, - registrations, - nonterminal_forward_tasks, - manual_repair_required, - })) - } - - /// Claims exactly the highest unsettled compensation sequence under its generation fence. - pub async fn claim_next_compensation( - &self, - scope: ExecutionScope, - run_uid: Uuid, - compensation_id: CompensationId, - expected_generation: u64, - ) -> Result { - let mut conn = scope.begin(&self.pool).await?; + let visible_run = run_from_row(&run_row)?; + lock_compensation_capacity( + &mut conn, + visible_run.tenant_id, + config.max_fleet_active_tasks, + config.max_tenant_active_tasks, + ) + .await?; let Some(run_row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) .bind(run_uid) .fetch_optional(conn.as_mut()) @@ -444,446 +768,4710 @@ impl ExecutionRepository { .map_err(sqlx_error)? else { conn.commit().await.map_err(storage_error)?; - return Ok(CompensationClaimOutcome::NotFound); + return Ok(CompensationAttemptAdmissionOutcome::NotFound); }; let run = run_from_row(&run_row)?; - if run.status != ExecutionRunStatus::Compensating + if run.tenant_id != visible_run.tenant_id + || run.status != ExecutionRunStatus::Compensating || run.manual_repair_required - || !load_nonterminal_tasks(&mut conn, run_uid).await?.is_empty() + || run.pending_terminal.is_none() + || nonterminal_task_exists(&mut conn, run_uid).await? { conn.commit().await.map_err(storage_error)?; - return Ok(CompensationClaimOutcome::Conflict); + return Ok(CompensationAttemptAdmissionOutcome::Conflict); } - let Some(compensation_row) = sqlx::query(LOAD_COMPENSATION_FOR_UPDATE_SQL) - .bind(run_uid) - .bind(compensation_id.as_uuid()) - .fetch_optional(conn.as_mut()) - .await - .map_err(sqlx_error)? - else { - conn.commit().await.map_err(storage_error)?; - return Ok(CompensationClaimOutcome::NotFound); - }; - let compensation = compensation_from_row(&compensation_row)?; - let highest_unsettled: Option = sqlx::query_scalar( - "SELECT compensation_id FROM moa.execution_compensation \ - WHERE run_uid = $1 AND status <> 'completed' \ - ORDER BY registered_sequence DESC LIMIT 1 FOR UPDATE", + let Some(row) = sqlx::query( + "SELECT * FROM moa.execution_compensation WHERE run_uid = $1 \ + AND status <> 'completed' ORDER BY registered_sequence DESC \ + LIMIT 1 FOR UPDATE", ) .bind(run_uid) .fetch_optional(conn.as_mut()) .await - .map_err(sqlx_error)?; - if highest_unsettled != Some(compensation_id.as_uuid()) { + .map_err(sqlx_error)? + else { + conn.commit().await.map_err(storage_error)?; + return Ok(CompensationAttemptAdmissionOutcome::Complete); + }; + let registration = compensation_from_row(&row)?; + let attempt_state = compensation_attempt_state_from_row(&row)?; + if matches!( + registration.status, + CompensationStatus::Failed | CompensationStatus::UnknownOutcome + ) { conn.commit().await.map_err(storage_error)?; - return Ok(CompensationClaimOutcome::Conflict); + return Ok(CompensationAttemptAdmissionOutcome::Conflict); } - if compensation.status == CompensationStatus::Running - && compensation.generation == expected_generation - { + if attempt_state == CompensationAttemptState::Dispatching { + let admission = + load_existing_compensation_admission(&mut conn, config, &run, &row, ®istration) + .await?; + conn.commit().await.map_err(storage_error)?; + return Ok(CompensationAttemptAdmissionOutcome::Replayed(Box::new( + admission, + ))); + } + if !compensation_capacity_available(&mut conn, visible_run.tenant_id).await? { conn.commit().await.map_err(storage_error)?; - return Ok(CompensationClaimOutcome::Replayed(compensation)); + return Ok(CompensationAttemptAdmissionOutcome::CapacityUnavailable { retry_at }); } - if compensation.status != CompensationStatus::Pending - || compensation.generation != expected_generation + if attempt_state != CompensationAttemptState::Idle + || !matches!( + registration.status, + CompensationStatus::Pending | CompensationStatus::Running + ) { conn.commit().await.map_err(storage_error)?; - return Ok(CompensationClaimOutcome::Conflict); + return Ok(CompensationAttemptAdmissionOutcome::Conflict); } - if compensation.outcome.is_none() { + if registration.status == CompensationStatus::Pending && registration.outcome.is_none() { let forward_task = - load_forward_task(&mut conn, run_uid, compensation.forward_task_id).await?; + load_forward_task(&mut conn, run_uid, registration.forward_task_id).await?; let reservation = - compensation_reservation(&run, &compensation, forward_task.retry.max_attempts)?; + compensation_reservation(&run, ®istration, forward_task.retry.max_attempts)?; let mut ledger = budget_ledger(&run); if ledger.try_reserve(reservation).is_err() { - let failed = terminalize_compensation_budget_rejection( + terminalize_compensation_budget_rejection( &mut conn, &run, - &compensation, + ®istration, reservation, ) .await?; + enqueue_current_compensation_controller_wake( + &mut conn, + &run, + json!({"reason": "compensation_budget_rejected"}), + now, + ) + .await?; conn.commit().await.map_err(storage_error)?; - return Ok(CompensationClaimOutcome::BudgetRejected(failed)); + return Ok(CompensationAttemptAdmissionOutcome::Conflict); } - persist_run_budget(&mut conn, run_uid, &ledger, true).await?; + persist_run_budget(&mut conn, run_uid, &ledger, false).await?; } - let row = sqlx::query(CLAIM_COMPENSATION_SQL) - .bind(run_uid) - .bind(compensation_id.as_uuid()) - .bind(to_i64(expected_generation, "compensation generation")?) - .fetch_optional(conn.as_mut()) + let attempt_generation = required_u64(&row, "attempt_generation")?; + let dispatch_uid = Uuid::now_v7(); + let reservation_uid = Uuid::now_v7(); + let watchdog_uid = Uuid::now_v7(); + insert_compensation_capacity_reservation( + &mut conn, + reservation_uid, + &run, + ®istration, + attempt_generation, + deadline, + ) + .await?; + increment_compensation_capacity(&mut conn, run.tenant_id).await?; + let watchdog = create_trigger_with_dispatch_in_conn( + conn.as_mut(), + config, + &compensation_trigger( + &run, + ®istration, + attempt_generation, + watchdog_uid, + ExecutionTriggerKind::CompensationWatchdog, + deadline, + json!({}), + ), + ) + .await?; + let attempt_request = ExecutionCompensationAttemptRequest { + dispatch_uid, + capacity_reservation_uid: reservation_uid, + watchdog_trigger_uid: watchdog.trigger.trigger_uid, + watchdog_dispatch_uid: watchdog.dispatch.dispatch_uid, + run_uid: run.run_uid, + compensation_id: registration.compensation_id, + compensation_generation: registration.generation, + compensation_attempt_generation: attempt_generation, + controller_generation: run.controller_generation, + attempt_deadline_at: deadline, + tenant_id: run.tenant_id, + }; + let dispatch_request = compensation_dispatch(&run, &attempt_request, now)?; + let dispatch = enqueue_dispatch_in_conn(conn.as_mut(), &dispatch_request).await?; + let updated = sqlx::query( + "UPDATE moa.execution_compensation SET status = 'running', \ + attempt_state = 'dispatching', attempt_started_at = $6, \ + last_progress_at = GREATEST(last_progress_at, $6), \ + attempt_deadline_at = $7, waiting_since = NULL, \ + active_dispatch_uid = $8, dispatch_sequence = dispatch_sequence + 1, \ + started_at = COALESCE(started_at, $6), updated_at = NOW() \ + WHERE run_uid = $1 AND compensation_id = $2 AND generation = $3 \ + AND attempt_generation = $4 AND attempt_state = 'idle' \ + AND status IN ('pending', 'running') AND EXISTS ( \ + SELECT 1 FROM moa.execution_run AS run \ + WHERE run.run_uid=$1 AND run.controller_generation=$5) \ + RETURNING *", + ) + .bind(run_uid) + .bind(registration.compensation_id.as_uuid()) + .bind(to_i64(registration.generation, "compensation generation")?) + .bind(to_i64( + attempt_generation, + "compensation attempt generation", + )?) + .bind(to_i64(run.controller_generation, "controller generation")?) + .bind(now) + .bind(deadline) + .bind(dispatch_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(updated) = updated else { + conn.rollback().await.map_err(storage_error)?; + return Ok(CompensationAttemptAdmissionOutcome::Conflict); + }; + let record = compensation_attempt_from_row(&updated, &run)?; + conn.commit().await.map_err(storage_error)?; + Ok(CompensationAttemptAdmissionOutcome::Admitted(Box::new( + CompensationAttemptAdmission { + attempt: record, + capacity_reservation_uid: reservation_uid, + dispatch, + watchdog, + }, + ))) + } + + /// Starts one exact immutable compensation slice dispatch. + pub async fn start_compensation_attempt( + &self, + scope: ExecutionScope, + fence: CompensationAttemptFence, + now: DateTime, + ) -> Result { + self.transition_active_compensation_attempt(scope, fence, now, true) .await - .map_err(sqlx_error)?; + } + + /// Records monotonic progress for one exact active compensation slice. + pub async fn record_compensation_attempt_progress( + &self, + scope: ExecutionScope, + fence: CompensationAttemptFence, + observed_at: DateTime, + ) -> Result { + self.transition_active_compensation_attempt(scope, fence, observed_at, false) + .await + } + + /// Claims one exact active compensation slice before sandbox checkpoint and release I/O. + pub async fn begin_compensation_attempt_release( + &self, + request: &ExecutionCompensationAttemptCancelRequest, + claimed_at: DateTime, + ) -> Result { + let mut conn = ExecutionScope::ControlPlane.begin(&self.pool).await?; + let Some((run, row)) = load_fenced_compensation_for_cancel(&mut conn, request).await? + else { + conn.commit().await.map_err(storage_error)?; + return Ok(CompensationAttemptReleaseClaimOutcome::NotFound); + }; + let current = compensation_attempt_from_row(&row, &run)?; + if run.tenant_id != request.tenant_id + || !compensation_attempt_resources_match(&mut conn, request).await? + { + conn.commit().await.map_err(storage_error)?; + return Ok(CompensationAttemptReleaseClaimOutcome::Stale); + } + if current.attempt_state == CompensationAttemptState::Cancelling { + conn.commit().await.map_err(storage_error)?; + return Ok(if current.release_intent == Some(request.intent) { + CompensationAttemptReleaseClaimOutcome::Replayed(current) + } else { + CompensationAttemptReleaseClaimOutcome::Stale + }); + } + if !matches!( + current.attempt_state, + CompensationAttemptState::Dispatching | CompensationAttemptState::Running + ) || current.registration.status != CompensationStatus::Running + { + conn.commit().await.map_err(storage_error)?; + return Ok(CompensationAttemptReleaseClaimOutcome::InvalidState); + } + let row = sqlx::query( + "UPDATE moa.execution_compensation SET attempt_state='cancelling', \ + release_intent=$7, last_progress_at=GREATEST(last_progress_at,$6), \ + updated_at=NOW() \ + WHERE run_uid=$1 AND compensation_id=$2 \ + AND generation=$3 AND attempt_generation=$4 AND active_dispatch_uid=$5 \ + AND attempt_state IN ('dispatching','running') RETURNING *", + ) + .bind(request.run_uid) + .bind(request.compensation_id.as_uuid()) + .bind(to_i64( + request.compensation_generation, + "compensation generation", + )?) + .bind(to_i64( + request.compensation_attempt_generation, + "compensation attempt generation", + )?) + .bind(request.active_dispatch_uid) + .bind(claimed_at) + .bind(compensation_release_intent_label(request.intent)) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; let Some(row) = row else { conn.rollback().await.map_err(storage_error)?; - return Ok(CompensationClaimOutcome::Conflict); + return Ok(CompensationAttemptReleaseClaimOutcome::Stale); }; - let claimed = compensation_from_row(&row)?; + let record = compensation_attempt_from_row(&row, &run)?; conn.commit().await.map_err(storage_error)?; - Ok(CompensationClaimOutcome::Claimed(claimed)) + Ok(CompensationAttemptReleaseClaimOutcome::Applied(record)) } - /// Records one compensation outcome and reconciles its separate bounded reservation. - pub async fn record_compensation_outcome( + async fn transition_active_compensation_attempt( &self, scope: ExecutionScope, - run_uid: Uuid, - compensation_id: CompensationId, - generation: u64, - outcome: ExecutionCompensationOutcome, - ) -> Result { + fence: CompensationAttemptFence, + observed_at: DateTime, + start: bool, + ) -> Result { let mut conn = scope.begin(&self.pool).await?; + let Some((run, row)) = load_fenced_compensation(&mut conn, fence).await? else { + conn.commit().await.map_err(storage_error)?; + return Ok(CompensationAttemptWriteOutcome::NotFound); + }; + let current = compensation_attempt_from_row(&row, &run)?; + let expected = if start { + CompensationAttemptState::Dispatching + } else { + CompensationAttemptState::Running + }; + if current.attempt_state == CompensationAttemptState::Running + && start + && current.last_progress_at >= observed_at + { + conn.commit().await.map_err(storage_error)?; + return Ok(CompensationAttemptWriteOutcome::Replayed(current)); + } + if current.attempt_state != expected + || observed_at < current.last_progress_at + || run.status != ExecutionRunStatus::Compensating + { + conn.commit().await.map_err(storage_error)?; + return Ok(CompensationAttemptWriteOutcome::Conflict); + } + let updated = sqlx::query( + "UPDATE moa.execution_compensation SET attempt_state = $7, \ + attempt_started_at = COALESCE(attempt_started_at, $6), \ + last_progress_at = $6, updated_at = $6 \ + WHERE run_uid = $1 AND compensation_id = $2 AND generation = $3 \ + AND attempt_generation = $4 AND active_dispatch_uid = $5 RETURNING *", + ) + .bind(fence.run_uid) + .bind(fence.compensation_id.as_uuid()) + .bind(to_i64( + fence.compensation_generation, + "compensation generation", + )?) + .bind(to_i64( + fence.attempt_generation, + "compensation attempt generation", + )?) + .bind(fence.dispatch_uid) + .bind(observed_at) + .bind(CompensationAttemptState::Running.as_str()) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let record = compensation_attempt_from_row(&updated, &run)?; + conn.commit().await.map_err(storage_error)?; + Ok(CompensationAttemptWriteOutcome::Applied(record)) + } + + /// Finalizes one cancelling compensation only after exact sandbox release proof. + pub async fn settle_released_compensation_attempt( + &self, + request: &ExecutionCompensationAttemptCancelRequest, + outcome: ExecutionCompensationOutcome, + now: DateTime, + workspace_release_receipt: Option, + ) -> Result { + validate_compensation_settlement_intent(request.intent, &outcome)?; + if !compensation_release_receipt_matches(request, workspace_release_receipt.as_ref()) { + return Ok(CompensationAttemptWriteOutcome::Conflict); + } + self.settle_compensation_attempt_from_state( + ExecutionScope::ControlPlane, + compensation_cancel_request_fence(request), + outcome, + now, + CompensationAttemptState::Cancelling, + Some(request), + workspace_release_receipt.as_ref(), + ) + .await + } + + /// Releases one paused compensation slice back to idle after exact sandbox teardown proof. + pub async fn yield_released_compensation_attempt( + &self, + request: &ExecutionCompensationAttemptCancelRequest, + now: DateTime, + workspace_release_receipt: Option, + ) -> Result { + if request.intent != ExecutionCompensationReleaseIntent::Pause { + return Err(Error::InvalidRepositoryInput { + message: "released compensation yield requires the pause intent".to_string(), + }); + } + if !compensation_release_receipt_matches(request, workspace_release_receipt.as_ref()) { + return Ok(CompensationAttemptWriteOutcome::Conflict); + } + let fence = compensation_cancel_request_fence(request); + let mut conn = ExecutionScope::ControlPlane.begin(&self.pool).await?; + let Some(tenant_id) = load_compensation_tenant(&mut conn, fence.run_uid).await? else { + conn.commit().await.map_err(storage_error)?; + return Ok(CompensationAttemptWriteOutcome::NotFound); + }; + lock_capacity_for_release(&mut conn, tenant_id).await?; let Some(run_row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) - .bind(run_uid) + .bind(request.run_uid) .fetch_optional(conn.as_mut()) .await .map_err(sqlx_error)? else { conn.commit().await.map_err(storage_error)?; - return Ok(CompensationOutcomeWrite::NotFound); + return Ok(CompensationAttemptWriteOutcome::NotFound); }; let run = run_from_row(&run_row)?; let Some(row) = sqlx::query(LOAD_COMPENSATION_FOR_UPDATE_SQL) - .bind(run_uid) - .bind(compensation_id.as_uuid()) + .bind(request.run_uid) + .bind(request.compensation_id.as_uuid()) .fetch_optional(conn.as_mut()) .await .map_err(sqlx_error)? else { conn.commit().await.map_err(storage_error)?; - return Ok(CompensationOutcomeWrite::NotFound); + return Ok(CompensationAttemptWriteOutcome::NotFound); }; - let compensation = compensation_from_row(&row)?; - let exact_settled_replay = compensation.generation == generation - && compensation.status.is_settled() - && compensation.outcome.as_ref() == Some(&outcome); - let exact_requeue_replay = compensation.status == CompensationStatus::Pending - && generation.checked_add(1) == Some(compensation.generation) - && compensation.attempt == compensation.generation - && compensation.outcome.as_ref() == Some(&outcome); - if exact_settled_replay || exact_requeue_replay { - conn.commit().await.map_err(storage_error)?; - return Ok(CompensationOutcomeWrite::Replayed(compensation)); - } - if run.status != ExecutionRunStatus::Compensating - || compensation.status != CompensationStatus::Running - || compensation.generation != generation + let current = compensation_attempt_from_row(&row, &run)?; + if !persisted_compensation_release_receipt_matches( + &mut conn, + request, + workspace_release_receipt.as_ref(), + ) + .await? { conn.commit().await.map_err(storage_error)?; - return Ok(CompensationOutcomeWrite::Conflict); + return Ok(CompensationAttemptWriteOutcome::Conflict); } - let forward_task = - load_forward_task(&mut conn, run_uid, compensation.forward_task_id).await?; - let full_reservation = - compensation_reservation(&run, &compensation, forward_task.retry.max_attempts)?; - let previous_usage = compensation - .outcome - .as_ref() - .map(ExecutionCompensationOutcome::usage) - .cloned() - .unwrap_or_else(zero_usage); - let remaining = remaining_compensation_reservation(full_reservation, &previous_usage); - let retryable = matches!( - outcome, - ExecutionCompensationOutcome::Failed { - retryable: true, - .. - } - ) && compensation.attempt < u64::from(forward_task.retry.max_attempts); - let terminal = !retryable; - let mut ledger = budget_ledger(&run); - let reconciliation = ledger.reconcile_cumulative_with_ceiling( - remaining, - &previous_usage, - outcome.usage(), - terminal, - i64::MAX as u64, - )?; - let (status, attempt, next_generation, manual_repair, error) = match &outcome { - ExecutionCompensationOutcome::Completed { .. } => ( - CompensationStatus::Completed, - compensation.attempt, - compensation.generation, - false, - None, - ), - ExecutionCompensationOutcome::Failed { message, .. } if retryable => ( - CompensationStatus::Pending, - compensation.attempt.checked_add(1).ok_or_else(|| { - Error::InvalidRepositoryData { - message: "compensation attempt overflow".to_string(), - } - })?, - compensation.generation.checked_add(1).ok_or_else(|| { - Error::InvalidRepositoryData { - message: "compensation generation overflow".to_string(), - } - })?, - false, - Some(json!({"class":"retryable", "message":message})), - ), - ExecutionCompensationOutcome::Failed { message, .. } => ( - CompensationStatus::Failed, - compensation.attempt, - compensation.generation, - true, - Some(json!({"class":"terminal", "message":message})), - ), - ExecutionCompensationOutcome::UnknownOutcome { message, .. } => ( - CompensationStatus::UnknownOutcome, - compensation.attempt, - compensation.generation, - true, - Some( - json!({"class":"unknown_outcome", "message":message, "manual_repair_required":true}), - ), - ), + if current.attempt_state == CompensationAttemptState::Idle + && current.attempt_generation == fence.attempt_generation.saturating_add(1) + { + conn.commit().await.map_err(storage_error)?; + return Ok(CompensationAttemptWriteOutcome::Replayed(current)); + } + if run.tenant_id != request.tenant_id + || current.attempt_state != CompensationAttemptState::Cancelling + || current.release_intent != Some(ExecutionCompensationReleaseIntent::Pause) + || !compensation_attempt_resources_match(&mut conn, request).await? + { + conn.commit().await.map_err(storage_error)?; + return Ok(CompensationAttemptWriteOutcome::Conflict); + } + release_unbound_compensation_external_intent_in_conn(&mut conn, &run, ¤t).await?; + let resource_fence = compensation_cancel_resource_fence(request); + release_compensation_capacity(&mut conn, &run, resource_fence).await?; + supersede_compensation_triggers(&mut conn, resource_fence, None).await?; + let updated = sqlx::query( + "UPDATE moa.execution_compensation SET attempt_state='idle', \ + attempt_generation=attempt_generation+1, attempt_started_at=NULL, \ + attempt_deadline_at=NULL, waiting_since=NULL, active_dispatch_uid=NULL, \ + external_job_uid=NULL, release_intent=NULL, \ + last_progress_at=GREATEST(last_progress_at,$6), updated_at=NOW() \ + WHERE run_uid=$1 AND compensation_id=$2 AND generation=$3 \ + AND attempt_generation=$4 AND active_dispatch_uid=$5 \ + AND attempt_state='cancelling' AND release_intent='pause' RETURNING *", + ) + .bind(fence.run_uid) + .bind(fence.compensation_id.as_uuid()) + .bind(to_i64( + fence.compensation_generation, + "compensation generation", + )?) + .bind(to_i64( + fence.attempt_generation, + "compensation attempt generation", + )?) + .bind(fence.dispatch_uid) + .bind(now) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(updated) = updated else { + conn.rollback().await.map_err(storage_error)?; + return Ok(CompensationAttemptWriteOutcome::Conflict); + }; + let run = reconcile_run_after_compensation_capacity_release(&mut conn, &run, now).await?; + if !matches!( + run.status, + ExecutionRunStatus::PauseRequested + | ExecutionRunStatus::Pausing + | ExecutionRunStatus::Paused + ) { + enqueue_current_compensation_controller_wake( + &mut conn, + &run, + json!({"reason": "compensation_pause_released"}), + now, + ) + .await?; + } + let record = compensation_attempt_from_row(&updated, &run)?; + conn.commit().await.map_err(storage_error)?; + Ok(CompensationAttemptWriteOutcome::Applied(record)) + } + + /// Requeues one compensation after recovery proved its provider job never started. + pub async fn yield_released_compensation_attempt_after_external_not_started( + &self, + request: &ExecutionCompensationAttemptCancelRequest, + now: DateTime, + workspace_release_receipt: Option, + ) -> Result { + if request.intent != ExecutionCompensationReleaseIntent::Retry { + return Err(Error::InvalidRepositoryInput { + message: "external NotStarted recovery requires the retry release intent" + .to_string(), + }); + } + if !compensation_release_receipt_matches(request, workspace_release_receipt.as_ref()) { + return Ok(CompensationAttemptWriteOutcome::Conflict); + } + let fence = compensation_cancel_request_fence(request); + let mut conn = ExecutionScope::ControlPlane.begin(&self.pool).await?; + let Some(tenant_id) = load_compensation_tenant(&mut conn, fence.run_uid).await? else { + conn.commit().await.map_err(storage_error)?; + return Ok(CompensationAttemptWriteOutcome::NotFound); + }; + lock_capacity_for_release(&mut conn, tenant_id).await?; + let Some(run_row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) + .bind(request.run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + conn.commit().await.map_err(storage_error)?; + return Ok(CompensationAttemptWriteOutcome::NotFound); + }; + let run = run_from_row(&run_row)?; + let Some(row) = sqlx::query(LOAD_COMPENSATION_FOR_UPDATE_SQL) + .bind(request.run_uid) + .bind(request.compensation_id.as_uuid()) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + conn.commit().await.map_err(storage_error)?; + return Ok(CompensationAttemptWriteOutcome::NotFound); }; - let persisted = persisted_compensation_outcome(&row, Some(outcome.clone()))?; + let current = compensation_attempt_from_row(&row, &run)?; + if !persisted_compensation_release_receipt_matches( + &mut conn, + request, + workspace_release_receipt.as_ref(), + ) + .await? + { + conn.commit().await.map_err(storage_error)?; + return Ok(CompensationAttemptWriteOutcome::Conflict); + } + if current.attempt_state == CompensationAttemptState::Idle + && current.attempt_generation == fence.attempt_generation.saturating_add(1) + { + conn.commit().await.map_err(storage_error)?; + return Ok(CompensationAttemptWriteOutcome::Replayed(current)); + } + if run.tenant_id != request.tenant_id + || run.controller_generation != request.controller_generation + || current.registration.generation != request.compensation_generation + || current.attempt_generation != request.compensation_attempt_generation + || current.active_dispatch_uid != Some(request.active_dispatch_uid) + || current.attempt_state != CompensationAttemptState::Cancelling + || current.release_intent != Some(ExecutionCompensationReleaseIntent::Retry) + || current.external_job_uid.is_some() + || !compensation_attempt_resources_match(&mut conn, request).await? + { + conn.commit().await.map_err(storage_error)?; + return Ok(CompensationAttemptWriteOutcome::Conflict); + } + let resource_fence = compensation_cancel_resource_fence(request); + release_compensation_capacity(&mut conn, &run, resource_fence).await?; + supersede_compensation_triggers(&mut conn, resource_fence, None).await?; let updated = sqlx::query( - "UPDATE moa.execution_compensation SET status = $3, attempt = $4, generation = $5, \ - outcome = $6, error = $7, updated_at = NOW(), \ - completed_at = CASE WHEN $8 THEN NOW() ELSE NULL END \ - WHERE run_uid = $1 AND compensation_id = $2 \ - RETURNING compensation_id, run_uid, forward_task_id, registered_sequence, \ - forward_generation, compensator, mapped_input, status, attempt, generation, \ - outcome, error, created_at, updated_at, started_at, completed_at", + "UPDATE moa.execution_compensation SET attempt_state='idle', \ + attempt_generation=attempt_generation+1, attempt_started_at=NULL, \ + attempt_deadline_at=NULL, waiting_since=NULL, active_dispatch_uid=NULL, \ + external_job_uid=NULL, release_intent=NULL, \ + last_progress_at=GREATEST(last_progress_at,$6), updated_at=NOW() \ + WHERE run_uid=$1 AND compensation_id=$2 AND generation=$3 \ + AND attempt_generation=$4 AND active_dispatch_uid=$5 \ + AND attempt_state='cancelling' AND release_intent='retry' \ + AND external_job_uid IS NULL RETURNING *", ) - .bind(run_uid) - .bind(compensation_id.as_uuid()) - .bind(status.as_str()) - .bind(to_i64(attempt, "compensation attempt")?) - .bind(to_i64(next_generation, "compensation generation")?) - .bind(serde_json::to_value(persisted)?) - .bind(error) - .bind(status.is_settled()) - .fetch_one(conn.as_mut()) + .bind(fence.run_uid) + .bind(fence.compensation_id.as_uuid()) + .bind(to_i64( + fence.compensation_generation, + "compensation generation", + )?) + .bind(to_i64( + fence.attempt_generation, + "compensation attempt generation", + )?) + .bind(fence.dispatch_uid) + .bind(now) + .fetch_optional(conn.as_mut()) .await .map_err(sqlx_error)?; - persist_run_budget_and_repair(&mut conn, run_uid, &reconciliation, manual_repair).await?; - let updated = compensation_from_row(&updated)?; + let Some(updated) = updated else { + conn.rollback().await.map_err(storage_error)?; + return Ok(CompensationAttemptWriteOutcome::Conflict); + }; + enqueue_current_compensation_controller_wake( + &mut conn, + &run, + json!({"reason": "compensation_external_not_started_released"}), + now, + ) + .await?; + let record = compensation_attempt_from_row(&updated, &run)?; conn.commit().await.map_err(storage_error)?; - Ok(match status { - CompensationStatus::Completed => CompensationOutcomeWrite::Completed(updated), - CompensationStatus::Pending => CompensationOutcomeWrite::Requeued(updated), - CompensationStatus::Failed => CompensationOutcomeWrite::Failed(updated), - CompensationStatus::UnknownOutcome => CompensationOutcomeWrite::UnknownOutcome(updated), - CompensationStatus::Running => CompensationOutcomeWrite::Conflict, - }) + Ok(CompensationAttemptWriteOutcome::Applied(record)) + } + + /// Parks one cancelling slice after its sandbox ownership was durably released. + pub async fn park_released_compensation_review( + &self, + request: &ExecutionCompensationAttemptCancelRequest, + review_uid: Uuid, + expires_at: DateTime, + now: DateTime, + workspace_release_receipt: Option, + ) -> Result { + if request.intent != ExecutionCompensationReleaseIntent::Review { + return Err(Error::InvalidRepositoryInput { + message: "compensation review park requires the review release intent".to_string(), + }); + } + if review_uid.is_nil() || expires_at <= now { + return Err(Error::InvalidRepositoryInput { + message: "compensation review requires a non-nil UID and future expiry".to_string(), + }); + } + if !compensation_release_receipt_matches(request, workspace_release_receipt.as_ref()) { + return Ok(CompensationAttemptWriteOutcome::Conflict); + } + let fence = compensation_cancel_request_fence(request); + let mut conn = ExecutionScope::ControlPlane.begin(&self.pool).await?; + let Some(tenant_id) = load_compensation_tenant(&mut conn, fence.run_uid).await? else { + conn.commit().await.map_err(storage_error)?; + return Ok(CompensationAttemptWriteOutcome::NotFound); + }; + lock_capacity_for_release(&mut conn, tenant_id).await?; + let Some((run, row)) = load_fenced_compensation_for_cancel(&mut conn, request).await? + else { + conn.commit().await.map_err(storage_error)?; + return Ok(CompensationAttemptWriteOutcome::NotFound); + }; + let current = compensation_attempt_from_row(&row, &run)?; + if !persisted_compensation_release_receipt_matches( + &mut conn, + request, + workspace_release_receipt.as_ref(), + ) + .await? + { + conn.commit().await.map_err(storage_error)?; + return Ok(CompensationAttemptWriteOutcome::Conflict); + } + if run.tenant_id != request.tenant_id { + conn.commit().await.map_err(storage_error)?; + return Ok(CompensationAttemptWriteOutcome::Conflict); + } + let mut persisted = + persisted_compensation_outcome(&row, current.registration.outcome.clone())?; + if current.attempt_state == CompensationAttemptState::WaitingReview { + let exact_review = persisted.review_audit.iter().any(|entry| { + entry.review_uid == review_uid + && entry.generation == fence.compensation_generation + && !entry.accepted + && entry.expires_at == Some(expires_at) + }); + conn.commit().await.map_err(storage_error)?; + return Ok(if exact_review { + CompensationAttemptWriteOutcome::Replayed(current) + } else { + CompensationAttemptWriteOutcome::Conflict + }); + } + if current.attempt_state != CompensationAttemptState::Cancelling { + conn.commit().await.map_err(storage_error)?; + return Ok(CompensationAttemptWriteOutcome::Conflict); + } + if current.release_intent != Some(ExecutionCompensationReleaseIntent::Review) + || !compensation_attempt_resources_match(&mut conn, request).await? + { + conn.commit().await.map_err(storage_error)?; + return Ok(CompensationAttemptWriteOutcome::Conflict); + } + release_unbound_compensation_external_intent_in_conn(&mut conn, &run, ¤t).await?; + let resource_fence = compensation_cancel_resource_fence(request); + release_compensation_capacity(&mut conn, &run, resource_fence).await?; + supersede_compensation_triggers(&mut conn, resource_fence, None).await?; + persisted.review_audit.push(CompensationReviewAuditEntry { + review_uid, + generation: fence.compensation_generation, + accepted: false, + resolution: None, + expires_at: Some(expires_at), + recorded_at: now, + }); + let updated = sqlx::query( + "UPDATE moa.execution_compensation SET attempt_state = 'waiting_review', \ + attempt_deadline_at = NULL, waiting_since = $6, active_dispatch_uid = NULL, \ + release_intent=NULL, outcome=$7, \ + last_progress_at=GREATEST(last_progress_at,$6), updated_at=NOW() \ + WHERE run_uid = $1 AND compensation_id = $2 AND generation = $3 \ + AND attempt_generation = $4 AND active_dispatch_uid = $5 RETURNING *", + ) + .bind(fence.run_uid) + .bind(fence.compensation_id.as_uuid()) + .bind(to_i64( + fence.compensation_generation, + "compensation generation", + )?) + .bind(to_i64( + fence.attempt_generation, + "compensation attempt generation", + )?) + .bind(fence.dispatch_uid) + .bind(now) + .bind(serde_json::to_value(persisted)?) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let run = reconcile_run_after_compensation_capacity_release(&mut conn, &run, now).await?; + let record = compensation_attempt_from_row(&updated, &run)?; + conn.commit().await.map_err(storage_error)?; + Ok(CompensationAttemptWriteOutcome::Applied(record)) + } + + /// Commits one provider job before parking a released compensation attempt. + pub async fn begin_compensation_external_release( + &self, + request: &ExecutionCompensationAttemptCancelRequest, + external_job_uid: Uuid, + claimed_at: DateTime, + ) -> Result { + if request.intent != ExecutionCompensationReleaseIntent::ExternalJob { + return Err(Error::InvalidRepositoryInput { + message: "external-job release requires the exact external-job intent".to_string(), + }); + } + let mut conn = ExecutionScope::ControlPlane.begin(&self.pool).await?; + let outcome = begin_compensation_external_release_in_conn( + &mut conn, + request, + external_job_uid, + claimed_at, + ) + .await?; + conn.commit().await.map_err(storage_error)?; + Ok(outcome) + } + + /// Finalizes a released compensation attempt into its already-durable provider wait. + pub async fn yield_released_compensation_attempt_to_external_job( + &self, + request: &ExecutionCompensationAttemptCancelRequest, + external_job_uid: Uuid, + workspace_release_receipt: Option, + yielded_at: DateTime, + ) -> Result { + if request.intent != ExecutionCompensationReleaseIntent::ExternalJob { + return Err(Error::InvalidRepositoryInput { + message: "external-job yield requires the exact external-job release intent" + .to_string(), + }); + } + if !compensation_release_receipt_matches(request, workspace_release_receipt.as_ref()) { + return Ok(CompensationAttemptExternalOutcome::Stale); + } + let fence = compensation_cancel_request_fence(request); + let mut conn = ExecutionScope::ControlPlane.begin(&self.pool).await?; + let Some(tenant_id) = load_compensation_tenant(&mut conn, fence.run_uid).await? else { + conn.commit().await.map_err(storage_error)?; + return Ok(CompensationAttemptExternalOutcome::NotFound); + }; + lock_capacity_for_release(&mut conn, tenant_id).await?; + let Some((run, row)) = load_fenced_compensation_for_cancel(&mut conn, request).await? + else { + conn.commit().await.map_err(storage_error)?; + return Ok(CompensationAttemptExternalOutcome::NotFound); + }; + let current = compensation_attempt_from_row(&row, &run)?; + if !persisted_compensation_release_receipt_matches( + &mut conn, + request, + workspace_release_receipt.as_ref(), + ) + .await? + { + conn.commit().await.map_err(storage_error)?; + return Ok(CompensationAttemptExternalOutcome::Stale); + } + let Some(persisted_job) = + load_external_job_for_update_in_conn(conn.as_mut(), external_job_uid).await? + else { + conn.rollback().await.map_err(storage_error)?; + return Ok(CompensationAttemptExternalOutcome::NotFound); + }; + let expected_owner = ExecutionExternalJobOwner::Compensation { + compensation_id: request.compensation_id.as_uuid(), + compensation_generation: request.compensation_generation, + compensation_attempt_generation: request.compensation_attempt_generation, + }; + if run.tenant_id != request.tenant_id + || persisted_job.tenant_id != request.tenant_id + || persisted_job.run_uid != request.run_uid + || persisted_job.owner != expected_owner + { + conn.commit().await.map_err(storage_error)?; + return Ok(CompensationAttemptExternalOutcome::Stale); + } + if matches!( + current.attempt_state, + CompensationAttemptState::WaitingExternal + | CompensationAttemptState::Terminal + | CompensationAttemptState::UnknownOutcome + ) && current.external_job_uid == Some(external_job_uid) + { + conn.commit().await.map_err(storage_error)?; + return Ok(CompensationAttemptExternalOutcome::Replayed { + attempt: current, + external_job: persisted_job, + }); + } + if current.attempt_state != CompensationAttemptState::Cancelling { + conn.commit().await.map_err(storage_error)?; + return Ok(CompensationAttemptExternalOutcome::InvalidState); + } + if !compensation_attempt_resources_match(&mut conn, request).await? { + conn.commit().await.map_err(storage_error)?; + return Ok(CompensationAttemptExternalOutcome::Stale); + } + let resource_fence = compensation_cancel_resource_fence(request); + release_compensation_capacity(&mut conn, &run, resource_fence).await?; + supersede_compensation_triggers(&mut conn, resource_fence, None).await?; + let row = sqlx::query( + "UPDATE moa.execution_compensation SET attempt_state='waiting_external', \ + waiting_since=$6, external_job_uid=$7, active_dispatch_uid=NULL, \ + attempt_deadline_at=NULL, release_intent=NULL, \ + last_progress_at=GREATEST(last_progress_at,$6), updated_at=NOW() \ + WHERE run_uid=$1 AND compensation_id=$2 AND generation=$3 \ + AND attempt_generation=$4 AND active_dispatch_uid=$5 \ + AND attempt_state='cancelling' RETURNING *", + ) + .bind(fence.run_uid) + .bind(fence.compensation_id.as_uuid()) + .bind(to_i64( + fence.compensation_generation, + "compensation generation", + )?) + .bind(to_i64( + fence.attempt_generation, + "compensation attempt generation", + )?) + .bind(fence.dispatch_uid) + .bind(yielded_at) + .bind(external_job_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(row) = row else { + conn.rollback().await.map_err(storage_error)?; + return Ok(CompensationAttemptExternalOutcome::Stale); + }; + let mut attempt = compensation_attempt_from_row(&row, &run)?; + if persisted_job.state.is_terminal() { + attempt = + match settle_external_job_terminal_in_conn(&mut conn, &persisted_job, yielded_at) + .await? + { + CompensationExternalJobSettlementOutcome::Applied(attempt) + | CompensationExternalJobSettlementOutcome::Replayed(attempt) => attempt, + CompensationExternalJobSettlementOutcome::DeferredRelease(_) => { + conn.rollback().await.map_err(storage_error)?; + return Ok(CompensationAttemptExternalOutcome::Stale); + } + CompensationExternalJobSettlementOutcome::Stale => { + conn.rollback().await.map_err(storage_error)?; + return Ok(CompensationAttemptExternalOutcome::Stale); + } + CompensationExternalJobSettlementOutcome::NotFound => { + conn.rollback().await.map_err(storage_error)?; + return Ok(CompensationAttemptExternalOutcome::NotFound); + } + }; + } + if !persisted_job.state.is_terminal() { + enqueue_run_activation_in_conn( + conn.as_mut(), + run.tenant_id, + run.run_uid, + run.controller_generation, + yielded_at, + json!({ + "source": "compensation_external_job_started", + "compensation_id": fence.compensation_id, + "external_job_uid": external_job_uid, + }), + ) + .await?; + } + conn.commit().await.map_err(storage_error)?; + Ok(CompensationAttemptExternalOutcome::Applied { + attempt, + external_job: persisted_job, + }) + } + + /// Resolves the current parked compensation review from its stable action-review owner. + #[allow(clippy::too_many_arguments)] + pub async fn resolve_current_compensation_review( + &self, + scope: ExecutionScope, + run_uid: Uuid, + compensation_id: CompensationId, + logical_generation: u64, + review_uid: Uuid, + resolution: &ExecutionActionReviewResolution, + now: DateTime, + ) -> Result { + if !matches!(scope, ExecutionScope::ControlPlane) + || logical_generation == 0 + || review_uid.is_nil() + { + return Err(Error::InvalidRepositoryInput { + message: "compensation review resolution requires control-plane scope and exact identities" + .to_string(), + }); + } + let mut conn = scope.begin(&self.pool).await?; + let Some(run_row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) + .bind(run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + conn.commit().await.map_err(storage_error)?; + return Ok(CompensationReviewResolutionOutcome::NotFound); + }; + let run = run_from_row(&run_row)?; + let Some(row) = sqlx::query(LOAD_COMPENSATION_FOR_UPDATE_SQL) + .bind(run_uid) + .bind(compensation_id.as_uuid()) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + conn.commit().await.map_err(storage_error)?; + return Ok(CompensationReviewResolutionOutcome::NotFound); + }; + let current = compensation_attempt_from_row(&row, &run)?; + let persisted = persisted_compensation_outcome(&row, current.registration.outcome.clone())?; + if let Some(existing) = persisted + .review_audit + .iter() + .find(|entry| entry.review_uid == review_uid && entry.generation == logical_generation) + { + if existing.accepted && existing.resolution.as_ref() != Some(resolution) { + return Err(Error::InvalidRepositoryData { + message: "compensation review UID replayed with different semantics" + .to_string(), + }); + } + if existing.accepted { + conn.commit().await.map_err(storage_error)?; + return Ok(CompensationReviewResolutionOutcome::Replayed(current)); + } + } + if current.registration.generation != logical_generation { + conn.commit().await.map_err(storage_error)?; + return Ok(CompensationReviewResolutionOutcome::Stale); + } + if matches!( + current.attempt_state, + CompensationAttemptState::Dispatching | CompensationAttemptState::Running + ) { + conn.commit().await.map_err(storage_error)?; + return Ok(CompensationReviewResolutionOutcome::NotReady); + } + if current.attempt_state != CompensationAttemptState::WaitingReview { + conn.commit().await.map_err(storage_error)?; + return Ok(CompensationReviewResolutionOutcome::Stale); + } + let parked_review = persisted.review_audit.iter().any(|entry| { + entry.review_uid == review_uid + && entry.generation == logical_generation + && !entry.accepted + }); + if !parked_review { + conn.commit().await.map_err(storage_error)?; + return Ok(CompensationReviewResolutionOutcome::Stale); + } + // Pausing advances the run controller generation, but the storage-only review remains + // owned by the immutable attempt dispatch that originally entered the wait. + let dispatch_owner: Option<(Uuid, i64)> = sqlx::query_as( + "SELECT dispatch_uid,controller_generation FROM moa.execution_dispatch_outbox \ + WHERE run_uid=$1 AND compensation_id=$2 AND compensation_generation=$3 \ + AND compensation_attempt_generation=$4 AND dispatch_kind='compensation_attempt' \ + ORDER BY created_at DESC,dispatch_uid DESC LIMIT 1", + ) + .bind(run_uid) + .bind(compensation_id.as_uuid()) + .bind(to_i64(logical_generation, "compensation generation")?) + .bind(to_i64( + current.attempt_generation, + "compensation attempt generation", + )?) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some((dispatch_uid, attempt_controller_generation)) = dispatch_owner else { + conn.rollback().await.map_err(storage_error)?; + return Ok(CompensationReviewResolutionOutcome::Stale); + }; + let fence = CompensationAttemptFence { + run_uid, + compensation_id, + controller_generation: to_u64( + attempt_controller_generation, + "attempt controller generation", + )?, + compensation_generation: logical_generation, + attempt_generation: current.attempt_generation, + dispatch_uid, + }; + if let ExecutionActionReviewResolution::ExternalJob { + external_job_uid, + job, + } = resolution + { + let write = Self::settle_reviewed_compensation_external_job_in_conn( + &mut conn, + &run, + &row, + fence, + review_uid, + resolution, + *external_job_uid, + job, + now, + ) + .await?; + conn.commit().await.map_err(storage_error)?; + return match write { + CompensationAttemptWriteOutcome::Applied(record) => { + Ok(CompensationReviewResolutionOutcome::Applied(record)) + } + CompensationAttemptWriteOutcome::Replayed(record) => { + Ok(CompensationReviewResolutionOutcome::Replayed(record)) + } + CompensationAttemptWriteOutcome::NotFound => { + Ok(CompensationReviewResolutionOutcome::NotFound) + } + CompensationAttemptWriteOutcome::Conflict => { + Ok(CompensationReviewResolutionOutcome::Stale) + } + }; + } + let outcome = compensation_outcome_from_review_resolution(resolution)?; + let write = Self::settle_reviewed_compensation_in_conn( + &mut conn, &run, &row, fence, review_uid, resolution, outcome, now, + ) + .await?; + conn.commit().await.map_err(storage_error)?; + match write { + CompensationAttemptWriteOutcome::Applied(record) => { + Ok(CompensationReviewResolutionOutcome::Applied(record)) + } + CompensationAttemptWriteOutcome::Replayed(record) => { + Ok(CompensationReviewResolutionOutcome::Replayed(record)) + } + CompensationAttemptWriteOutcome::NotFound => { + Ok(CompensationReviewResolutionOutcome::NotFound) + } + CompensationAttemptWriteOutcome::Conflict => { + Ok(CompensationReviewResolutionOutcome::Stale) + } + } + } + + #[allow(clippy::too_many_arguments)] + async fn settle_compensation_attempt_from_state( + &self, + scope: ExecutionScope, + fence: CompensationAttemptFence, + outcome: ExecutionCompensationOutcome, + now: DateTime, + expected_state: CompensationAttemptState, + cancellation_request: Option<&ExecutionCompensationAttemptCancelRequest>, + workspace_release_receipt: Option<&ExecutionHandReleaseReceipt>, + ) -> Result { + let mut conn = scope.begin(&self.pool).await?; + let Some(tenant_id) = load_compensation_tenant(&mut conn, fence.run_uid).await? else { + conn.commit().await.map_err(storage_error)?; + return Ok(CompensationAttemptWriteOutcome::NotFound); + }; + lock_capacity_for_release(&mut conn, tenant_id).await?; + let loaded = if let Some(request) = cancellation_request { + load_fenced_compensation_for_cancel(&mut conn, request).await? + } else { + load_fenced_compensation(&mut conn, fence).await? + }; + let Some((run, row)) = loaded else { + conn.commit().await.map_err(storage_error)?; + return Ok(CompensationAttemptWriteOutcome::NotFound); + }; + let current = compensation_attempt_from_row(&row, &run)?; + if current.attempt_state != expected_state + || cancellation_request.is_some_and(|request| run.tenant_id != request.tenant_id) + { + conn.commit().await.map_err(storage_error)?; + return Ok(CompensationAttemptWriteOutcome::Conflict); + } + if let Some(request) = cancellation_request + && (current.release_intent != Some(request.intent) + || !compensation_attempt_resources_match(&mut conn, request).await? + || !persisted_compensation_release_receipt_matches( + &mut conn, + request, + workspace_release_receipt, + ) + .await?) + { + conn.commit().await.map_err(storage_error)?; + return Ok(CompensationAttemptWriteOutcome::Conflict); + } + let forward_task = load_forward_task( + &mut conn, + fence.run_uid, + current.registration.forward_task_id, + ) + .await?; + let full_reservation = + compensation_reservation(&run, ¤t.registration, forward_task.retry.max_attempts)?; + let previous_usage = current + .registration + .outcome + .as_ref() + .map(ExecutionCompensationOutcome::usage) + .cloned() + .unwrap_or_else(zero_usage); + let remaining = remaining_compensation_reservation(full_reservation, &previous_usage); + let retry = matches!( + outcome, + ExecutionCompensationOutcome::Failed { + retryable: true, + .. + } + ) && current.registration.attempt < u64::from(forward_task.retry.max_attempts); + let accepted_outcome = if retry { + outcome + } else { + force_terminal_failure_if_exhausted(outcome) + }; + let mut ledger = budget_ledger(&run); + let reconciliation = ledger.reconcile_cumulative_with_ceiling( + remaining, + &previous_usage, + accepted_outcome.usage(), + !retry, + i64::MAX as u64, + )?; + let (status, attempt_state, attempt, generation, next_attempt_generation, repair, error) = + compensation_settlement_fields(¤t, &accepted_outcome, retry)?; + let persisted = persisted_compensation_outcome(&row, Some(accepted_outcome))?; + release_unbound_compensation_external_intent_in_conn(&mut conn, &run, ¤t).await?; + let resource_fence = cancellation_request + .map(compensation_cancel_resource_fence) + .unwrap_or(fence); + release_compensation_capacity(&mut conn, &run, resource_fence).await?; + supersede_compensation_triggers(&mut conn, resource_fence, None).await?; + let updated = sqlx::query( + "UPDATE moa.execution_compensation SET status=$6, attempt_state=$7, attempt=$8, \ + generation=$9, attempt_generation=$10, outcome=$11, error=$12, \ + attempt_started_at=NULL, attempt_deadline_at=NULL, waiting_since=NULL, \ + active_dispatch_uid=NULL, release_intent=NULL, \ + last_progress_at=GREATEST(last_progress_at,$13), updated_at=NOW(), \ + completed_at=CASE WHEN $14 THEN $13 ELSE NULL END \ + WHERE run_uid=$1 AND compensation_id=$2 AND generation=$3 \ + AND attempt_generation=$4 AND active_dispatch_uid=$5 \ + AND attempt_state=$15 RETURNING *", + ) + .bind(fence.run_uid) + .bind(fence.compensation_id.as_uuid()) + .bind(to_i64( + fence.compensation_generation, + "compensation generation", + )?) + .bind(to_i64( + fence.attempt_generation, + "compensation attempt generation", + )?) + .bind(fence.dispatch_uid) + .bind(status.as_str()) + .bind(attempt_state.as_str()) + .bind(to_i64(attempt, "compensation attempt")?) + .bind(to_i64(generation, "compensation generation")?) + .bind(to_i64( + next_attempt_generation, + "compensation attempt generation", + )?) + .bind(serde_json::to_value(persisted)?) + .bind(error) + .bind(now) + .bind(status.is_settled()) + .bind(expected_state.as_str()) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + persist_run_budget_and_repair(&mut conn, fence.run_uid, &reconciliation, repair).await?; + enqueue_current_compensation_controller_wake( + &mut conn, + &run, + json!({"reason": "compensation_settled"}), + now, + ) + .await?; + let record = compensation_attempt_from_row(&updated, &run)?; + conn.commit().await.map_err(storage_error)?; + Ok(CompensationAttemptWriteOutcome::Applied(record)) + } + + #[allow(clippy::too_many_arguments)] + async fn settle_reviewed_compensation_in_conn( + conn: &mut ScopedConn<'_>, + run: &ExecutionRunRecord, + row: &PgRow, + fence: CompensationAttemptFence, + review_uid: Uuid, + resolution: &ExecutionActionReviewResolution, + outcome: ExecutionCompensationOutcome, + now: DateTime, + ) -> Result { + let current = compensation_attempt_from_row(row, run)?; + let replay = persisted_compensation_outcome(row, current.registration.outcome.clone())?; + if let Some(existing) = replay.review_audit.iter().find(|entry| { + entry.review_uid == review_uid && entry.generation == fence.compensation_generation + }) { + if existing.accepted { + if existing.resolution.as_ref() != Some(resolution) + || replay.result.as_ref() != Some(&outcome) + { + return Err(Error::InvalidRepositoryData { + message: "compensation review UID replayed with different semantics" + .to_string(), + }); + } + return Ok(CompensationAttemptWriteOutcome::Replayed(current)); + } + } else { + return Ok(CompensationAttemptWriteOutcome::Conflict); + } + if current.attempt_state != CompensationAttemptState::WaitingReview { + return Ok(CompensationAttemptWriteOutcome::Conflict); + } + let mut persisted = persisted_compensation_outcome(row, Some(outcome.clone()))?; + let entry = persisted + .review_audit + .iter_mut() + .find(|entry| { + entry.review_uid == review_uid + && entry.generation == fence.compensation_generation + && !entry.accepted + }) + .ok_or_else(|| Error::InvalidRepositoryData { + message: "parked compensation review lost its persisted audit entry".to_string(), + })?; + entry.accepted = true; + entry.resolution = Some(resolution.clone()); + entry.recorded_at = now; + let forward_task = + load_forward_task(conn, fence.run_uid, current.registration.forward_task_id).await?; + let full_reservation = + compensation_reservation(run, ¤t.registration, forward_task.retry.max_attempts)?; + let previous_usage = current + .registration + .outcome + .as_ref() + .map(ExecutionCompensationOutcome::usage) + .cloned() + .unwrap_or_else(zero_usage); + let remaining = remaining_compensation_reservation(full_reservation, &previous_usage); + let retry = matches!( + outcome, + ExecutionCompensationOutcome::Failed { + retryable: true, + .. + } + ) && current.registration.attempt < u64::from(forward_task.retry.max_attempts); + let accepted_outcome = if retry { + outcome + } else { + force_terminal_failure_if_exhausted(outcome) + }; + persisted.result = Some(accepted_outcome.clone()); + let mut ledger = budget_ledger(run); + let reconciliation = ledger.reconcile_cumulative_with_ceiling( + remaining, + &previous_usage, + accepted_outcome.usage(), + !retry, + i64::MAX as u64, + )?; + let (status, attempt_state, attempt, generation, next_attempt_generation, repair, error) = + compensation_settlement_fields(¤t, &accepted_outcome, retry)?; + release_unbound_compensation_external_intent_in_conn(conn, run, ¤t).await?; + supersede_compensation_triggers(conn, fence, None).await?; + let updated = sqlx::query( + "UPDATE moa.execution_compensation SET status=$5, attempt_state=$6, attempt=$7, \ + generation=$8, attempt_generation=$9, outcome=$10, error=$11, \ + attempt_started_at=NULL, attempt_deadline_at=NULL, waiting_since=NULL, \ + active_dispatch_uid=NULL, \ + last_progress_at=GREATEST(last_progress_at,$12), updated_at=NOW(), \ + completed_at=CASE WHEN $13 THEN $12 ELSE NULL END \ + WHERE run_uid=$1 AND compensation_id=$2 AND generation=$3 \ + AND attempt_generation=$4 AND attempt_state='waiting_review' RETURNING *", + ) + .bind(fence.run_uid) + .bind(fence.compensation_id.as_uuid()) + .bind(to_i64( + fence.compensation_generation, + "compensation generation", + )?) + .bind(to_i64( + fence.attempt_generation, + "compensation attempt generation", + )?) + .bind(status.as_str()) + .bind(attempt_state.as_str()) + .bind(to_i64(attempt, "compensation attempt")?) + .bind(to_i64(generation, "compensation generation")?) + .bind(to_i64( + next_attempt_generation, + "compensation attempt generation", + )?) + .bind(serde_json::to_value(persisted)?) + .bind(error) + .bind(now) + .bind(status.is_settled()) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + persist_run_budget_and_repair(conn, fence.run_uid, &reconciliation, repair).await?; + if !matches!( + run.status, + ExecutionRunStatus::PauseRequested + | ExecutionRunStatus::Pausing + | ExecutionRunStatus::Paused + ) { + enqueue_current_compensation_controller_wake( + conn, + run, + json!({"reason": "compensation_review_resolved", "review_uid": review_uid}), + now, + ) + .await?; + } + let record = compensation_attempt_from_row(&updated, run)?; + Ok(CompensationAttemptWriteOutcome::Applied(record)) + } + + #[allow(clippy::too_many_arguments)] + async fn settle_reviewed_compensation_external_job_in_conn( + conn: &mut ScopedConn<'_>, + run: &ExecutionRunRecord, + row: &PgRow, + fence: CompensationAttemptFence, + review_uid: Uuid, + resolution: &ExecutionActionReviewResolution, + external_job_uid: Uuid, + job: &moa_core::types::tools::AsyncToolJob, + now: DateTime, + ) -> Result { + let current = compensation_attempt_from_row(row, run)?; + if current.attempt_state != CompensationAttemptState::WaitingReview { + return Ok(CompensationAttemptWriteOutcome::Conflict); + } + let mut persisted = + persisted_compensation_outcome(row, current.registration.outcome.clone())?; + let entry = persisted + .review_audit + .iter_mut() + .find(|entry| { + entry.review_uid == review_uid + && entry.generation == fence.compensation_generation + && !entry.accepted + }) + .ok_or_else(|| Error::InvalidRepositoryData { + message: "parked compensation review lost its persisted audit entry".to_string(), + })?; + entry.accepted = true; + entry.resolution = Some(resolution.clone()); + entry.recorded_at = now; + let bound_job_exists = sqlx::query_scalar::<_, bool>( + "SELECT TRUE FROM moa.execution_external_job \ + WHERE external_job_uid=$1 AND tenant_id=$2 AND run_uid=$3 \ + AND compensation_id=$4 AND compensation_generation=$5 \ + AND compensation_attempt_generation=$6 AND idempotency_key=$7 \ + AND provider=$8 AND provider_job_id=$9 AND state <> 'unbound' FOR UPDATE", + ) + .bind(external_job_uid) + .bind(run.tenant_id.0) + .bind(run.run_uid) + .bind(fence.compensation_id.as_uuid()) + .bind(to_i64( + fence.compensation_generation, + "compensation generation", + )?) + .bind(to_i64( + fence.attempt_generation, + "compensation attempt generation", + )?) + .bind(&job.idempotency_key) + .bind(&job.provider) + .bind(&job.provider_job_id) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + .unwrap_or(false); + if !bound_job_exists { + return Err(Error::InvalidRepositoryData { + message: + "reviewed compensation external result requires its exact bound provider job" + .to_string(), + }); + } + let updated = sqlx::query( + "UPDATE moa.execution_compensation SET attempt_state='waiting_external', \ + external_job_uid=$5, waiting_since=$6, outcome=$7, \ + last_progress_at=GREATEST(last_progress_at,$6), updated_at=NOW() \ + WHERE run_uid=$1 AND compensation_id=$2 AND generation=$3 \ + AND attempt_generation=$4 AND attempt_state='waiting_review' \ + AND active_dispatch_uid IS NULL RETURNING *", + ) + .bind(run.run_uid) + .bind(fence.compensation_id.as_uuid()) + .bind(to_i64( + fence.compensation_generation, + "compensation generation", + )?) + .bind(to_i64( + fence.attempt_generation, + "compensation attempt generation", + )?) + .bind(external_job_uid) + .bind(now) + .bind(serde_json::to_value(persisted)?) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(updated) = updated else { + return Ok(CompensationAttemptWriteOutcome::Conflict); + }; + if !matches!( + run.status, + ExecutionRunStatus::PauseRequested + | ExecutionRunStatus::Pausing + | ExecutionRunStatus::Paused + ) { + enqueue_current_compensation_controller_wake( + conn, + run, + json!({ + "reason": "compensation_review_external_job_started", + "review_uid": review_uid, + "external_job_uid": external_job_uid, + }), + now, + ) + .await?; + } + Ok(CompensationAttemptWriteOutcome::Applied( + compensation_attempt_from_row(&updated, run)?, + )) + } +} + +fn checked_attempt_deadline(config: &ExecutionConfig, now: DateTime) -> Result> { + let seconds = i64::try_from(config.active_attempt_timeout_seconds).map_err(|_| { + Error::InvalidRepositoryInput { + message: "active compensation attempt timeout exceeds chrono duration".to_string(), + } + })?; + now.checked_add_signed(Duration::seconds(seconds)) + .ok_or_else(|| Error::InvalidRepositoryInput { + message: "active compensation attempt deadline is not representable".to_string(), + }) +} + +fn checked_retry_at(config: &ExecutionConfig, now: DateTime) -> Result> { + let seconds = i64::try_from(config.trigger_reconciliation_cadence_seconds).map_err(|_| { + Error::InvalidRepositoryInput { + message: "compensation admission retry cadence exceeds chrono duration".to_string(), + } + })?; + now.checked_add_signed(Duration::seconds(seconds)) + .ok_or_else(|| Error::InvalidRepositoryInput { + message: "compensation admission retry time is not representable".to_string(), + }) +} + +async fn lock_compensation_capacity( + conn: &mut ScopedConn<'_>, + tenant_id: TenantId, + fleet_limit: u32, + tenant_limit: u32, +) -> Result<()> { + for (scope_kind, owner, limit) in [ + ("fleet", None, fleet_limit), + ("tenant", Some(tenant_id.0), tenant_limit), + ] { + sqlx::query( + "INSERT INTO moa.execution_capacity_bucket (capacity_bucket_uid, scope_kind, \ + tenant_id, resource_dimension, limit_value) VALUES ($1, $2, $3, \ + 'active_tasks', $4) ON CONFLICT DO NOTHING", + ) + .bind(Uuid::now_v7()) + .bind(scope_kind) + .bind(owner) + .bind(i64::from(limit)) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + sqlx::query( + "UPDATE moa.execution_capacity_bucket SET limit_value=$4, version=version+1, \ + updated_at=NOW() WHERE scope_kind=$1 AND tenant_id IS NOT DISTINCT FROM $2 \ + AND resource_dimension=$3 RETURNING capacity_bucket_uid", + ) + .bind(scope_kind) + .bind(owner) + .bind("active_tasks") + .bind(i64::from(limit)) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + } + Ok(()) +} + +async fn lock_capacity_for_release(conn: &mut ScopedConn<'_>, tenant_id: TenantId) -> Result<()> { + for (scope_kind, owner) in [("fleet", None), ("tenant", Some(tenant_id.0))] { + sqlx::query( + "SELECT capacity_bucket_uid FROM moa.execution_capacity_bucket \ + WHERE scope_kind=$1 AND tenant_id IS NOT DISTINCT FROM $2 \ + AND resource_dimension='active_tasks' FOR UPDATE", + ) + .bind(scope_kind) + .bind(owner) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + } + Ok(()) +} + +async fn compensation_capacity_available( + conn: &mut ScopedConn<'_>, + tenant_id: TenantId, +) -> Result { + let rows: Vec<(String, i64, i64)> = sqlx::query_as( + "SELECT scope_kind, limit_value, reserved_quantity \ + FROM moa.execution_capacity_bucket WHERE resource_dimension='active_tasks' \ + AND ((scope_kind='fleet' AND tenant_id IS NULL) \ + OR (scope_kind='tenant' AND tenant_id=$1))", + ) + .bind(tenant_id.0) + .fetch_all(conn.as_mut()) + .await + .map_err(sqlx_error)?; + Ok(rows.len() == 2 && rows.iter().all(|(_, limit, reserved)| reserved < limit)) +} + +async fn increment_compensation_capacity( + conn: &mut ScopedConn<'_>, + tenant_id: TenantId, +) -> Result<()> { + for (scope_kind, owner) in [("fleet", None), ("tenant", Some(tenant_id.0))] { + let updated = sqlx::query( + "UPDATE moa.execution_capacity_bucket SET reserved_quantity=reserved_quantity+1, \ + version=version+1, updated_at=NOW() WHERE scope_kind=$1 \ + AND tenant_id IS NOT DISTINCT FROM $2 AND resource_dimension='active_tasks' \ + AND reserved_quantity < limit_value", + ) + .bind(scope_kind) + .bind(owner) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if updated.rows_affected() != 1 { + return Err(Error::InvalidRepositoryData { + message: format!("locked {scope_kind} compensation capacity was over-admitted"), + }); + } + } + Ok(()) +} + +async fn insert_compensation_capacity_reservation( + conn: &mut ScopedConn<'_>, + reservation_uid: Uuid, + run: &ExecutionRunRecord, + registration: &CompensationRegistrationProjection, + attempt_generation: u64, + deadline: DateTime, +) -> Result<()> { + sqlx::query( + "INSERT INTO moa.execution_capacity_reservation (reservation_uid, tenant_id, run_uid, \ + compensation_id, controller_generation, compensation_generation, \ + compensation_attempt_generation, resource_dimension, quantity, expires_at) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,'active_tasks',1,$8)", + ) + .bind(reservation_uid) + .bind(run.tenant_id.0) + .bind(run.run_uid) + .bind(registration.compensation_id.as_uuid()) + .bind(to_i64(run.controller_generation, "controller generation")?) + .bind(to_i64(registration.generation, "compensation generation")?) + .bind(to_i64( + attempt_generation, + "compensation attempt generation", + )?) + .bind(deadline) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + Ok(()) +} + +async fn release_compensation_capacity( + conn: &mut ScopedConn<'_>, + run: &ExecutionRunRecord, + fence: CompensationAttemptFence, +) -> Result<()> { + let reservation_uid: Option = sqlx::query_scalar( + "SELECT reservation_uid FROM moa.execution_capacity_reservation \ + WHERE tenant_id=$1 AND run_uid=$2 AND compensation_id=$3 \ + AND controller_generation=$4 AND compensation_generation=$5 \ + AND compensation_attempt_generation=$6 AND resource_dimension='active_tasks' \ + AND state IN ('reserved','reconciling') FOR UPDATE", + ) + .bind(run.tenant_id.0) + .bind(fence.run_uid) + .bind(fence.compensation_id.as_uuid()) + .bind(to_i64( + fence.controller_generation, + "controller generation", + )?) + .bind(to_i64( + fence.compensation_generation, + "compensation generation", + )?) + .bind(to_i64( + fence.attempt_generation, + "compensation attempt generation", + )?) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(reservation_uid) = reservation_uid else { + return Err(Error::InvalidRepositoryData { + message: "active compensation slice lost its capacity reservation".to_string(), + }); + }; + for (scope_kind, owner) in [("fleet", None), ("tenant", Some(run.tenant_id.0))] { + let updated = sqlx::query( + "UPDATE moa.execution_capacity_bucket SET reserved_quantity=reserved_quantity-1, \ + version=version+1, updated_at=NOW() WHERE scope_kind=$1 \ + AND tenant_id IS NOT DISTINCT FROM $2 AND resource_dimension='active_tasks' \ + AND reserved_quantity >= 1", + ) + .bind(scope_kind) + .bind(owner) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if updated.rows_affected() != 1 { + return Err(Error::InvalidRepositoryData { + message: format!("{scope_kind} active compensation capacity underflow"), + }); + } + } + sqlx::query( + "UPDATE moa.execution_capacity_reservation SET state='released', released_at=NOW(), \ + updated_at=NOW() WHERE reservation_uid=$1", + ) + .bind(reservation_uid) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + Ok(()) +} + +fn compensation_dispatch( + run: &ExecutionRunRecord, + request: &ExecutionCompensationAttemptRequest, + now: DateTime, +) -> Result { + Ok(NewExecutionDispatch { + dispatch_uid: request.dispatch_uid, + tenant_id: run.tenant_id, + run_uid: Some(run.run_uid), + task_id: None, + compensation_id: Some(request.compensation_id.as_uuid()), + trigger_uid: None, + external_job_uid: None, + kind: ExecutionDispatchKind::CompensationAttempt, + controller_generation: Some(run.controller_generation), + wake_epoch: None, + attempt_generation: None, + compensation_generation: Some(request.compensation_generation), + compensation_attempt_generation: Some(request.compensation_attempt_generation), + not_before_at: now, + payload: serde_json::to_value(request)?, + }) +} + +fn compensation_trigger( + run: &ExecutionRunRecord, + registration: &CompensationRegistrationProjection, + attempt_generation: u64, + trigger_uid: Uuid, + kind: ExecutionTriggerKind, + due_at: DateTime, + payload: Value, +) -> NewExecutionTrigger { + NewExecutionTrigger { + trigger_uid, + tenant_id: run.tenant_id, + run_uid: Some(run.run_uid), + task_id: None, + compensation_id: Some(registration.compensation_id.as_uuid()), + schedule_uid: None, + kind, + controller_generation: Some(run.controller_generation), + attempt_generation: None, + compensation_generation: Some(registration.generation), + compensation_attempt_generation: Some(attempt_generation), + occurrence_sequence: None, + schedule_incarnation: None, + due_at, + payload, + } +} + +async fn load_existing_compensation_admission( + conn: &mut ScopedConn<'_>, + config: &ExecutionConfig, + run: &ExecutionRunRecord, + row: &PgRow, + registration: &CompensationRegistrationProjection, +) -> Result { + let attempt_generation = required_u64(row, "attempt_generation")?; + let dispatch_uid: Uuid = row + .try_get::, _>("active_dispatch_uid") + .map_err(row_error)? + .ok_or_else(|| Error::InvalidRepositoryData { + message: "dispatching compensation has no active dispatch UID".to_string(), + })?; + let reservation_uid: Uuid = sqlx::query_scalar( + "SELECT reservation_uid FROM moa.execution_capacity_reservation \ + WHERE run_uid=$1 AND compensation_id=$2 AND controller_generation=$3 \ + AND compensation_generation=$4 AND compensation_attempt_generation=$5 \ + AND resource_dimension='active_tasks' AND state='reserved'", + ) + .bind(run.run_uid) + .bind(registration.compensation_id.as_uuid()) + .bind(to_i64(run.controller_generation, "controller generation")?) + .bind(to_i64(registration.generation, "compensation generation")?) + .bind(to_i64( + attempt_generation, + "compensation attempt generation", + )?) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let deadline: DateTime = row.try_get("attempt_deadline_at").map_err(row_error)?; + let watchdog_row = sqlx::query( + "SELECT trigger_uid, due_at, payload FROM moa.execution_trigger \ + WHERE run_uid=$1 AND compensation_id=$2 AND trigger_kind='compensation_watchdog' \ + AND controller_generation=$3 AND compensation_generation=$4 \ + AND compensation_attempt_generation=$5 AND state IN ('pending','dispatching')", + ) + .bind(run.run_uid) + .bind(registration.compensation_id.as_uuid()) + .bind(to_i64(run.controller_generation, "controller generation")?) + .bind(to_i64(registration.generation, "compensation generation")?) + .bind(to_i64( + attempt_generation, + "compensation attempt generation", + )?) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let watchdog_uid: Uuid = watchdog_row.try_get("trigger_uid").map_err(row_error)?; + let payload: Value = watchdog_row.try_get("payload").map_err(row_error)?; + let dispatch_row = sqlx::query( + "SELECT not_before_at, payload FROM moa.execution_dispatch_outbox \ + WHERE dispatch_uid=$1", + ) + .bind(dispatch_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let dispatch_payload: Value = dispatch_row.try_get("payload").map_err(row_error)?; + let attempt_request: ExecutionCompensationAttemptRequest = + serde_json::from_value(dispatch_payload)?; + let dispatch_request = compensation_dispatch( + run, + &attempt_request, + dispatch_row.try_get("not_before_at").map_err(row_error)?, + )?; + let dispatch = enqueue_dispatch_in_conn(conn.as_mut(), &dispatch_request).await?; + let watchdog = create_trigger_with_dispatch_in_conn( + conn.as_mut(), + config, + &compensation_trigger( + run, + registration, + attempt_generation, + watchdog_uid, + ExecutionTriggerKind::CompensationWatchdog, + deadline, + payload, + ), + ) + .await?; + Ok(CompensationAttemptAdmission { + attempt: compensation_attempt_from_row(row, run)?, + capacity_reservation_uid: reservation_uid, + dispatch, + watchdog, + }) +} + +enum PendingCompensationDrive { + Admitted(Box), + Replayed(Box), + CapacityUnavailable { retry_at: DateTime }, + ExternalCancellation(ExecutionDispatchRecord), + Parked, + Complete, + ManualRepair(CompensationRegistrationProjection), +} + +async fn drive_pending_terminal_compensation_in_conn( + conn: &mut ScopedConn<'_>, + config: &ExecutionConfig, + run: &ExecutionRunRecord, + now: DateTime, +) -> Result { + let nonterminal_forward_exists: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM moa.execution_task WHERE run_uid=$1 \ + AND status NOT IN ('completed','skipped','failed','cancelled','unknown_outcome'))", + ) + .bind(run.run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if run.status != ExecutionRunStatus::Compensating + || run.manual_repair_required + || run.pending_terminal.is_none() + || nonterminal_forward_exists + { + return Err(Error::InvalidRepositoryData { + message: "bounded compensation driver entered from an invalid run state".to_string(), + }); + } + let Some(row) = sqlx::query( + "SELECT * FROM moa.execution_compensation WHERE run_uid=$1 \ + AND status <> 'completed' ORDER BY registered_sequence DESC \ + LIMIT 1 FOR UPDATE", + ) + .bind(run.run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + return Ok(PendingCompensationDrive::Complete); + }; + let registration = compensation_from_row(&row)?; + let attempt_state = compensation_attempt_state_from_row(&row)?; + if matches!( + registration.status, + CompensationStatus::Failed | CompensationStatus::UnknownOutcome + ) { + return Ok(PendingCompensationDrive::ManualRepair(registration)); + } + if attempt_state == CompensationAttemptState::Dispatching { + return Ok(PendingCompensationDrive::Replayed(Box::new( + load_existing_compensation_admission(conn, config, run, &row, ®istration).await?, + ))); + } + if attempt_state == CompensationAttemptState::WaitingExternal { + let external_job_uid = row + .try_get::, _>("external_job_uid") + .map_err(row_error)? + .ok_or_else(|| Error::InvalidRepositoryData { + message: "waiting-external compensation lost its exact external job UID" + .to_string(), + })?; + let owner = ExecutionExternalJobOwner::Compensation { + compensation_id: registration.compensation_id.as_uuid(), + compensation_generation: registration.generation, + compensation_attempt_generation: required_u64(&row, "attempt_generation")?, + }; + return match request_external_job_cancellation_in_conn( + conn, + config, + external_job_uid, + owner, + now, + ) + .await? + { + ExecutionExternalJobCancellationRequestOutcome::Applied(dispatch) + | ExecutionExternalJobCancellationRequestOutcome::Replayed(dispatch) => { + Ok(PendingCompensationDrive::ExternalCancellation(dispatch)) + } + ExecutionExternalJobCancellationRequestOutcome::UnboundPendingRecovery => { + Ok(PendingCompensationDrive::Parked) + } + ExecutionExternalJobCancellationRequestOutcome::AlreadyTerminal => { + let job = load_external_job_for_update_in_conn(conn.as_mut(), external_job_uid) + .await? + .ok_or_else(|| Error::InvalidRepositoryData { + message: "terminal compensation external job disappeared".to_string(), + })?; + settle_external_job_terminal_in_conn(conn, &job, now).await?; + Ok(PendingCompensationDrive::Parked) + } + ExecutionExternalJobCancellationRequestOutcome::NotFound + | ExecutionExternalJobCancellationRequestOutcome::Stale => { + Err(Error::InvalidRepositoryData { + message: "waiting-external compensation has a stale external job owner" + .to_string(), + }) + } + }; + } + if matches!( + attempt_state, + CompensationAttemptState::Running + | CompensationAttemptState::Cancelling + | CompensationAttemptState::WaitingReview + ) { + return Ok(PendingCompensationDrive::Parked); + } + if attempt_state != CompensationAttemptState::Idle + || !matches!( + registration.status, + CompensationStatus::Pending | CompensationStatus::Running + ) + { + return Err(Error::InvalidRepositoryData { + message: "highest reverse-order compensation is not dispatchable or settled" + .to_string(), + }); + } + let retry_at = checked_retry_at(config, now)?; + if !compensation_capacity_available(conn, run.tenant_id).await? { + return Ok(PendingCompensationDrive::CapacityUnavailable { retry_at }); + } + if registration.status == CompensationStatus::Pending && registration.outcome.is_none() { + let forward_task = + load_forward_task(conn, run.run_uid, registration.forward_task_id).await?; + let reservation = + compensation_reservation(run, ®istration, forward_task.retry.max_attempts)?; + let mut ledger = budget_ledger(run); + if ledger.try_reserve(reservation).is_err() { + let failed = + terminalize_compensation_budget_rejection(conn, run, ®istration, reservation) + .await?; + return Ok(PendingCompensationDrive::ManualRepair(failed)); + } + persist_run_budget(conn, run.run_uid, &ledger, false).await?; + } + let deadline = checked_attempt_deadline(config, now)?; + let attempt_generation = required_u64(&row, "attempt_generation")?; + let dispatch_uid = Uuid::now_v7(); + let reservation_uid = Uuid::now_v7(); + let watchdog_uid = Uuid::now_v7(); + insert_compensation_capacity_reservation( + conn, + reservation_uid, + run, + ®istration, + attempt_generation, + deadline, + ) + .await?; + increment_compensation_capacity(conn, run.tenant_id).await?; + let watchdog = create_trigger_with_dispatch_in_conn( + conn.as_mut(), + config, + &compensation_trigger( + run, + ®istration, + attempt_generation, + watchdog_uid, + ExecutionTriggerKind::CompensationWatchdog, + deadline, + json!({}), + ), + ) + .await?; + let attempt_request = ExecutionCompensationAttemptRequest { + dispatch_uid, + capacity_reservation_uid: reservation_uid, + watchdog_trigger_uid: watchdog.trigger.trigger_uid, + watchdog_dispatch_uid: watchdog.dispatch.dispatch_uid, + run_uid: run.run_uid, + compensation_id: registration.compensation_id, + compensation_generation: registration.generation, + compensation_attempt_generation: attempt_generation, + controller_generation: run.controller_generation, + attempt_deadline_at: deadline, + tenant_id: run.tenant_id, + }; + let dispatch = enqueue_dispatch_in_conn( + conn.as_mut(), + &compensation_dispatch(run, &attempt_request, now)?, + ) + .await?; + let updated = sqlx::query( + "UPDATE moa.execution_compensation SET status='running', \ + attempt_state='dispatching', attempt_started_at=$6, \ + last_progress_at=GREATEST(last_progress_at,$6), \ + attempt_deadline_at=$7, waiting_since=NULL, active_dispatch_uid=$8, \ + dispatch_sequence=dispatch_sequence+1, started_at=COALESCE(started_at,$6), \ + updated_at=NOW() WHERE run_uid=$1 AND compensation_id=$2 AND generation=$3 \ + AND attempt_generation=$4 AND attempt_state='idle' \ + AND status IN ('pending','running') AND EXISTS ( \ + SELECT 1 FROM moa.execution_run AS current_run \ + WHERE current_run.run_uid=$1 AND current_run.controller_generation=$5) \ + RETURNING *", + ) + .bind(run.run_uid) + .bind(registration.compensation_id.as_uuid()) + .bind(to_i64(registration.generation, "compensation generation")?) + .bind(to_i64( + attempt_generation, + "compensation attempt generation", + )?) + .bind(to_i64(run.controller_generation, "controller generation")?) + .bind(now) + .bind(deadline) + .bind(dispatch_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + .ok_or_else(|| Error::InvalidRepositoryData { + message: "bounded compensation admission lost its exact row lock".to_string(), + })?; + Ok(PendingCompensationDrive::Admitted(Box::new( + CompensationAttemptAdmission { + attempt: compensation_attempt_from_row(&updated, run)?, + capacity_reservation_uid: reservation_uid, + dispatch, + watchdog, + }, + ))) +} + +fn compensation_attempt_state_from_row(row: &PgRow) -> Result { + row.try_get::("attempt_state") + .map_err(row_error)? + .parse() +} + +fn compensation_release_intent_from_row( + row: &PgRow, +) -> Result> { + row.try_get::, _>("release_intent") + .map_err(row_error)? + .map(|label| match label.as_str() { + "outcome" => Ok(ExecutionCompensationReleaseIntent::Outcome), + "retry" => Ok(ExecutionCompensationReleaseIntent::Retry), + "review" => Ok(ExecutionCompensationReleaseIntent::Review), + "external_job" => Ok(ExecutionCompensationReleaseIntent::ExternalJob), + "pause" => Ok(ExecutionCompensationReleaseIntent::Pause), + "watchdog" => Ok(ExecutionCompensationReleaseIntent::Watchdog), + "deadline" => Ok(ExecutionCompensationReleaseIntent::Deadline), + "run_terminal" => Ok(ExecutionCompensationReleaseIntent::RunTerminal), + _ => Err(Error::InvalidRepositoryData { + message: format!("unknown compensation release intent `{label}`"), + }), + }) + .transpose() +} + +fn compensation_cancel_request_fence( + request: &ExecutionCompensationAttemptCancelRequest, +) -> CompensationAttemptFence { + CompensationAttemptFence { + run_uid: request.run_uid, + compensation_id: request.compensation_id, + controller_generation: request.controller_generation, + compensation_generation: request.compensation_generation, + attempt_generation: request.compensation_attempt_generation, + dispatch_uid: request.active_dispatch_uid, + } +} + +fn compensation_cancel_resource_fence( + request: &ExecutionCompensationAttemptCancelRequest, +) -> CompensationAttemptFence { + CompensationAttemptFence { + run_uid: request.run_uid, + compensation_id: request.compensation_id, + controller_generation: request.attempt_controller_generation, + compensation_generation: request.compensation_generation, + attempt_generation: request.compensation_attempt_generation, + dispatch_uid: request.active_dispatch_uid, + } +} + +fn compensation_release_receipt_matches( + request: &ExecutionCompensationAttemptCancelRequest, + receipt: Option<&ExecutionHandReleaseReceipt>, +) -> bool { + receipt.is_some_and(|receipt| { + receipt.tenant_id == request.tenant_id + && receipt.run_id.0 == request.run_uid + && matches!( + receipt.owner, + ExecutionHandReleaseOwner::Compensation { + compensation_id, + logical_generation, + } if compensation_id.0 == request.compensation_id.as_uuid() + && logical_generation == request.compensation_generation + ) + && receipt.attempt_generation == request.compensation_attempt_generation + }) +} + +async fn persisted_compensation_release_receipt_matches( + conn: &mut ScopedConn<'_>, + request: &ExecutionCompensationAttemptCancelRequest, + receipt: Option<&ExecutionHandReleaseReceipt>, +) -> Result { + let Some(receipt) = receipt else { + return Ok(false); + }; + if !compensation_release_receipt_matches(request, Some(receipt)) { + return Ok(false); + } + let matches: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM moa.sandbox_execution_hand_release_receipts \ + WHERE receipt_id=$1 AND tenant_id=$2 AND run_uid=$3 \ + AND owner_kind='compensation' AND task_id IS NULL AND compensation_id=$4 \ + AND logical_generation=$5 AND attempt_generation=$6 \ + AND receipt_state='released' AND destroy_outcome='verified_absent' \ + AND released_at IS NOT NULL \ + AND hand_provisioning_operation_id IS NOT DISTINCT FROM $7 \ + AND hand_lease_generation IS NOT DISTINCT FROM $8)", + ) + .bind(receipt.receipt_id) + .bind(request.tenant_id.0) + .bind(request.run_uid) + .bind(request.compensation_id.as_uuid()) + .bind(to_i64( + request.compensation_generation, + "compensation generation", + )?) + .bind(to_i64( + request.compensation_attempt_generation, + "compensation attempt generation", + )?) + .bind( + receipt + .hand_provisioning_operation_id + .map(|operation_id| operation_id.0), + ) + .bind( + receipt + .hand_lease_generation + .map(|generation| to_i64(generation, "hand lease generation")) + .transpose()?, + ) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + Ok(matches) +} + +async fn compensation_attempt_resources_match( + conn: &mut ScopedConn<'_>, + request: &ExecutionCompensationAttemptCancelRequest, +) -> Result { + let matches: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM moa.execution_capacity_reservation AS reservation \ + WHERE reservation.reservation_uid=$1 AND reservation.tenant_id=$2 \ + AND reservation.run_uid=$3 AND reservation.compensation_id=$4 \ + AND reservation.controller_generation=$5 \ + AND reservation.compensation_generation=$6 \ + AND reservation.compensation_attempt_generation=$7 \ + AND reservation.resource_dimension='active_tasks' \ + AND reservation.state IN ('reserved','reconciling')) \ + AND EXISTS (SELECT 1 FROM moa.execution_trigger AS trigger \ + WHERE trigger.trigger_uid=$8 AND trigger.tenant_id=$2 AND trigger.run_uid=$3 \ + AND trigger.compensation_id=$4 AND trigger.controller_generation=$5 \ + AND trigger.compensation_generation=$6 \ + AND trigger.compensation_attempt_generation=$7 \ + AND trigger.trigger_kind='compensation_watchdog' \ + AND trigger.state IN ('pending','dispatching'))", + ) + .bind(request.capacity_reservation_uid) + .bind(request.tenant_id.0) + .bind(request.run_uid) + .bind(request.compensation_id.as_uuid()) + .bind(to_i64( + request.attempt_controller_generation, + "attempt controller generation", + )?) + .bind(to_i64( + request.compensation_generation, + "compensation generation", + )?) + .bind(to_i64( + request.compensation_attempt_generation, + "compensation attempt generation", + )?) + .bind(request.watchdog_trigger_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + Ok(matches) +} + +async fn canonical_active_compensation_release_request( + conn: &mut ScopedConn<'_>, + run: &ExecutionRunRecord, + attempt: &CompensationAttemptRecord, + intent: ExecutionCompensationReleaseIntent, +) -> Result> { + let Some(active_dispatch_uid) = attempt.active_dispatch_uid else { + return Ok(None); + }; + let resource = sqlx::query( + "SELECT reservation.reservation_uid, trigger.trigger_uid \ + FROM moa.execution_capacity_reservation AS reservation \ + JOIN moa.execution_trigger AS trigger ON trigger.run_uid=reservation.run_uid \ + AND trigger.compensation_id=reservation.compensation_id \ + AND trigger.controller_generation=reservation.controller_generation \ + AND trigger.compensation_generation=reservation.compensation_generation \ + AND trigger.compensation_attempt_generation=reservation.compensation_attempt_generation \ + AND trigger.trigger_kind='compensation_watchdog' \ + AND trigger.state IN ('pending','dispatching') \ + WHERE reservation.tenant_id=$1 AND reservation.run_uid=$2 \ + AND reservation.compensation_id=$3 AND reservation.controller_generation=$4 \ + AND reservation.compensation_generation=$5 \ + AND reservation.compensation_attempt_generation=$6 \ + AND reservation.resource_dimension='active_tasks' \ + AND reservation.state IN ('reserved','reconciling') \ + FOR UPDATE OF reservation, trigger", + ) + .bind(run.tenant_id.0) + .bind(run.run_uid) + .bind(attempt.registration.compensation_id.as_uuid()) + .bind(to_i64( + attempt.controller_generation, + "attempt controller generation", + )?) + .bind(to_i64( + attempt.registration.generation, + "compensation generation", + )?) + .bind(to_i64( + attempt.attempt_generation, + "compensation attempt generation", + )?) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(resource) = resource else { + return Ok(None); + }; + Ok(Some(ExecutionCompensationAttemptCancelRequest { + cancellation_dispatch_uid: Uuid::new_v5( + &active_dispatch_uid, + b"compensation-attempt-release-v1", + ), + tenant_id: run.tenant_id, + run_uid: run.run_uid, + compensation_id: attempt.registration.compensation_id, + controller_generation: run.controller_generation, + attempt_controller_generation: attempt.controller_generation, + compensation_generation: attempt.registration.generation, + compensation_attempt_generation: attempt.attempt_generation, + active_dispatch_uid, + capacity_reservation_uid: resource.try_get("reservation_uid").map_err(row_error)?, + watchdog_trigger_uid: resource.try_get("trigger_uid").map_err(row_error)?, + intent, + })) +} + +async fn release_unbound_compensation_external_intent_in_conn( + conn: &mut ScopedConn<'_>, + run: &ExecutionRunRecord, + attempt: &CompensationAttemptRecord, +) -> Result<()> { + let rows = sqlx::query( + "SELECT job.external_job_uid, job.job_generation, job.provider, job.idempotency_key, \ + capacity.expires_at \ + FROM moa.execution_external_job AS job \ + JOIN moa.execution_capacity_reservation AS capacity \ + ON capacity.tenant_id=job.tenant_id \ + AND capacity.external_job_uid=job.external_job_uid \ + AND capacity.resource_dimension='external_jobs' \ + AND capacity.state IN ('reserved','reconciling') \ + WHERE job.tenant_id=$1 AND job.run_uid=$2 AND job.compensation_id=$3 \ + AND job.compensation_generation=$4 \ + AND job.compensation_attempt_generation=$5 AND job.state='unbound' \ + ORDER BY job.external_job_uid LIMIT 2 FOR UPDATE OF job, capacity", + ) + .bind(run.tenant_id.0) + .bind(run.run_uid) + .bind(attempt.registration.compensation_id.as_uuid()) + .bind(to_i64( + attempt.registration.generation, + "compensation generation", + )?) + .bind(to_i64( + attempt.attempt_generation, + "compensation attempt generation", + )?) + .fetch_all(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if rows.len() > 1 { + return Err(Error::InvalidRepositoryData { + message: "compensation attempt owns multiple unbound external-job intents".to_string(), + }); + } + let Some(row) = rows.first() else { + return Ok(()); + }; + let intent = NewExecutionExternalJobIntent { + external_job_uid: row.try_get("external_job_uid").map_err(row_error)?, + tenant_id: run.tenant_id, + run_uid: run.run_uid, + owner: ExecutionExternalJobOwner::Compensation { + compensation_id: attempt.registration.compensation_id.as_uuid(), + compensation_generation: attempt.registration.generation, + compensation_attempt_generation: attempt.attempt_generation, + }, + job_generation: required_u64(row, "job_generation")?, + provider: row.try_get("provider").map_err(row_error)?, + idempotency_key: row.try_get("idempotency_key").map_err(row_error)?, + expires_at: row.try_get("expires_at").map_err(row_error)?, + }; + match release_external_job_intent_in_conn(conn.as_mut(), &intent).await? { + ExecutionExternalJobIntentReleaseOutcome::Released + | ExecutionExternalJobIntentReleaseOutcome::AlreadyReleased => Ok(()), + ExecutionExternalJobIntentReleaseOutcome::Stale + | ExecutionExternalJobIntentReleaseOutcome::AlreadyBound => { + Err(Error::InvalidRepositoryData { + message: + "compensation finalizer found a stale or already-bound external-job intent" + .to_string(), + }) + } + } +} + +fn compensation_attempt_from_row( + row: &PgRow, + run: &ExecutionRunRecord, +) -> Result { + Ok(CompensationAttemptRecord { + registration: compensation_from_row(row)?, + run: run.clone(), + controller_generation: run.controller_generation, + attempt_generation: required_u64(row, "attempt_generation")?, + attempt_state: compensation_attempt_state_from_row(row)?, + attempt_started_at: row.try_get("attempt_started_at").map_err(row_error)?, + last_progress_at: row.try_get("last_progress_at").map_err(row_error)?, + attempt_deadline_at: row.try_get("attempt_deadline_at").map_err(row_error)?, + waiting_since: row.try_get("waiting_since").map_err(row_error)?, + active_dispatch_uid: row.try_get("active_dispatch_uid").map_err(row_error)?, + external_job_uid: row.try_get("external_job_uid").map_err(row_error)?, + release_intent: compensation_release_intent_from_row(row)?, + dispatch_sequence: required_u64(row, "dispatch_sequence")?, + }) +} + +async fn load_compensation_tenant( + conn: &mut ScopedConn<'_>, + run_uid: Uuid, +) -> Result> { + sqlx::query_scalar::<_, Uuid>("SELECT tenant_id FROM moa.execution_run WHERE run_uid=$1") + .bind(run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error) + .map(|tenant| tenant.map(TenantId)) +} + +async fn load_fenced_compensation( + conn: &mut ScopedConn<'_>, + fence: CompensationAttemptFence, +) -> Result> { + load_fenced_compensation_with_attempt_controller(conn, fence, fence.controller_generation).await +} + +async fn load_fenced_compensation_for_cancel( + conn: &mut ScopedConn<'_>, + request: &ExecutionCompensationAttemptCancelRequest, +) -> Result> { + load_fenced_compensation_with_attempt_controller( + conn, + compensation_cancel_request_fence(request), + request.attempt_controller_generation, + ) + .await +} + +/// Atomically adopts one already-bound provider job into exact compensation teardown. +pub(super) async fn begin_compensation_external_release_in_conn( + conn: &mut ScopedConn<'_>, + request: &ExecutionCompensationAttemptCancelRequest, + external_job_uid: Uuid, + claimed_at: DateTime, +) -> Result { + if request.intent != ExecutionCompensationReleaseIntent::ExternalJob { + return Err(Error::InvalidRepositoryInput { + message: "external-job release requires the exact external-job intent".to_string(), + }); + } + let Some((run, row)) = load_fenced_compensation_for_cancel(conn, request).await? else { + return Ok(CompensationAttemptExternalOutcome::NotFound); + }; + let current = compensation_attempt_from_row(&row, &run)?; + let Some(persisted_job) = + load_external_job_for_update_in_conn(conn.as_mut(), external_job_uid).await? + else { + return Ok(CompensationAttemptExternalOutcome::NotFound); + }; + let expected_owner = ExecutionExternalJobOwner::Compensation { + compensation_id: request.compensation_id.as_uuid(), + compensation_generation: request.compensation_generation, + compensation_attempt_generation: request.compensation_attempt_generation, + }; + if run.tenant_id != request.tenant_id + || persisted_job.tenant_id != request.tenant_id + || persisted_job.run_uid != request.run_uid + || persisted_job.owner != expected_owner + || persisted_job.state == ExecutionExternalJobState::Unbound + { + return Ok(CompensationAttemptExternalOutcome::Stale); + } + if matches!( + current.attempt_state, + CompensationAttemptState::Cancelling + | CompensationAttemptState::WaitingExternal + | CompensationAttemptState::Terminal + | CompensationAttemptState::UnknownOutcome + ) && current.external_job_uid == Some(external_job_uid) + && (current.attempt_state != CompensationAttemptState::Cancelling + || current.release_intent == Some(ExecutionCompensationReleaseIntent::ExternalJob)) + { + return Ok(CompensationAttemptExternalOutcome::Replayed { + attempt: current, + external_job: persisted_job, + }); + } + if !compensation_attempt_resources_match(conn, request).await? { + return Ok(CompensationAttemptExternalOutcome::Stale); + } + if current.attempt_state != CompensationAttemptState::Running + || current.registration.status != CompensationStatus::Running + { + return Ok(CompensationAttemptExternalOutcome::InvalidState); + } + let row = sqlx::query( + "UPDATE moa.execution_compensation SET attempt_state='cancelling', \ + external_job_uid=$6, release_intent=$8, \ + last_progress_at=GREATEST(last_progress_at,$7), updated_at=NOW() \ + WHERE run_uid=$1 AND compensation_id=$2 AND generation=$3 \ + AND attempt_generation=$4 AND active_dispatch_uid=$5 \ + AND attempt_state='running' RETURNING *", + ) + .bind(request.run_uid) + .bind(request.compensation_id.as_uuid()) + .bind(to_i64( + request.compensation_generation, + "compensation generation", + )?) + .bind(to_i64( + request.compensation_attempt_generation, + "compensation attempt generation", + )?) + .bind(request.active_dispatch_uid) + .bind(external_job_uid) + .bind(claimed_at) + .bind(compensation_release_intent_label(request.intent)) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(row) = row else { + return Ok(CompensationAttemptExternalOutcome::Stale); + }; + Ok(CompensationAttemptExternalOutcome::Applied { + attempt: compensation_attempt_from_row(&row, &run)?, + external_job: persisted_job, + }) +} + +/// Adopts one recovered, already-bound provider job using canonical attempt resources. +pub(super) async fn begin_recovered_compensation_external_release_in_conn( + conn: &mut ScopedConn<'_>, + job: &ExecutionExternalJobRecord, + claimed_at: DateTime, +) -> Result { + let Some(persisted_job) = + load_external_job_for_update_in_conn(conn.as_mut(), job.external_job_uid).await? + else { + return Ok(CompensationRecoveredExternalReleaseClaimOutcome::NotFound); + }; + if persisted_job != *job { + return Ok(CompensationRecoveredExternalReleaseClaimOutcome::Stale); + } + let ExecutionExternalJobOwner::Compensation { + compensation_id, + compensation_generation, + compensation_attempt_generation, + } = persisted_job.owner + else { + return Err(Error::InvalidRepositoryInput { + message: "recovered compensation external release requires a compensation owner" + .to_string(), + }); + }; + if persisted_job.state == ExecutionExternalJobState::Unbound { + return Ok(CompensationRecoveredExternalReleaseClaimOutcome::InvalidState); + } + let Some(run_row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) + .bind(persisted_job.run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + return Ok(CompensationRecoveredExternalReleaseClaimOutcome::NotFound); + }; + let run = run_from_row(&run_row)?; + let Some(row) = sqlx::query(LOAD_COMPENSATION_FOR_UPDATE_SQL) + .bind(persisted_job.run_uid) + .bind(compensation_id) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + return Ok(CompensationRecoveredExternalReleaseClaimOutcome::NotFound); + }; + let current = compensation_attempt_from_row(&row, &run)?; + if run.tenant_id != persisted_job.tenant_id + || current.registration.generation != compensation_generation + || current.attempt_generation != compensation_attempt_generation + { + return Ok(CompensationRecoveredExternalReleaseClaimOutcome::Stale); + } + if matches!( + current.attempt_state, + CompensationAttemptState::WaitingExternal + | CompensationAttemptState::Terminal + | CompensationAttemptState::UnknownOutcome + ) && current.external_job_uid == Some(persisted_job.external_job_uid) + { + return Ok(CompensationRecoveredExternalReleaseClaimOutcome::AlreadySettled); + } + let Some(request) = canonical_active_compensation_release_request( + conn, + &run, + ¤t, + ExecutionCompensationReleaseIntent::ExternalJob, + ) + .await? + else { + return Ok(CompensationRecoveredExternalReleaseClaimOutcome::Stale); + }; + if current.attempt_state == CompensationAttemptState::Cancelling { + return Ok( + if current.release_intent == Some(ExecutionCompensationReleaseIntent::ExternalJob) + && current.external_job_uid == Some(persisted_job.external_job_uid) + { + CompensationRecoveredExternalReleaseClaimOutcome::Replayed { + request, + attempt: current, + } + } else { + CompensationRecoveredExternalReleaseClaimOutcome::Stale + }, + ); + } + if current.attempt_state != CompensationAttemptState::Running + || current.registration.status != CompensationStatus::Running + || current.external_job_uid.is_some() + { + return Ok(CompensationRecoveredExternalReleaseClaimOutcome::InvalidState); + } + let updated = sqlx::query( + "UPDATE moa.execution_compensation SET attempt_state='cancelling', \ + external_job_uid=$6, release_intent='external_job', \ + last_progress_at=GREATEST(last_progress_at,$7), updated_at=NOW() \ + WHERE run_uid=$1 AND compensation_id=$2 AND generation=$3 \ + AND attempt_generation=$4 AND active_dispatch_uid=$5 \ + AND attempt_state='running' AND external_job_uid IS NULL RETURNING *", + ) + .bind(persisted_job.run_uid) + .bind(compensation_id) + .bind(to_i64(compensation_generation, "compensation generation")?) + .bind(to_i64( + compensation_attempt_generation, + "compensation attempt generation", + )?) + .bind(request.active_dispatch_uid) + .bind(persisted_job.external_job_uid) + .bind(claimed_at) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(updated) = updated else { + return Ok(CompensationRecoveredExternalReleaseClaimOutcome::Stale); + }; + Ok(CompensationRecoveredExternalReleaseClaimOutcome::Applied { + request, + attempt: compensation_attempt_from_row(&updated, &run)?, + }) +} + +/// Fences one compensation for verified teardown after start recovery proved NotStarted. +/// +/// The caller must release the exact unbound external-job intent in the same transaction +/// before invoking this helper. That ordering proves there is no bound provider owner while +/// keeping intent release and compensation teardown fencing atomic. +pub(super) async fn begin_compensation_external_not_started_release_in_conn( + conn: &mut ScopedConn<'_>, + intent: &NewExecutionExternalJobIntent, + claimed_at: DateTime, +) -> Result { + let ExecutionExternalJobOwner::Compensation { + compensation_id, + compensation_generation, + compensation_attempt_generation, + } = intent.owner + else { + return Err(Error::InvalidRepositoryInput { + message: "compensation NotStarted recovery requires a compensation owner".to_string(), + }); + }; + let provider_owner_remains: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM moa.execution_external_job WHERE external_job_uid=$1) \ + OR EXISTS (SELECT 1 FROM moa.execution_capacity_reservation \ + WHERE external_job_uid=$1 AND resource_dimension='external_jobs' \ + AND state IN ('reserved','reconciling'))", + ) + .bind(intent.external_job_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if provider_owner_remains { + return Ok(CompensationExternalNotStartedReleaseClaimOutcome::Stale); + } + let Some(run_row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) + .bind(intent.run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + return Ok(CompensationExternalNotStartedReleaseClaimOutcome::NotFound); + }; + let run = run_from_row(&run_row)?; + let Some(row) = sqlx::query(LOAD_COMPENSATION_FOR_UPDATE_SQL) + .bind(intent.run_uid) + .bind(compensation_id) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + return Ok(CompensationExternalNotStartedReleaseClaimOutcome::NotFound); + }; + let current = compensation_attempt_from_row(&row, &run)?; + if run.tenant_id != intent.tenant_id + || current.registration.generation != compensation_generation + { + return Ok(CompensationExternalNotStartedReleaseClaimOutcome::Stale); + } + if current.attempt_state == CompensationAttemptState::Idle + && current.attempt_generation == compensation_attempt_generation.saturating_add(1) + { + return Ok(CompensationExternalNotStartedReleaseClaimOutcome::AlreadySettled); + } + if current.attempt_generation != compensation_attempt_generation { + return Ok(CompensationExternalNotStartedReleaseClaimOutcome::Stale); + } + let Some(active_dispatch_uid) = current.active_dispatch_uid else { + return Ok(CompensationExternalNotStartedReleaseClaimOutcome::InvalidState); + }; + let resource = sqlx::query( + "SELECT reservation.reservation_uid, trigger.trigger_uid \ + FROM moa.execution_capacity_reservation AS reservation \ + JOIN moa.execution_trigger AS trigger ON trigger.run_uid=reservation.run_uid \ + AND trigger.compensation_id=reservation.compensation_id \ + AND trigger.controller_generation=reservation.controller_generation \ + AND trigger.compensation_generation=reservation.compensation_generation \ + AND trigger.compensation_attempt_generation=reservation.compensation_attempt_generation \ + AND trigger.trigger_kind='compensation_watchdog' \ + AND trigger.state IN ('pending','dispatching') \ + WHERE reservation.tenant_id=$1 AND reservation.run_uid=$2 \ + AND reservation.compensation_id=$3 AND reservation.controller_generation=$4 \ + AND reservation.compensation_generation=$5 \ + AND reservation.compensation_attempt_generation=$6 \ + AND reservation.resource_dimension='active_tasks' \ + AND reservation.state IN ('reserved','reconciling') \ + FOR UPDATE OF reservation, trigger", + ) + .bind(intent.tenant_id.0) + .bind(intent.run_uid) + .bind(compensation_id) + .bind(to_i64( + current.controller_generation, + "attempt controller generation", + )?) + .bind(to_i64(compensation_generation, "compensation generation")?) + .bind(to_i64( + compensation_attempt_generation, + "compensation attempt generation", + )?) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(resource) = resource else { + return Ok(CompensationExternalNotStartedReleaseClaimOutcome::Stale); + }; + let request = ExecutionCompensationAttemptCancelRequest { + cancellation_dispatch_uid: Uuid::new_v5( + &active_dispatch_uid, + b"compensation-attempt-release-v1", + ), + tenant_id: intent.tenant_id, + run_uid: intent.run_uid, + compensation_id: CompensationId::from_uuid(compensation_id), + controller_generation: run.controller_generation, + attempt_controller_generation: current.controller_generation, + compensation_generation, + compensation_attempt_generation, + active_dispatch_uid, + capacity_reservation_uid: resource.try_get("reservation_uid").map_err(row_error)?, + watchdog_trigger_uid: resource.try_get("trigger_uid").map_err(row_error)?, + intent: ExecutionCompensationReleaseIntent::Retry, + }; + if current.attempt_state == CompensationAttemptState::Cancelling { + return Ok( + if current.release_intent == Some(ExecutionCompensationReleaseIntent::Retry) + && current.external_job_uid.is_none() + { + CompensationExternalNotStartedReleaseClaimOutcome::Replayed { + request, + attempt: current, + } + } else { + CompensationExternalNotStartedReleaseClaimOutcome::Stale + }, + ); + } + if current.attempt_state != CompensationAttemptState::Running + || current.registration.status != CompensationStatus::Running + || current.external_job_uid.is_some() + { + return Ok(CompensationExternalNotStartedReleaseClaimOutcome::InvalidState); + } + let updated = sqlx::query( + "UPDATE moa.execution_compensation SET attempt_state='cancelling', \ + release_intent='retry', last_progress_at=GREATEST(last_progress_at,$6), \ + updated_at=NOW() \ + WHERE run_uid=$1 AND compensation_id=$2 AND generation=$3 \ + AND attempt_generation=$4 AND active_dispatch_uid=$5 \ + AND attempt_state='running' AND external_job_uid IS NULL RETURNING *", + ) + .bind(intent.run_uid) + .bind(compensation_id) + .bind(to_i64(compensation_generation, "compensation generation")?) + .bind(to_i64( + compensation_attempt_generation, + "compensation attempt generation", + )?) + .bind(active_dispatch_uid) + .bind(claimed_at) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(updated) = updated else { + return Ok(CompensationExternalNotStartedReleaseClaimOutcome::Stale); + }; + Ok(CompensationExternalNotStartedReleaseClaimOutcome::Applied { + request, + attempt: compensation_attempt_from_row(&updated, &run)?, + }) +} + +async fn load_fenced_compensation_with_attempt_controller( + conn: &mut ScopedConn<'_>, + fence: CompensationAttemptFence, + attempt_controller_generation: u64, +) -> Result> { + let Some(run_row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) + .bind(fence.run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + return Ok(None); + }; + let run = run_from_row(&run_row)?; + if run.controller_generation != fence.controller_generation { + return Ok(None); + } + let Some(row) = sqlx::query(LOAD_COMPENSATION_FOR_UPDATE_SQL) + .bind(fence.run_uid) + .bind(fence.compensation_id.as_uuid()) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + return Ok(None); + }; + if required_u64(&row, "generation")? != fence.compensation_generation + || required_u64(&row, "attempt_generation")? != fence.attempt_generation + { + return Ok(None); + } + let dispatch_exists: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM moa.execution_dispatch_outbox WHERE dispatch_uid=$1 \ + AND run_uid=$2 AND compensation_id=$3 AND controller_generation=$4 \ + AND compensation_generation=$5 AND compensation_attempt_generation=$6 \ + AND dispatch_kind='compensation_attempt')", + ) + .bind(fence.dispatch_uid) + .bind(fence.run_uid) + .bind(fence.compensation_id.as_uuid()) + .bind(to_i64( + attempt_controller_generation, + "attempt controller generation", + )?) + .bind(to_i64( + fence.compensation_generation, + "compensation generation", + )?) + .bind(to_i64( + fence.attempt_generation, + "compensation attempt generation", + )?) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + Ok(dispatch_exists.then_some((run, row))) +} + +/// Repairs one exact compensation dispatch that dead-lettered before provider start. +pub(super) async fn settle_unstarted_compensation_attempt_in_conn( + conn: &mut ScopedConn<'_>, + request: &ExecutionCompensationAttemptRequest, + settled_at: DateTime, +) -> Result { + let Some(tenant_id) = load_compensation_tenant(conn, request.run_uid).await? else { + return Ok(CompensationAttemptWriteOutcome::NotFound); + }; + if tenant_id != request.tenant_id { + return Ok(CompensationAttemptWriteOutcome::Conflict); + } + lock_capacity_for_release(conn, tenant_id).await?; + let run_row = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) + .bind(request.run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let run = run_from_row(&run_row)?; + let row = sqlx::query(LOAD_COMPENSATION_FOR_UPDATE_SQL) + .bind(request.run_uid) + .bind(request.compensation_id.as_uuid()) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(row) = row else { + return Ok(CompensationAttemptWriteOutcome::NotFound); + }; + let current = compensation_attempt_from_row(&row, &run)?; + if current.registration.generation == request.compensation_generation + && current.attempt_generation == request.compensation_attempt_generation.saturating_add(1) + && current.attempt_state == CompensationAttemptState::Idle + && current.active_dispatch_uid.is_none() + { + return Ok(CompensationAttemptWriteOutcome::Replayed(current)); + } + let exact_resources: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM moa.execution_capacity_reservation \ + WHERE reservation_uid=$1 AND tenant_id=$2 AND run_uid=$3 AND compensation_id=$4 \ + AND controller_generation=$5 AND compensation_generation=$6 \ + AND compensation_attempt_generation=$7 AND resource_dimension='active_tasks' \ + AND state IN ('reserved','reconciling')) \ + AND EXISTS (SELECT 1 FROM moa.execution_trigger WHERE trigger_uid=$8 AND run_uid=$3 \ + AND compensation_id=$4 AND controller_generation=$5 AND compensation_generation=$6 \ + AND compensation_attempt_generation=$7 AND trigger_kind='compensation_watchdog' \ + AND state IN ('pending','dispatching'))", + ) + .bind(request.capacity_reservation_uid) + .bind(request.tenant_id.0) + .bind(request.run_uid) + .bind(request.compensation_id.as_uuid()) + .bind(to_i64( + request.controller_generation, + "controller generation", + )?) + .bind(to_i64( + request.compensation_generation, + "compensation generation", + )?) + .bind(to_i64( + request.compensation_attempt_generation, + "compensation attempt generation", + )?) + .bind(request.watchdog_trigger_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if run.controller_generation != request.controller_generation + || current.registration.generation != request.compensation_generation + || current.attempt_generation != request.compensation_attempt_generation + || current.active_dispatch_uid != Some(request.dispatch_uid) + || current.attempt_state != CompensationAttemptState::Dispatching + || !exact_resources + { + return Ok(CompensationAttemptWriteOutcome::Conflict); + } + let fence = CompensationAttemptFence { + run_uid: request.run_uid, + compensation_id: request.compensation_id, + controller_generation: request.controller_generation, + compensation_generation: request.compensation_generation, + attempt_generation: request.compensation_attempt_generation, + dispatch_uid: request.dispatch_uid, + }; + release_unbound_compensation_external_intent_in_conn(conn, &run, ¤t).await?; + release_compensation_capacity(conn, &run, fence).await?; + supersede_compensation_triggers(conn, fence, None).await?; + let updated = sqlx::query( + "UPDATE moa.execution_compensation SET attempt_state='idle', \ + attempt_generation=attempt_generation+1, attempt_started_at=NULL, \ + attempt_deadline_at=NULL, waiting_since=NULL, active_dispatch_uid=NULL, \ + external_job_uid=NULL, release_intent=NULL, \ + last_progress_at=GREATEST(last_progress_at,$6), updated_at=NOW() \ + WHERE run_uid=$1 AND compensation_id=$2 AND generation=$3 \ + AND attempt_generation=$4 AND active_dispatch_uid=$5 \ + AND attempt_state='dispatching' RETURNING *", + ) + .bind(request.run_uid) + .bind(request.compensation_id.as_uuid()) + .bind(to_i64( + request.compensation_generation, + "compensation generation", + )?) + .bind(to_i64( + request.compensation_attempt_generation, + "compensation attempt generation", + )?) + .bind(request.dispatch_uid) + .bind(settled_at) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + enqueue_current_compensation_controller_wake( + conn, + &run, + json!({"reason":"compensation_dispatch_delivery_lost"}), + settled_at, + ) + .await?; + Ok(CompensationAttemptWriteOutcome::Applied( + compensation_attempt_from_row(&updated, &run)?, + )) +} + +async fn supersede_compensation_triggers( + conn: &mut ScopedConn<'_>, + fence: CompensationAttemptFence, + except: Option, +) -> Result<()> { + let except = except.map(ExecutionTriggerKind::as_str); + sqlx::query( + "UPDATE moa.execution_dispatch_outbox SET state='superseded', claim_owner=NULL, \ + claimed_at=NULL, claim_expires_at=NULL, updated_at=NOW() WHERE trigger_uid IN ( \ + SELECT trigger_uid FROM moa.execution_trigger WHERE run_uid=$1 \ + AND compensation_id=$2 AND controller_generation=$3 \ + AND compensation_generation=$4 AND compensation_attempt_generation=$5 \ + AND ($6::TEXT IS NULL OR trigger_kind <> $6) \ + ) AND state IN ('pending','dispatching')", + ) + .bind(fence.run_uid) + .bind(fence.compensation_id.as_uuid()) + .bind(to_i64( + fence.controller_generation, + "controller generation", + )?) + .bind(to_i64( + fence.compensation_generation, + "compensation generation", + )?) + .bind(to_i64( + fence.attempt_generation, + "compensation attempt generation", + )?) + .bind(except) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + sqlx::query( + "UPDATE moa.execution_trigger SET state='superseded', claim_owner=NULL, claimed_at=NULL, \ + claim_expires_at=NULL, updated_at=NOW() WHERE run_uid=$1 AND compensation_id=$2 \ + AND controller_generation=$3 AND compensation_generation=$4 \ + AND compensation_attempt_generation=$5 AND ($6::TEXT IS NULL OR trigger_kind <> $6) \ + AND state IN ('pending','dispatching')", + ) + .bind(fence.run_uid) + .bind(fence.compensation_id.as_uuid()) + .bind(to_i64( + fence.controller_generation, + "controller generation", + )?) + .bind(to_i64( + fence.compensation_generation, + "compensation generation", + )?) + .bind(to_i64( + fence.attempt_generation, + "compensation attempt generation", + )?) + .bind(except) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + Ok(()) +} + +async fn enqueue_current_compensation_controller_wake( + conn: &mut ScopedConn<'_>, + run: &ExecutionRunRecord, + payload: Value, + now: DateTime, +) -> Result<()> { + enqueue_run_activation_in_conn( + conn.as_mut(), + run.tenant_id, + run.run_uid, + run.controller_generation, + now, + payload, + ) + .await?; + Ok(()) +} + +type CompensationSettlementFields = ( + CompensationStatus, + CompensationAttemptState, + u64, + u64, + u64, + bool, + Option, +); + +fn compensation_settlement_fields( + current: &CompensationAttemptRecord, + outcome: &ExecutionCompensationOutcome, + retry: bool, +) -> Result { + if retry { + let message = match outcome { + ExecutionCompensationOutcome::Failed { message, .. } => message, + ExecutionCompensationOutcome::Completed { .. } + | ExecutionCompensationOutcome::UnknownOutcome { .. } => { + return Err(Error::InvalidRepositoryInput { + message: "only failed compensation outcomes may retry".to_string(), + }); + } + }; + return Ok(( + CompensationStatus::Pending, + CompensationAttemptState::Idle, + current.registration.attempt.checked_add(1).ok_or_else(|| { + Error::InvalidRepositoryData { + message: "compensation attempt overflow".to_string(), + } + })?, + current + .registration + .generation + .checked_add(1) + .ok_or_else(|| Error::InvalidRepositoryData { + message: "compensation generation overflow".to_string(), + })?, + current.attempt_generation.checked_add(1).ok_or_else(|| { + Error::InvalidRepositoryData { + message: "compensation attempt generation overflow".to_string(), + } + })?, + false, + Some(json!({"class":"retryable", "message":message})), + )); + } + match outcome { + ExecutionCompensationOutcome::Completed { .. } => Ok(( + CompensationStatus::Completed, + CompensationAttemptState::Terminal, + current.registration.attempt, + current.registration.generation, + current.attempt_generation, + false, + None, + )), + ExecutionCompensationOutcome::Failed { message, .. } => Ok(( + CompensationStatus::Failed, + CompensationAttemptState::Terminal, + current.registration.attempt, + current.registration.generation, + current.attempt_generation, + true, + Some(json!({"class":"terminal", "message":message})), + )), + ExecutionCompensationOutcome::UnknownOutcome { message, .. } => Ok(( + CompensationStatus::UnknownOutcome, + CompensationAttemptState::UnknownOutcome, + current.registration.attempt, + current.registration.generation, + current.attempt_generation, + true, + Some(json!({ + "class":"unknown_outcome", + "message":message, + "manual_repair_required":true + })), + )), + } +} + +fn force_terminal_failure_if_exhausted( + outcome: ExecutionCompensationOutcome, +) -> ExecutionCompensationOutcome { + match outcome { + ExecutionCompensationOutcome::Failed { + message, + retryable: true, + usage, + } => ExecutionCompensationOutcome::Failed { + message, + retryable: false, + usage, + }, + other => other, + } +} + +fn validate_pending_terminal_page_limit(page_limit: u32) -> Result<()> { + if page_limit == 0 || page_limit > MAX_PENDING_TERMINAL_PAGE_SIZE { + return Err(Error::InvalidRepositoryInput { + message: format!( + "pending-terminal page limit must be between 1 and {MAX_PENDING_TERMINAL_PAGE_SIZE}" + ), + }); + } + Ok(()) +} + +/// Settles one terminal provider job into its exact waiting compensation attempt. +pub(super) async fn settle_external_job_terminal_in_conn( + conn: &mut ScopedConn<'_>, + job: &ExecutionExternalJobRecord, + settled_at: DateTime, +) -> Result { + if !job.state.is_terminal() { + return Err(Error::InvalidRepositoryInput { + message: "compensation external-job settlement requires a terminal job".to_string(), + }); + } + let ExecutionExternalJobOwner::Compensation { + compensation_id, + compensation_generation, + compensation_attempt_generation, + } = job.owner + else { + return Ok(CompensationExternalJobSettlementOutcome::Stale); + }; + let Some(run_row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) + .bind(job.run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + return Ok(CompensationExternalJobSettlementOutcome::NotFound); + }; + let run = run_from_row(&run_row)?; + let Some(row) = sqlx::query(LOAD_COMPENSATION_FOR_UPDATE_SQL) + .bind(job.run_uid) + .bind(compensation_id) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + return Ok(CompensationExternalJobSettlementOutcome::NotFound); + }; + let current = compensation_attempt_from_row(&row, &run)?; + if current.registration.status.is_settled() { + return Ok(if current.external_job_uid == Some(job.external_job_uid) { + CompensationExternalJobSettlementOutcome::Replayed(current) + } else { + CompensationExternalJobSettlementOutcome::Stale + }); + } + if run.tenant_id == job.tenant_id + && current.registration.generation == compensation_generation + && current.attempt_generation == compensation_attempt_generation + && current.attempt_state == CompensationAttemptState::Cancelling + && current.release_intent == Some(ExecutionCompensationReleaseIntent::ExternalJob) + && current.external_job_uid == Some(job.external_job_uid) + { + return Ok(CompensationExternalJobSettlementOutcome::DeferredRelease( + current, + )); + } + if run.tenant_id != job.tenant_id + || current.registration.generation != compensation_generation + || current.attempt_generation != compensation_attempt_generation + || current.attempt_state != CompensationAttemptState::WaitingExternal + || current.external_job_uid != Some(job.external_job_uid) + { + return Ok(CompensationExternalJobSettlementOutcome::Stale); + } + let previous_usage = current + .registration + .outcome + .as_ref() + .map(ExecutionCompensationOutcome::usage) + .cloned() + .unwrap_or_else(zero_usage); + let outcome = match job.state { + ExecutionExternalJobState::Completed => ExecutionCompensationOutcome::Completed { + output: job.output.clone().unwrap_or(Value::Null), + usage: previous_usage.clone(), + }, + ExecutionExternalJobState::Failed => ExecutionCompensationOutcome::Failed { + message: job + .error + .as_ref() + .map(Value::to_string) + .unwrap_or_else(|| "asynchronous compensation job failed".to_string()), + retryable: true, + usage: previous_usage.clone(), + }, + ExecutionExternalJobState::Cancelled => ExecutionCompensationOutcome::Failed { + message: "asynchronous compensation job was cancelled".to_string(), + retryable: false, + usage: previous_usage.clone(), + }, + ExecutionExternalJobState::UnknownOutcome => ExecutionCompensationOutcome::UnknownOutcome { + message: job.error.as_ref().map(Value::to_string).unwrap_or_else(|| { + "asynchronous compensation job has an unknown outcome".to_string() + }), + usage: previous_usage.clone(), + }, + ExecutionExternalJobState::Unbound + | ExecutionExternalJobState::Starting + | ExecutionExternalJobState::Running + | ExecutionExternalJobState::WaitingReconcile + | ExecutionExternalJobState::CancelRequested => { + return Err(Error::InvalidRepositoryInput { + message: "compensation external-job settlement observed nonterminal state" + .to_string(), + }); + } + }; + let forward_task = + load_forward_task(conn, job.run_uid, current.registration.forward_task_id).await?; + let full_reservation = + compensation_reservation(&run, ¤t.registration, forward_task.retry.max_attempts)?; + let remaining = remaining_compensation_reservation(full_reservation, &previous_usage); + let retry = matches!( + outcome, + ExecutionCompensationOutcome::Failed { + retryable: true, + .. + } + ) && run.pending_terminal.is_none() + && current.registration.attempt < u64::from(forward_task.retry.max_attempts); + let accepted_outcome = if retry { + outcome + } else { + force_terminal_failure_if_exhausted(outcome) + }; + let mut ledger = budget_ledger(&run); + let reconciliation = ledger.reconcile_cumulative_with_ceiling( + remaining, + &previous_usage, + accepted_outcome.usage(), + !retry, + i64::MAX as u64, + )?; + let (status, attempt_state, attempt, generation, next_attempt_generation, repair, error) = + compensation_settlement_fields(¤t, &accepted_outcome, retry)?; + let persisted = persisted_compensation_outcome(&row, Some(accepted_outcome))?; + let updated = sqlx::query( + "UPDATE moa.execution_compensation SET status=$6, attempt_state=$7, attempt=$8, \ + generation=$9, attempt_generation=$10, outcome=$11, error=$12, \ + attempt_started_at=NULL, attempt_deadline_at=NULL, waiting_since=NULL, \ + active_dispatch_uid=NULL, \ + last_progress_at=GREATEST(last_progress_at,$13), updated_at=NOW(), \ + completed_at=CASE WHEN $14 THEN $13 ELSE NULL END \ + WHERE run_uid=$1 AND compensation_id=$2 AND generation=$3 \ + AND attempt_generation=$4 AND external_job_uid=$5 \ + AND attempt_state='waiting_external' RETURNING *", + ) + .bind(job.run_uid) + .bind(compensation_id) + .bind(to_i64(compensation_generation, "compensation generation")?) + .bind(to_i64( + compensation_attempt_generation, + "compensation attempt generation", + )?) + .bind(job.external_job_uid) + .bind(status.as_str()) + .bind(attempt_state.as_str()) + .bind(to_i64(attempt, "compensation attempt")?) + .bind(to_i64(generation, "compensation generation")?) + .bind(to_i64( + next_attempt_generation, + "compensation attempt generation", + )?) + .bind(serde_json::to_value(persisted)?) + .bind(error) + .bind(settled_at) + .bind(status.is_settled()) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(updated) = updated else { + return Ok(CompensationExternalJobSettlementOutcome::Stale); + }; + persist_run_budget_and_repair(conn, job.run_uid, &reconciliation, repair).await?; + if !matches!( + run.status, + ExecutionRunStatus::PauseRequested + | ExecutionRunStatus::Pausing + | ExecutionRunStatus::Paused + ) { + enqueue_current_compensation_controller_wake( + conn, + &run, + json!({ + "reason": "compensation_external_job_settled", + "external_job_uid": job.external_job_uid, + }), + settled_at, + ) + .await?; + } + Ok(CompensationExternalJobSettlementOutcome::Applied( + compensation_attempt_from_row(&updated, &run)?, + )) +} + +async fn load_and_lock_pending_terminal_run( + conn: &mut ScopedConn<'_>, + config: &ExecutionConfig, + run_uid: Uuid, +) -> Result> { + let Some(visible_row) = sqlx::query(LOAD_RUN_SQL) + .bind(run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + return Ok(None); + }; + let visible = run_from_row(&visible_row)?; + prelock_capacity_dimensions_in_tx( + conn.as_mut(), + config, + visible.tenant_id, + &[ + ExecutionCapacityDimension::ActiveRuns, + ExecutionCapacityDimension::ActiveTasks, + ExecutionCapacityDimension::ParkedRuns, + ExecutionCapacityDimension::ScheduledTriggers, + ExecutionCapacityDimension::ExternalJobs, + ], + ) + .await?; + let row = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) + .bind(run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let run = run_from_row(&row)?; + if run.tenant_id != visible.tenant_id { + return Err(Error::InvalidRepositoryData { + message: "execution run tenant changed while acquiring compensation capacity locks" + .to_string(), + }); + } + Ok(Some(run)) +} + +async fn replayed_pending_terminal_commit( + conn: &mut ScopedConn<'_>, + config: &ExecutionConfig, + run: ExecutionRunRecord, +) -> Result { + let work_remaining: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM moa.execution_task WHERE run_uid=$1 \ + AND status NOT IN ('completed','skipped','failed','cancelled','unknown_outcome')) \ + OR EXISTS (SELECT 1 FROM moa.execution_compensation WHERE run_uid=$1 \ + AND status <> 'completed') \ + OR EXISTS (SELECT 1 FROM moa.execution_capacity_reservation WHERE run_uid=$1 \ + AND resource_dimension IN ('active_tasks','scheduled_triggers','external_jobs') \ + AND state IN ('reserved','reconciling'))", + ) + .bind(run.run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let compensation_admission = if run.status == ExecutionRunStatus::Compensating { + let row = sqlx::query( + "SELECT * FROM moa.execution_compensation WHERE run_uid=$1 \ + AND status <> 'completed' ORDER BY registered_sequence DESC LIMIT 1 FOR UPDATE", + ) + .bind(run.run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if let Some(row) = row { + if compensation_attempt_state_from_row(&row)? == CompensationAttemptState::Dispatching { + let registration = compensation_from_row(&row)?; + Some(Box::new( + load_existing_compensation_admission(conn, config, &run, &row, ®istration) + .await?, + )) + } else { + None + } + } else { + None + } + } else { + None + }; + let continuation = load_pending_terminal_continuation(conn, &run).await?; + let stage = if run.status.is_terminal() { + if run.manual_repair_required { + PendingTerminalAdvanceStage::ManualRepairRequired + } else { + PendingTerminalAdvanceStage::Finalized + } + } else if compensation_admission.is_some() { + PendingTerminalAdvanceStage::CompensationQueued + } else if work_remaining { + PendingTerminalAdvanceStage::Draining + } else { + PendingTerminalAdvanceStage::EnqueuedPage + }; + Ok(PendingTerminalAdvanceCommit { + run, + stage, + settled_task_count: 0, + drained_trigger_count: 0, + cancellation_dispatches: Vec::new(), + compensation_admission, + continuation: continuation.map(Box::new), + work_remaining, + }) +} + +async fn load_pending_terminal_continuation( + conn: &mut ScopedConn<'_>, + run: &ExecutionRunRecord, +) -> Result> { + let row = sqlx::query( + "SELECT dispatch_uid, not_before_at, payload, wake_epoch \ + FROM moa.execution_dispatch_outbox WHERE run_uid=$1 \ + AND dispatch_kind='run_activation' AND controller_generation=$2 \ + AND payload->>'source_wake_epoch'=$3 ORDER BY created_at DESC LIMIT 1", + ) + .bind(run.run_uid) + .bind(to_i64(run.controller_generation, "controller generation")?) + .bind(run.processed_wake_epoch.to_string()) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(row) = row else { + return Ok(None); + }; + let wake_epoch = required_u64(&row, "wake_epoch")?; + let request = NewExecutionDispatch { + dispatch_uid: row.try_get("dispatch_uid").map_err(row_error)?, + tenant_id: run.tenant_id, + run_uid: Some(run.run_uid), + task_id: None, + compensation_id: None, + trigger_uid: None, + external_job_uid: None, + kind: ExecutionDispatchKind::RunActivation, + controller_generation: Some(run.controller_generation), + wake_epoch: Some(wake_epoch), + attempt_generation: None, + compensation_generation: None, + compensation_attempt_generation: None, + not_before_at: row.try_get("not_before_at").map_err(row_error)?, + payload: row.try_get("payload").map_err(row_error)?, + }; + enqueue_dispatch_in_conn(conn.as_mut(), &request) + .await + .map(Some) +} + +#[allow(clippy::too_many_arguments)] +async fn advance_pending_terminal_page_in_conn( + mut conn: ScopedConn<'_>, + config: &ExecutionConfig, + mut run: ExecutionRunRecord, + controller_generation: u64, + expected_wake_epoch: u64, + new_pending: Option, + now: DateTime, + page_limit: u32, +) -> Result { + if let Some(pending) = new_pending { + if let Some(current) = &run.pending_terminal { + if current != &pending { + conn.commit().await.map_err(storage_error)?; + return Ok(PendingTerminalAdvanceOutcome::Conflict); + } + } else { + let row = sqlx::query( + "UPDATE moa.execution_run SET pending_terminal_status=$4, \ + pending_terminal_reason=$5, pending_terminal_cause=$6, \ + pending_terminal_output=$7, cancellation_reason=$8, \ + waiting_reasons='[]'::JSONB, next_wake_at=NULL, waiting_since=NULL, \ + waiting_task_count=0, waiting_input_task_count=0, \ + waiting_review_task_count=0, waiting_signal_task_count=0, \ + waiting_timer_task_count=0, waiting_external_task_count=0, \ + waiting_replan_task_count=0, waiting_input_user_task_count=0, \ + waiting_input_tenant_admin_task_count=0, \ + waiting_input_external_task_count=0, waiting_reasons_truncated=FALSE, \ + updated_at=$9 WHERE run_uid=$1 AND controller_generation=$2 \ + AND wake_epoch=$3 AND pending_terminal_status IS NULL \ + AND status NOT IN ('completed','partial','blocked','unsupported', \ + 'failed','cancelled','compensating') RETURNING *", + ) + .bind(run.run_uid) + .bind(to_i64(controller_generation, "controller generation")?) + .bind(to_i64(expected_wake_epoch, "expected wake epoch")?) + .bind(pending.status.as_str()) + .bind(pending.reason.as_str()) + .bind(serde_json::to_value(PendingTerminalEvidencePayload { + terminal_evidence: pending.terminal_evidence.clone(), + completion_check_results: pending.completion_check_results.clone(), + terminal_gaps: pending.terminal_gaps.clone(), + })?) + .bind(&pending.output) + .bind(&pending.cancellation_reason) + .bind(now) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(row) = row else { + conn.rollback().await.map_err(storage_error)?; + return Ok(PendingTerminalAdvanceOutcome::Conflict); + }; + run = run_from_row(&row)?; + } + } + let pending = run + .pending_terminal + .clone() + .ok_or_else(|| Error::InvalidRepositoryData { + message: "terminal drain lost its pending terminal intent".to_string(), + })?; + let cancel_reason = if pending.reason == ExecutionTerminalReason::DeadlineExceeded { + ExecutionAttemptCancelReason::DeadlineExceeded + } else { + ExecutionAttemptCancelReason::RunTerminal + }; + let task_rows = sqlx::query( + "SELECT task.* FROM moa.execution_task AS task WHERE task.run_uid=$1 \ + AND task.status NOT IN ('completed','skipped','failed','cancelled','unknown_outcome') \ + AND task.attempt_state <> 'cancelling' \ + ORDER BY CASE WHEN task.attempt_state IN ('dispatching','running') THEN 0 ELSE 1 END, \ + task.task_id LIMIT $2 FOR UPDATE", + ) + .bind(run.run_uid) + .bind(i64::from(page_limit)) + .fetch_all(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let processed_task_count = + u32::try_from(task_rows.len()).map_err(|_| Error::InvalidRepositoryData { + message: "terminal drain task page exceeds u32".to_string(), + })?; + let mut settled_task_count = 0_u64; + let mut cancellation_dispatches = Vec::with_capacity(task_rows.len()); + for row in task_rows { + let task = task_from_row(&row)?; + if task.status == ExecutionTaskStatus::WaitingExternal { + let external_job_uid = + task.external_job_uid + .ok_or_else(|| Error::InvalidRepositoryData { + message: "waiting-external task lost its exact external job UID" + .to_string(), + })?; + let owner = ExecutionExternalJobOwner::Task { + task_id: task.task_id.as_uuid(), + attempt_generation: task.attempt_generation, + }; + match request_external_job_cancellation_in_conn( + &mut conn, + config, + external_job_uid, + owner, + now, + ) + .await? + { + ExecutionExternalJobCancellationRequestOutcome::Applied(dispatch) + | ExecutionExternalJobCancellationRequestOutcome::Replayed(dispatch) => { + cancellation_dispatches.push(dispatch); + } + ExecutionExternalJobCancellationRequestOutcome::UnboundPendingRecovery => {} + ExecutionExternalJobCancellationRequestOutcome::AlreadyTerminal => { + let job = load_external_job_for_update_in_conn(conn.as_mut(), external_job_uid) + .await? + .ok_or_else(|| Error::InvalidRepositoryData { + message: "terminal external job disappeared under its owner fence" + .to_string(), + })?; + settle_task_external_job_terminal_in_conn(&mut conn, &job, now).await?; + } + ExecutionExternalJobCancellationRequestOutcome::NotFound + | ExecutionExternalJobCancellationRequestOutcome::Stale => { + return Err(Error::InvalidRepositoryData { + message: "waiting-external task has a stale external job owner".to_string(), + }); + } + } + continue; + } + if matches!( + task.attempt_state, + ExecutionAttemptState::Dispatching | ExecutionAttemptState::Running + ) { + cancellation_dispatches.push( + enqueue_pending_terminal_task_cancellation( + &mut conn, + &run, + &task, + cancel_reason, + pending.reason, + now, + ) + .await?, + ); + continue; + } + supersede_storage_task_waits(&mut conn, &task).await?; + let original_status = task.status; + match record_task_outcome_in_conn( + &mut conn, + run.run_uid, + task.task_id, + task.generation, + cancelled_task_outcome( + format!("run terminal fence: {}", pending.reason.as_str()), + task.actual.clone(), + ), + ) + .await? + { + TaskOutcomeWrite::Applied { task, .. } | TaskOutcomeWrite::Replayed { task, .. } => { + transition_node_counters_in_tx( + &mut conn, + run.run_uid, + &task.node_id, + &task.item_key, + original_status, + ExecutionTaskStatus::Cancelled, + ) + .await?; + } + TaskOutcomeWrite::Rejected { reason, .. } => { + return Err(Error::InvalidRepositoryData { + message: format!("terminal drain task settlement was rejected: {reason:?}"), + }); + } + TaskOutcomeWrite::NotFound => { + return Err(Error::InvalidRepositoryData { + message: "terminal drain lost a row-locked task".to_string(), + }); + } + } + settled_task_count = + settled_task_count + .checked_add(1) + .ok_or_else(|| Error::InvalidRepositoryData { + message: "terminal drain settled-task count overflow".to_string(), + })?; + } + + let task_dispatch_count = cancellation_dispatches.len(); + let remaining_slots = page_limit.saturating_sub(processed_task_count); + if remaining_slots > 0 && run.status != ExecutionRunStatus::Compensating { + let compensation_rows = sqlx::query( + "SELECT compensation.* FROM moa.execution_compensation AS compensation \ + WHERE compensation.run_uid=$1 \ + AND compensation.attempt_state IN ('dispatching','running') \ + ORDER BY compensation.registered_sequence DESC LIMIT $2 FOR UPDATE", + ) + .bind(run.run_uid) + .bind(i64::from(remaining_slots)) + .fetch_all(conn.as_mut()) + .await + .map_err(sqlx_error)?; + for row in compensation_rows { + cancellation_dispatches.push( + enqueue_pending_terminal_compensation_cancellation( + &mut conn, + &run, + &row, + cancel_reason, + pending.reason, + now, + ) + .await?, + ); + } + } + let compensation_cancellation_count = cancellation_dispatches + .len() + .checked_sub(task_dispatch_count) + .and_then(|count| u32::try_from(count).ok()) + .ok_or_else(|| Error::InvalidRepositoryData { + message: "terminal drain compensation cancellation count overflow".to_string(), + })?; + let charged_after_cancellations = processed_task_count + .checked_add(compensation_cancellation_count) + .ok_or_else(|| Error::InvalidRepositoryData { + message: "terminal drain page accounting overflow after cancellation".to_string(), + })?; + let trigger_slots = page_limit.saturating_sub(charged_after_cancellations); + + let nonterminal_forward_count: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM moa.execution_task WHERE run_uid=$1 \ + AND status NOT IN ('completed','skipped','failed','cancelled','unknown_outcome')", + ) + .bind(run.run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let actionable_forward_exists: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM moa.execution_task WHERE run_uid=$1 \ + AND status NOT IN ('completed','skipped','failed','cancelled','unknown_outcome') \ + AND attempt_state <> 'cancelling' AND status <> 'waiting_external')", + ) + .bind(run.run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let active_count = active_attempt_capacity_count(&mut conn, run.run_uid).await?; + let has_registrations: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM moa.execution_compensation WHERE run_uid=$1)", + ) + .bind(run.run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let retain_cancelled_effects = pending.status == ExecutionRunStatus::Cancelled + && run.active_plan.definition.cancel_policy == ExecutionCancelPolicy::RetainEffects; + let should_compensate = has_registrations && !retain_cancelled_effects; + let active_trigger_exists: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM moa.execution_trigger WHERE run_uid=$1 \ + AND state IN ('pending','dispatching'))", + ) + .bind(run.run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let cleanup_triggers_now = nonterminal_forward_count == 0 + && active_count == 0 + && !should_compensate + && run.status != ExecutionRunStatus::Compensating; + let (mut drained_trigger_count, mut trigger_work_remaining) = + if cleanup_triggers_now && trigger_slots > 0 { + let page = drain_run_triggers_page_in_conn(&mut conn, &run, trigger_slots).await?; + (page.drained_trigger_count, page.work_remaining) + } else if cleanup_triggers_now { + (0, active_trigger_exists) + } else { + (0, false) + }; + let ready_count: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM moa.execution_task WHERE run_uid=$1 AND status='ready'", + ) + .bind(run.run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + + let mut stage = PendingTerminalAdvanceStage::Draining; + let mut work_remaining = nonterminal_forward_count > 0 || active_count > 0; + let mut continuation_payload = None; + let mut continuation_not_before = now; + let mut checkpoint_status = run.status; + let mut checkpoint_active_count = active_count; + let mut compensation_admission = None; + if actionable_forward_exists || trigger_work_remaining { + stage = PendingTerminalAdvanceStage::EnqueuedPage; + work_remaining = true; + continuation_payload = Some(json!({ + "reason":"pending_terminal_page", + "source_wake_epoch": expected_wake_epoch, + })); + } else if nonterminal_forward_count == 0 && active_count == 0 { + if should_compensate && run.status != ExecutionRunStatus::Compensating { + stage = PendingTerminalAdvanceStage::EnqueuedPage; + checkpoint_status = ExecutionRunStatus::Compensating; + work_remaining = true; + continuation_payload = Some(json!({ + "reason":"pending_terminal_compensation", + "source_wake_epoch": expected_wake_epoch, + })); + } else if run.status == ExecutionRunStatus::Compensating { + match drive_pending_terminal_compensation_in_conn(&mut conn, config, &run, now).await? { + PendingCompensationDrive::Admitted(admission) + | PendingCompensationDrive::Replayed(admission) => { + stage = PendingTerminalAdvanceStage::CompensationQueued; + work_remaining = true; + compensation_admission = Some(admission); + checkpoint_active_count = + active_attempt_capacity_count(&mut conn, run.run_uid).await?; + } + PendingCompensationDrive::CapacityUnavailable { retry_at } => { + stage = PendingTerminalAdvanceStage::EnqueuedPage; + work_remaining = true; + continuation_not_before = retry_at; + continuation_payload = Some(json!({ + "reason":"pending_terminal_compensation_capacity", + "source_wake_epoch": expected_wake_epoch, + })); + } + PendingCompensationDrive::ExternalCancellation(dispatch) => { + cancellation_dispatches.push(dispatch); + work_remaining = true; + } + PendingCompensationDrive::Parked => { + work_remaining = true; + } + PendingCompensationDrive::ManualRepair(registration) => { + if trigger_slots > 0 { + let page = + drain_run_triggers_page_in_conn(&mut conn, &run, trigger_slots).await?; + drained_trigger_count = page.drained_trigger_count; + trigger_work_remaining = page.work_remaining; + } else { + trigger_work_remaining = active_trigger_exists; + } + if trigger_work_remaining { + stage = PendingTerminalAdvanceStage::EnqueuedPage; + work_remaining = true; + continuation_payload = Some(json!({ + "reason":"pending_terminal_manual_repair_cleanup", + "source_wake_epoch": expected_wake_epoch, + })); + } else { + let non_lifetime_capacity_exists: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM moa.execution_capacity_reservation \ + WHERE run_uid=$1 AND resource_dimension IN \ + ('active_tasks','scheduled_triggers','external_jobs') \ + AND state IN ('reserved','reconciling'))", + ) + .bind(run.run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if non_lifetime_capacity_exists { + return Err(Error::InvalidRepositoryData { + message: "failed compensation retained non-lifetime capacity" + .to_string(), + }); + } + let failure = compensation_failure_pending(&pending, ®istration)?; + replace_pending_terminal_exact( + &mut conn, + &run, + &pending, + &failure, + controller_generation, + expected_wake_epoch, + now, + ) + .await?; + let finalized = finalize_pending_terminal_exact( + &mut conn, + &run, + &failure, + controller_generation, + expected_wake_epoch, + now, + ) + .await?; + conn.commit().await.map_err(storage_error)?; + return Ok(PendingTerminalAdvanceOutcome::Applied(Box::new( + PendingTerminalAdvanceCommit { + run: finalized, + stage: PendingTerminalAdvanceStage::ManualRepairRequired, + settled_task_count, + drained_trigger_count, + cancellation_dispatches, + compensation_admission: None, + continuation: None, + work_remaining: false, + }, + ))); + } + } + PendingCompensationDrive::Complete => { + if trigger_slots > 0 { + let page = + drain_run_triggers_page_in_conn(&mut conn, &run, trigger_slots).await?; + drained_trigger_count = page.drained_trigger_count; + trigger_work_remaining = page.work_remaining; + } else { + trigger_work_remaining = active_trigger_exists; + } + if trigger_work_remaining { + stage = PendingTerminalAdvanceStage::EnqueuedPage; + work_remaining = true; + continuation_payload = Some(json!({ + "reason":"pending_terminal_trigger_cleanup", + "source_wake_epoch": expected_wake_epoch, + })); + } else { + let non_lifetime_capacity_exists: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM moa.execution_capacity_reservation \ + WHERE run_uid=$1 AND resource_dimension IN \ + ('active_tasks','scheduled_triggers','external_jobs') \ + AND state IN ('reserved','reconciling'))", + ) + .bind(run.run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if non_lifetime_capacity_exists { + return Err(Error::InvalidRepositoryData { + message: "completed compensation retained non-lifetime capacity" + .to_string(), + }); + } + let finalized = finalize_pending_terminal_exact( + &mut conn, + &run, + &pending, + controller_generation, + expected_wake_epoch, + now, + ) + .await?; + conn.commit().await.map_err(storage_error)?; + return Ok(PendingTerminalAdvanceOutcome::Applied(Box::new( + PendingTerminalAdvanceCommit { + run: finalized, + stage: PendingTerminalAdvanceStage::Finalized, + settled_task_count, + drained_trigger_count, + cancellation_dispatches, + compensation_admission: None, + continuation: None, + work_remaining: false, + }, + ))); + } + } + } + } else { + let non_lifetime_capacity_exists: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM moa.execution_capacity_reservation WHERE run_uid=$1 \ + AND resource_dimension IN ('active_tasks','scheduled_triggers','external_jobs') \ + AND state IN ('reserved','reconciling'))", + ) + .bind(run.run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if non_lifetime_capacity_exists { + work_remaining = true; + } else { + let finalized = finalize_pending_terminal_exact( + &mut conn, + &run, + &pending, + controller_generation, + expected_wake_epoch, + now, + ) + .await?; + stage = if finalized.manual_repair_required { + PendingTerminalAdvanceStage::ManualRepairRequired + } else { + PendingTerminalAdvanceStage::Finalized + }; + conn.commit().await.map_err(storage_error)?; + return Ok(PendingTerminalAdvanceOutcome::Applied(Box::new( + PendingTerminalAdvanceCommit { + run: finalized, + stage, + settled_task_count, + drained_trigger_count, + cancellation_dispatches, + compensation_admission: None, + continuation: None, + work_remaining: false, + }, + ))); + } + } + } + + let checkpointed = checkpoint_pending_terminal_wake( + &mut conn, + run.run_uid, + controller_generation, + expected_wake_epoch, + checkpoint_status, + u64::try_from(ready_count).map_err(|_| Error::InvalidRepositoryData { + message: "terminal drain ready-task count is negative".to_string(), + })?, + checkpoint_active_count, + now, + ) + .await?; + let continuation = if let Some(payload) = continuation_payload { + Some(Box::new( + enqueue_run_activation_in_conn( + conn.as_mut(), + checkpointed.tenant_id, + checkpointed.run_uid, + checkpointed.controller_generation, + continuation_not_before, + payload, + ) + .await?, + )) + } else { + None + }; + let row = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) + .bind(run.run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + run = run_from_row(&row)?; + conn.commit().await.map_err(storage_error)?; + Ok(PendingTerminalAdvanceOutcome::Applied(Box::new( + PendingTerminalAdvanceCommit { + run, + stage, + settled_task_count, + drained_trigger_count, + cancellation_dispatches, + compensation_admission, + continuation, + work_remaining, + }, + ))) +} + +async fn enqueue_pending_terminal_task_cancellation( + conn: &mut ScopedConn<'_>, + run: &ExecutionRunRecord, + task: &ExecutionTaskRecord, + reason: ExecutionAttemptCancelReason, + terminal_reason: ExecutionTerminalReason, + now: DateTime, +) -> Result { + let row = sqlx::query( + "SELECT reservation.reservation_uid, trigger.trigger_uid \ + FROM moa.execution_capacity_reservation AS reservation \ + JOIN moa.execution_trigger AS trigger ON trigger.run_uid=reservation.run_uid \ + AND trigger.task_id=reservation.task_id \ + AND trigger.controller_generation=reservation.controller_generation \ + AND trigger.attempt_generation=reservation.attempt_generation \ + AND trigger.trigger_kind='task_watchdog' \ + AND trigger.state IN ('pending','dispatching') \ + WHERE reservation.run_uid=$1 AND reservation.task_id=$2 \ + AND reservation.controller_generation=$3 AND reservation.attempt_generation=$4 \ + AND reservation.resource_dimension='active_tasks' \ + AND reservation.state IN ('reserved','reconciling') FOR UPDATE OF reservation, trigger", + ) + .bind(run.run_uid) + .bind(task.task_id.as_uuid()) + .bind(to_i64(run.controller_generation, "controller generation")?) + .bind(to_i64(task.attempt_generation, "task attempt generation")?) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + .ok_or_else(|| Error::InvalidRepositoryData { + message: format!( + "active task {} is missing its exact capacity or watchdog receipt", + task.task_id + ), + })?; + let active_dispatch_uid = + task.active_dispatch_uid + .ok_or_else(|| Error::InvalidRepositoryData { + message: format!( + "active task {} is missing its dispatch identity", + task.task_id + ), + })?; + let capacity_reservation_uid: Uuid = row.try_get("reservation_uid").map_err(row_error)?; + let watchdog_trigger_uid: Uuid = row.try_get("trigger_uid").map_err(row_error)?; + let cancellation_dispatch_uid = pending_terminal_cancel_dispatch_uid( + active_dispatch_uid, + run.controller_generation, + terminal_reason, + ); + let cancelling = sqlx::query( + "UPDATE moa.execution_task SET attempt_state='cancelling', \ + last_progress_at=GREATEST(last_progress_at,$6), updated_at=NOW() \ + WHERE run_uid=$1 AND task_id=$2 \ + AND generation=$3 AND attempt_generation=$4 AND active_dispatch_uid=$5 \ + AND attempt_state IN ('dispatching','running')", + ) + .bind(run.run_uid) + .bind(task.task_id.as_uuid()) + .bind(to_i64(task.generation, "task generation")?) + .bind(to_i64(task.attempt_generation, "task attempt generation")?) + .bind(active_dispatch_uid) + .bind(now) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if cancelling.rows_affected() != 1 { + return Err(Error::InvalidRepositoryData { + message: format!("task {} lost its terminal cancellation fence", task.task_id), + }); + } + let reconciling = sqlx::query( + "UPDATE moa.execution_capacity_reservation SET state='reconciling', updated_at=$2 \ + WHERE reservation_uid=$1 AND state IN ('reserved','reconciling')", + ) + .bind(capacity_reservation_uid) + .bind(now) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if reconciling.rows_affected() != 1 { + return Err(Error::InvalidRepositoryData { + message: format!("task {} lost its active-capacity receipt", task.task_id), + }); + } + let payload = serde_json::to_value(ExecutionTaskAttemptCancelRequest { + cancellation_dispatch_uid, + tenant_id: run.tenant_id, + run_uid: run.run_uid, + task_id: task.task_id, + controller_generation: run.controller_generation, + attempt_controller_generation: run.controller_generation, + task_generation: task.generation, + attempt_generation: task.attempt_generation, + active_dispatch_uid, + capacity_reservation_uid, + watchdog_trigger_uid, + reason, + })?; + enqueue_dispatch_in_conn( + conn.as_mut(), + &NewExecutionDispatch { + dispatch_uid: cancellation_dispatch_uid, + tenant_id: run.tenant_id, + run_uid: Some(run.run_uid), + task_id: Some(task.task_id.as_uuid()), + compensation_id: None, + trigger_uid: None, + external_job_uid: task.external_job_uid, + kind: ExecutionDispatchKind::TaskAttemptCancel, + controller_generation: Some(run.controller_generation), + wake_epoch: None, + attempt_generation: Some(task.attempt_generation), + compensation_generation: None, + compensation_attempt_generation: None, + not_before_at: now, + payload, + }, + ) + .await +} + +async fn enqueue_pending_terminal_compensation_cancellation( + conn: &mut ScopedConn<'_>, + run: &ExecutionRunRecord, + compensation_row: &PgRow, + reason: ExecutionAttemptCancelReason, + terminal_reason: ExecutionTerminalReason, + now: DateTime, +) -> Result { + let registration = compensation_from_row(compensation_row)?; + let attempt_generation = required_u64(compensation_row, "attempt_generation")?; + let active_dispatch_uid: Uuid = compensation_row + .try_get("active_dispatch_uid") + .map_err(row_error)?; + let receipt = sqlx::query( + "SELECT reservation.reservation_uid, trigger.trigger_uid \ + FROM moa.execution_capacity_reservation AS reservation \ + JOIN moa.execution_trigger AS trigger ON trigger.run_uid=reservation.run_uid \ + AND trigger.compensation_id=reservation.compensation_id \ + AND trigger.controller_generation=reservation.controller_generation \ + AND trigger.compensation_generation=reservation.compensation_generation \ + AND trigger.compensation_attempt_generation=reservation.compensation_attempt_generation \ + AND trigger.trigger_kind='compensation_watchdog' \ + AND trigger.state IN ('pending','dispatching') \ + WHERE reservation.run_uid=$1 AND reservation.compensation_id=$2 \ + AND reservation.controller_generation=$3 AND reservation.compensation_generation=$4 \ + AND reservation.compensation_attempt_generation=$5 \ + AND reservation.resource_dimension='active_tasks' \ + AND reservation.state IN ('reserved','reconciling') FOR UPDATE OF reservation, trigger", + ) + .bind(run.run_uid) + .bind(registration.compensation_id.as_uuid()) + .bind(to_i64(run.controller_generation, "controller generation")?) + .bind(to_i64(registration.generation, "compensation generation")?) + .bind(to_i64( + attempt_generation, + "compensation attempt generation", + )?) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + .ok_or_else(|| Error::InvalidRepositoryData { + message: format!( + "active compensation {} is missing its exact capacity or watchdog receipt", + registration.compensation_id + ), + })?; + let capacity_reservation_uid: Uuid = receipt.try_get("reservation_uid").map_err(row_error)?; + let watchdog_trigger_uid: Uuid = receipt.try_get("trigger_uid").map_err(row_error)?; + let cancellation_dispatch_uid = pending_terminal_cancel_dispatch_uid( + active_dispatch_uid, + run.controller_generation, + terminal_reason, + ); + let intent = compensation_release_intent(reason); + let cancelling = sqlx::query( + "UPDATE moa.execution_compensation SET attempt_state='cancelling', \ + release_intent=$7, last_progress_at=GREATEST(last_progress_at,$6), \ + updated_at=NOW() \ + WHERE run_uid=$1 AND compensation_id=$2 \ + AND generation=$3 AND attempt_generation=$4 AND active_dispatch_uid=$5 \ + AND attempt_state IN ('dispatching','running')", + ) + .bind(run.run_uid) + .bind(registration.compensation_id.as_uuid()) + .bind(to_i64(registration.generation, "compensation generation")?) + .bind(to_i64( + attempt_generation, + "compensation attempt generation", + )?) + .bind(active_dispatch_uid) + .bind(now) + .bind(compensation_release_intent_label(intent)) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if cancelling.rows_affected() != 1 { + return Err(Error::InvalidRepositoryData { + message: format!( + "compensation {} lost its terminal cancellation fence", + registration.compensation_id + ), + }); + } + let reconciling = sqlx::query( + "UPDATE moa.execution_capacity_reservation SET state='reconciling', updated_at=$2 \ + WHERE reservation_uid=$1 AND state IN ('reserved','reconciling')", + ) + .bind(capacity_reservation_uid) + .bind(now) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if reconciling.rows_affected() != 1 { + return Err(Error::InvalidRepositoryData { + message: format!( + "compensation {} lost its active-capacity receipt", + registration.compensation_id + ), + }); + } + let payload = serde_json::to_value(ExecutionCompensationAttemptCancelRequest { + cancellation_dispatch_uid, + tenant_id: run.tenant_id, + run_uid: run.run_uid, + compensation_id: registration.compensation_id, + controller_generation: run.controller_generation, + attempt_controller_generation: run.controller_generation, + compensation_generation: registration.generation, + compensation_attempt_generation: attempt_generation, + active_dispatch_uid, + capacity_reservation_uid, + watchdog_trigger_uid, + intent, + })?; + enqueue_dispatch_in_conn( + conn.as_mut(), + &NewExecutionDispatch { + dispatch_uid: cancellation_dispatch_uid, + tenant_id: run.tenant_id, + run_uid: Some(run.run_uid), + task_id: None, + compensation_id: Some(registration.compensation_id.as_uuid()), + trigger_uid: None, + external_job_uid: None, + kind: ExecutionDispatchKind::CompensationAttemptCancel, + controller_generation: Some(run.controller_generation), + wake_epoch: None, + attempt_generation: None, + compensation_generation: Some(registration.generation), + compensation_attempt_generation: Some(attempt_generation), + not_before_at: now, + payload, + }, + ) + .await +} + +fn pending_terminal_cancel_dispatch_uid( + active_dispatch_uid: Uuid, + controller_generation: u64, + terminal_reason: ExecutionTerminalReason, +) -> Uuid { + let name = format!( + "{active_dispatch_uid}:{controller_generation}:{}", + terminal_reason.as_str() + ); + Uuid::new_v5(&PENDING_TERMINAL_CANCEL_NAMESPACE, name.as_bytes()) +} + +fn compensation_release_intent( + reason: ExecutionAttemptCancelReason, +) -> ExecutionCompensationReleaseIntent { + match reason { + ExecutionAttemptCancelReason::DeadlineExceeded => { + ExecutionCompensationReleaseIntent::Deadline + } + ExecutionAttemptCancelReason::RunTerminal => { + ExecutionCompensationReleaseIntent::RunTerminal + } + ExecutionAttemptCancelReason::PauseRequested => ExecutionCompensationReleaseIntent::Pause, + ExecutionAttemptCancelReason::ExternalJobStarted => { + ExecutionCompensationReleaseIntent::ExternalJob + } } +} - /// Audits one action-review resolution under a compensation generation fence. - pub async fn record_compensation_action_review_resolution( - &self, - scope: ExecutionScope, - run_uid: Uuid, - compensation_id: CompensationId, - generation: u64, - review_uid: Uuid, - resolution: &ExecutionActionReviewResolution, - ) -> Result { - let mut conn = scope.begin(&self.pool).await?; - let Some(row) = sqlx::query(LOAD_COMPENSATION_FOR_UPDATE_SQL) - .bind(run_uid) - .bind(compensation_id.as_uuid()) - .fetch_optional(conn.as_mut()) - .await - .map_err(sqlx_error)? - else { - conn.commit().await.map_err(storage_error)?; - return Ok(ActionReviewResolutionWrite::NotFound); - }; - let compensation = compensation_from_row(&row)?; - let mut persisted = persisted_compensation_outcome(&row, compensation.outcome.clone())?; - if let Some(existing) = persisted - .review_audit - .iter() - .find(|entry| entry.review_uid == review_uid && entry.generation == generation) - { - if existing.resolution != *resolution { - return Err(Error::InvalidRepositoryData { - message: "compensation review UID was replayed with a different resolution" - .to_string(), - }); - } - conn.commit().await.map_err(storage_error)?; - return Ok(ActionReviewResolutionWrite::Replayed); +fn compensation_release_intent_label(intent: ExecutionCompensationReleaseIntent) -> &'static str { + match intent { + ExecutionCompensationReleaseIntent::Outcome => "outcome", + ExecutionCompensationReleaseIntent::Retry => "retry", + ExecutionCompensationReleaseIntent::Review => "review", + ExecutionCompensationReleaseIntent::ExternalJob => "external_job", + ExecutionCompensationReleaseIntent::Pause => "pause", + ExecutionCompensationReleaseIntent::Watchdog => "watchdog", + ExecutionCompensationReleaseIntent::Deadline => "deadline", + ExecutionCompensationReleaseIntent::RunTerminal => "run_terminal", + } +} + +fn validate_compensation_settlement_intent( + intent: ExecutionCompensationReleaseIntent, + outcome: &ExecutionCompensationOutcome, +) -> Result<()> { + let retryable_failure = matches!( + outcome, + ExecutionCompensationOutcome::Failed { + retryable: true, + .. } - let accepted = compensation.status == CompensationStatus::Running - && compensation.generation == generation; - persisted.review_audit.push(CompensationReviewAuditEntry { - review_uid, - generation, - accepted, - resolution: resolution.clone(), - recorded_at: Utc::now(), - }); - sqlx::query( - "UPDATE moa.execution_compensation SET outcome = $3, updated_at = NOW() \ - WHERE run_uid = $1 AND compensation_id = $2", - ) - .bind(run_uid) - .bind(compensation_id.as_uuid()) - .bind(serde_json::to_value(persisted)?) - .execute(conn.as_mut()) - .await - .map_err(sqlx_error)?; - conn.commit().await.map_err(storage_error)?; - Ok(if accepted { - ActionReviewResolutionWrite::Applied - } else { - ActionReviewResolutionWrite::AuditedStale + ); + let valid = match intent { + ExecutionCompensationReleaseIntent::Outcome => !retryable_failure, + ExecutionCompensationReleaseIntent::Retry + | ExecutionCompensationReleaseIntent::Watchdog => retryable_failure, + ExecutionCompensationReleaseIntent::Deadline + | ExecutionCompensationReleaseIntent::RunTerminal => !retryable_failure, + ExecutionCompensationReleaseIntent::Review + | ExecutionCompensationReleaseIntent::ExternalJob + | ExecutionCompensationReleaseIntent::Pause => false, + }; + if valid { + Ok(()) + } else { + Err(Error::InvalidRepositoryInput { + message: format!( + "compensation release intent `{}` does not match its settlement path", + compensation_release_intent_label(intent) + ), }) } +} - /// Finalizes the original terminal intent or a typed compensation-failure outcome. - pub async fn finalize_compensation( - &self, - scope: ExecutionScope, - run_uid: Uuid, - expected_wake_epoch: u64, - ) -> Result { - let mut conn = scope.begin(&self.pool).await?; - let Some(run_row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) - .bind(run_uid) - .fetch_optional(conn.as_mut()) - .await - .map_err(sqlx_error)? - else { - conn.commit().await.map_err(storage_error)?; - return Ok(CompensationFinalizationOutcome::NotFound); - }; - let run = run_from_row(&run_row)?; - if run.status.is_terminal() && run.pending_terminal.is_none() { - let manual = run.manual_repair_required - && run.status == ExecutionRunStatus::Failed - && run.terminal_reason == Some(ExecutionTerminalReason::CompensationFailed); - conn.commit().await.map_err(storage_error)?; - return Ok(if manual { - CompensationFinalizationOutcome::ManualRepairRequired(run) - } else { - CompensationFinalizationOutcome::Replayed(run) - }); +fn compensation_outcome_from_review_resolution( + resolution: &ExecutionActionReviewResolution, +) -> Result { + Ok(match resolution { + ExecutionActionReviewResolution::Completed { tool_output } => { + ExecutionCompensationOutcome::Completed { + output: tool_output.clone(), + usage: zero_usage(), + } } - if run.status != ExecutionRunStatus::Compensating - || run.wake_epoch != expected_wake_epoch - || run.pending_terminal.is_none() - { - conn.commit().await.map_err(storage_error)?; - return Ok(CompensationFinalizationOutcome::Conflict); + ExecutionActionReviewResolution::UnknownOutcome { message } => { + ExecutionCompensationOutcome::UnknownOutcome { + message: message.clone(), + usage: zero_usage(), + } } - let registrations = load_compensations(&mut conn, run_uid).await?; - if registrations - .iter() - .any(|registration| registration.status == CompensationStatus::Running) - { - conn.commit().await.map_err(storage_error)?; - return Ok(CompensationFinalizationOutcome::Conflict); + ExecutionActionReviewResolution::ExternalJob { .. } => { + return Err(Error::InvalidRepositoryInput { + message: "compensation external-job review requires the durable external-job owner handoff" + .to_string(), + }); } - let pending = run - .pending_terminal - .clone() - .ok_or_else(|| Error::InvalidRepositoryData { - message: "compensating run lost pending terminal intent".to_string(), - })?; - if run.manual_repair_required - || registrations.iter().any(|registration| { - matches!( - registration.status, - CompensationStatus::Failed | CompensationStatus::UnknownOutcome - ) - }) - { - let (compensation_id, outcome) = - compensation_failure_evidence(&mut conn, run_uid, ®istrations).await?; - let terminal_evidence = ExecutionTerminalEvidence { - cause: ExecutionTerminalCause::CompensationFailure { - original_status: pending.status, - original_reason: pending.reason, - original_cause: Box::new(pending.terminal_evidence.cause.clone()), - compensation_id, - outcome, - }, - satisfied_requirement_count: pending.terminal_evidence.satisfied_requirement_count, - requirement_count: pending.terminal_evidence.requirement_count, - }; - let row = finalize_compensation_run( - &mut conn, - run_uid, - CompensationTerminalWrite { - status: ExecutionRunStatus::Failed, - reason: ExecutionTerminalReason::CompensationFailed, - evidence: &terminal_evidence, - completion_check_results: &pending.completion_check_results, - terminal_gaps: &pending.terminal_gaps, - output: pending.output, - manual_repair_required: true, - }, - ) - .await?; - let run = run_from_row(&row)?; - conn.commit().await.map_err(storage_error)?; - return Ok(CompensationFinalizationOutcome::ManualRepairRequired(run)); + ExecutionActionReviewResolution::Failed { message, .. } => { + ExecutionCompensationOutcome::Failed { + message: message.clone(), + retryable: false, + usage: zero_usage(), + } } - if registrations - .iter() - .any(|registration| registration.status != CompensationStatus::Completed) - { - conn.commit().await.map_err(storage_error)?; - return Ok(CompensationFinalizationOutcome::Conflict); + ExecutionActionReviewResolution::NotDispatched { reason } => { + ExecutionCompensationOutcome::Failed { + message: format!("compensation was not dispatched: {reason:?}"), + retryable: true, + usage: zero_usage(), + } } - let row = finalize_compensation_run( - &mut conn, - run_uid, - CompensationTerminalWrite { - status: pending.status, - reason: pending.reason, - evidence: &pending.terminal_evidence, - completion_check_results: &pending.completion_check_results, - terminal_gaps: &pending.terminal_gaps, - output: pending.output, - manual_repair_required: false, - }, - ) - .await?; - let run = run_from_row(&row)?; - conn.commit().await.map_err(storage_error)?; - Ok(CompensationFinalizationOutcome::Finalized(run)) - } + ExecutionActionReviewResolution::Denied { reason } + | ExecutionActionReviewResolution::TimedOut { reason } => { + ExecutionCompensationOutcome::Failed { + message: reason.clone(), + retryable: false, + usage: zero_usage(), + } + } + }) } -async fn load_nonterminal_tasks( +async fn supersede_storage_task_waits( conn: &mut ScopedConn<'_>, - run_uid: Uuid, -) -> Result> { - let rows = sqlx::query(LIST_ALL_TASKS_SQL) - .bind(run_uid) - .fetch_all(conn.as_mut()) + task: &ExecutionTaskRecord, +) -> Result<()> { + let trigger_uids = sqlx::query_scalar::<_, Uuid>( + "UPDATE moa.execution_trigger SET state='superseded', claimed_at=NULL, \ + claimed_by=NULL, updated_at=NOW() WHERE run_uid=$1 AND task_id=$2 \ + AND trigger_kind <> 'task_watchdog' AND state IN ('pending','dispatching') \ + RETURNING trigger_uid", + ) + .bind(task.run_uid) + .bind(task.task_id.as_uuid()) + .fetch_all(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if !trigger_uids.is_empty() { + sqlx::query( + "UPDATE moa.execution_dispatch_outbox SET state='superseded', claimed_at=NULL, \ + claimed_by=NULL, updated_at=NOW() WHERE trigger_uid=ANY($1::UUID[]) \ + AND state IN ('pending','dispatching')", + ) + .bind(&trigger_uids) + .execute(conn.as_mut()) .await .map_err(sqlx_error)?; - Ok(rows - .iter() - .map(task_from_row) - .collect::>>()? - .into_iter() - .filter(|task| !task.status.is_terminal()) - .collect()) + } + Ok(()) +} + +async fn active_attempt_capacity_count(conn: &mut ScopedConn<'_>, run_uid: Uuid) -> Result { + let count: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM moa.execution_capacity_reservation WHERE run_uid=$1 \ + AND resource_dimension='active_tasks' AND state IN ('reserved','reconciling')", + ) + .bind(run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + u64::try_from(count).map_err(|_| Error::InvalidRepositoryData { + message: "active-attempt capacity count is negative".to_string(), + }) +} + +async fn reconcile_run_after_compensation_capacity_release( + conn: &mut ScopedConn<'_>, + run: &ExecutionRunRecord, + now: DateTime, +) -> Result { + let active_count = active_attempt_capacity_count(conn, run.run_uid).await?; + let row = sqlx::query( + "UPDATE moa.execution_run SET active_task_count=$3, \ + status=CASE WHEN status IN ('pause_requested','pausing') AND $3=0 \ + THEN 'paused' ELSE status END, \ + activation_state=CASE WHEN status IN ('pause_requested','pausing') AND $3=0 \ + THEN 'paused' ELSE activation_state END, \ + paused_at=CASE WHEN status IN ('pause_requested','pausing') AND $3=0 \ + THEN COALESCE(paused_at,$4) ELSE paused_at END, \ + last_progress_at=GREATEST(last_progress_at,$4),updated_at=NOW() \ + WHERE run_uid=$1 AND controller_generation=$2 RETURNING *", + ) + .bind(run.run_uid) + .bind(to_i64(run.controller_generation, "controller generation")?) + .bind(to_i64(active_count, "active task count")?) + .bind(now) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + .ok_or_else(|| Error::InvalidRepositoryData { + message: "compensation release lost its current run generation".to_string(), + })?; + run_from_row(&row) } -async fn load_compensations( +#[allow(clippy::too_many_arguments)] +async fn checkpoint_pending_terminal_wake( conn: &mut ScopedConn<'_>, run_uid: Uuid, -) -> Result> { - sqlx::query(LIST_COMPENSATIONS_SQL) - .bind(run_uid) - .fetch_all(conn.as_mut()) - .await - .map_err(sqlx_error)? - .iter() - .map(compensation_from_row) - .collect() + controller_generation: u64, + expected_wake_epoch: u64, + status: ExecutionRunStatus, + ready_task_count: u64, + active_task_count: u64, + now: DateTime, +) -> Result { + let row = sqlx::query( + "UPDATE moa.execution_run SET status=$4, activation_state='idle', \ + next_wake_at=NULL, waiting_since=NULL, ready_task_count=$5, \ + active_task_count=$6, processed_wake_epoch=$3, \ + last_progress_at=GREATEST(last_progress_at,$7), updated_at=NOW() \ + WHERE run_uid=$1 AND controller_generation=$2 AND wake_epoch >= $3 \ + AND processed_wake_epoch < $3 \ + AND activation_state IN ('queued','advancing','paused') RETURNING *", + ) + .bind(run_uid) + .bind(to_i64(controller_generation, "controller generation")?) + .bind(to_i64(expected_wake_epoch, "expected wake epoch")?) + .bind(status.as_str()) + .bind(to_i64(ready_task_count, "ready task count")?) + .bind(to_i64(active_task_count, "active task count")?) + .bind(now) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + .ok_or_else(|| Error::InvalidRepositoryData { + message: "terminal drain lost its controller wake checkpoint fence".to_string(), + })?; + run_from_row(&row) +} + +fn compensation_failure_pending( + original: &PendingExecutionTerminal, + registration: &CompensationRegistrationProjection, +) -> Result { + let outcome = registration + .outcome + .clone() + .ok_or_else(|| Error::InvalidRepositoryData { + message: "failed compensation is missing its terminal outcome".to_string(), + })?; + let pending = PendingExecutionTerminal { + status: ExecutionRunStatus::Failed, + reason: ExecutionTerminalReason::CompensationFailed, + terminal_evidence: ExecutionTerminalEvidence { + cause: ExecutionTerminalCause::CompensationFailure { + original_status: original.status, + original_reason: original.reason, + original_cause: Box::new(original.terminal_evidence.cause.clone()), + compensation_id: registration.compensation_id, + outcome, + }, + satisfied_requirement_count: original.terminal_evidence.satisfied_requirement_count, + requirement_count: original.terminal_evidence.requirement_count, + }, + completion_check_results: original.completion_check_results.clone(), + terminal_gaps: original.terminal_gaps.clone(), + output: original.output.clone(), + cancellation_reason: None, + }; + pending.validate()?; + Ok(pending) +} + +async fn replace_pending_terminal_exact( + conn: &mut ScopedConn<'_>, + run: &ExecutionRunRecord, + expected: &PendingExecutionTerminal, + replacement: &PendingExecutionTerminal, + controller_generation: u64, + expected_wake_epoch: u64, + now: DateTime, +) -> Result<()> { + let expected_payload = serde_json::to_value(PendingTerminalEvidencePayload { + terminal_evidence: expected.terminal_evidence.clone(), + completion_check_results: expected.completion_check_results.clone(), + terminal_gaps: expected.terminal_gaps.clone(), + })?; + let replacement_payload = serde_json::to_value(PendingTerminalEvidencePayload { + terminal_evidence: replacement.terminal_evidence.clone(), + completion_check_results: replacement.completion_check_results.clone(), + terminal_gaps: replacement.terminal_gaps.clone(), + })?; + let updated = sqlx::query( + "UPDATE moa.execution_run SET pending_terminal_status=$5, \ + pending_terminal_reason=$6, pending_terminal_cause=$7, pending_terminal_output=$8, \ + cancellation_reason=NULL, manual_repair_required=TRUE, updated_at=$9 \ + WHERE run_uid=$1 AND controller_generation=$2 AND wake_epoch >= $3 \ + AND pending_terminal_cause=$4 AND status='compensating'", + ) + .bind(run.run_uid) + .bind(to_i64(controller_generation, "controller generation")?) + .bind(to_i64(expected_wake_epoch, "expected wake epoch")?) + .bind(expected_payload) + .bind(replacement.status.as_str()) + .bind(replacement.reason.as_str()) + .bind(replacement_payload) + .bind(&replacement.output) + .bind(now) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if updated.rows_affected() != 1 { + return Err(Error::InvalidRepositoryData { + message: "compensation failure lost its exact pending-terminal replacement fence" + .to_string(), + }); + } + Ok(()) +} + +async fn finalize_pending_terminal_exact( + conn: &mut ScopedConn<'_>, + run: &ExecutionRunRecord, + pending: &PendingExecutionTerminal, + controller_generation: u64, + expected_wake_epoch: u64, + now: DateTime, +) -> Result { + release_owned_run_capacity_in_tx( + conn.as_mut(), + run.tenant_id, + run.run_uid, + run.controller_generation, + ) + .await?; + let evidence_payload = serde_json::to_value(PendingTerminalEvidencePayload { + terminal_evidence: pending.terminal_evidence.clone(), + completion_check_results: pending.completion_check_results.clone(), + terminal_gaps: pending.terminal_gaps.clone(), + })?; + let row = sqlx::query( + "UPDATE moa.execution_run SET status=$4, terminal_reason=$5, terminal_cause=$6, \ + terminal_satisfied_requirement_count=$7, terminal_requirement_count=$8, \ + completion_check_results=$9, terminal_gaps=$10, output=$11, \ + pending_terminal_status=NULL, pending_terminal_reason=NULL, \ + pending_terminal_cause=NULL, pending_terminal_output=NULL, \ + reserved_cost_microusd=0, reserved_tokens=0, reserved_tasks=0, \ + reserved_tool_calls=0, reserved_retrieved_bytes=0, \ + activation_state='terminal', waiting_reasons='[]'::JSONB, next_wake_at=NULL, \ + waiting_task_count=0, waiting_input_task_count=0, waiting_review_task_count=0, \ + waiting_signal_task_count=0, waiting_timer_task_count=0, \ + waiting_external_task_count=0, waiting_replan_task_count=0, \ + waiting_input_user_task_count=0, waiting_input_tenant_admin_task_count=0, \ + waiting_input_external_task_count=0, waiting_reasons_truncated=FALSE, \ + waiting_since=NULL, ready_task_count=0, active_task_count=0, \ + processed_wake_epoch=$3, completed_at=$12, \ + last_progress_at=GREATEST(last_progress_at,$12), updated_at=NOW() \ + WHERE run_uid=$1 AND controller_generation=$2 AND wake_epoch >= $3 \ + AND processed_wake_epoch < $3 AND pending_terminal_cause=$13 \ + AND NOT EXISTS (SELECT 1 FROM moa.execution_task WHERE run_uid=$1 \ + AND status NOT IN ('completed','skipped','failed','cancelled','unknown_outcome')) \ + AND NOT EXISTS (SELECT 1 FROM moa.execution_capacity_reservation WHERE run_uid=$1 \ + AND resource_dimension IN ('active_tasks','scheduled_triggers','external_jobs') \ + AND state IN ('reserved','reconciling')) \ + RETURNING *", + ) + .bind(run.run_uid) + .bind(to_i64(controller_generation, "controller generation")?) + .bind(to_i64(expected_wake_epoch, "expected wake epoch")?) + .bind(pending.status.as_str()) + .bind(pending.reason.as_str()) + .bind(serde_json::to_value(&pending.terminal_evidence.cause)?) + .bind(to_i64( + pending.terminal_evidence.satisfied_requirement_count, + "terminal satisfied requirement count", + )?) + .bind(to_i64( + pending.terminal_evidence.requirement_count, + "terminal requirement count", + )?) + .bind(serde_json::to_value(&pending.completion_check_results)?) + .bind(serde_json::to_value(&pending.terminal_gaps)?) + .bind(&pending.output) + .bind(now) + .bind(evidence_payload) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + .ok_or_else(|| Error::InvalidRepositoryData { + message: "terminal drain lost its final exact fence".to_string(), + })?; + run_from_row(&row) +} + +async fn nonterminal_task_exists(conn: &mut ScopedConn<'_>, run_uid: Uuid) -> Result { + sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM moa.execution_task WHERE run_uid=$1 \ + AND status NOT IN ('completed','skipped','failed','cancelled','unknown_outcome'))", + ) + .bind(run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error) } async fn load_forward_task( @@ -1060,9 +5648,12 @@ async fn terminalize_compensation_budget_rejection( .map_err(sqlx_error)?; let persisted = persisted_compensation_outcome(&row, Some(outcome))?; let row = sqlx::query( - "UPDATE moa.execution_compensation SET status='failed', outcome=$3, \ + "UPDATE moa.execution_compensation SET status='failed', attempt_state='terminal', outcome=$3, \ error=jsonb_build_object('class','budget_exceeded','message','approved execution budget cannot reserve compensation'), \ - started_at=COALESCE(started_at,NOW()), completed_at=NOW(), updated_at=NOW() WHERE run_uid=$1 AND compensation_id=$2 \ + started_at=COALESCE(started_at,NOW()), attempt_started_at=COALESCE(attempt_started_at,NOW()), \ + last_progress_at=GREATEST(last_progress_at,NOW()), attempt_deadline_at=NULL, \ + waiting_since=NULL, active_dispatch_uid=NULL, \ + completed_at=NOW(), updated_at=NOW() WHERE run_uid=$1 AND compensation_id=$2 \ RETURNING compensation_id, run_uid, forward_task_id, registered_sequence, \ forward_generation, compensator, mapped_input, status, attempt, generation, \ outcome, error, created_at, updated_at, started_at, completed_at", @@ -1083,111 +5674,6 @@ async fn terminalize_compensation_budget_rejection( compensation_from_row(&row) } -async fn compensation_failure_evidence( - conn: &mut ScopedConn<'_>, - run_uid: Uuid, - registrations: &[CompensationRegistrationProjection], -) -> Result<(CompensationId, ExecutionCompensationOutcome)> { - if let Some(registration) = registrations.iter().find(|registration| { - matches!( - registration.status, - CompensationStatus::Failed | CompensationStatus::UnknownOutcome - ) - }) && let Some(outcome) = registration.outcome.clone() - { - return Ok((registration.compensation_id, outcome)); - } - let tasks = load_nonterminal_or_terminal_tasks(conn, run_uid).await?; - let task = tasks - .into_iter() - .find(|task| { - matches!( - task.current_outcome.as_ref().map(|outcome| &outcome.result), - Some(ExecutionTaskResult::UnknownOutcome { .. }) - ) - }) - .ok_or_else(|| Error::InvalidRepositoryData { - message: "manual repair fence has no failed compensation or ambiguous forward task" - .to_string(), - })?; - let current = task - .current_outcome - .as_ref() - .ok_or_else(|| Error::InvalidRepositoryData { - message: "ambiguous forward task lost its outcome".to_string(), - })?; - let ExecutionTaskResult::UnknownOutcome { message } = ¤t.result else { - return Err(Error::InvalidRepositoryData { - message: "manual repair task is not an unknown outcome".to_string(), - }); - }; - Ok(( - CompensationId::derive(task.task_id), - ExecutionCompensationOutcome::UnknownOutcome { - message: message.clone(), - usage: current.usage.clone(), - }, - )) -} - -async fn load_nonterminal_or_terminal_tasks( - conn: &mut ScopedConn<'_>, - run_uid: Uuid, -) -> Result> { - sqlx::query(LIST_ALL_TASKS_SQL) - .bind(run_uid) - .fetch_all(conn.as_mut()) - .await - .map_err(sqlx_error)? - .iter() - .map(task_from_row) - .collect() -} - -struct CompensationTerminalWrite<'a> { - status: ExecutionRunStatus, - reason: ExecutionTerminalReason, - evidence: &'a ExecutionTerminalEvidence, - completion_check_results: &'a [Value], - terminal_gaps: &'a [String], - output: Option, - manual_repair_required: bool, -} - -async fn finalize_compensation_run( - conn: &mut ScopedConn<'_>, - run_uid: Uuid, - write: CompensationTerminalWrite<'_>, -) -> Result { - sqlx::query( - "UPDATE moa.execution_run SET status=$2, terminal_reason=$3, terminal_cause=$4, \ - terminal_satisfied_requirement_count=$5, terminal_requirement_count=$6, \ - completion_check_results=$7, terminal_gaps=$8, output=$9, \ - pending_terminal_status=NULL, pending_terminal_reason=NULL, pending_terminal_cause=NULL, \ - pending_terminal_output=NULL, manual_repair_required=$10, waiting_reasons='[]'::JSONB, \ - wake_epoch=wake_epoch+1, completed_at=NOW(), updated_at=NOW() WHERE run_uid=$1 RETURNING *", - ) - .bind(run_uid) - .bind(write.status.as_str()) - .bind(write.reason.as_str()) - .bind(serde_json::to_value(&write.evidence.cause)?) - .bind(to_i64( - write.evidence.satisfied_requirement_count, - "terminal satisfied requirements", - )?) - .bind(to_i64( - write.evidence.requirement_count, - "terminal requirements", - )?) - .bind(serde_json::to_value(write.completion_check_results)?) - .bind(serde_json::to_value(write.terminal_gaps)?) - .bind(write.output) - .bind(write.manual_repair_required) - .fetch_one(conn.as_mut()) - .await - .map_err(sqlx_error) -} - fn zero_usage() -> ExecutionUsage { ExecutionUsage { cost_microusd: 0, diff --git a/crates/moa-execution/src/repository/completion.rs b/crates/moa-execution/src/repository/completion.rs new file mode 100644 index 000000000..1cc9c1429 --- /dev/null +++ b/crates/moa-execution/src/repository/completion.rs @@ -0,0 +1,1571 @@ +//! Bounded persisted completion scanning and exact verifier materialization. + +use std::collections::{BTreeMap, BTreeSet}; + +use moa_artifacts::execution_plan::{CompletionCheckKind, ExecutionFailureClass, RetryPolicy}; +use moa_config::ExecutionConfig; +use serde::{Deserialize, Serialize}; + +use super::*; +use super::{ + materialize::prepare_task_materialization_batch, + rows::*, + run::enqueue_run_activation_in_conn, + sql::*, + terminal::{ReplanStopReceipt, RunFinalizationRequest}, +}; +use crate::{ + completion::{ + CompletionCheckResult, CompletionEvaluation, CompletionStatus, execution_terminal_reason, + terminal_evidence_from_evaluation, + }, + interpreter::verifier_turn_reservation, + replan::{replan_stop_gaps, replan_stop_status}, + repository::replan_stop::ExecutionReplanStopIntentRecord, + schema::validate_instance, + state::{ExecutionLimitStop, ExecutionTaskFailure}, +}; + +const MAX_COMPLETION_PAGE_SIZE: u32 = 1_000; +const MAX_EVIDENCE_SAMPLES_PER_CHECK: usize = 20; + +/// One generation- and wake-fenced bounded completion advance. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct CompletionAdvanceRequest { + /// Run whose ordinary terminal projection is being evaluated. + pub run_uid: Uuid, + /// Exact run-controller generation owning the scan. + pub controller_generation: u64, + /// Exact claimed wake that may produce terminal intent. + pub wake_epoch: u64, + /// Maximum forward task rows inspected by this call. + pub page_size: u32, + /// Deterministic evaluation time supplied by the controller. + pub now: DateTime, +} + +/// Result of one bounded persisted completion advance. +#[derive(Clone, Debug, PartialEq)] +pub enum CompletionAdvanceOutcome { + /// More bounded evidence or verifier settlement is required. + Continue { + /// Forward task rows durably scanned by this call. + scanned_tasks: u32, + /// Plan-node aggregate rows durably scanned by this call. + scanned_nodes: u32, + }, + /// A ReplanStop page committed together with the only valid next controller wake. + ReplanStopContinue { + /// Forward task rows durably scanned by this call. + scanned_tasks: u32, + /// Plan-node aggregate rows durably scanned by this call. + scanned_nodes: u32, + /// Exact new run-activation dispatch bound to the persisted intent. + continuation: Box, + }, + /// Exact verifier tasks were inserted into the ordinary ready queue. + VerifiersMaterialized { + /// Persisted verifier tasks in declared check order. + tasks: Vec, + }, + /// Verifier tasks exist but have not settled; their task wake will reactivate the run. + WaitingForVerifiers, + /// Every ordinary completion gate passed and can be finalized atomically. + FinalizationReady(Box), + /// The same scan already observed a non-success terminal intent. + NonSuccessTerminal { + /// Fully prepared deterministic terminal intent for the compensation boundary. + pending_terminal: PendingExecutionTerminal, + }, + /// Bounded ReplanStop evaluation is ready for its exact terminal fence and receipt. + ReplanStopReady { + /// Fully prepared deterministic terminal intent. + pending_terminal: PendingExecutionTerminal, + /// Exact task/revision/amendment receipt owned by the persisted intent. + receipt: ReplanStopReceipt, + }, + /// Run, generation, wake, or ordinary terminal boundary is no longer current. + NotReady, +} + +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +#[serde(default, deny_unknown_fields)] +struct CompletionTaskEvidence { + authorization_denied: bool, + unsupported_by_requirement: BTreeMap, + citation_failures: BTreeMap, +} + +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct UnsupportedRequirementEvidence { + task_count: u64, + unsupported_task_count: u64, +} + +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct CitationFailureEvidence { + failure_count: u64, + samples: Vec, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct CitationFailureSample { + node_id: String, + item_key: String, + count: u64, +} + +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +#[serde(default, deny_unknown_fields)] +struct CompletionNodeEvidence { + terminal_output: Option, + requirements: BTreeMap, + required_checks: BTreeMap, + coverage_passed: BTreeMap, +} + +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct RequirementNodeEvidence { + eligible_node_count: u64, + completed_node_count: u64, +} + +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct RequiredNodeCheckEvidence { + observed_node_count: u64, + failed_node_ids: Vec, +} + +impl ExecutionRepository { + /// Advances at most one bounded page of terminal evidence and materializes verifiers exactly once. + pub async fn advance_completion_projection( + &self, + scope: ExecutionScope, + config: &ExecutionConfig, + request: CompletionAdvanceRequest, + ) -> Result { + self.advance_completion_projection_inner(scope, config, request, None) + .await + } + + /// Advances one bounded ReplanStop completion page from its exact persisted controller intent. + pub async fn advance_replan_stop_completion_projection( + &self, + scope: ExecutionScope, + config: &ExecutionConfig, + request: CompletionAdvanceRequest, + intent: &ExecutionReplanStopIntentRecord, + ) -> Result { + self.advance_completion_projection_inner(scope, config, request, Some(intent)) + .await + } + + async fn advance_completion_projection_inner( + &self, + scope: ExecutionScope, + config: &ExecutionConfig, + request: CompletionAdvanceRequest, + replan_stop: Option<&ExecutionReplanStopIntentRecord>, + ) -> Result { + let page_size = request.page_size.clamp(1, MAX_COMPLETION_PAGE_SIZE); + let mut scanned_tasks = 0_u32; + let mut scanned_nodes = 0_u32; + let generation = to_i64( + request.controller_generation, + "completion controller generation", + )?; + let mut conn = scope.begin(&self.pool).await?; + let Some(run_row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) + .bind(request.run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + conn.commit().await.map_err(storage_error)?; + return Ok(CompletionAdvanceOutcome::NotReady); + }; + let run = run_from_row(&run_row)?; + if let Some(intent) = replan_stop + && (intent.tenant_id != run.tenant_id + || intent.run_uid != run.run_uid + || intent.controller_generation != request.controller_generation + || intent.wake_epoch != request.wake_epoch + || intent.base_plan_revision != run.plan_revision) + { + conn.commit().await.map_err(storage_error)?; + return Ok(CompletionAdvanceOutcome::NotReady); + } + if let Some(intent) = replan_stop { + let persisted_intent = sqlx::query_scalar::<_, i32>( + "SELECT 1 FROM moa.execution_replan_stop_intent \ + WHERE tenant_id=$1 AND run_uid=$2 AND controller_generation=$3 \ + AND wake_epoch=$4 AND origin_task_id=$5 AND task_generation=$6 \ + AND base_plan_revision=$7 AND stop_reason=$8 AND detail=$9 \ + AND amendment_hash=$10 FOR UPDATE", + ) + .bind(run.tenant_id.0) + .bind(run.run_uid) + .bind(to_i64( + request.controller_generation, + "replan-stop controller generation", + )?) + .bind(to_i64(request.wake_epoch, "replan-stop wake epoch")?) + .bind(intent.origin_task_id.as_uuid()) + .bind(to_i64( + intent.task_generation, + "replan-stop task generation", + )?) + .bind(to_i64( + intent.base_plan_revision, + "replan-stop plan revision", + )?) + .bind(intent.stop_reason.as_str()) + .bind(&intent.detail) + .bind(intent.amendment_hash.to_string()) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if persisted_intent.is_none() { + conn.commit().await.map_err(storage_error)?; + return Ok(CompletionAdvanceOutcome::NotReady); + } + } + validate_completion_runtime_bounds(&run, config)?; + if run.controller_generation != request.controller_generation + || run.wake_epoch != request.wake_epoch + || run.status.is_terminal() + || run.pending_terminal.is_some() + { + conn.commit().await.map_err(storage_error)?; + return Ok(CompletionAdvanceOutcome::NotReady); + } + let excluded_task_id = replan_stop.map(|intent| intent.origin_task_id.as_uuid()); + let excluded_node_id = if let Some(intent) = replan_stop { + let row = sqlx::query( + "SELECT node_id,generation,status FROM moa.execution_task \ + WHERE run_uid=$1 AND task_id=$2", + ) + .bind(run.run_uid) + .bind(intent.origin_task_id.as_uuid()) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(row) = row else { + conn.commit().await.map_err(storage_error)?; + return Ok(CompletionAdvanceOutcome::NotReady); + }; + if required_u64(&row, "generation")? != intent.task_generation + || row.try_get::("status").map_err(row_error)? != "waiting_replan" + { + conn.commit().await.map_err(storage_error)?; + return Ok(CompletionAdvanceOutcome::NotReady); + } + Some(row.try_get::("node_id").map_err(row_error)?) + } else { + None + }; + let unfinished = sqlx::query_scalar::<_, bool>( + "SELECT EXISTS (SELECT 1 FROM moa.execution_node_state \ + WHERE run_uid=$1 AND node_id NOT LIKE '@check/%' \ + AND ($2::TEXT IS NULL OR node_id<>$2) \ + AND node_status NOT IN ('completed','skipped','failed','cancelled')) \ + OR EXISTS (SELECT 1 FROM moa.execution_task \ + WHERE run_uid=$1 AND node_id NOT LIKE '@check/%' \ + AND ($3::UUID IS NULL OR task_id<>$3) \ + AND status NOT IN ('completed','skipped','failed','cancelled','unknown_outcome'))", + ) + .bind(request.run_uid) + .bind(&excluded_node_id) + .bind(excluded_task_id) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if unfinished { + conn.commit().await.map_err(storage_error)?; + return Ok(CompletionAdvanceOutcome::NotReady); + } + + let scan_kind = if replan_stop.is_some() { + "replan_stop" + } else { + "ordinary" + }; + sqlx::query( + "DELETE FROM moa.execution_completion_scan WHERE run_uid=$1 \ + AND (scan_kind<>$2 OR excluded_task_id IS DISTINCT FROM $3)", + ) + .bind(run.run_uid) + .bind(scan_kind) + .bind(excluded_task_id) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + sqlx::query( + "INSERT INTO moa.execution_completion_scan (tenant_id, run_uid, plan_revision, \ + controller_generation,scan_kind,excluded_task_id,source_progress_at) \ + VALUES ($1,$2,$3,$4,$5,$6,$7) \ + ON CONFLICT (tenant_id,run_uid) DO UPDATE SET plan_revision=EXCLUDED.plan_revision, \ + controller_generation=EXCLUDED.controller_generation, task_cursor=NULL, \ + scanned_task_count=0, task_evidence='{}'::JSONB, scan_complete=FALSE, \ + node_cursor=NULL, completion_evidence='{}'::JSONB, \ + node_scan_complete=FALSE, verifiers_materialized=FALSE, \ + scan_kind=EXCLUDED.scan_kind,excluded_task_id=EXCLUDED.excluded_task_id, \ + source_progress_at=EXCLUDED.source_progress_at,updated_at=NOW() \ + WHERE execution_completion_scan.plan_revision <> EXCLUDED.plan_revision \ + OR execution_completion_scan.controller_generation \ + <> EXCLUDED.controller_generation \ + OR execution_completion_scan.source_progress_at \ + <> EXCLUDED.source_progress_at", + ) + .bind(run.tenant_id.0) + .bind(run.run_uid) + .bind(to_i64(run.plan_revision, "completion plan revision")?) + .bind(generation) + .bind(scan_kind) + .bind(excluded_task_id) + .bind(run.last_progress_at) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let scan = sqlx::query( + "SELECT plan_revision, controller_generation,scan_kind,excluded_task_id, \ + source_progress_at,task_cursor, scanned_task_count, \ + task_evidence, scan_complete, node_cursor, completion_evidence, \ + node_scan_complete, verifiers_materialized \ + FROM moa.execution_completion_scan WHERE run_uid = $1 FOR UPDATE", + ) + .bind(run.run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if required_u64(&scan, "plan_revision")? != run.plan_revision + || required_u64(&scan, "controller_generation")? != request.controller_generation + || scan.try_get::("scan_kind").map_err(row_error)? != scan_kind + || scan + .try_get::, _>("excluded_task_id") + .map_err(row_error)? + != excluded_task_id + || scan + .try_get::, _>("source_progress_at") + .map_err(row_error)? + != run.last_progress_at + { + conn.commit().await.map_err(storage_error)?; + return Ok(CompletionAdvanceOutcome::NotReady); + } + let mut evidence: CompletionTaskEvidence = serde_json::from_value( + scan.try_get::("task_evidence") + .map_err(row_error)?, + )?; + let scan_complete: bool = scan.try_get("scan_complete").map_err(row_error)?; + let mut node_evidence: CompletionNodeEvidence = serde_json::from_value( + scan.try_get::("completion_evidence") + .map_err(row_error)?, + )?; + let node_scan_complete: bool = scan.try_get("node_scan_complete").map_err(row_error)?; + let verifiers_materialized: bool = + scan.try_get("verifiers_materialized").map_err(row_error)?; + if !scan_complete { + let cursor: Option = scan.try_get("task_cursor").map_err(row_error)?; + let rows = sqlx::query( + "SELECT * FROM moa.execution_task WHERE run_uid = $1 \ + AND node_id NOT LIKE '@check/%' AND ($2::UUID IS NULL OR task_id > $2) \ + ORDER BY task_id LIMIT $3", + ) + .bind(run.run_uid) + .bind(cursor) + .bind(i64::from(page_size) + 1) + .fetch_all(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let page_size_usize = + usize::try_from(page_size).map_err(|_| Error::ArithmeticOverflow { + context: "completion page size".to_string(), + })?; + let has_more = rows.len() > page_size_usize; + let page = rows.into_iter().take(page_size_usize).collect::>(); + let tasks = page.iter().map(task_from_row).collect::>>()?; + scanned_tasks = u32::try_from(tasks.len()).map_err(|_| Error::ArithmeticOverflow { + context: "completion page task count".to_string(), + })?; + for task in &tasks { + accumulate_task_evidence(&run, task, &mut evidence)?; + } + let next_cursor = tasks.last().map(|task| task.task_id.as_uuid()).or(cursor); + let scanned_delta = + u64::try_from(tasks.len()).map_err(|_| Error::ArithmeticOverflow { + context: "completion scanned task count".to_string(), + })?; + let scanned = required_u64(&scan, "scanned_task_count")? + .checked_add(scanned_delta) + .ok_or_else(|| Error::ArithmeticOverflow { + context: "completion scanned task count".to_string(), + })?; + sqlx::query( + "UPDATE moa.execution_completion_scan SET task_cursor=$2, scanned_task_count=$3, \ + task_evidence=$4, scan_complete=$5, updated_at=NOW() \ + WHERE run_uid=$1 AND plan_revision=$6 AND controller_generation=$7", + ) + .bind(run.run_uid) + .bind(next_cursor) + .bind(to_i64(scanned, "completion scanned task count")?) + .bind(serde_json::to_value(&evidence)?) + .bind(!has_more) + .bind(to_i64(run.plan_revision, "completion plan revision")?) + .bind(generation) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if has_more || !tasks.is_empty() { + return commit_completion_page( + conn, + &run, + &request, + replan_stop, + scanned_tasks, + scanned_nodes, + ) + .await; + } + } + + if !node_scan_complete { + let cursor: Option = scan.try_get("node_cursor").map_err(row_error)?; + let rows = sqlx::query( + "SELECT node_id,node_order,node_status,total_task_count,succeeded_task_count, \ + failed_task_count,cancelled_task_count,aggregate_output \ + FROM moa.execution_node_state WHERE run_uid=$1 \ + AND node_id NOT LIKE '@check/%' AND ($2::BIGINT IS NULL OR node_order > $2) \ + ORDER BY node_order,node_state_uid LIMIT $3", + ) + .bind(run.run_uid) + .bind(cursor) + .bind(i64::from(page_size) + 1) + .fetch_all(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let page_size_usize = + usize::try_from(page_size).map_err(|_| Error::ArithmeticOverflow { + context: "completion node page size".to_string(), + })?; + let has_more = rows.len() > page_size_usize; + let page = rows.into_iter().take(page_size_usize).collect::>(); + scanned_nodes = u32::try_from(page.len()).map_err(|_| Error::ArithmeticOverflow { + context: "completion page node count".to_string(), + })?; + for row in &page { + accumulate_node_evidence( + &run, + row, + &mut node_evidence, + excluded_node_id.as_deref(), + )?; + } + let next_cursor = page + .last() + .map(|row| row.try_get::("node_order").map_err(row_error)) + .transpose()? + .or(cursor); + sqlx::query( + "UPDATE moa.execution_completion_scan SET node_cursor=$2, \ + completion_evidence=$3,node_scan_complete=$4,updated_at=NOW() \ + WHERE run_uid=$1 AND plan_revision=$5 AND controller_generation=$6", + ) + .bind(run.run_uid) + .bind(next_cursor) + .bind(serde_json::to_value(&node_evidence)?) + .bind(!has_more) + .bind(to_i64(run.plan_revision, "completion plan revision")?) + .bind(generation) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if has_more || !page.is_empty() { + return commit_completion_page( + conn, + &run, + &request, + replan_stop, + scanned_tasks, + scanned_nodes, + ) + .await; + } + } + + let verifier_checks = run + .goal + .completion_checks + .iter() + .filter(|check| matches!(check.kind, CompletionCheckKind::AgentVerifier { .. })) + .count(); + if replan_stop.is_none() && verifier_checks > 0 && !verifiers_materialized { + let (tasks, all_materialized) = materialize_verifiers_in_tx( + conn.as_mut(), + config, + &run, + &evidence, + &node_evidence, + page_size, + ) + .await?; + sqlx::query( + "UPDATE moa.execution_completion_scan SET verifiers_materialized=$4, \ + updated_at=NOW() WHERE run_uid=$1 AND plan_revision=$2 \ + AND controller_generation=$3", + ) + .bind(run.run_uid) + .bind(to_i64(run.plan_revision, "completion plan revision")?) + .bind(generation) + .bind(all_materialized) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + conn.commit().await.map_err(storage_error)?; + return Ok(CompletionAdvanceOutcome::VerifiersMaterialized { tasks }); + } + + if replan_stop.is_some() && !verifiers_materialized { + sqlx::query( + "UPDATE moa.execution_completion_scan SET verifiers_materialized=TRUE, \ + updated_at=NOW() WHERE run_uid=$1 AND plan_revision=$2 \ + AND controller_generation=$3 AND scan_kind='replan_stop'", + ) + .bind(run.run_uid) + .bind(to_i64(run.plan_revision, "completion plan revision")?) + .bind(generation) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + } + let verifier_tasks = if replan_stop.is_some() { + load_existing_verifier_tasks(conn.as_mut(), config, &run).await? + } else { + load_verifier_tasks(conn.as_mut(), &run).await? + }; + if replan_stop.is_none() && verifier_tasks.iter().any(|task| !task.status.is_terminal()) { + conn.commit().await.map_err(storage_error)?; + return Ok(CompletionAdvanceOutcome::WaitingForVerifiers); + } + let (mut evaluation, terminal_output) = evaluate_persisted_completion( + &run, + &node_evidence, + &evidence, + &verifier_tasks, + request.now, + )?; + if let Some(intent) = replan_stop { + evaluation.status = replan_stop_status( + terminal_output.is_some(), + evaluation.satisfied_requirement_ids.len(), + ); + evaluation + .gaps + .extend(replan_stop_gaps(intent.stop_reason, Some(&intent.detail))); + evaluation.gaps.sort(); + evaluation.gaps.dedup(); + let terminal_projection = + terminal_projection_for_evaluation(&evaluation, terminal_output)?; + let cause = ExecutionTerminalCause::ReplanStop { + reason: intent.stop_reason, + }; + let terminal_evidence = terminal_evidence_from_evaluation(cause, &evaluation)?; + let reason = execution_terminal_reason( + &terminal_evidence.cause, + &terminal_projection, + &evaluation, + )?; + let pending_terminal = PendingExecutionTerminal { + status: crate::completion::run_status_from_completion(evaluation.status), + reason, + terminal_evidence, + completion_check_results: evaluation + .checks + .iter() + .map(serde_json::to_value) + .collect::, _>>()?, + terminal_gaps: evaluation.gaps, + output: node_evidence.terminal_output, + cancellation_reason: None, + }; + pending_terminal.validate()?; + conn.commit().await.map_err(storage_error)?; + return Ok(CompletionAdvanceOutcome::ReplanStopReady { + pending_terminal, + receipt: intent.receipt(), + }); + } + let terminal_projection = terminal_projection_for_evaluation(&evaluation, terminal_output)?; + if evaluation.status != CompletionStatus::Completed { + let cause = ExecutionTerminalCause::Completion { + limit_stop: evaluation.limit_stop, + }; + let terminal_evidence = terminal_evidence_from_evaluation(cause, &evaluation)?; + let reason = execution_terminal_reason( + &terminal_evidence.cause, + &terminal_projection, + &evaluation, + )?; + let pending_terminal = PendingExecutionTerminal { + status: crate::completion::run_status_from_completion(evaluation.status), + reason, + terminal_evidence, + completion_check_results: evaluation + .checks + .iter() + .map(serde_json::to_value) + .collect::, _>>()?, + terminal_gaps: evaluation.gaps.clone(), + output: node_evidence.terminal_output.clone(), + cancellation_reason: None, + }; + pending_terminal.validate()?; + conn.commit().await.map_err(storage_error)?; + return Ok(CompletionAdvanceOutcome::NonSuccessTerminal { pending_terminal }); + } + let terminal_evidence = terminal_evidence_from_evaluation( + ExecutionTerminalCause::Completion { + limit_stop: evaluation.limit_stop, + }, + &evaluation, + )?; + let terminal_reason = + execution_terminal_reason(&terminal_evidence.cause, &terminal_projection, &evaluation)?; + let finalization = RunFinalizationRequest { + run_uid: run.run_uid, + expected_revision: run.plan_revision, + expected_wake_epoch: request.wake_epoch, + terminal_projection, + completion_evaluation: evaluation, + terminal_evidence, + terminal_reason, + }; + conn.commit().await.map_err(storage_error)?; + Ok(CompletionAdvanceOutcome::FinalizationReady(Box::new( + finalization, + ))) + } +} + +async fn commit_completion_page( + mut conn: ScopedConn<'_>, + run: &ExecutionRunRecord, + request: &CompletionAdvanceRequest, + replan_stop: Option<&ExecutionReplanStopIntentRecord>, + scanned_tasks: u32, + scanned_nodes: u32, +) -> Result { + let Some(intent) = replan_stop else { + conn.commit().await.map_err(storage_error)?; + return Ok(CompletionAdvanceOutcome::Continue { + scanned_tasks, + scanned_nodes, + }); + }; + let acknowledged = sqlx::query( + "UPDATE moa.execution_run SET processed_wake_epoch=$3, \ + activation_state='idle',updated_at=NOW() \ + WHERE run_uid=$1 AND controller_generation=$2 AND wake_epoch=$3 \ + AND processed_wake_epoch<$3 AND activation_state='advancing' \ + RETURNING tenant_id", + ) + .bind(run.run_uid) + .bind(to_i64( + request.controller_generation, + "completion controller generation", + )?) + .bind(to_i64(request.wake_epoch, "completion wake epoch")?) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if acknowledged.is_none() { + conn.rollback().await.map_err(storage_error)?; + return Ok(CompletionAdvanceOutcome::NotReady); + } + let continuation = enqueue_run_activation_in_conn( + conn.as_mut(), + run.tenant_id, + run.run_uid, + request.controller_generation, + request.now, + json!({ + "reason": "replan_stop_completion_continue", + "origin_task_id": intent.origin_task_id, + "base_plan_revision": intent.base_plan_revision, + }), + ) + .await?; + let new_wake_epoch = continuation + .wake_epoch + .ok_or_else(|| Error::InvalidRepositoryData { + message: "replan-stop continuation is missing its wake epoch".to_string(), + })?; + let rebound = sqlx::query( + "UPDATE moa.execution_replan_stop_intent SET wake_epoch=$4,updated_at=NOW() \ + WHERE run_uid=$1 AND controller_generation=$2 AND wake_epoch=$3 \ + AND origin_task_id=$5 AND task_generation=$6 AND base_plan_revision=$7 \ + AND amendment_hash=$8", + ) + .bind(run.run_uid) + .bind(to_i64( + request.controller_generation, + "replan-stop controller generation", + )?) + .bind(to_i64(request.wake_epoch, "replan-stop wake epoch")?) + .bind(to_i64( + new_wake_epoch, + "replan-stop continuation wake epoch", + )?) + .bind(intent.origin_task_id.as_uuid()) + .bind(to_i64( + intent.task_generation, + "replan-stop task generation", + )?) + .bind(to_i64( + intent.base_plan_revision, + "replan-stop plan revision", + )?) + .bind(intent.amendment_hash.to_string()) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if rebound.rows_affected() != 1 { + conn.rollback().await.map_err(storage_error)?; + return Ok(CompletionAdvanceOutcome::NotReady); + } + conn.commit().await.map_err(storage_error)?; + Ok(CompletionAdvanceOutcome::ReplanStopContinue { + scanned_tasks, + scanned_nodes, + continuation: Box::new(continuation), + }) +} + +fn accumulate_task_evidence( + run: &ExecutionRunRecord, + task: &ExecutionTaskRecord, + evidence: &mut CompletionTaskEvidence, +) -> Result<()> { + let mut unsupported = false; + if let Some(outcome) = &task.current_outcome + && let ExecutionTaskResult::Failed { class, .. } = &outcome.result + { + if *class == ExecutionFailureClass::AuthorizationDenied { + evidence.authorization_denied = true; + } + if *class == ExecutionFailureClass::Unsupported { + unsupported = true; + } + } + for requirement_id in &task.requirement_ids { + let requirement = evidence + .unsupported_by_requirement + .entry(requirement_id.clone()) + .or_default(); + requirement.task_count = + requirement + .task_count + .checked_add(1) + .ok_or_else(|| Error::ArithmeticOverflow { + context: "completion requirement task count".to_string(), + })?; + if unsupported { + requirement.unsupported_task_count = requirement + .unsupported_task_count + .checked_add(1) + .ok_or_else(|| Error::ArithmeticOverflow { + context: "completion unsupported task count".to_string(), + })?; + } + } + let citation_count = u64::try_from( + task.citations + .iter() + .filter(|citation| !citation.source_id.trim().is_empty()) + .count(), + ) + .map_err(|_| Error::ArithmeticOverflow { + context: "completion citation count".to_string(), + })?; + for check in &run.goal.completion_checks { + let CompletionCheckKind::Citations { + node_ids, + min_per_task, + } = &check.kind + else { + continue; + }; + if node_ids.contains(&task.node_id) && citation_count < u64::from(*min_per_task) { + let failed = evidence + .citation_failures + .entry(check.id.clone()) + .or_default(); + failed.failure_count = + failed + .failure_count + .checked_add(1) + .ok_or_else(|| Error::ArithmeticOverflow { + context: "completion citation failure count".to_string(), + })?; + if failed.samples.len() < MAX_EVIDENCE_SAMPLES_PER_CHECK { + failed.samples.push(CitationFailureSample { + node_id: task.node_id.clone(), + item_key: task.item_key.clone(), + count: citation_count, + }); + } + } + } + Ok(()) +} + +async fn materialize_verifiers_in_tx( + conn: &mut PgConnection, + config: &ExecutionConfig, + run: &ExecutionRunRecord, + evidence: &CompletionTaskEvidence, + nodes: &CompletionNodeEvidence, + limit: u32, +) -> Result<(Vec, bool)> { + let coverage_by_node = coverage_by_node(run, nodes); + let unresolved = unsatisfied_requirements(run, nodes, &coverage_by_node); + let terminal_output = nodes.terminal_output.clone(); + let existing = sqlx::query_scalar::<_, String>( + "SELECT node_id FROM moa.execution_task WHERE run_uid=$1 \ + AND node_id LIKE '@check/%' ORDER BY node_id LIMIT $2", + ) + .bind(run.run_uid) + .bind( + i64::try_from(config.dispatch_batch_size) + .map_err(|_| Error::ArithmeticOverflow { + context: "completion verifier dispatch bound".to_string(), + })? + .checked_add(1) + .ok_or_else(|| Error::ArithmeticOverflow { + context: "completion verifier dispatch bound".to_string(), + })?, + ) + .fetch_all(&mut *conn) + .await + .map_err(sqlx_error)?; + if existing.len() > config.dispatch_batch_size { + return Err(Error::InvalidRepositoryData { + message: "persisted verifier tasks exceed the compiler-validated dispatch bound" + .to_string(), + }); + } + let existing = existing.into_iter().collect::>(); + let declared = run + .goal + .completion_checks + .iter() + .filter(|check| matches!(check.kind, CompletionCheckKind::AgentVerifier { .. })) + .map(|check| format!("@check/{}", check.id)) + .collect::>(); + if !existing.is_subset(&declared) { + return Err(Error::InvalidRepositoryData { + message: "persisted verifier task is not declared by the completion contract" + .to_string(), + }); + } + let limit = usize::try_from(limit).map_err(|_| Error::ArithmeticOverflow { + context: "completion verifier page limit".to_string(), + })?; + let mut tasks = Vec::new(); + let base_order = u64::try_from(run.active_plan.definition.nodes.len()).map_err(|_| { + Error::ArithmeticOverflow { + context: "completion verifier node order".to_string(), + } + })?; + for (index, check) in run.goal.completion_checks.iter().enumerate() { + let CompletionCheckKind::AgentVerifier { + instructions, + max_turns, + } = &check.kind + else { + continue; + }; + let node_id = format!("@check/{}", check.id); + if existing.contains(&node_id) || tasks.len() >= limit { + continue; + } + let item_key = format!("check:{}", check.id); + tasks.push(LogicalTask { + task_id: ExecutionTaskId::derive(run.run_uid, &node_id, &item_key)?, + node_id: node_id.clone(), + item_key, + requirement_ids: unresolved.clone(), + plan_revision: run.plan_revision, + generation: 1, + input: json!({ + "goal": &run.goal, + "check_id": &check.id, + "description": &check.description, + "terminal_output": &terminal_output, + "bounded_task_evidence": evidence, + }), + kind: LogicalTaskKind::CompletionVerifier { + check_id: check.id.clone(), + instructions: instructions.clone(), + max_turns: *max_turns, + }, + compensation: None, + retry: RetryPolicy { + max_attempts: 1, + initial_backoff_ms: 0, + max_backoff_ms: 0, + }, + reservation: verifier_turn_reservation(config, *max_turns)?, + }); + let node_order = base_order + .checked_add(u64::try_from(index).map_err(|_| Error::ArithmeticOverflow { + context: "completion verifier node order".to_string(), + })?) + .ok_or_else(|| Error::ArithmeticOverflow { + context: "completion verifier node order".to_string(), + })?; + sqlx::query( + "INSERT INTO moa.execution_node_state (node_state_uid,tenant_id,run_uid,node_id, \ + node_order,node_status,materialization_cursor,materialization_complete, \ + total_task_count,ready_task_count) \ + VALUES ($1,$2,$3,$4,$5,'ready',1,TRUE,1,1) \ + ON CONFLICT (run_uid,node_id) DO NOTHING", + ) + .bind(Uuid::now_v7()) + .bind(run.tenant_id.0) + .bind(run.run_uid) + .bind(node_id) + .bind(to_i64(node_order, "completion verifier node order")?) + .execute(&mut *conn) + .await + .map_err(sqlx_error)?; + } + let batch = prepare_task_materialization_batch(run.run_uid, run.plan_revision, &tasks)?; + let inserted = sqlx::query(INSERT_TASK_BATCH_SQL) + .bind(&batch) + .bind(run.run_uid) + .bind(run.tenant_id.0) + .bind(run.contact_id.map(|contact| contact.0)) + .bind(to_i64(run.plan_revision, "completion plan revision")?) + .fetch_all(&mut *conn) + .await + .map_err(sqlx_error)?; + if inserted.len() != tasks.len() { + return Err(Error::InvalidRepositoryData { + message: "completion verifier materialization lost its exact task-key fence" + .to_string(), + }); + } + let task_ids = tasks + .iter() + .map(|task| task.task_id.as_uuid()) + .collect::>(); + sqlx::query( + "UPDATE moa.execution_task SET status='ready', ready_at=NOW(), updated_at=NOW() \ + WHERE run_uid=$1 AND task_id=ANY($2::UUID[]) AND status='pending'", + ) + .bind(run.run_uid) + .bind(task_ids) + .execute(&mut *conn) + .await + .map_err(sqlx_error)?; + sqlx::query( + "UPDATE moa.execution_run SET progress_total_tasks=progress_total_tasks+$2, \ + ready_task_count=ready_task_count+$2, updated_at=NOW() WHERE run_uid=$1", + ) + .bind(run.run_uid) + .bind(to_i64( + u64::try_from(tasks.len()).map_err(|_| Error::ArithmeticOverflow { + context: "completion verifier task count".to_string(), + })?, + "completion verifier task count", + )?) + .execute(&mut *conn) + .await + .map_err(sqlx_error)?; + let rows = sqlx::query(LOAD_TASK_BATCH_SQL) + .bind(&batch) + .bind(run.run_uid) + .fetch_all(&mut *conn) + .await + .map_err(sqlx_error)?; + let records = rows.iter().map(task_from_row).collect::>>()?; + let verifier_count = run + .goal + .completion_checks + .iter() + .filter(|check| matches!(check.kind, CompletionCheckKind::AgentVerifier { .. })) + .count(); + let materialized_count = + existing + .len() + .checked_add(records.len()) + .ok_or_else(|| Error::ArithmeticOverflow { + context: "completion verifier materialized count".to_string(), + })?; + Ok((records, materialized_count == verifier_count)) +} + +async fn load_verifier_tasks( + conn: &mut PgConnection, + run: &ExecutionRunRecord, +) -> Result> { + let declared = run + .goal + .completion_checks + .iter() + .filter(|check| matches!(check.kind, CompletionCheckKind::AgentVerifier { .. })) + .map(|check| format!("@check/{}", check.id)) + .collect::>(); + let expected = declared.len(); + let fetch_limit = i64::try_from(expected) + .map_err(|_| Error::ArithmeticOverflow { + context: "completion verifier load bound".to_string(), + })? + .checked_add(1) + .ok_or_else(|| Error::ArithmeticOverflow { + context: "completion verifier load bound".to_string(), + })?; + let rows = sqlx::query( + "SELECT * FROM moa.execution_task WHERE run_uid=$1 AND node_id LIKE '@check/%' \ + ORDER BY node_id,task_id LIMIT $2", + ) + .bind(run.run_uid) + .bind(fetch_limit) + .fetch_all(&mut *conn) + .await + .map_err(sqlx_error)?; + let persisted = rows + .iter() + .map(|row| row.try_get::("node_id").map_err(row_error)) + .collect::>>()?; + if rows.len() != expected || persisted != declared { + return Err(Error::InvalidRepositoryData { + message: "persisted verifier tasks do not exactly match declared completion checks" + .to_string(), + }); + } + rows.iter().map(task_from_row).collect() +} + +async fn load_existing_verifier_tasks( + conn: &mut PgConnection, + config: &ExecutionConfig, + run: &ExecutionRunRecord, +) -> Result> { + let declared = run + .goal + .completion_checks + .iter() + .filter(|check| matches!(check.kind, CompletionCheckKind::AgentVerifier { .. })) + .map(|check| format!("@check/{}", check.id)) + .collect::>(); + let limit = i64::try_from(config.dispatch_batch_size) + .map_err(|_| Error::ArithmeticOverflow { + context: "replan-stop verifier load bound".to_string(), + })? + .checked_add(1) + .ok_or_else(|| Error::ArithmeticOverflow { + context: "replan-stop verifier load bound".to_string(), + })?; + let rows = sqlx::query( + "SELECT * FROM moa.execution_task WHERE run_uid=$1 AND node_id LIKE '@check/%' \ + ORDER BY node_id,task_id LIMIT $2", + ) + .bind(run.run_uid) + .bind(limit) + .fetch_all(&mut *conn) + .await + .map_err(sqlx_error)?; + if rows.len() > config.dispatch_batch_size { + return Err(Error::InvalidRepositoryData { + message: "persisted verifier tasks exceed the compiler-validated dispatch bound" + .to_string(), + }); + } + let persisted = rows + .iter() + .map(|row| row.try_get::("node_id").map_err(row_error)) + .collect::>>()?; + if !persisted.is_subset(&declared) { + return Err(Error::InvalidRepositoryData { + message: "persisted verifier task is not declared by the completion contract" + .to_string(), + }); + } + rows.iter().map(task_from_row).collect() +} + +fn accumulate_node_evidence( + run: &ExecutionRunRecord, + row: &PgRow, + evidence: &mut CompletionNodeEvidence, + forced_failed_node_id: Option<&str>, +) -> Result<()> { + let node_id: String = row.try_get("node_id").map_err(row_error)?; + let status: String = row.try_get("node_status").map_err(row_error)?; + let force_failed = forced_failed_node_id == Some(node_id.as_str()); + if !force_failed + && !matches!( + status.as_str(), + "completed" | "skipped" | "failed" | "cancelled" + ) + { + return Err(Error::InvalidRepositoryData { + message: format!("completion scan observed nonterminal node `{node_id}`"), + }); + } + let total = required_u64(row, "total_task_count")?; + let succeeded = required_u64(row, "succeeded_task_count")?; + let failed = required_u64(row, "failed_task_count")?; + let cancelled = required_u64(row, "cancelled_task_count")?; + let passed = !force_failed + && status == "completed" + && failed == 0 + && cancelled == 0 + && (total == 0 || succeeded > 0); + let plan_node = run + .active_plan + .definition + .nodes + .iter() + .find(|node| node.id == node_id) + .ok_or_else(|| Error::InvalidRepositoryData { + message: format!("completion node `{node_id}` is absent from the active plan"), + })?; + if force_failed || status != "skipped" { + for requirement_id in &plan_node.requirement_ids { + let requirement = evidence + .requirements + .entry(requirement_id.clone()) + .or_default(); + requirement.eligible_node_count = requirement + .eligible_node_count + .checked_add(1) + .ok_or_else(|| Error::ArithmeticOverflow { + context: "completion eligible requirement node count".to_string(), + })?; + if passed { + requirement.completed_node_count = requirement + .completed_node_count + .checked_add(1) + .ok_or_else(|| Error::ArithmeticOverflow { + context: "completion completed requirement node count".to_string(), + })?; + } + } + } + for check in &run.goal.completion_checks { + let CompletionCheckKind::RequiredNodes { node_ids } = &check.kind else { + continue; + }; + if node_ids.contains(&node_id) { + let check_evidence = evidence + .required_checks + .entry(check.id.clone()) + .or_default(); + check_evidence.observed_node_count = check_evidence + .observed_node_count + .checked_add(1) + .ok_or_else(|| Error::ArithmeticOverflow { + context: "completion required-node observation count".to_string(), + })?; + if !passed { + check_evidence.failed_node_ids.push(node_id.clone()); + } + } + } + for coverage in run + .goal + .coverage + .iter() + .filter(|coverage| coverage.map_node_id == node_id) + { + let coverage_passed = passed && (coverage.require_all || total == 0 || succeeded > 0); + evidence + .coverage_passed + .entry(coverage.id.clone()) + .and_modify(|current| *current &= coverage_passed) + .or_insert(coverage_passed); + } + if matches!(plan_node.operation, ExecutionOperation::Output { .. }) { + evidence.terminal_output = row.try_get("aggregate_output").map_err(row_error)?; + } + Ok(()) +} + +fn evaluate_persisted_completion( + run: &ExecutionRunRecord, + nodes: &CompletionNodeEvidence, + evidence: &CompletionTaskEvidence, + verifier_tasks: &[ExecutionTaskRecord], + now: DateTime, +) -> Result<(CompletionEvaluation, Option)> { + let terminal_output = nodes.terminal_output.clone(); + let mut checks = Vec::new(); + let mut coverage_by_node = BTreeMap::new(); + let mut failed_coverage = Vec::new(); + for coverage in &run.goal.coverage { + let passed = nodes + .coverage_passed + .get(&coverage.id) + .copied() + .unwrap_or(false); + coverage_by_node + .entry(coverage.map_node_id.clone()) + .and_modify(|node_passed| *node_passed &= passed) + .or_insert(passed); + if !passed { + failed_coverage.push(coverage.id.clone()); + } + } + for check in &run.goal.completion_checks { + let (passed, check_evidence) = match &check.kind { + CompletionCheckKind::OutputSchema => { + let passed = terminal_output.as_ref().is_some_and(|output| { + validate_instance( + &run.active_plan.definition.output_schema, + output, + "plan.output", + ) + .is_ok() + && run + .active_plan + .definition + .nodes + .iter() + .find(|node| { + matches!(node.operation, ExecutionOperation::Output { .. }) + }) + .is_some_and(|node| { + validate_instance(&node.output_schema, output, "output_node.output") + .is_ok() + }) + }); + ( + passed, + json!({"terminal_output_present": terminal_output.is_some()}), + ) + } + CompletionCheckKind::RequiredNodes { node_ids } => { + let persisted = nodes + .required_checks + .get(&check.id) + .cloned() + .unwrap_or_default(); + let incomplete = persisted.failed_node_ids; + let observed = usize::try_from(persisted.observed_node_count).map_err(|_| { + Error::ArithmeticOverflow { + context: "completion required-node observation count".to_string(), + } + })?; + ( + incomplete.is_empty() && observed == node_ids.len(), + json!({"incomplete_node_ids": incomplete}), + ) + } + CompletionCheckKind::MapCoverage { map_node_id } => { + let matching = run + .goal + .coverage + .iter() + .filter(|coverage| coverage.map_node_id == *map_node_id) + .collect::>(); + let passed = !matching.is_empty() + && coverage_by_node.get(map_node_id).copied().unwrap_or(false); + ( + passed, + json!({"map_node_id": map_node_id, "persisted_materialization_complete": passed}), + ) + } + CompletionCheckKind::Citations { .. } => { + let failed = evidence + .citation_failures + .get(&check.id) + .cloned() + .unwrap_or_default(); + (failed.failure_count == 0, serde_json::to_value(failed)?) + } + CompletionCheckKind::AgentVerifier { .. } => { + let node_id = format!("@check/{}", check.id); + let output = verifier_tasks + .iter() + .find(|task| { + task.node_id == node_id && task.status == ExecutionTaskStatus::Completed + }) + .and_then(|task| task.output.as_ref()); + let object = output.and_then(Value::as_object); + let valid = object.is_some_and(|object| { + object.len() == 2 + && object.get("passed").and_then(Value::as_bool).is_some() + && object.contains_key("evidence") + }); + let verdict = object + .and_then(|object| object.get("passed")) + .and_then(Value::as_bool) + .unwrap_or(false); + ( + valid && verdict, + json!({"verdict": verdict, "valid_shape": valid, "evidence": object.and_then(|object| object.get("evidence")).cloned().unwrap_or(Value::Null)}), + ) + } + }; + checks.push(CompletionCheckResult { + check_id: check.id.clone(), + passed, + evidence: check_evidence, + }); + } + let (satisfied, unsatisfied) = partition_requirements(run, nodes, &coverage_by_node); + let mut gaps = checks + .iter() + .filter(|check| !check.passed) + .map(|check| format!("completion check {} failed", check.check_id)) + .collect::>(); + gaps.extend( + failed_coverage + .iter() + .map(|id| format!("coverage {id} failed")), + ); + gaps.extend( + unsatisfied + .iter() + .map(|id| format!("requirement {id} is unsatisfied")), + ); + let deliverables_pass = run.goal.deliverables.iter().all(|deliverable| { + terminal_output + .as_ref() + .and_then(|output| output.pointer(&deliverable.output_pointer)) + .is_some_and(|value| { + validate_instance( + &deliverable.schema, + value, + &format!("goal.deliverables.{}", deliverable.id), + ) + .is_ok() + }) + }); + if !deliverables_pass { + gaps.push("one or more deliverables are missing or invalid".to_string()); + } + let constraints_pass = run.goal.constraints.iter().all(|constraint| { + run.goal + .completion_checks + .iter() + .enumerate() + .filter(|(_, check)| check.constraint_ids.contains(&constraint.id)) + .all(|(index, _)| checks.get(index).is_some_and(|result| result.passed)) + }); + if !constraints_pass { + gaps.push("one or more constraint-linked checks failed".to_string()); + } + let deadline_exceeded = run + .approved_budget + .deadline_at + .is_some_and(|deadline| now > deadline); + let limit_stop = if deadline_exceeded { + Some(ExecutionLimitStop::DeadlineExceeded) + } else if run.budget_overrun { + Some(ExecutionLimitStop::BudgetExceeded) + } else { + None + }; + if deadline_exceeded { + gaps.push("execution deadline exceeded".to_string()); + } + if run.budget_overrun { + gaps.push("execution budget overrun".to_string()); + } + gaps.sort(); + gaps.dedup(); + let all_pass = checks.iter().all(|check| check.passed) + && failed_coverage.is_empty() + && unsatisfied.is_empty() + && deliverables_pass + && constraints_pass + && terminal_output.is_some(); + let useful = terminal_output.is_some() || !satisfied.is_empty(); + let fully_unsupported = unsatisfied.iter().any(|requirement_id| { + evidence + .unsupported_by_requirement + .get(requirement_id) + .is_some_and(|unsupported| { + unsupported.task_count > 0 + && unsupported.task_count == unsupported.unsupported_task_count + }) + }); + let status = if all_pass && limit_stop.is_none() { + CompletionStatus::Completed + } else if evidence.authorization_denied { + CompletionStatus::Blocked + } else if fully_unsupported { + CompletionStatus::Unsupported + } else if useful { + CompletionStatus::Partial + } else { + CompletionStatus::Failed + }; + Ok(( + CompletionEvaluation { + status, + limit_stop, + checks, + satisfied_requirement_ids: satisfied, + unsatisfied_requirement_ids: unsatisfied, + gaps, + }, + terminal_output, + )) +} + +fn partition_requirements( + run: &ExecutionRunRecord, + nodes: &CompletionNodeEvidence, + coverage: &BTreeMap, +) -> (Vec, Vec) { + let mut satisfied = Vec::new(); + let mut unsatisfied = Vec::new(); + for requirement in &run.goal.requirements { + let requirement_nodes = nodes + .requirements + .get(&requirement.id) + .cloned() + .unwrap_or_default(); + let coverage_passed = run + .active_plan + .definition + .nodes + .iter() + .filter(|node| { + node.requirement_ids.contains(&requirement.id) + && matches!(node.operation, ExecutionOperation::Map { .. }) + }) + .all(|node| coverage.get(&node.id).copied().unwrap_or(true)); + let passed = requirement_nodes.eligible_node_count > 0 + && requirement_nodes.eligible_node_count == requirement_nodes.completed_node_count + && coverage_passed; + if passed { + satisfied.push(requirement.id.clone()); + } else { + unsatisfied.push(requirement.id.clone()); + } + } + satisfied.sort(); + unsatisfied.sort(); + (satisfied, unsatisfied) +} + +fn unsatisfied_requirements( + run: &ExecutionRunRecord, + nodes: &CompletionNodeEvidence, + coverage: &BTreeMap, +) -> Vec { + partition_requirements(run, nodes, coverage).1 +} + +fn coverage_by_node( + run: &ExecutionRunRecord, + nodes: &CompletionNodeEvidence, +) -> BTreeMap { + let mut by_node = BTreeMap::new(); + for coverage in &run.goal.coverage { + let passed = nodes + .coverage_passed + .get(&coverage.id) + .copied() + .unwrap_or(false); + by_node + .entry(coverage.map_node_id.clone()) + .and_modify(|current| *current &= passed) + .or_insert(passed); + } + by_node +} + +fn validate_completion_runtime_bounds( + run: &ExecutionRunRecord, + config: &ExecutionConfig, +) -> Result<()> { + let metadata_count = run + .goal + .requirements + .len() + .saturating_add(run.goal.constraints.len()) + .saturating_add(run.goal.deliverables.len()) + .saturating_add(run.goal.coverage.len()) + .saturating_add(run.goal.completion_checks.len()); + let referenced_nodes = run + .goal + .completion_checks + .iter() + .map(|check| match &check.kind { + CompletionCheckKind::RequiredNodes { node_ids } + | CompletionCheckKind::Citations { node_ids, .. } => node_ids.len(), + CompletionCheckKind::MapCoverage { .. } => 1, + CompletionCheckKind::OutputSchema | CompletionCheckKind::AgentVerifier { .. } => 0, + }) + .fold(0_usize, usize::saturating_add); + let verifier_count = run + .goal + .completion_checks + .iter() + .filter(|check| matches!(check.kind, CompletionCheckKind::AgentVerifier { .. })) + .count(); + if metadata_count > config.maximum_activation_steps + || referenced_nodes > config.maximum_activation_steps + || verifier_count > config.dispatch_batch_size + { + return Err(Error::InvalidRepositoryData { + message: + "persisted completion contract exceeds its compiler-validated activation bounds" + .to_string(), + }); + } + Ok(()) +} + +fn terminal_projection_for_evaluation( + evaluation: &CompletionEvaluation, + output: Option, +) -> Result { + crate::completion::terminal_projection_from_evaluation( + evaluation, + output, + None, + (evaluation.status == CompletionStatus::Failed).then(|| ExecutionTaskFailure { + class: ExecutionFailureClass::Terminal, + message: evaluation.gaps.join("; "), + capability_ref: None, + }), + (evaluation.status == CompletionStatus::Unsupported) + .then(|| "required execution paths are unsupported".to_string()), + ) +} + +#[cfg(test)] +mod tests { + use super::{CompletionNodeEvidence, CompletionTaskEvidence}; + + #[test] + fn empty_persisted_completion_evidence_uses_the_canonical_defaults() { + // Pins: a fresh completion scan persists `{}` for both evidence documents, and + // the first controller page must decode them before accumulating any evidence. + let task: CompletionTaskEvidence = + serde_json::from_str("{}").expect("empty task evidence should decode"); + let node: CompletionNodeEvidence = + serde_json::from_str("{}").expect("empty node evidence should decode"); + + assert!(!task.authorization_denied); + assert!(task.unsupported_by_requirement.is_empty()); + assert!(task.citation_failures.is_empty()); + assert!(node.terminal_output.is_none()); + assert!(node.requirements.is_empty()); + assert!(node.required_checks.is_empty()); + assert!(node.coverage_passed.is_empty()); + } +} diff --git a/crates/moa-execution/src/repository/external_job.rs b/crates/moa-execution/src/repository/external_job.rs new file mode 100644 index 000000000..054ff83fc --- /dev/null +++ b/crates/moa-execution/src/repository/external_job.rs @@ -0,0 +1,2450 @@ +//! Generation-fenced asynchronous provider-job persistence and callback deduplication. + +use std::str::FromStr; + +use crate::wire::{ + ExecutionCompensationAttemptCancelRequest, ExecutionExternalJobCancelRequest, + ExecutionExternalJobStartRecoveryOwner, ExecutionExternalJobStartRecoveryRequest, +}; +use chrono::{DateTime, Utc}; +use moa_config::ExecutionConfig; +use moa_core::types::identifiers::TenantId; +use moa_core::types::tools::{AsyncToolJobCallbackOutcome, AsyncToolJobTerminalOutcome}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use sqlx::{PgConnection, Row}; +use uuid::Uuid; + +use super::{ + Error, ExecutionRepository, ExecutionScope, Result, + capacity::{ + CapacityReserveOutcome, ExecutionCapacityDimension, ExecutionCapacityOwner, + ExecutionCapacityRequest, execution_capacity_reservation_uid, + prelock_capacity_dimensions_in_tx, prelock_existing_capacity_dimensions_in_tx, + release_capacity_in_tx, reserve_capacity_in_tx, + }, + compensation::{ + CompensationExternalJobSettlementOutcome, + CompensationExternalNotStartedReleaseClaimOutcome, + CompensationRecoveredExternalReleaseClaimOutcome, + begin_compensation_external_not_started_release_in_conn, + begin_recovered_compensation_external_release_in_conn, + settle_external_job_terminal_in_conn as settle_compensation_external_job_terminal_in_conn, + }, + outbox::{ + ExecutionDispatchKind, ExecutionDispatchRecord, NewExecutionDispatch, + enqueue_dispatch_in_conn, + }, + run::enqueue_run_activation_in_conn, + sqlx_error, storage_error, + task::{ + ExternalJobTaskSettlementOutcome, TaskAttemptExternalOutcome, + TaskExternalStartRetryOutcome, + settle_external_job_terminal_in_conn as settle_task_external_job_terminal_in_conn, + }, + to_i64, + trigger::{ + ExecutionTriggerKind, NewExecutionTrigger, create_trigger_with_dispatch_in_conn, + supersede_trigger_in_conn, + }, +}; + +const MAX_RECONCILE_BATCH_SIZE: u32 = 1_000; +const EXTERNAL_RECONCILE_TRIGGER_NAMESPACE: Uuid = + Uuid::from_u128(0xf01c_6bd5_175f_581c_8f53_e2e3_c69a_0eab); +const EXTERNAL_START_RECOVERY_TRIGGER_NAMESPACE: Uuid = + Uuid::from_u128(0x3551_037a_2e09_55fa_a23f_e1b9_3717_2172); +const EXTERNAL_CANCEL_DISPATCH_NAMESPACE: Uuid = + Uuid::from_u128(0x8445_6cf7_3f09_5c27_9eb2_d9e8_1fc0_fa8c); + +/// Durable asynchronous provider-job lifecycle state. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ExecutionExternalJobState { + /// Capacity is reserved, but no provider call has been made or bound yet. + Unbound, + /// Provider start has been durably admitted but not confirmed running. + Starting, + /// Provider confirms active asynchronous work. + Running, + /// MOA is waiting for a sparse reconciliation wake. + WaitingReconcile, + /// Cancellation was requested but its provider outcome is unresolved. + CancelRequested, + /// Provider work completed successfully. + Completed, + /// Provider work failed definitively. + Failed, + /// Provider work was cancelled definitively. + Cancelled, + /// Provider outcome cannot safely be inferred. + UnknownOutcome, +} + +impl ExecutionExternalJobState { + /// Returns the canonical database label. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Unbound => "unbound", + Self::Starting => "starting", + Self::Running => "running", + Self::WaitingReconcile => "waiting_reconcile", + Self::CancelRequested => "cancel_requested", + Self::Completed => "completed", + Self::Failed => "failed", + Self::Cancelled => "cancelled", + Self::UnknownOutcome => "unknown_outcome", + } + } + + /// Returns whether the state is terminal. + #[must_use] + pub const fn is_terminal(self) -> bool { + matches!( + self, + Self::Completed | Self::Failed | Self::Cancelled | Self::UnknownOutcome + ) + } +} + +impl FromStr for ExecutionExternalJobState { + type Err = Error; + + fn from_str(value: &str) -> Result { + match value { + "unbound" => Ok(Self::Unbound), + "starting" => Ok(Self::Starting), + "running" => Ok(Self::Running), + "waiting_reconcile" => Ok(Self::WaitingReconcile), + "cancel_requested" => Ok(Self::CancelRequested), + "completed" => Ok(Self::Completed), + "failed" => Ok(Self::Failed), + "cancelled" => Ok(Self::Cancelled), + "unknown_outcome" => Ok(Self::UnknownOutcome), + _ => Err(Error::InvalidRepositoryData { + message: format!("unknown execution external-job state `{value}`"), + }), + } + } +} + +/// Exact durable owner of one asynchronous provider job. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ExecutionExternalJobOwner { + /// One exact active forward-task attempt. + Task { + /// Stable logical task identity. + task_id: Uuid, + /// Exact active task-attempt generation. + attempt_generation: u64, + }, + /// One exact active compensation attempt. + Compensation { + /// Stable compensation registration identity. + compensation_id: Uuid, + /// Exact compensation logical generation. + compensation_generation: u64, + /// Exact active compensation-attempt generation. + compensation_attempt_generation: u64, + }, +} + +/// Immutable pre-provider intent that reserves external-job capacity. +#[derive(Clone, Debug, PartialEq)] +pub struct NewExecutionExternalJobIntent { + /// Stable MOA job identity. + pub external_job_uid: Uuid, + /// Tenant that owns the task and job. + pub tenant_id: TenantId, + /// Owning execution run. + pub run_uid: Uuid, + /// Exact task or compensation attempt that owns the provider job. + pub owner: ExecutionExternalJobOwner, + /// Provider-job generation, incremented when replacement is explicit. + pub job_generation: u64, + /// Declared adapter/provider key used for crash-safe start recovery. + pub provider: String, + /// Stable provider idempotency key. + pub idempotency_key: String, + /// Deadline after which an unbound intent can be reclaimed safely. + pub expires_at: DateTime, +} + +/// Generation-fenced provider identity bound after an asynchronous start response. +#[derive(Clone, Debug, PartialEq)] +pub struct ExecutionExternalJobBinding { + /// Stable MOA job identity selected before provider dispatch. + pub external_job_uid: Uuid, + /// Tenant that owns the task and job. + pub tenant_id: TenantId, + /// Owning execution run. + pub run_uid: Uuid, + /// Exact task or compensation attempt that owns the provider job. + pub owner: ExecutionExternalJobOwner, + /// Exact pre-reserved provider-job generation. + pub job_generation: u64, + /// Stable provider idempotency key used for the provider call. + pub idempotency_key: String, + /// Provider name, which must match the pre-reserved declared provider. + pub provider: String, + /// Provider-issued job identity. + pub provider_job_id: String, + /// Reference used by the callback-authentication boundary. + pub callback_auth_reference: String, + /// Initial bound, nonterminal provider-job state. + pub state: ExecutionExternalJobState, + /// Optional provider progress phase. + pub progress_phase: Option, + /// Whether provider cancellation is supported. + pub cancel_supported: bool, + /// Next sparse reconciliation time. + pub next_reconcile_at: Option>, + /// Bounded evidence when adapter output violated its reserved start contract. + pub provider_contract_violation: Option, +} + +/// One persisted asynchronous provider job. +#[derive(Clone, Debug, PartialEq)] +pub struct ExecutionExternalJobRecord { + /// Stable MOA job identity. + pub external_job_uid: Uuid, + /// Tenant that owns the row. + pub tenant_id: TenantId, + /// Owning execution run. + pub run_uid: Uuid, + /// Exact task or compensation attempt that owns the provider job. + pub owner: ExecutionExternalJobOwner, + /// Provider-job generation. + pub job_generation: u64, + /// Declared adapter/provider key reserved before provider dispatch. + pub declared_provider: String, + /// Bound provider name; absent only while the intent is unbound. + pub provider: Option, + /// Provider-issued job identity. + pub provider_job_id: Option, + /// Stable provider idempotency key. + pub idempotency_key: String, + /// Callback-authentication reference. + pub callback_auth_reference: Option, + /// Current lifecycle state. + pub state: ExecutionExternalJobState, + /// Latest progress phase. + pub progress_phase: Option, + /// Whether provider cancellation is supported. + pub cancel_supported: bool, + /// Next sparse reconciliation time. + pub next_reconcile_at: Option>, + /// Last accepted provider callback event identity. + pub last_provider_event_id: Option, + /// Terminal provider output. + pub output: Option, + /// Terminal provider error. + pub error: Option, + /// Creation time. + pub created_at: DateTime, + /// Last mutation time. + pub updated_at: DateTime, + /// Terminal time. + pub completed_at: Option>, + /// Durable provider contract-violation evidence, if recovery containment was required. + pub provider_contract_violation: Option, +} + +/// Idempotent disposition of an exact unbound intent release. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ExecutionExternalJobIntentReleaseOutcome { + /// The exact unbound intent and its capacity receipt were removed. + Released, + /// Neither the intent nor an active capacity receipt remains. + AlreadyReleased, + /// The stable UID exists with different immutable coordinates. + Stale, + /// Provider identity was already bound and must never be reclaimed as an intent. + AlreadyBound, +} + +/// Owner adoption committed after crash-safe provider start recovery. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub enum ExecutionExternalJobStartRecoveryAdoptionOutcome { + /// Provider ownership or the no-start retry transition was committed. + Applied { + /// Compensation teardown request that must obtain its verified hand-release receipt. + compensation_release: Option>, + }, + /// The same exact owner transition was already committed. + Replayed { + /// Compensation teardown request replayed until its verified finalizer commits. + compensation_release: Option>, + }, + /// Another exact recovery already settled the intent and owner. + AlreadySettled, + /// No exact intent or owner exists. + NotFound, + /// An immutable owner, generation, checkpoint, capacity, or watchdog fence differed. + Stale, + /// The owner cannot adopt this recovery result from its current state. + InvalidState, +} + +/// Authenticated callback for one provider-job generation. +#[derive(Clone, Debug, PartialEq)] +pub struct ExecutionExternalJobCallback { + /// Stable MOA job identity selected after callback authentication. + pub external_job_uid: Uuid, + /// Expected provider-job generation. + pub job_generation: u64, + /// Expected provider name. + pub provider: String, + /// Expected provider-issued job identity. + pub provider_job_id: String, + /// Provider event identity used for deduplication. + pub provider_event_id: String, + /// Typed progress or terminal mutation. + pub update: ExecutionExternalJobCallbackUpdate, +} + +/// Mutation carried by one authenticated provider callback. +#[derive(Clone, Debug, PartialEq)] +pub enum ExecutionExternalJobCallbackUpdate { + /// Advances observable progress without terminalizing the provider job. + Progress { + /// Current nonterminal provider lifecycle state. + state: ExecutionExternalJobState, + /// Latest bounded provider progress phase. + progress_phase: Option, + /// Next sparse reconciliation time, if provider polling remains necessary. + next_reconcile_at: Option>, + }, + /// Records the definitive provider outcome. + Terminal { + /// Terminal provider lifecycle state. + state: ExecutionExternalJobState, + /// Final bounded provider progress phase. + progress_phase: Option, + /// Structured terminal output. + output: Option, + /// Structured terminal error. + error: Option, + }, +} + +impl From for ExecutionExternalJobCallbackUpdate { + fn from(outcome: AsyncToolJobCallbackOutcome) -> Self { + match outcome { + AsyncToolJobCallbackOutcome::Progress { + progress_phase, + next_reconcile_at, + } => Self::Progress { + state: ExecutionExternalJobState::WaitingReconcile, + progress_phase: Some(progress_phase), + next_reconcile_at: Some(next_reconcile_at), + }, + AsyncToolJobCallbackOutcome::Terminal { outcome } => match outcome { + AsyncToolJobTerminalOutcome::Completed { output } => Self::Terminal { + state: ExecutionExternalJobState::Completed, + progress_phase: Some("completed".to_string()), + output: Some(output), + error: None, + }, + AsyncToolJobTerminalOutcome::Failed { error } => Self::Terminal { + state: ExecutionExternalJobState::Failed, + progress_phase: Some("failed".to_string()), + output: None, + error: Some(error), + }, + AsyncToolJobTerminalOutcome::Cancelled => Self::Terminal { + state: ExecutionExternalJobState::Cancelled, + progress_phase: Some("cancelled".to_string()), + output: None, + error: None, + }, + AsyncToolJobTerminalOutcome::UnknownOutcome { error } => Self::Terminal { + state: ExecutionExternalJobState::UnknownOutcome, + progress_phase: Some("unknown_outcome".to_string()), + output: None, + error: Some(error), + }, + }, + } + } +} + +/// Result of applying an authenticated provider callback. +#[derive(Clone, Debug, PartialEq)] +pub enum ExecutionExternalJobCallbackOutcome { + /// Callback advanced the exact current job generation. + Applied(Box), + /// The exact provider event was already accepted. + Duplicate, + /// Callback generation or provider identity is stale. + StaleGeneration, + /// A terminal outcome already fenced later callbacks. + AlreadyTerminal, + /// No visible job has the supplied MOA identity. + NotFound, +} + +/// Atomic callback settlement and its persisted controller wake. +#[derive(Clone, Debug, PartialEq)] +pub struct ExecutionExternalJobCallbackWrite { + /// Generation-fenced callback disposition. + pub outcome: ExecutionExternalJobCallbackOutcome, + /// Exact controller activation committed with an applied callback. + /// + /// Terminal runs retain the callback receipt without creating an + /// unreachable activation. + pub activation: Option, +} + +/// Exact asynchronous-provider cancellation settlement. +#[derive(Clone, Debug, PartialEq)] +pub struct ExecutionExternalJobCancellation { + /// Stable MOA job identity. + pub external_job_uid: Uuid, + /// Expected provider-job generation. + pub job_generation: u64, + /// Expected provider name. + pub provider: String, + /// Expected provider-issued job identity. + pub provider_job_id: String, + /// Cancellation lifecycle result. + pub state: ExecutionExternalJobState, + /// Next sparse reconciliation time when cancellation remains unresolved. + pub next_reconcile_at: Option>, + /// Structured uncertainty evidence for an unknown outcome. + pub error: Option, +} + +/// Result of settling one provider cancellation response. +#[derive(Clone, Debug, PartialEq)] +pub enum ExecutionExternalJobCancellationOutcome { + /// Cancellation state advanced for the exact current generation. + Applied(Box), + /// Generation or provider identity no longer names the current job. + StaleGeneration, + /// A definitive provider outcome already fenced cancellation settlement. + AlreadyTerminal, + /// No visible job has the supplied MOA identity. + NotFound, +} + +/// Pending-terminal request to durably cancel one bound external owner. +#[derive(Clone, Debug, PartialEq)] +pub enum ExecutionExternalJobCancellationRequestOutcome { + /// Cancellation was requested and its exact dispatch is durable. + Applied(ExecutionDispatchRecord), + /// The exact cancellation request was already durable. + Replayed(ExecutionDispatchRecord), + /// The intent is still unbound and must be resolved by start recovery first. + UnboundPendingRecovery, + /// Provider work is already terminal. + AlreadyTerminal, + /// No matching external job exists. + NotFound, + /// Stable job identity no longer belongs to the expected owner generation. + Stale, +} + +impl ExecutionRepository { + /// Loads one visible asynchronous provider job by its stable MOA identity. + pub async fn load_external_job( + &self, + scope: ExecutionScope, + external_job_uid: Uuid, + ) -> Result> { + let mut conn = scope.begin(&self.pool).await?; + let row = + sqlx::query("SELECT * FROM moa.execution_external_job WHERE external_job_uid = $1") + .bind(external_job_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let record = row.as_ref().map(external_job_from_row).transpose()?; + conn.commit().await.map_err(storage_error)?; + Ok(record) + } + + /// Reserves one exact unbound external-job intent before provider dispatch. + pub async fn reserve_external_job_intent( + &self, + scope: ExecutionScope, + config: &ExecutionConfig, + intent: NewExecutionExternalJobIntent, + ) -> Result { + let mut conn = scope.begin(&self.pool).await?; + prelock_capacity_dimensions_in_tx( + conn.as_mut(), + config, + intent.tenant_id, + &[ + ExecutionCapacityDimension::ScheduledTriggers, + ExecutionCapacityDimension::ExternalJobs, + ], + ) + .await?; + let record = reserve_external_job_intent_in_conn(conn.as_mut(), config, &intent).await?; + conn.commit().await.map_err(storage_error)?; + Ok(record) + } + + /// Binds one live pre-reserved intent to the provider's asynchronous response. + pub async fn bind_external_job( + &self, + scope: ExecutionScope, + config: &ExecutionConfig, + binding: ExecutionExternalJobBinding, + ) -> Result { + let mut conn = scope.begin(&self.pool).await?; + if let Some(tenant_id) = sqlx::query_scalar::<_, Uuid>( + "SELECT tenant_id FROM moa.execution_external_job WHERE external_job_uid=$1", + ) + .bind(binding.external_job_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + { + prelock_existing_capacity_dimensions_in_tx( + conn.as_mut(), + TenantId(tenant_id), + &[ + ExecutionCapacityDimension::ScheduledTriggers, + ExecutionCapacityDimension::ExternalJobs, + ], + ) + .await?; + } + let record = bind_external_job_in_conn(conn.as_mut(), config, &binding).await?; + conn.commit().await.map_err(storage_error)?; + Ok(record) + } + + /// Releases one exact unbound intent when no provider work was dispatched. + pub async fn release_external_job_intent( + &self, + scope: ExecutionScope, + intent: NewExecutionExternalJobIntent, + ) -> Result { + let mut conn = scope.begin(&self.pool).await?; + let capacity_owner_tenant = sqlx::query_scalar::<_, Uuid>( + "SELECT tenant_id FROM moa.execution_external_job WHERE external_job_uid=$1 \ + UNION ALL SELECT tenant_id FROM moa.execution_capacity_reservation \ + WHERE reservation_uid=$2 AND released_at IS NULL LIMIT 1", + ) + .bind(intent.external_job_uid) + .bind(execution_capacity_reservation_uid( + ExecutionCapacityDimension::ExternalJobs, + intent.external_job_uid, + None, + )) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if let Some(tenant_id) = capacity_owner_tenant { + prelock_existing_capacity_dimensions_in_tx( + conn.as_mut(), + TenantId(tenant_id), + &[ + ExecutionCapacityDimension::ScheduledTriggers, + ExecutionCapacityDimension::ExternalJobs, + ], + ) + .await?; + } + let outcome = release_external_job_intent_in_conn(conn.as_mut(), &intent).await?; + conn.commit().await.map_err(storage_error)?; + Ok(outcome) + } + + /// Commits a provider `NotStarted` recovery and adopts its exact owner in one transaction. + pub async fn recover_external_job_start_not_started( + &self, + request: &ExecutionExternalJobStartRecoveryRequest, + recovered_at: DateTime, + ) -> Result { + let intent = external_job_intent_from_recovery_request(request); + let mut conn = ExecutionScope::ControlPlane.begin(&self.pool).await?; + prelock_existing_capacity_dimensions_in_tx( + conn.as_mut(), + request.tenant_id, + &[ + ExecutionCapacityDimension::ActiveTasks, + ExecutionCapacityDimension::ScheduledTriggers, + ExecutionCapacityDimension::ExternalJobs, + ], + ) + .await?; + match release_external_job_intent_in_conn(conn.as_mut(), &intent).await? { + ExecutionExternalJobIntentReleaseOutcome::Released + | ExecutionExternalJobIntentReleaseOutcome::AlreadyReleased => {} + ExecutionExternalJobIntentReleaseOutcome::Stale + | ExecutionExternalJobIntentReleaseOutcome::AlreadyBound => { + conn.commit().await.map_err(storage_error)?; + return Ok(ExecutionExternalJobStartRecoveryAdoptionOutcome::Stale); + } + } + let outcome = match intent.owner { + ExecutionExternalJobOwner::Task { .. } => { + let outcome = ExecutionRepository::requeue_task_external_start_not_started_in_conn( + &mut conn, + &intent, + recovered_at, + ) + .await?; + map_task_not_started_recovery(outcome) + } + ExecutionExternalJobOwner::Compensation { .. } => { + let outcome = begin_compensation_external_not_started_release_in_conn( + &mut conn, + &intent, + recovered_at, + ) + .await?; + map_compensation_not_started_recovery(outcome) + } + }; + if recovery_adoption_committed(&outcome) { + conn.commit().await.map_err(storage_error)?; + } else { + conn.rollback().await.map_err(storage_error)?; + } + Ok(outcome) + } + + /// Binds a recovered provider start and adopts its exact owner in one transaction. + pub async fn recover_external_job_start_started( + &self, + config: &ExecutionConfig, + request: &ExecutionExternalJobStartRecoveryRequest, + binding: ExecutionExternalJobBinding, + recovered_at: DateTime, + ) -> Result { + if binding.external_job_uid != request.external_job_uid + || binding.job_generation != request.job_generation + || binding.tenant_id != request.tenant_id + || binding.run_uid != request.run_uid + || binding.owner != external_job_owner_from_recovery_request(request) + { + return Err(Error::InvalidRepositoryInput { + message: "recovered external-job binding lost its trigger owner fences".to_string(), + }); + } + let mut conn = ExecutionScope::ControlPlane.begin(&self.pool).await?; + prelock_existing_capacity_dimensions_in_tx( + conn.as_mut(), + request.tenant_id, + &[ + ExecutionCapacityDimension::ActiveTasks, + ExecutionCapacityDimension::ScheduledTriggers, + ExecutionCapacityDimension::ExternalJobs, + ], + ) + .await?; + // Expiry fences the unresolved provider start, not a provider job that + // recover_start has now proven exists. The bound job keeps this receipt + // until its normal terminal or cancellation settlement releases it. + sqlx::query( + "UPDATE moa.execution_capacity_reservation SET expires_at=NULL, updated_at=NOW() \ + WHERE tenant_id=$1 AND external_job_uid=$2 \ + AND resource_dimension='external_jobs' AND state='reserved' \ + AND released_at IS NULL", + ) + .bind(request.tenant_id.0) + .bind(request.external_job_uid) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let job = bind_external_job_in_conn(conn.as_mut(), config, &binding).await?; + let outcome = match job.owner { + ExecutionExternalJobOwner::Task { .. } => { + let outcome = ExecutionRepository::adopt_recovered_task_external_job_in_conn( + &mut conn, + &job, + recovered_at, + ) + .await?; + map_task_started_recovery(outcome) + } + ExecutionExternalJobOwner::Compensation { .. } => { + let outcome = begin_recovered_compensation_external_release_in_conn( + &mut conn, + &job, + recovered_at, + ) + .await?; + map_compensation_started_recovery(outcome) + } + }; + if !recovery_adoption_committed(&outcome) { + let _ = request_external_job_cancellation_in_conn( + &mut conn, + config, + job.external_job_uid, + job.owner, + recovered_at, + ) + .await?; + } + conn.commit().await.map_err(storage_error)?; + Ok(outcome) + } + + /// Lists a bounded page of expired intents that require provider start recovery. + /// + /// This method never deletes an intent: the provider may have started work + /// before a crash. The caller may release only after `recover_start` proves + /// `NotStarted`; `Started` must bind and `Unknown` must remain durable. + pub async fn list_expired_external_job_intents( + &self, + scope: ExecutionScope, + batch_size: u32, + ) -> Result> { + if batch_size == 0 || batch_size > MAX_RECONCILE_BATCH_SIZE { + return Err(Error::InvalidRepositoryInput { + message: format!( + "external job intent reclaim batch must be 1..={MAX_RECONCILE_BATCH_SIZE}" + ), + }); + } + let mut conn = scope.begin(&self.pool).await?; + let rows = sqlx::query( + r#" + SELECT job.*, capacity.expires_at AS capacity_expires_at + FROM moa.execution_external_job AS job + JOIN moa.execution_capacity_reservation AS capacity + ON capacity.tenant_id=job.tenant_id + AND capacity.external_job_uid=job.external_job_uid + AND capacity.resource_dimension='external_jobs' + AND capacity.released_at IS NULL + WHERE job.state='unbound' AND capacity.expires_at <= now() + ORDER BY capacity.expires_at, job.tenant_id, job.external_job_uid + FOR UPDATE OF job, capacity SKIP LOCKED + LIMIT $1 + "#, + ) + .bind(i64::from(batch_size)) + .fetch_all(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let mut intents = Vec::with_capacity(rows.len()); + for row in rows { + let record = external_job_from_row(&row)?; + let expires_at = row + .try_get::>, _>("capacity_expires_at") + .map_err(super::row_error)? + .ok_or_else(|| Error::InvalidRepositoryData { + message: "unbound external job capacity receipt has no expiry".to_string(), + })?; + intents.push(NewExecutionExternalJobIntent { + external_job_uid: record.external_job_uid, + tenant_id: record.tenant_id, + run_uid: record.run_uid, + owner: record.owner, + job_generation: record.job_generation, + provider: record.declared_provider, + idempotency_key: record.idempotency_key, + expires_at, + }); + } + conn.commit().await.map_err(storage_error)?; + Ok(intents) + } + + /// Atomically applies one authenticated callback and wakes its live run controller. + pub async fn apply_external_job_callback_and_activate( + &self, + scope: ExecutionScope, + config: &ExecutionConfig, + callback: ExecutionExternalJobCallback, + ) -> Result { + let mut conn = scope.begin(&self.pool).await?; + if let Some(tenant_id) = sqlx::query_scalar::<_, Uuid>( + "SELECT tenant_id FROM moa.execution_external_job WHERE external_job_uid=$1", + ) + .bind(callback.external_job_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + { + prelock_capacity_dimensions_in_tx( + conn.as_mut(), + config, + TenantId(tenant_id), + &[ + ExecutionCapacityDimension::ActiveRuns, + ExecutionCapacityDimension::ParkedRuns, + ExecutionCapacityDimension::ScheduledTriggers, + ExecutionCapacityDimension::ExternalJobs, + ], + ) + .await?; + } + let outcome = apply_external_job_callback_in_conn(conn.as_mut(), &callback).await?; + let activation = if let ExecutionExternalJobCallbackOutcome::Applied(job) = &outcome { + let deferred_release = if job.state.is_terminal() { + settle_external_job_owner_terminal_in_conn(&mut conn, job, Utc::now()).await? + } else { + false + }; + let run = sqlx::query_as::<_, (i64, String)>( + "SELECT controller_generation, status \ + FROM moa.execution_run WHERE tenant_id = $1 AND run_uid = $2", + ) + .bind(job.tenant_id.0) + .bind(job.run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + .ok_or_else(|| Error::InvalidRepositoryData { + message: "external job callback references a missing execution run".to_string(), + })?; + let run_is_terminal = matches!( + run.1.as_str(), + "completed" | "partial" | "blocked" | "unsupported" | "failed" | "cancelled" + ); + let run_is_paused = matches!(run.1.as_str(), "pause_requested" | "pausing" | "paused"); + replace_external_reconcile_trigger_in_conn( + conn.as_mut(), + config, + job, + super::to_u64(run.0, "controller generation")?, + ) + .await?; + if job.state.is_terminal() && !deferred_release && !run_is_terminal && !run_is_paused { + Some( + enqueue_run_activation_in_conn( + conn.as_mut(), + job.tenant_id, + job.run_uid, + super::to_u64(run.0, "controller generation")?, + Utc::now(), + json!({ + "source": "external_job_callback", + "external_job_uid": job.external_job_uid, + "provider_event_id": callback.provider_event_id, + }), + ) + .await?, + ) + } else { + None + } + } else { + None + }; + conn.commit().await.map_err(storage_error)?; + Ok(ExecutionExternalJobCallbackWrite { + outcome, + activation, + }) + } + + /// Settles one provider cancellation response under its exact job generation. + pub async fn settle_external_job_cancellation( + &self, + scope: ExecutionScope, + config: &ExecutionConfig, + cancellation: ExecutionExternalJobCancellation, + ) -> Result { + let mut conn = scope.begin(&self.pool).await?; + if let Some(tenant_id) = sqlx::query_scalar::<_, Uuid>( + "SELECT tenant_id FROM moa.execution_external_job WHERE external_job_uid=$1", + ) + .bind(cancellation.external_job_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + { + prelock_capacity_dimensions_in_tx( + conn.as_mut(), + config, + TenantId(tenant_id), + &[ + ExecutionCapacityDimension::ActiveRuns, + ExecutionCapacityDimension::ParkedRuns, + ExecutionCapacityDimension::ScheduledTriggers, + ExecutionCapacityDimension::ExternalJobs, + ], + ) + .await?; + } + let outcome = + settle_external_job_cancellation_in_conn(conn.as_mut(), &cancellation).await?; + if let ExecutionExternalJobCancellationOutcome::Applied(job) = &outcome { + let (controller_generation, status) = sqlx::query_as::<_, (i64, String)>( + "SELECT controller_generation,status FROM moa.execution_run \ + WHERE tenant_id=$1 AND run_uid=$2", + ) + .bind(job.tenant_id.0) + .bind(job.run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + replace_external_reconcile_trigger_in_conn( + conn.as_mut(), + config, + job, + super::to_u64(controller_generation, "controller generation")?, + ) + .await?; + let deferred_release = if job.state.is_terminal() { + settle_external_job_owner_terminal_in_conn(&mut conn, job, Utc::now()).await? + } else { + false + }; + if job.state.is_terminal() + && !deferred_release + && !matches!( + status.as_str(), + "pause_requested" + | "pausing" + | "paused" + | "completed" + | "partial" + | "blocked" + | "unsupported" + | "failed" + | "cancelled" + ) + { + enqueue_run_activation_in_conn( + conn.as_mut(), + job.tenant_id, + job.run_uid, + super::to_u64(controller_generation, "controller generation")?, + Utc::now(), + json!({ + "source": "external_job_cancellation", + "external_job_uid": job.external_job_uid, + }), + ) + .await?; + } + } + conn.commit().await.map_err(storage_error)?; + Ok(outcome) + } + + /// Lists a bounded indexed page of active provider jobs due for reconciliation. + pub async fn list_due_external_jobs( + &self, + scope: ExecutionScope, + batch_size: u32, + ) -> Result> { + if batch_size == 0 || batch_size > MAX_RECONCILE_BATCH_SIZE { + return Err(Error::InvalidRepositoryInput { + message: format!( + "execution external-job reconciliation batch must be 1..={MAX_RECONCILE_BATCH_SIZE}" + ), + }); + } + let mut conn = scope.begin(&self.pool).await?; + let rows = sqlx::query( + r#" + SELECT * + FROM moa.execution_external_job + WHERE state IN ('starting', 'running', 'waiting_reconcile', 'cancel_requested') + AND next_reconcile_at IS NOT NULL + AND next_reconcile_at <= now() + ORDER BY next_reconcile_at, tenant_id, external_job_uid + LIMIT $1 + "#, + ) + .bind(i64::from(batch_size)) + .fetch_all(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let records = rows + .iter() + .map(external_job_from_row) + .collect::>()?; + conn.commit().await.map_err(storage_error)?; + Ok(records) + } +} + +async fn replace_external_reconcile_trigger_in_conn( + conn: &mut PgConnection, + config: &ExecutionConfig, + job: &ExecutionExternalJobRecord, + controller_generation: u64, +) -> Result<()> { + let ( + task_id, + attempt_generation, + compensation_id, + compensation_generation, + compensation_attempt_generation, + ) = external_job_trigger_owner(job.owner); + let existing = sqlx::query_as::< + _, + ( + Uuid, + Option, + Option, + Option, + Option, + DateTime, + Value, + ), + >( + "SELECT trigger_uid, controller_generation, attempt_generation, \ + compensation_generation, compensation_attempt_generation, due_at, payload \ + FROM moa.execution_trigger WHERE tenant_id=$1 AND run_uid=$2 \ + AND task_id IS NOT DISTINCT FROM $3 \ + AND compensation_id IS NOT DISTINCT FROM $4 \ + AND trigger_kind='external_reconcile' AND state IN ('pending','dispatching') \ + FOR UPDATE", + ) + .bind(job.tenant_id.0) + .bind(job.run_uid) + .bind(task_id) + .bind(compensation_id) + .fetch_optional(&mut *conn) + .await + .map_err(sqlx_error)?; + if let Some(( + trigger_uid, + trigger_generation, + existing_attempt_generation, + existing_compensation_generation, + existing_compensation_attempt_generation, + due_at, + payload, + )) = existing + { + let exact_replay = job.next_reconcile_at == Some(due_at) + && payload + .get("external_job_uid") + .and_then(Value::as_str) + .and_then(|value| Uuid::parse_str(value).ok()) + == Some(job.external_job_uid) + && payload.get("job_generation").and_then(Value::as_u64) == Some(job.job_generation); + if exact_replay { + return Ok(()); + } + supersede_trigger_in_conn( + conn, + trigger_uid, + ExecutionTriggerKind::ExternalReconcile, + trigger_generation + .map(|value| super::to_u64(value, "controller generation")) + .transpose()?, + existing_attempt_generation + .map(|value| super::to_u64(value, "attempt generation")) + .transpose()?, + existing_compensation_generation + .map(|value| super::to_u64(value, "compensation generation")) + .transpose()?, + existing_compensation_attempt_generation + .map(|value| super::to_u64(value, "compensation attempt generation")) + .transpose()?, + ) + .await?; + } + let Some(next_reconcile_at) = job.next_reconcile_at else { + return Ok(()); + }; + if job.state.is_terminal() { + return Err(Error::InvalidRepositoryData { + message: "terminal external job retained a reconciliation deadline".to_string(), + }); + } + let identity = format!( + "{}:{}:{}", + job.external_job_uid, + job.job_generation, + job.updated_at.timestamp_micros() + ); + create_trigger_with_dispatch_in_conn( + conn, + config, + &NewExecutionTrigger { + trigger_uid: Uuid::new_v5(&EXTERNAL_RECONCILE_TRIGGER_NAMESPACE, identity.as_bytes()), + tenant_id: job.tenant_id, + run_uid: Some(job.run_uid), + task_id, + compensation_id, + schedule_uid: None, + schedule_incarnation: None, + kind: ExecutionTriggerKind::ExternalReconcile, + controller_generation: Some(controller_generation), + attempt_generation, + compensation_generation, + compensation_attempt_generation, + occurrence_sequence: None, + due_at: next_reconcile_at, + payload: json!({ + "external_job_uid": job.external_job_uid, + "job_generation": job.job_generation, + }), + }, + ) + .await?; + Ok(()) +} + +fn external_start_recovery_trigger_uid(external_job_uid: Uuid, job_generation: u64) -> Uuid { + Uuid::new_v5( + &EXTERNAL_START_RECOVERY_TRIGGER_NAMESPACE, + format!("{external_job_uid}:{job_generation}").as_bytes(), + ) +} + +async fn create_external_start_recovery_trigger_in_conn( + conn: &mut PgConnection, + config: &ExecutionConfig, + intent: &NewExecutionExternalJobIntent, + controller_generation: u64, +) -> Result<()> { + let ( + task_id, + attempt_generation, + compensation_id, + compensation_generation, + compensation_attempt_generation, + ) = external_job_trigger_owner(intent.owner); + create_trigger_with_dispatch_in_conn( + conn, + config, + &NewExecutionTrigger { + trigger_uid: external_start_recovery_trigger_uid( + intent.external_job_uid, + intent.job_generation, + ), + tenant_id: intent.tenant_id, + run_uid: Some(intent.run_uid), + task_id, + compensation_id, + schedule_uid: None, + schedule_incarnation: None, + kind: ExecutionTriggerKind::ExternalStartRecovery, + controller_generation: Some(controller_generation), + attempt_generation, + compensation_generation, + compensation_attempt_generation, + occurrence_sequence: None, + due_at: intent.expires_at, + payload: json!({ + "external_job_uid": intent.external_job_uid, + "job_generation": intent.job_generation, + "declared_provider": intent.provider, + "idempotency_key": intent.idempotency_key, + }), + }, + ) + .await?; + Ok(()) +} + +async fn settle_external_start_recovery_trigger_in_conn( + conn: &mut PgConnection, + job: &ExecutionExternalJobRecord, +) -> Result<()> { + let trigger_uid = external_start_recovery_trigger_uid(job.external_job_uid, job.job_generation); + let row = sqlx::query_as::<_, (Option, Option, Option, Option)>( + "SELECT controller_generation,attempt_generation,compensation_generation, \ + compensation_attempt_generation \ + FROM moa.execution_trigger WHERE trigger_uid=$1", + ) + .bind(trigger_uid) + .fetch_optional(&mut *conn) + .await + .map_err(sqlx_error)?; + let Some((controller, attempt, compensation, compensation_attempt)) = row else { + return Err(Error::InvalidRepositoryData { + message: "external job intent is missing its start-recovery trigger".to_string(), + }); + }; + supersede_trigger_in_conn( + conn, + trigger_uid, + ExecutionTriggerKind::ExternalStartRecovery, + controller + .map(|value| super::to_u64(value, "controller generation")) + .transpose()?, + attempt + .map(|value| super::to_u64(value, "attempt generation")) + .transpose()?, + compensation + .map(|value| super::to_u64(value, "compensation generation")) + .transpose()?, + compensation_attempt + .map(|value| super::to_u64(value, "compensation attempt generation")) + .transpose()?, + ) + .await?; + Ok(()) +} + +/// Reserves an unbound external-job intent without committing the caller transaction. +pub async fn reserve_external_job_intent_in_conn( + conn: &mut PgConnection, + config: &ExecutionConfig, + intent: &NewExecutionExternalJobIntent, +) -> Result { + validate_external_job_intent(intent, true)?; + lock_external_job_intent_owner_in_conn(conn, intent).await?; + let ( + task_id, + attempt_generation, + compensation_id, + compensation_generation, + compensation_attempt_generation, + ) = external_job_owner_columns(intent.owner)?; + let inserted = sqlx::query( + r#" + INSERT INTO moa.execution_external_job ( + external_job_uid, tenant_id, run_uid, task_id, attempt_generation, + compensation_id, compensation_generation, compensation_attempt_generation, + job_generation, declared_provider, idempotency_key, state, cancel_supported + ) + SELECT $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, 'unbound', FALSE + WHERE ( + ($4 IS NOT NULL AND EXISTS ( + SELECT 1 FROM moa.execution_task AS task + WHERE task.task_id = $4 AND task.run_uid = $3 AND task.tenant_id = $2 + AND task.attempt_generation = $5 + AND ( + (task.status='running' AND task.attempt_state IN ('running','cancelling')) + OR (task.status='waiting_review' AND task.attempt_state='waiting') + ) + )) + OR + ($6 IS NOT NULL AND EXISTS ( + SELECT 1 FROM moa.execution_compensation AS compensation + WHERE compensation.compensation_id = $6 + AND compensation.run_uid = $3 AND compensation.tenant_id = $2 + AND compensation.generation = $7 + AND compensation.attempt_generation = $8 + AND compensation.status = 'running' + AND compensation.attempt_state IN ('running','cancelling','waiting_review') + )) + ) + ON CONFLICT (external_job_uid) DO NOTHING + RETURNING * + "#, + ) + .bind(intent.external_job_uid) + .bind(intent.tenant_id.0) + .bind(intent.run_uid) + .bind(task_id) + .bind(attempt_generation) + .bind(compensation_id) + .bind(compensation_generation) + .bind(compensation_attempt_generation) + .bind(to_i64(intent.job_generation, "external job generation")?) + .bind(&intent.provider) + .bind(&intent.idempotency_key) + .fetch_optional(&mut *conn) + .await + .map_err(sqlx_error)?; + let row = match inserted { + Some(row) => row, + None => sqlx::query("SELECT * FROM moa.execution_external_job WHERE external_job_uid = $1") + .bind(intent.external_job_uid) + .fetch_optional(&mut *conn) + .await + .map_err(sqlx_error)? + .ok_or_else(|| Error::InvalidRepositoryInput { + message: "external job owner generation is not current".to_string(), + })?, + }; + let record = external_job_from_row(&row)?; + if !external_job_matches_intent(&record, intent) { + return Err(Error::InvalidRepositoryInput { + message: "external job intent UID is already bound to different immutable semantics" + .to_string(), + }); + } + let controller_generation = sqlx::query_scalar::<_, i64>( + "SELECT controller_generation FROM moa.execution_run \ + WHERE tenant_id = $1 AND run_uid = $2 \ + AND status NOT IN ('completed','partial','blocked','unsupported','failed','cancelled')", + ) + .bind(record.tenant_id.0) + .bind(record.run_uid) + .fetch_optional(&mut *conn) + .await + .map_err(sqlx_error)? + .ok_or_else(|| Error::InvalidRepositoryInput { + message: "external job intent run is missing or terminal".to_string(), + })?; + let controller_generation = super::to_u64(controller_generation, "controller generation")?; + let capacity = + external_job_capacity_request(&record, controller_generation, Some(intent.expires_at)); + if reserve_capacity_in_tx(conn, config, capacity).await? == CapacityReserveOutcome::Saturated { + return Err(Error::CapacitySaturated { + dimension: ExecutionCapacityDimension::ExternalJobs.as_str(), + }); + } + create_external_start_recovery_trigger_in_conn(conn, config, intent, controller_generation) + .await?; + Ok(record) +} + +/// Binds provider identity to a live intent without reserving capacity again. +pub async fn bind_external_job_in_conn( + conn: &mut PgConnection, + config: &ExecutionConfig, + binding: &ExecutionExternalJobBinding, +) -> Result { + validate_external_job_binding(binding)?; + let row = sqlx::query( + "SELECT job.*, COALESCE(capacity.expires_at > now(), TRUE) AS capacity_live, \ + now() AS observed_at \ + FROM moa.execution_external_job AS job \ + JOIN moa.execution_capacity_reservation AS capacity \ + ON capacity.tenant_id=job.tenant_id \ + AND capacity.external_job_uid=job.external_job_uid \ + AND capacity.resource_dimension='external_jobs' AND capacity.released_at IS NULL \ + WHERE job.external_job_uid=$1 FOR UPDATE OF job, capacity", + ) + .bind(binding.external_job_uid) + .fetch_optional(&mut *conn) + .await + .map_err(sqlx_error)? + .ok_or_else(|| Error::InvalidRepositoryInput { + message: "external job binding requires a live reserved intent".to_string(), + })?; + let current = external_job_from_row(&row)?; + if !external_job_matches_binding_identity(¤t, binding) { + return Err(Error::InvalidRepositoryInput { + message: "external job binding does not match its reserved intent".to_string(), + }); + } + if current.state != ExecutionExternalJobState::Unbound { + if external_job_matches_provider_result(¤t, binding) { + return Ok(current); + } + return Err(Error::InvalidRepositoryInput { + message: "external job intent is already bound to different provider semantics" + .to_string(), + }); + } + let capacity_live = row + .try_get::("capacity_live") + .map_err(super::row_error)?; + let observed_at = row + .try_get::, _>("observed_at") + .map_err(super::row_error)?; + let (controller_generation, run_status) = sqlx::query_as::<_, (i64, String)>( + "SELECT controller_generation,status FROM moa.execution_run \ + WHERE tenant_id=$1 AND run_uid=$2 FOR UPDATE", + ) + .bind(current.tenant_id.0) + .bind(current.run_uid) + .fetch_optional(&mut *conn) + .await + .map_err(sqlx_error)? + .ok_or_else(|| Error::InvalidRepositoryData { + message: "reserved external job intent references a missing run".to_string(), + })?; + let run_is_terminal = matches!( + run_status.as_str(), + "completed" | "partial" | "blocked" | "unsupported" | "failed" | "cancelled" + ); + let provider_mismatch = binding.provider != current.declared_provider; + let contract_violation = binding.provider_contract_violation.clone().or_else(|| { + provider_mismatch.then(|| { + format!( + "adapter returned provider `{}` for declared provider `{}`", + binding.provider, current.declared_provider + ) + }) + }); + let violation_audit = contract_violation.as_ref().map(|detail| { + json!({ + "kind": "provider_contract_mismatch", + "observed_at": observed_at.to_rfc3339(), + "detail": detail, + }) + }); + let requires_recovery = + !capacity_live || run_is_terminal || provider_mismatch || contract_violation.is_some(); + let state = if requires_recovery { + ExecutionExternalJobState::CancelRequested + } else { + binding.state + }; + let next_reconcile_at = if requires_recovery { + Some(observed_at) + } else { + binding.next_reconcile_at + }; + settle_external_start_recovery_trigger_in_conn(conn, ¤t).await?; + let row = sqlx::query( + r#" + UPDATE moa.execution_external_job + SET provider=$2, provider_job_id=$3, callback_auth_reference=$4, + state=$5, progress_phase=$6, cancel_supported=$7, + next_reconcile_at=$8, provider_contract_violation=$9, updated_at=now() + WHERE external_job_uid=$1 AND state='unbound' AND job_generation=$10 + RETURNING * + "#, + ) + .bind(binding.external_job_uid) + .bind(¤t.declared_provider) + .bind(&binding.provider_job_id) + .bind(&binding.callback_auth_reference) + .bind(state.as_str()) + .bind(&binding.progress_phase) + .bind(binding.cancel_supported) + .bind(next_reconcile_at) + .bind(&violation_audit) + .bind(to_i64(binding.job_generation, "external job generation")?) + .fetch_one(&mut *conn) + .await + .map_err(sqlx_error)?; + let record = external_job_from_row(&row)?; + let controller_generation = super::to_u64(controller_generation, "controller generation")?; + replace_external_reconcile_trigger_in_conn(conn, config, &record, controller_generation) + .await?; + if requires_recovery { + enqueue_external_cancel_in_conn(conn, &record, controller_generation, observed_at).await?; + } + Ok(record) +} + +/// Releases an exact unbound intent and its capacity receipt transactionally. +pub async fn release_external_job_intent_in_conn( + conn: &mut PgConnection, + intent: &NewExecutionExternalJobIntent, +) -> Result { + validate_external_job_intent(intent, false)?; + let row = sqlx::query( + "SELECT * FROM moa.execution_external_job WHERE external_job_uid=$1 FOR UPDATE", + ) + .bind(intent.external_job_uid) + .fetch_optional(&mut *conn) + .await + .map_err(sqlx_error)?; + let Some(row) = row else { + let receipt_exists = sqlx::query_scalar::<_, bool>( + "SELECT TRUE FROM moa.execution_capacity_reservation \ + WHERE reservation_uid=$1 AND released_at IS NULL", + ) + .bind(execution_capacity_reservation_uid( + ExecutionCapacityDimension::ExternalJobs, + intent.external_job_uid, + None, + )) + .fetch_optional(&mut *conn) + .await + .map_err(sqlx_error)? + .unwrap_or(false); + return Ok(if receipt_exists { + ExecutionExternalJobIntentReleaseOutcome::Stale + } else { + ExecutionExternalJobIntentReleaseOutcome::AlreadyReleased + }); + }; + let record = external_job_from_row(&row)?; + if !external_job_matches_intent(&record, intent) { + return Ok(ExecutionExternalJobIntentReleaseOutcome::Stale); + } + if record.state != ExecutionExternalJobState::Unbound { + return Ok(ExecutionExternalJobIntentReleaseOutcome::AlreadyBound); + } + settle_external_start_recovery_trigger_in_conn(conn, &record).await?; + release_external_job_capacity_in_conn(conn, &record).await?; + let deleted = sqlx::query( + "DELETE FROM moa.execution_external_job \ + WHERE external_job_uid=$1 AND job_generation=$2 AND state='unbound'", + ) + .bind(intent.external_job_uid) + .bind(to_i64(intent.job_generation, "external job generation")?) + .execute(&mut *conn) + .await + .map_err(sqlx_error)?; + if deleted.rows_affected() != 1 { + return Err(Error::InvalidRepositoryData { + message: "external job intent release lost its exact state fence".to_string(), + }); + } + Ok(ExecutionExternalJobIntentReleaseOutcome::Released) +} + +/// Loads and locks one external job inside a caller-owned transaction. +pub(super) async fn load_external_job_for_update_in_conn( + conn: &mut PgConnection, + external_job_uid: Uuid, +) -> Result> { + let row = sqlx::query( + "SELECT * FROM moa.execution_external_job WHERE external_job_uid=$1 FOR UPDATE", + ) + .bind(external_job_uid) + .fetch_optional(&mut *conn) + .await + .map_err(sqlx_error)?; + row.as_ref().map(external_job_from_row).transpose() +} + +/// Applies a callback without committing the caller-owned transaction. +pub async fn apply_external_job_callback_in_conn( + conn: &mut PgConnection, + callback: &ExecutionExternalJobCallback, +) -> Result { + validate_callback(callback)?; + let row = sqlx::query( + "SELECT * FROM moa.execution_external_job WHERE external_job_uid = $1 FOR UPDATE", + ) + .bind(callback.external_job_uid) + .fetch_optional(&mut *conn) + .await + .map_err(sqlx_error)?; + let Some(row) = row else { + return Ok(ExecutionExternalJobCallbackOutcome::NotFound); + }; + let record = external_job_from_row(&row)?; + if record.state == ExecutionExternalJobState::Unbound + || record.job_generation != callback.job_generation + || record.provider.as_deref() != Some(callback.provider.as_str()) + || record.provider_job_id.as_deref() != Some(callback.provider_job_id.as_str()) + { + return Ok(ExecutionExternalJobCallbackOutcome::StaleGeneration); + } + if record.last_provider_event_id.as_deref() == Some(callback.provider_event_id.as_str()) { + return Ok(ExecutionExternalJobCallbackOutcome::Duplicate); + } + if record.state.is_terminal() { + return Ok(ExecutionExternalJobCallbackOutcome::AlreadyTerminal); + } + let receipt_inserted = sqlx::query_scalar::<_, bool>( + r#" + INSERT INTO moa.execution_external_job_callback_receipt ( + tenant_id, external_job_uid, provider, provider_event_id, job_generation + ) VALUES ($1, $2, $3, $4, $5) + ON CONFLICT DO NOTHING + RETURNING TRUE + "#, + ) + .bind(record.tenant_id.0) + .bind(record.external_job_uid) + .bind(&callback.provider) + .bind(&callback.provider_event_id) + .bind(to_i64(callback.job_generation, "external job generation")?) + .fetch_optional(&mut *conn) + .await + .map_err(sqlx_error)? + .is_some(); + if !receipt_inserted { + return Ok(ExecutionExternalJobCallbackOutcome::Duplicate); + } + let row = match &callback.update { + ExecutionExternalJobCallbackUpdate::Progress { + state, + progress_phase, + next_reconcile_at, + } => sqlx::query( + r#" + UPDATE moa.execution_external_job + SET state = $2, progress_phase = $3, next_reconcile_at = $4, + last_provider_event_id = $5, updated_at = now() + WHERE external_job_uid = $1 AND job_generation = $6 + AND state IN ('starting', 'running', 'waiting_reconcile', 'cancel_requested') + RETURNING * + "#, + ) + .bind(callback.external_job_uid) + .bind(state.as_str()) + .bind(progress_phase) + .bind(next_reconcile_at) + .bind(&callback.provider_event_id) + .bind(to_i64(callback.job_generation, "external job generation")?) + .fetch_one(&mut *conn) + .await + .map_err(sqlx_error)?, + ExecutionExternalJobCallbackUpdate::Terminal { + state, + progress_phase, + output, + error, + } => sqlx::query( + r#" + UPDATE moa.execution_external_job + SET state = $2, progress_phase = $3, next_reconcile_at = NULL, + last_provider_event_id = $4, output = $5, error = $6, + completed_at = now(), updated_at = now() + WHERE external_job_uid = $1 AND job_generation = $7 + AND state IN ('starting', 'running', 'waiting_reconcile', 'cancel_requested') + RETURNING * + "#, + ) + .bind(callback.external_job_uid) + .bind(state.as_str()) + .bind(progress_phase) + .bind(&callback.provider_event_id) + .bind(output) + .bind(error) + .bind(to_i64(callback.job_generation, "external job generation")?) + .fetch_one(&mut *conn) + .await + .map_err(sqlx_error)?, + }; + let record = external_job_from_row(&row)?; + if record.state.is_terminal() { + release_external_job_capacity_in_conn(conn, &record).await?; + } + Ok(ExecutionExternalJobCallbackOutcome::Applied(Box::new( + record, + ))) +} + +/// Settles provider cancellation without committing the caller-owned transaction. +pub async fn settle_external_job_cancellation_in_conn( + conn: &mut PgConnection, + cancellation: &ExecutionExternalJobCancellation, +) -> Result { + validate_cancellation(cancellation)?; + let row = sqlx::query( + "SELECT * FROM moa.execution_external_job WHERE external_job_uid = $1 FOR UPDATE", + ) + .bind(cancellation.external_job_uid) + .fetch_optional(&mut *conn) + .await + .map_err(sqlx_error)?; + let Some(row) = row else { + return Ok(ExecutionExternalJobCancellationOutcome::NotFound); + }; + let record = external_job_from_row(&row)?; + if record.state == ExecutionExternalJobState::Unbound + || record.job_generation != cancellation.job_generation + || record.provider.as_deref() != Some(cancellation.provider.as_str()) + || record.provider_job_id.as_deref() != Some(cancellation.provider_job_id.as_str()) + { + return Ok(ExecutionExternalJobCancellationOutcome::StaleGeneration); + } + if record.state.is_terminal() { + return Ok(ExecutionExternalJobCancellationOutcome::AlreadyTerminal); + } + let completed_at = cancellation.state.is_terminal().then(Utc::now); + let row = sqlx::query( + r#" + UPDATE moa.execution_external_job + SET state = $2, next_reconcile_at = $3, error = $4, completed_at = $5, + updated_at = now() + WHERE external_job_uid = $1 AND job_generation = $6 + AND state IN ('starting', 'running', 'waiting_reconcile', 'cancel_requested') + RETURNING * + "#, + ) + .bind(cancellation.external_job_uid) + .bind(cancellation.state.as_str()) + .bind(cancellation.next_reconcile_at) + .bind(&cancellation.error) + .bind(completed_at) + .bind(to_i64( + cancellation.job_generation, + "external job generation", + )?) + .fetch_one(&mut *conn) + .await + .map_err(sqlx_error)?; + let record = external_job_from_row(&row)?; + if record.state.is_terminal() { + release_external_job_capacity_in_conn(conn, &record).await?; + } + Ok(ExecutionExternalJobCancellationOutcome::Applied(Box::new( + record, + ))) +} + +async fn settle_external_job_owner_terminal_in_conn( + conn: &mut super::ScopedConn<'_>, + job: &ExecutionExternalJobRecord, + settled_at: DateTime, +) -> Result { + match job.owner { + ExecutionExternalJobOwner::Task { .. } => { + match settle_task_external_job_terminal_in_conn(conn, job, settled_at).await? { + ExternalJobTaskSettlementOutcome::Applied(_) + | ExternalJobTaskSettlementOutcome::Replayed(_) => Ok(false), + ExternalJobTaskSettlementOutcome::DeferredRelease(_) => Ok(true), + ExternalJobTaskSettlementOutcome::Stale + | ExternalJobTaskSettlementOutcome::NotFound => Err(Error::InvalidRepositoryData { + message: "terminal external job lost its exact task-attempt owner fence" + .to_string(), + }), + } + } + ExecutionExternalJobOwner::Compensation { .. } => { + match settle_compensation_external_job_terminal_in_conn(conn, job, settled_at).await? { + CompensationExternalJobSettlementOutcome::Applied(_) + | CompensationExternalJobSettlementOutcome::Replayed(_) => Ok(false), + CompensationExternalJobSettlementOutcome::DeferredRelease(_) => Ok(true), + CompensationExternalJobSettlementOutcome::Stale + | CompensationExternalJobSettlementOutcome::NotFound => { + Err(Error::InvalidRepositoryData { + message: + "terminal external job lost its exact compensation-attempt owner fence" + .to_string(), + }) + } + } + } + } +} + +async fn enqueue_external_cancel_in_conn( + conn: &mut PgConnection, + job: &ExecutionExternalJobRecord, + controller_generation: u64, + not_before_at: DateTime, +) -> Result { + let provider = job + .provider + .as_ref() + .ok_or_else(|| Error::InvalidRepositoryData { + message: "bound external job is missing provider identity".to_string(), + })?; + let provider_job_id = + job.provider_job_id + .as_ref() + .ok_or_else(|| Error::InvalidRepositoryData { + message: "bound external job is missing provider job identity".to_string(), + })?; + let (task_id, attempt_generation, compensation_id, compensation_generation, comp_attempt) = + external_job_trigger_owner(job.owner); + let payload = ExecutionExternalJobCancelRequest { + tenant_id: job.tenant_id, + external_job_uid: job.external_job_uid, + job_generation: job.job_generation, + provider: provider.clone(), + provider_job_id: provider_job_id.clone(), + idempotency_key: job.idempotency_key.clone(), + }; + enqueue_dispatch_in_conn( + conn, + &NewExecutionDispatch { + dispatch_uid: Uuid::new_v5( + &EXTERNAL_CANCEL_DISPATCH_NAMESPACE, + format!("{}:{}", job.external_job_uid, job.job_generation).as_bytes(), + ), + tenant_id: job.tenant_id, + run_uid: Some(job.run_uid), + task_id, + compensation_id, + trigger_uid: None, + external_job_uid: Some(job.external_job_uid), + kind: ExecutionDispatchKind::ExternalCancel, + controller_generation: Some(controller_generation), + wake_epoch: None, + attempt_generation, + compensation_generation, + compensation_attempt_generation: comp_attempt, + not_before_at, + payload: serde_json::to_value(payload).map_err(|error| { + Error::InvalidRepositoryInput { + message: format!("failed to encode external cancel request: {error}"), + } + })?, + }, + ) + .await +} + +/// Requests cancellation for one exact bound owner inside a caller transaction. +pub(super) async fn request_external_job_cancellation_in_conn( + conn: &mut super::ScopedConn<'_>, + config: &ExecutionConfig, + external_job_uid: Uuid, + expected_owner: ExecutionExternalJobOwner, + requested_at: DateTime, +) -> Result { + let Some(job) = load_external_job_for_update_in_conn(conn.as_mut(), external_job_uid).await? + else { + return Ok(ExecutionExternalJobCancellationRequestOutcome::NotFound); + }; + if job.owner != expected_owner { + return Ok(ExecutionExternalJobCancellationRequestOutcome::Stale); + } + if job.state == ExecutionExternalJobState::Unbound { + return Ok(ExecutionExternalJobCancellationRequestOutcome::UnboundPendingRecovery); + } + if job.state.is_terminal() { + return Ok(ExecutionExternalJobCancellationRequestOutcome::AlreadyTerminal); + } + let replayed = job.state == ExecutionExternalJobState::CancelRequested; + let row = if replayed { + job + } else { + let row = sqlx::query( + "UPDATE moa.execution_external_job SET state='cancel_requested', \ + next_reconcile_at=COALESCE(next_reconcile_at,$2),updated_at=now() \ + WHERE external_job_uid=$1 AND job_generation=$3 \ + AND state IN ('starting','running','waiting_reconcile') RETURNING *", + ) + .bind(external_job_uid) + .bind(requested_at) + .bind(to_i64(job.job_generation, "external job generation")?) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + external_job_from_row(&row)? + }; + let controller_generation = sqlx::query_scalar::<_, i64>( + "SELECT controller_generation FROM moa.execution_run \ + WHERE tenant_id=$1 AND run_uid=$2 FOR UPDATE", + ) + .bind(row.tenant_id.0) + .bind(row.run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let controller_generation = super::to_u64(controller_generation, "controller generation")?; + replace_external_reconcile_trigger_in_conn(conn.as_mut(), config, &row, controller_generation) + .await?; + let dispatch = + enqueue_external_cancel_in_conn(conn.as_mut(), &row, controller_generation, requested_at) + .await?; + Ok(if replayed { + ExecutionExternalJobCancellationRequestOutcome::Replayed(dispatch) + } else { + ExecutionExternalJobCancellationRequestOutcome::Applied(dispatch) + }) +} + +fn external_job_capacity_request( + job: &ExecutionExternalJobRecord, + controller_generation: u64, + expires_at: Option>, +) -> ExecutionCapacityRequest { + ExecutionCapacityRequest { + reservation_uid: execution_capacity_reservation_uid( + ExecutionCapacityDimension::ExternalJobs, + job.external_job_uid, + None, + ), + tenant_id: job.tenant_id, + run_uid: Some(job.run_uid), + controller_generation: Some(controller_generation), + dimension: ExecutionCapacityDimension::ExternalJobs, + owner: ExecutionCapacityOwner::ExternalJob { + external_job_uid: job.external_job_uid, + }, + expires_at, + } +} + +fn external_job_owner_from_recovery_request( + request: &ExecutionExternalJobStartRecoveryRequest, +) -> ExecutionExternalJobOwner { + match request.owner { + ExecutionExternalJobStartRecoveryOwner::Task { + task_id, + attempt_generation, + } => ExecutionExternalJobOwner::Task { + task_id, + attempt_generation, + }, + ExecutionExternalJobStartRecoveryOwner::Compensation { + compensation_id, + compensation_generation, + compensation_attempt_generation, + } => ExecutionExternalJobOwner::Compensation { + compensation_id, + compensation_generation, + compensation_attempt_generation, + }, + } +} + +fn recovery_adoption_committed(outcome: &ExecutionExternalJobStartRecoveryAdoptionOutcome) -> bool { + matches!( + outcome, + ExecutionExternalJobStartRecoveryAdoptionOutcome::Applied { .. } + | ExecutionExternalJobStartRecoveryAdoptionOutcome::Replayed { .. } + | ExecutionExternalJobStartRecoveryAdoptionOutcome::AlreadySettled + ) +} + +fn external_job_intent_from_recovery_request( + request: &ExecutionExternalJobStartRecoveryRequest, +) -> NewExecutionExternalJobIntent { + NewExecutionExternalJobIntent { + external_job_uid: request.external_job_uid, + tenant_id: request.tenant_id, + run_uid: request.run_uid, + owner: external_job_owner_from_recovery_request(request), + job_generation: request.job_generation, + provider: request.provider.clone(), + idempotency_key: request.idempotency_key.clone(), + // Expiry is not an immutable intent coordinate. Recovery authorization comes from the + // exact trigger payload and the provider's journaled NotStarted/Started disposition. + expires_at: DateTime::::MAX_UTC, + } +} + +fn map_task_not_started_recovery( + outcome: TaskExternalStartRetryOutcome, +) -> ExecutionExternalJobStartRecoveryAdoptionOutcome { + match outcome { + TaskExternalStartRetryOutcome::Applied { .. } => { + ExecutionExternalJobStartRecoveryAdoptionOutcome::Applied { + compensation_release: None, + } + } + TaskExternalStartRetryOutcome::Replayed { .. } => { + ExecutionExternalJobStartRecoveryAdoptionOutcome::Replayed { + compensation_release: None, + } + } + TaskExternalStartRetryOutcome::NotFound => { + ExecutionExternalJobStartRecoveryAdoptionOutcome::NotFound + } + TaskExternalStartRetryOutcome::Stale => { + ExecutionExternalJobStartRecoveryAdoptionOutcome::Stale + } + TaskExternalStartRetryOutcome::InvalidState => { + ExecutionExternalJobStartRecoveryAdoptionOutcome::InvalidState + } + } +} + +fn map_task_started_recovery( + outcome: TaskAttemptExternalOutcome, +) -> ExecutionExternalJobStartRecoveryAdoptionOutcome { + match outcome { + TaskAttemptExternalOutcome::Applied { .. } => { + ExecutionExternalJobStartRecoveryAdoptionOutcome::Applied { + compensation_release: None, + } + } + TaskAttemptExternalOutcome::Replayed { .. } => { + ExecutionExternalJobStartRecoveryAdoptionOutcome::Replayed { + compensation_release: None, + } + } + TaskAttemptExternalOutcome::NotFound => { + ExecutionExternalJobStartRecoveryAdoptionOutcome::NotFound + } + TaskAttemptExternalOutcome::Stale => { + ExecutionExternalJobStartRecoveryAdoptionOutcome::Stale + } + TaskAttemptExternalOutcome::InvalidState => { + ExecutionExternalJobStartRecoveryAdoptionOutcome::InvalidState + } + } +} + +fn map_compensation_not_started_recovery( + outcome: CompensationExternalNotStartedReleaseClaimOutcome, +) -> ExecutionExternalJobStartRecoveryAdoptionOutcome { + match outcome { + CompensationExternalNotStartedReleaseClaimOutcome::Applied { request, .. } => { + ExecutionExternalJobStartRecoveryAdoptionOutcome::Applied { + compensation_release: Some(Box::new(request)), + } + } + CompensationExternalNotStartedReleaseClaimOutcome::Replayed { request, .. } => { + ExecutionExternalJobStartRecoveryAdoptionOutcome::Replayed { + compensation_release: Some(Box::new(request)), + } + } + CompensationExternalNotStartedReleaseClaimOutcome::AlreadySettled => { + ExecutionExternalJobStartRecoveryAdoptionOutcome::AlreadySettled + } + CompensationExternalNotStartedReleaseClaimOutcome::NotFound => { + ExecutionExternalJobStartRecoveryAdoptionOutcome::NotFound + } + CompensationExternalNotStartedReleaseClaimOutcome::Stale => { + ExecutionExternalJobStartRecoveryAdoptionOutcome::Stale + } + CompensationExternalNotStartedReleaseClaimOutcome::InvalidState => { + ExecutionExternalJobStartRecoveryAdoptionOutcome::InvalidState + } + } +} + +fn map_compensation_started_recovery( + outcome: CompensationRecoveredExternalReleaseClaimOutcome, +) -> ExecutionExternalJobStartRecoveryAdoptionOutcome { + match outcome { + CompensationRecoveredExternalReleaseClaimOutcome::Applied { request, .. } => { + ExecutionExternalJobStartRecoveryAdoptionOutcome::Applied { + compensation_release: Some(Box::new(request)), + } + } + CompensationRecoveredExternalReleaseClaimOutcome::Replayed { request, .. } => { + ExecutionExternalJobStartRecoveryAdoptionOutcome::Replayed { + compensation_release: Some(Box::new(request)), + } + } + CompensationRecoveredExternalReleaseClaimOutcome::AlreadySettled => { + ExecutionExternalJobStartRecoveryAdoptionOutcome::AlreadySettled + } + CompensationRecoveredExternalReleaseClaimOutcome::NotFound => { + ExecutionExternalJobStartRecoveryAdoptionOutcome::NotFound + } + CompensationRecoveredExternalReleaseClaimOutcome::Stale => { + ExecutionExternalJobStartRecoveryAdoptionOutcome::Stale + } + CompensationRecoveredExternalReleaseClaimOutcome::InvalidState => { + ExecutionExternalJobStartRecoveryAdoptionOutcome::InvalidState + } + } +} + +type ExternalJobTriggerOwnerColumns = ( + Option, + Option, + Option, + Option, + Option, +); + +type ExternalJobSqlOwnerColumns = ( + Option, + Option, + Option, + Option, + Option, +); + +fn external_job_trigger_owner(owner: ExecutionExternalJobOwner) -> ExternalJobTriggerOwnerColumns { + match owner { + ExecutionExternalJobOwner::Task { + task_id, + attempt_generation, + } => (Some(task_id), Some(attempt_generation), None, None, None), + ExecutionExternalJobOwner::Compensation { + compensation_id, + compensation_generation, + compensation_attempt_generation, + } => ( + None, + None, + Some(compensation_id), + Some(compensation_generation), + Some(compensation_attempt_generation), + ), + } +} + +fn external_job_owner_columns( + owner: ExecutionExternalJobOwner, +) -> Result { + let (task_id, attempt, compensation_id, generation, compensation_attempt) = + external_job_trigger_owner(owner); + Ok(( + task_id, + attempt + .map(|value| to_i64(value, "attempt generation")) + .transpose()?, + compensation_id, + generation + .map(|value| to_i64(value, "compensation generation")) + .transpose()?, + compensation_attempt + .map(|value| to_i64(value, "compensation attempt generation")) + .transpose()?, + )) +} + +async fn lock_external_job_intent_owner_in_conn( + conn: &mut PgConnection, + intent: &NewExecutionExternalJobIntent, +) -> Result<()> { + let current = match intent.owner { + ExecutionExternalJobOwner::Task { + task_id, + attempt_generation, + } => sqlx::query_scalar::<_, bool>( + r#" + SELECT TRUE FROM moa.execution_task + WHERE tenant_id=$1 AND run_uid=$2 AND task_id=$3 + AND attempt_generation=$4 + AND ( + (status='running' AND attempt_state IN ('running','cancelling')) + OR (status='waiting_review' AND attempt_state='waiting') + ) + FOR UPDATE + "#, + ) + .bind(intent.tenant_id.0) + .bind(intent.run_uid) + .bind(task_id) + .bind(to_i64(attempt_generation, "attempt generation")?) + .fetch_optional(&mut *conn) + .await + .map_err(sqlx_error)? + .unwrap_or(false), + ExecutionExternalJobOwner::Compensation { + compensation_id, + compensation_generation, + compensation_attempt_generation, + } => sqlx::query_scalar::<_, bool>( + r#" + SELECT TRUE FROM moa.execution_compensation + WHERE tenant_id=$1 AND run_uid=$2 AND compensation_id=$3 + AND generation=$4 AND attempt_generation=$5 + AND status='running' + AND attempt_state IN ('running','cancelling','waiting_review') + FOR UPDATE + "#, + ) + .bind(intent.tenant_id.0) + .bind(intent.run_uid) + .bind(compensation_id) + .bind(to_i64(compensation_generation, "compensation generation")?) + .bind(to_i64( + compensation_attempt_generation, + "compensation attempt generation", + )?) + .fetch_optional(&mut *conn) + .await + .map_err(sqlx_error)? + .unwrap_or(false), + }; + if !current { + return Err(Error::InvalidRepositoryInput { + message: "external job intent owner generation is not dispatchable".to_string(), + }); + } + Ok(()) +} + +async fn release_external_job_capacity_in_conn( + conn: &mut PgConnection, + job: &ExecutionExternalJobRecord, +) -> Result<()> { + let reservation_uid = execution_capacity_reservation_uid( + ExecutionCapacityDimension::ExternalJobs, + job.external_job_uid, + None, + ); + let controller_generation = sqlx::query_scalar::<_, i64>( + "SELECT controller_generation FROM moa.execution_capacity_reservation \ + WHERE reservation_uid = $1 AND tenant_id = $2 AND external_job_uid = $3", + ) + .bind(reservation_uid) + .bind(job.tenant_id.0) + .bind(job.external_job_uid) + .fetch_optional(&mut *conn) + .await + .map_err(sqlx_error)? + .ok_or_else(|| Error::InvalidRepositoryData { + message: "terminal external job is missing its capacity reservation".to_string(), + })?; + let request = external_job_capacity_request( + job, + super::to_u64(controller_generation, "controller generation")?, + None, + ); + match release_capacity_in_tx(conn, request).await? { + super::capacity::CapacityReleaseOutcome::Released + | super::capacity::CapacityReleaseOutcome::AlreadyReleased => Ok(()), + super::capacity::CapacityReleaseOutcome::NotFound + | super::capacity::CapacityReleaseOutcome::Stale => Err(Error::InvalidRepositoryData { + message: "external-job capacity release lost its exact owner fence".to_string(), + }), + } +} + +fn validate_external_job_intent( + intent: &NewExecutionExternalJobIntent, + require_future_expiry: bool, +) -> Result<()> { + let owner_is_valid = match intent.owner { + ExecutionExternalJobOwner::Task { + task_id, + attempt_generation, + } => !task_id.is_nil() && attempt_generation > 0, + ExecutionExternalJobOwner::Compensation { + compensation_id, + compensation_generation, + compensation_attempt_generation, + } => { + !compensation_id.is_nil() + && compensation_generation > 0 + && compensation_attempt_generation > 0 + } + }; + if intent.external_job_uid.is_nil() + || intent.idempotency_key.trim().is_empty() + || intent.idempotency_key.len() > 256 + || intent.provider.trim().is_empty() + || intent.provider.len() > 128 + || !owner_is_valid + || intent.job_generation == 0 + || (require_future_expiry && intent.expires_at <= Utc::now()) + { + return Err(Error::InvalidRepositoryInput { + message: "external job intent requires exact owner identity and a future expiry" + .to_string(), + }); + } + Ok(()) +} + +fn validate_external_job_binding(binding: &ExecutionExternalJobBinding) -> Result<()> { + validate_external_job_binding_identity_fields(binding)?; + if matches!( + binding.state, + ExecutionExternalJobState::Unbound + | ExecutionExternalJobState::CancelRequested + | ExecutionExternalJobState::Completed + | ExecutionExternalJobState::Failed + | ExecutionExternalJobState::Cancelled + | ExecutionExternalJobState::UnknownOutcome + ) { + return Err(Error::InvalidRepositoryInput { + message: "external job binding requires a live provider state".to_string(), + }); + } + Ok(()) +} + +fn validate_external_job_binding_identity_fields( + binding: &ExecutionExternalJobBinding, +) -> Result<()> { + let intent_shape = NewExecutionExternalJobIntent { + external_job_uid: binding.external_job_uid, + tenant_id: binding.tenant_id, + run_uid: binding.run_uid, + owner: binding.owner, + job_generation: binding.job_generation, + provider: binding.provider.clone(), + idempotency_key: binding.idempotency_key.clone(), + expires_at: DateTime::::MAX_UTC, + }; + validate_external_job_intent(&intent_shape, true)?; + if binding.provider.trim().is_empty() + || binding.provider.len() > 128 + || binding.provider_job_id.trim().is_empty() + || binding.provider_job_id.len() > 512 + || binding.callback_auth_reference.trim().is_empty() + || binding.callback_auth_reference.len() > 2_048 + || !bounded_optional_text(&binding.progress_phase, 256) + || !bounded_optional_text(&binding.provider_contract_violation, 4_096) + { + return Err(Error::InvalidRepositoryInput { + message: "external job binding requires bounded provider identity".to_string(), + }); + } + Ok(()) +} + +fn validate_callback(callback: &ExecutionExternalJobCallback) -> Result<()> { + if callback.external_job_uid.is_nil() + || callback.job_generation == 0 + || callback.provider.trim().is_empty() + || callback.provider.len() > 128 + || callback.provider_job_id.trim().is_empty() + || callback.provider_job_id.len() > 512 + || callback.provider_event_id.trim().is_empty() + || callback.provider_event_id.len() > 512 + { + return Err(Error::InvalidRepositoryInput { + message: "external job callback requires an exact bounded provider event identity" + .to_string(), + }); + } + let update_shape_is_valid = match &callback.update { + ExecutionExternalJobCallbackUpdate::Progress { + state, + progress_phase, + .. + } => { + !state.is_terminal() + && !matches!(state, ExecutionExternalJobState::Starting) + && bounded_optional_text(progress_phase, 256) + } + ExecutionExternalJobCallbackUpdate::Terminal { + state, + progress_phase, + output, + error, + } => { + let outcome_is_object = output.as_ref().is_none_or(Value::is_object) + && error.as_ref().is_none_or(Value::is_object); + let outcome_matches_state = match state { + ExecutionExternalJobState::Completed => output.is_some() && error.is_none(), + ExecutionExternalJobState::Failed | ExecutionExternalJobState::UnknownOutcome => { + output.is_none() && error.is_some() + } + ExecutionExternalJobState::Cancelled => output.is_none() && error.is_none(), + ExecutionExternalJobState::Unbound + | ExecutionExternalJobState::Starting + | ExecutionExternalJobState::Running + | ExecutionExternalJobState::WaitingReconcile + | ExecutionExternalJobState::CancelRequested => false, + }; + outcome_is_object && outcome_matches_state && bounded_optional_text(progress_phase, 256) + } + }; + if !update_shape_is_valid { + return Err(Error::InvalidRepositoryInput { + message: "external job callback update does not match its lifecycle state".to_string(), + }); + } + Ok(()) +} + +fn validate_cancellation(cancellation: &ExecutionExternalJobCancellation) -> Result<()> { + let identity_is_valid = !cancellation.external_job_uid.is_nil() + && cancellation.job_generation > 0 + && !cancellation.provider.trim().is_empty() + && cancellation.provider.len() <= 128 + && !cancellation.provider_job_id.trim().is_empty() + && cancellation.provider_job_id.len() <= 512; + let state_is_valid = match cancellation.state { + ExecutionExternalJobState::CancelRequested => { + cancellation.next_reconcile_at.is_some() && cancellation.error.is_none() + } + ExecutionExternalJobState::Cancelled => { + cancellation.next_reconcile_at.is_none() && cancellation.error.is_none() + } + ExecutionExternalJobState::UnknownOutcome => { + cancellation.next_reconcile_at.is_none() + && cancellation.error.as_ref().is_some_and(Value::is_object) + } + ExecutionExternalJobState::Unbound + | ExecutionExternalJobState::Starting + | ExecutionExternalJobState::Running + | ExecutionExternalJobState::WaitingReconcile + | ExecutionExternalJobState::Completed + | ExecutionExternalJobState::Failed => false, + }; + if !identity_is_valid || !state_is_valid { + return Err(Error::InvalidRepositoryInput { + message: + "external job cancellation requires exact identity and a typed cancellation state" + .to_string(), + }); + } + Ok(()) +} + +fn bounded_optional_text(value: &Option, max_len: usize) -> bool { + value + .as_ref() + .is_none_or(|value| !value.trim().is_empty() && value.len() <= max_len) +} + +fn external_job_matches_intent( + record: &ExecutionExternalJobRecord, + intent: &NewExecutionExternalJobIntent, +) -> bool { + record.external_job_uid == intent.external_job_uid + && record.tenant_id == intent.tenant_id + && record.run_uid == intent.run_uid + && record.owner == intent.owner + && record.job_generation == intent.job_generation + && record.declared_provider == intent.provider + && record.idempotency_key == intent.idempotency_key +} + +fn external_job_matches_binding_identity( + record: &ExecutionExternalJobRecord, + binding: &ExecutionExternalJobBinding, +) -> bool { + record.external_job_uid == binding.external_job_uid + && record.tenant_id == binding.tenant_id + && record.run_uid == binding.run_uid + && record.owner == binding.owner + && record.job_generation == binding.job_generation + && record.idempotency_key == binding.idempotency_key +} + +fn external_job_matches_provider_result( + record: &ExecutionExternalJobRecord, + binding: &ExecutionExternalJobBinding, +) -> bool { + external_job_matches_binding_identity(record, binding) + && record.provider.as_deref() == Some(record.declared_provider.as_str()) + && record.provider_job_id.as_deref() == Some(binding.provider_job_id.as_str()) + && record.callback_auth_reference.as_deref() + == Some(binding.callback_auth_reference.as_str()) + && matches!( + record.state, + ExecutionExternalJobState::Starting + | ExecutionExternalJobState::Running + | ExecutionExternalJobState::WaitingReconcile + | ExecutionExternalJobState::CancelRequested + ) + && record.progress_phase == binding.progress_phase + && record.cancel_supported == binding.cancel_supported +} + +fn external_job_from_row(row: &sqlx::postgres::PgRow) -> Result { + let task_id = row + .try_get::, _>("task_id") + .map_err(super::row_error)?; + let attempt_generation = row + .try_get::, _>("attempt_generation") + .map_err(super::row_error)?; + let compensation_id = row + .try_get::, _>("compensation_id") + .map_err(super::row_error)?; + let compensation_generation = row + .try_get::, _>("compensation_generation") + .map_err(super::row_error)?; + let compensation_attempt_generation = row + .try_get::, _>("compensation_attempt_generation") + .map_err(super::row_error)?; + let owner = match ( + task_id, + attempt_generation, + compensation_id, + compensation_generation, + compensation_attempt_generation, + ) { + (Some(task_id), Some(attempt_generation), None, None, None) => { + ExecutionExternalJobOwner::Task { + task_id, + attempt_generation: super::to_u64(attempt_generation, "attempt generation")?, + } + } + (None, None, Some(compensation_id), Some(generation), Some(attempt_generation)) => { + ExecutionExternalJobOwner::Compensation { + compensation_id, + compensation_generation: super::to_u64(generation, "compensation generation")?, + compensation_attempt_generation: super::to_u64( + attempt_generation, + "compensation attempt generation", + )?, + } + } + _ => { + return Err(Error::InvalidRepositoryData { + message: "external job has an invalid task/compensation owner shape".to_string(), + }); + } + }; + let job_generation = row + .try_get::("job_generation") + .map_err(super::row_error)?; + Ok(ExecutionExternalJobRecord { + external_job_uid: row.try_get("external_job_uid").map_err(super::row_error)?, + tenant_id: TenantId(row.try_get("tenant_id").map_err(super::row_error)?), + run_uid: row.try_get("run_uid").map_err(super::row_error)?, + owner, + job_generation: super::to_u64(job_generation, "external job generation")?, + declared_provider: row.try_get("declared_provider").map_err(super::row_error)?, + provider: row.try_get("provider").map_err(super::row_error)?, + provider_job_id: row.try_get("provider_job_id").map_err(super::row_error)?, + idempotency_key: row.try_get("idempotency_key").map_err(super::row_error)?, + callback_auth_reference: row + .try_get("callback_auth_reference") + .map_err(super::row_error)?, + state: row + .try_get::("state") + .map_err(super::row_error)? + .parse()?, + progress_phase: row.try_get("progress_phase").map_err(super::row_error)?, + cancel_supported: row.try_get("cancel_supported").map_err(super::row_error)?, + next_reconcile_at: row.try_get("next_reconcile_at").map_err(super::row_error)?, + last_provider_event_id: row + .try_get("last_provider_event_id") + .map_err(super::row_error)?, + output: row.try_get("output").map_err(super::row_error)?, + error: row.try_get("error").map_err(super::row_error)?, + created_at: row.try_get("created_at").map_err(super::row_error)?, + updated_at: row.try_get("updated_at").map_err(super::row_error)?, + completed_at: row.try_get("completed_at").map_err(super::row_error)?, + provider_contract_violation: row + .try_get("provider_contract_violation") + .map_err(super::row_error)?, + }) +} diff --git a/crates/moa-execution/src/repository/materialize.rs b/crates/moa-execution/src/repository/materialize.rs index 8ead79753..db063e52d 100644 --- a/crates/moa-execution/src/repository/materialize.rs +++ b/crates/moa-execution/src/repository/materialize.rs @@ -1,7 +1,9 @@ //! Run confirmation and idempotent logical-task materialization. use super::*; -use super::{projection::budget_ledger, rows::*, sql::*}; +use super::{projection::budget_ledger, rows::*, run::enqueue_run_activation_in_conn, sql::*}; + +const MAX_MATERIALIZATION_PAGE_SIZE: usize = 1_000; impl ExecutionRepository { /// Confirms the exact displayed active-plan hash and atomically persists its budget. @@ -65,6 +67,21 @@ impl ExecutionRepository { )); }; let confirmed = run_from_row(&row)?; + enqueue_run_activation_in_conn( + conn.as_mut(), + confirmed.tenant_id, + confirmed.run_uid, + confirmed.controller_generation, + confirmed.updated_at, + json!({"reason": "run_confirmed"}), + ) + .await?; + let row = sqlx::query(LOAD_RUN_SQL) + .bind(run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let confirmed = run_from_row(&row)?; conn.commit().await.map_err(storage_error)?; Ok(ConfirmationOutcome::Confirmed(confirmed)) } @@ -99,6 +116,13 @@ impl ExecutionRepository { marker: Option, tasks: Vec, ) -> Result { + if tasks.len() > MAX_MATERIALIZATION_PAGE_SIZE { + return Err(Error::InvalidRepositoryInput { + message: format!( + "task materialization exceeds the bounded page size of {MAX_MATERIALIZATION_PAGE_SIZE}" + ), + }); + } let plan_revision_db = to_i64(plan_revision, "plan revision")?; if let Some(marker) = marker.as_ref() && tasks.iter().any(|task| task.node_id != marker.node_id()) diff --git a/crates/moa-execution/src/repository/mod.rs b/crates/moa-execution/src/repository/mod.rs index bfdcabc47..f8918ba29 100644 --- a/crates/moa-execution/src/repository/mod.rs +++ b/crates/moa-execution/src/repository/mod.rs @@ -1,19 +1,32 @@ //! Scoped PostgreSQL persistence for durable execution runs and logical tasks. mod admission; -mod audit; +pub mod amendment; +/// Immutable execution planning-context and normalized audit persistence. +pub mod audit; mod audit_codec; -mod compensation; +pub mod capacity; +pub mod compensation; +pub mod completion; +pub mod external_job; mod materialize; +pub mod outbox; mod outcome; mod outcome_support; mod projection; +pub mod ready; +/// Durable bounded replan-stop intent handoff. +pub mod replan_stop; +pub mod retention; mod rows; -mod run; +pub mod run; +pub mod schedule; mod sql; -mod task; -mod terminal; +pub mod task; +/// Bounded terminal fencing, trigger drain, compensation, and finalization persistence. +pub mod terminal; mod transition; +pub mod trigger; use std::{collections::BTreeMap, str::FromStr}; @@ -23,6 +36,7 @@ use moa_artifacts::execution_plan::{ ExecutionOperation, ExecutionTaskOutcome, ExecutionTaskResult, ExecutionUsage, PlanAmendment, }; use moa_core::{ + traits::Identity, types::contact::ContactId, types::execution_planning::{ ExecutionCompileOutcome, ExecutionCompileSource, ExecutionPlannerCallKind, @@ -36,9 +50,11 @@ use moa_core::{ use moa_db::ScopedConn; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; -use sqlx::{PgPool, Postgres, Row, Transaction, postgres::PgRow}; +use sqlx::{PgConnection, PgPool, Row, postgres::PgRow}; use uuid::Uuid; +use self::outbox::ExecutionDispatchRecord; + use crate::{ Error, Result, budget::{BudgetLedger, BudgetReconciliation}, @@ -65,7 +81,6 @@ use crate::{ ExecutionActionReviewResolution, ExecutionPlanningContextSnapshot, ExecutionTemplateAdmissionRequest, ExecutionTerminalDelivery, ExecutionToolDispatchRejection, PinnedInstructionSkill, - execution_terminal_delivery_from_state, }, }; @@ -75,22 +90,42 @@ const DEFAULT_TASK_PAGE_LIMIT: u32 = 100; const MAX_TASK_PAGE_LIMIT: u32 = 1_000; const EXECUTION_AUDIT_NAMESPACE: Uuid = Uuid::from_u128(0x7b83_c5c2_5cf7_5fa0_8eb6_2d7c_6e0f_1d11); +/// Phase of one execution-scoped external effect. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ExecutionEffectPhase { + /// The active attempt is invoking an effect without an action-review handoff. + Direct, + /// The exact parked action review was cleared and claimed for execution. + Reviewed { + /// Stable action-review identity persisted by both the attempt and review owner. + review_uid: Uuid, + }, +} + /// Persisted execution operation seeking permission to begin one external effect. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum ExecutionEffectOwner { - /// Forward task coordinates fenced by the current task generation. + /// Forward task coordinates fenced by the current task and bounded-attempt generations. Task { /// Stable forward task identifier. task_id: ExecutionTaskId, - /// Exact current task generation. + /// Exact current logical task generation. generation: u64, + /// Exact current bounded-attempt generation. + attempt_generation: u64, + /// Exact direct or reviewed invocation phase. + phase: ExecutionEffectPhase, }, - /// Compensation coordinates fenced by the current compensation generation. + /// Compensation coordinates fenced by the current logical and bounded-attempt generations. Compensation { /// Stable compensation identifier. compensation_id: CompensationId, - /// Exact current compensation generation. + /// Exact current logical compensation generation. generation: u64, + /// Exact current bounded-attempt generation. + attempt_generation: u64, + /// Exact direct or reviewed invocation phase. + phase: ExecutionEffectPhase, }, } @@ -116,7 +151,8 @@ struct CompensationReviewAuditEntry { review_uid: Uuid, generation: u64, accepted: bool, - resolution: ExecutionActionReviewResolution, + resolution: Option, + expires_at: Option>, recorded_at: DateTime, } @@ -185,51 +221,6 @@ impl ExecutionScope { } } -async fn install_execution_scope( - tx: &mut Transaction<'_, Postgres>, - scope: ExecutionScope, -) -> Result<()> { - let (tenant_id, storage_partition_id, contact_id, control_plane) = match scope { - ExecutionScope::ControlPlane => (None, None, None, true), - ExecutionScope::Tenant { tenant_id } => ( - Some(tenant_id.to_string()), - Some( - moa_core::types::identifiers::StoragePartitionId::for_tenant(tenant_id).to_string(), - ), - Some(String::new()), - false, - ), - ExecutionScope::Contact { - tenant_id, - contact_id, - } => ( - Some(tenant_id.to_string()), - Some( - moa_core::types::identifiers::StoragePartitionId::for_tenant(tenant_id).to_string(), - ), - Some(contact_id.to_string()), - false, - ), - }; - sqlx::query( - r#" - SELECT - pg_catalog.set_config('moa.tenant_id', $1, true), - pg_catalog.set_config('moa.storage_partition_id', $2, true), - pg_catalog.set_config('moa.contact_id', $3, true), - pg_catalog.set_config('moa.control_plane', $4, true) - "#, - ) - .bind(tenant_id.as_deref().unwrap_or("")) - .bind(storage_partition_id.as_deref().unwrap_or("")) - .bind(contact_id.as_deref().unwrap_or("")) - .bind(if control_plane { "true" } else { "false" }) - .execute(&mut **tx) - .await - .map_err(sqlx_error)?; - Ok(()) -} - /// Input used to create one immutable execution run snapshot. #[derive(Clone, Debug)] pub struct NewExecutionRun { @@ -247,6 +238,8 @@ pub struct NewExecutionRun { pub planning_context_hash: ExecutionHash, /// Authenticated tenant user that owns the run. pub owner_user_id: UserId, + /// Exact authenticated principal admitted to create this durable run. + pub admitted_identity: Identity, /// Immutable user-derived goal contract. pub goal: ExecutionGoalContract, /// Initial canonical plan, also installed as revision one. @@ -301,6 +294,8 @@ pub struct ExecutionRunRecord { pub planning_context_hash: ExecutionHash, /// Originating tenant user. pub owner_user_id: UserId, + /// Exact authenticated principal admitted when the run was created. + pub admitted_identity: Identity, /// Immutable goal contract. pub goal: ExecutionGoalContract, /// Immutable initial canonical plan. @@ -345,6 +340,44 @@ pub struct ExecutionRunRecord { pub terminal_reason: Option, /// Current durable run status. pub status: ExecutionRunStatus, + /// Monotonic generation fencing controller activations and delayed wakes. + pub controller_generation: u64, + /// Current bounded-controller activation lifecycle. + pub activation_state: ExecutionActivationState, + /// Earliest exact time at which the controller should be reactivated. + pub next_wake_at: Option>, + /// Time at which the run entered its current storage-only wait. + pub waiting_since: Option>, + /// Latest durable scheduler progress timestamp. + pub last_progress_at: DateTime, + /// Time at which an authorized pause was first requested. + pub pause_requested_at: Option>, + /// Time at which the run became fully paused. + pub paused_at: Option>, + /// Number of tasks currently admitted to the durable ready queue. + pub ready_task_count: u64, + /// Number of task attempts currently consuming active capacity. + pub active_task_count: u64, + /// Exact number of logical tasks parked on durable waits. + pub waiting_task_count: u64, + /// Exact number of tasks waiting for user input. + pub waiting_input_task_count: u64, + /// Exact number of tasks waiting for governed review. + pub waiting_review_task_count: u64, + /// Exact number of tasks waiting for a named signal. + pub waiting_signal_task_count: u64, + /// Exact number of tasks waiting for an absolute timer. + pub waiting_timer_task_count: u64, + /// Exact number of tasks waiting for an external job. + pub waiting_external_task_count: u64, + /// Exact number of tasks waiting for bounded replanning. + pub waiting_replan_task_count: u64, + /// Exact input waits whose authorized audience is the owning user. + pub waiting_input_user_task_count: u64, + /// Exact input waits whose authorized audience is a tenant administrator. + pub waiting_input_tenant_admin_task_count: u64, + /// Exact input waits whose authorized audience is an external system. + pub waiting_input_external_task_count: u64, /// Approved resource limits. pub approved_budget: ExecutionBudgetLimit, /// Resources held by nonterminal tasks. @@ -361,8 +394,10 @@ pub struct ExecutionRunRecord { pub progress_failed_tasks: u64, /// Number of cancelled tasks. pub progress_cancelled_tasks: u64, - /// Exact current scheduler wait reasons. + /// Bounded deterministic sample of current scheduler wait reasons. pub waiting_reasons: Vec, + /// Whether exact waiting tasks exist outside the bounded reason sample. + pub waiting_reasons_truncated: bool, /// Monotonic epoch incremented by scheduling-relevant mutations. pub wake_epoch: u64, /// Last scheduler epoch acknowledged by compare-and-set. @@ -391,128 +426,6 @@ pub struct ExecutionRunRecord { pub confirmed_at: Option>, } -/// Input used to insert one immutable origin-bound planning-context snapshot. -#[derive(Clone, Debug)] -pub struct NewExecutionPlanningContext { - /// Exact immutable snapshot whose canonical bytes are hashed. - pub snapshot: ExecutionPlanningContextSnapshot, - /// Domain-separated hash of the canonical snapshot bytes. - pub planning_context_hash: ExecutionHash, -} - -/// Persisted immutable planning-context projection. -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -pub struct ExecutionPlanningContextRecord { - /// Durable planning-context identifier. - pub planning_context_uid: Uuid, - /// Exact immutable snapshot. - pub snapshot: ExecutionPlanningContextSnapshot, - /// Domain-separated hash of the canonical snapshot bytes. - pub planning_context_hash: ExecutionHash, - /// Database-owned creation timestamp. - pub created_at: DateTime, -} - -/// Result of inserting or replaying one unique origin-bound planning context. -#[derive(Clone, Debug, PartialEq)] -pub enum PlanningContextWriteOutcome { - /// The immutable snapshot was inserted. - Created(ExecutionPlanningContextRecord), - /// The exact immutable snapshot already existed for the origin. - Replayed(ExecutionPlanningContextRecord), - /// The unique origin already exists with different immutable bytes or scope. - Conflict, -} - -/// Persisted low-cardinality evidence for one route-audit insertion. -#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] -pub struct RouteAuditEvidence { - /// Deterministic UUIDv5 audit identifier. - pub audit_uid: Uuid, - /// Respond, Execute, or NeedsInput decision. - pub decision: ExecutionRouteKind, - /// Selected strategy, present exactly for Execute. - pub strategy: Option, - /// Redacted trusted-bypass or classifier provenance. - pub provenance: ExecutionRouteProvenance, - /// First durable acceptance timestamp. - pub accepted_at: DateTime, -} - -/// Durable result of inserting one normalized route audit. -#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] -pub enum RouteAuditWriteOutcome { - /// This transaction inserted the first route row. - Applied(RouteAuditEvidence), - /// The exact semantic route row already existed. - Replayed(RouteAuditEvidence), - /// The logical key already carries different route semantics. - Conflict { - /// Deterministic audit identifier for the conflicting logical key. - audit_uid: Uuid, - }, -} - -/// Persisted low-cardinality evidence for one planner-call audit insertion. -#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] -pub struct PlannerCallAuditEvidence { - /// Deterministic UUIDv5 audit identifier. - pub audit_uid: Uuid, - /// Exact closed planner call kind. - pub call: ExecutionPlannerCallKind, - /// Exact closed planner outcome. - pub outcome: ExecutionPlannerOutcome, - /// First persisted measured duration. - pub duration_micros: u64, - /// Candidate hash when required by the outcome. - pub candidate_hash: Option, -} - -/// Durable result of inserting one normalized planner-call audit. -#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] -pub enum PlannerCallAuditWriteOutcome { - /// This transaction inserted the first planner-call row. - Applied(PlannerCallAuditEvidence), - /// The exact semantic planner-call row already existed. - Replayed(PlannerCallAuditEvidence), - /// The logical key already carries different planner-call semantics. - Conflict { - /// Deterministic audit identifier for the conflicting logical key. - audit_uid: Uuid, - }, -} - -/// Persisted low-cardinality evidence for one compiler-audit insertion. -#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] -pub struct CompileAuditEvidence { - /// Deterministic UUIDv5 audit identifier. - pub audit_uid: Uuid, - /// Exact closed compiler source. - pub source: ExecutionCompileSource, - /// Exact closed compiler outcome. - pub outcome: ExecutionCompileOutcome, - /// First persisted measured duration. - pub duration_micros: u64, - /// Hash of the strict compile candidate. - pub candidate_hash: String, - /// Accepted final plan hash, when compilation succeeded. - pub final_plan_hash: Option, -} - -/// Durable result of inserting one normalized compiler audit. -#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] -pub enum CompileAuditWriteOutcome { - /// This transaction inserted the first compiler row. - Applied(CompileAuditEvidence), - /// The exact semantic compiler row already existed. - Replayed(CompileAuditEvidence), - /// The logical key already carries different compiler semantics. - Conflict { - /// Deterministic audit identifier for the conflicting logical key. - audit_uid: Uuid, - }, -} - /// Persisted logical-task projection. #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub struct ExecutionTaskRecord { @@ -538,6 +451,26 @@ pub struct ExecutionTaskRecord { pub attempt: u32, /// One-based dispatch generation fence. pub generation: u64, + /// One-based generation fencing the currently persisted attempt lifecycle. + pub attempt_generation: u64, + /// Current bounded-attempt lifecycle state. + pub attempt_state: ExecutionAttemptState, + /// Time at which the current active attempt began. + pub attempt_started_at: Option>, + /// Latest durable progress timestamp for this logical task. + pub last_progress_at: DateTime, + /// Absolute watchdog deadline for the current active attempt. + pub attempt_deadline_at: Option>, + /// Time at which the task entered its current storage-only wait. + pub waiting_since: Option>, + /// Time at which the task entered the ready queue. + pub ready_at: Option>, + /// Current asynchronous provider job, when the task is waiting externally. + pub external_job_uid: Option, + /// Dispatch lease currently owning this attempt, when one is active. + pub active_dispatch_uid: Option, + /// Monotonic dispatch fence for this logical task. + pub dispatch_sequence: u64, /// Resolved structured task input. pub input: Value, /// Append-only ordered payloads supplied by input resumes. @@ -580,6 +513,237 @@ pub struct ExecutionTaskRecord { pub completed_at: Option>, } +/// Bounded controller activation state persisted independently from run status. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ExecutionActivationState { + /// No controller activation is queued or running. + Idle, + /// One generation-fenced activation is ready for durable dispatch. + Queued, + /// The current generation is executing one bounded activation. + Advancing, + /// The run is explicitly paused and owns no controller activation. + Paused, + /// The run is terminal and cannot be activated again. + Terminal, +} + +impl ExecutionActivationState { + /// Returns the canonical database label. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Idle => "idle", + Self::Queued => "queued", + Self::Advancing => "advancing", + Self::Paused => "paused", + Self::Terminal => "terminal", + } + } +} + +impl FromStr for ExecutionActivationState { + type Err = Error; + + fn from_str(value: &str) -> Result { + match value { + "idle" => Ok(Self::Idle), + "queued" => Ok(Self::Queued), + "advancing" => Ok(Self::Advancing), + "paused" => Ok(Self::Paused), + "terminal" => Ok(Self::Terminal), + _ => Err(Error::InvalidRepositoryData { + message: format!("unknown execution activation state `{value}`"), + }), + } + } +} + +/// Bounded task-attempt state persisted independently from logical task status. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ExecutionAttemptState { + /// No attempt is currently dispatched or running. + Idle, + /// The attempt is committed for durable dispatch. + Dispatching, + /// The attempt currently consumes active task capacity. + Running, + /// Provider teardown was claimed and capacity remains owned until verified release. + Cancelling, + /// The logical task is parked without an active attempt. + Waiting, + /// The logical task settled terminally. + Terminal, + /// A non-idempotent attempt has an ambiguous external outcome. + UnknownOutcome, +} + +impl ExecutionAttemptState { + /// Returns the canonical database label. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Idle => "idle", + Self::Dispatching => "dispatching", + Self::Running => "running", + Self::Cancelling => "cancelling", + Self::Waiting => "waiting", + Self::Terminal => "terminal", + Self::UnknownOutcome => "unknown_outcome", + } + } +} + +impl FromStr for ExecutionAttemptState { + type Err = Error; + + fn from_str(value: &str) -> Result { + match value { + "idle" => Ok(Self::Idle), + "dispatching" => Ok(Self::Dispatching), + "running" => Ok(Self::Running), + "cancelling" => Ok(Self::Cancelling), + "waiting" => Ok(Self::Waiting), + "terminal" => Ok(Self::Terminal), + "unknown_outcome" => Ok(Self::UnknownOutcome), + _ => Err(Error::InvalidRepositoryData { + message: format!("unknown execution attempt state `{value}`"), + }), + } + } +} + +/// Exact durable checkpoint written when one bounded controller activation returns. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ExecutionRunActivationCheckpoint { + /// Product-visible run state after the bounded activation. + pub status: ExecutionRunStatus, + /// Whether another activation is queued or the run is parked/terminal. + pub activation_state: ExecutionActivationState, + /// Earliest exact wake time, when time can make more work ready. + pub next_wake_at: Option>, + /// Start of the current storage-only wait, when parked. + pub waiting_since: Option>, + /// Exact number of ready logical tasks. + pub ready_task_count: u64, + /// Exact number of active task attempts. + pub active_task_count: u64, +} + +/// Generation-fenced result of claiming or checkpointing a run activation. +#[derive(Clone, Debug, PartialEq)] +pub enum RunActivationWriteOutcome { + /// The exact generation mutation was committed. + Applied(ExecutionRunRecord), + /// The requested state was already durably present. + AlreadyApplied(ExecutionRunRecord), + /// No visible run exists under the supplied scope. + NotFound, + /// The supplied controller generation is stale or from the future. + GenerationMismatch, + /// The run is not in a lifecycle state that accepts this mutation. + InvalidState, +} + +/// Exact generation-and-wake claim made by one bounded run-controller activation. +#[derive(Clone, Debug, PartialEq)] +pub enum RunControllerClaimOutcome { + /// The queued wake was claimed and the run is now advancing. + Claimed(ExecutionRunRecord), + /// The same wake is already being advanced by a replay of the same durable invocation. + Resumed(ExecutionRunRecord), + /// The requested wake was already durably acknowledged. + Replayed(ExecutionRunRecord), + /// The run is terminal, so the activation is a successful no-op. + Terminal(ExecutionRunRecord), + /// No visible run exists under the supplied scope. + NotFound, + /// The request did not name the current controller generation. + StaleGeneration { + /// Current persisted controller generation. + current_generation: u64, + }, + /// The request did not name the current unprocessed wake. + StaleWake { + /// Current persisted wake epoch. + current_wake_epoch: u64, + /// Greatest wake epoch already acknowledged by the controller. + processed_wake_epoch: u64, + }, + /// The run lifecycle cannot accept an activation claim. + InvalidState, +} + +/// Atomic checkpoint request for one bounded run-controller activation. +#[derive(Clone, Debug, PartialEq)] +pub struct RunControllerCompletionRequest { + /// Exact controller generation claimed by the invocation. + pub controller_generation: u64, + /// Exact wake epoch claimed by the invocation. + pub wake_epoch: u64, + /// Durable run checkpoint produced by bounded scheduler work. + pub checkpoint: ExecutionRunActivationCheckpoint, + /// Structured activation payload when bounded work requires one continuation. + pub continuation_payload: Option, + /// Earliest time at which the continuation may be dispatched. + pub continuation_not_before_at: DateTime, +} + +/// Atomic checkpoint, wake acknowledgement, and optional continuation result. +#[derive(Clone, Debug, PartialEq)] +pub enum RunControllerCompletionOutcome { + /// The checkpoint and exact wake acknowledgement committed. + Applied { + /// Current run after the commit. + run: Box, + /// Exactly one continuation outbox record, when requested. + continuation: Option>, + }, + /// The exact wake had already completed and changed nothing. + Replayed(Box), + /// The storage-only checkpoint could not reserve its parked-run capacity. + CapacitySaturated { + /// Exact durable capacity dimension that rejected the checkpoint. + dimension: capacity::ExecutionCapacityDimension, + }, + /// No visible run exists under the supplied scope. + NotFound, + /// The request did not name the current controller generation. + StaleGeneration { + /// Current persisted controller generation. + current_generation: u64, + }, + /// The activation lost its exact wake fence. + StaleWake { + /// Current persisted wake epoch. + current_wake_epoch: u64, + /// Greatest wake epoch already acknowledged by the controller. + processed_wake_epoch: u64, + }, + /// The run lifecycle cannot accept this completion. + InvalidState, +} + +/// Result of materializing the exact current run-deadline trigger. +#[derive(Clone, Debug, PartialEq)] +pub enum RunDeadlineArmOutcome { + /// The current generation's immutable deadline trigger is durable. + Armed(Box), + /// The run has no approved deadline. + NoDeadline, + /// No visible run exists under the supplied scope. + NotFound, + /// The supplied generation is no longer current. + StaleGeneration { + /// Current persisted controller generation. + current_generation: u64, + }, + /// The run is terminal and owns no new deadline trigger. + Terminal, +} + /// Result of confirming an awaiting execution run. #[derive(Clone, Debug, PartialEq)] pub enum ConfirmationOutcome { @@ -813,146 +977,6 @@ pub enum TaskOutcomeRejection { UnsupportedSchemaVersion, } -/// Exact amendment identity that caused a compensation-safe replan-stop fence. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub struct ReplanStopReceipt { - /// Waiting-replan task whose current outcome triggered amendment evaluation. - pub task_id: ExecutionTaskId, - /// Exact generation of the waiting-replan task. - pub task_generation: u64, - /// Plan revision against which the amendment was evaluated. - pub base_plan_revision: u64, - /// Domain-separated hash of the exact amendment request. - pub amendment_hash: ExecutionHash, -} - -/// Durable result of installing the pre-compensation admission fence. -#[derive(Clone, Debug, PartialEq)] -pub struct TerminalFenceCommit { - /// Run projection with a persisted pending terminal intent. - pub run: ExecutionRunRecord, - /// Exact nonterminal forward task projections that must settle before undo begins. - pub tasks_to_settle: Vec, -} - -/// Result of fencing new forward admission before cancellation settlement. -#[derive(Clone, Debug, PartialEq)] -pub enum TerminalFenceOutcome { - /// The terminal intent was persisted and forward admission was fenced. - Applied(Box), - /// The exact terminal intent and fence already exist. - Replayed(Box), - /// No visible run exists. - NotFound, - /// Revision, wake epoch, run state, or prior terminal intent differed. - Conflict, -} - -/// Durable handoff after all forward work settled and compensation began. -#[derive(Clone, Debug, PartialEq)] -pub struct BeginCompensationCommit { - /// Run projection in the nonterminal `compensating` state. - pub run: ExecutionRunRecord, - /// Registered undo work in strict descending commit sequence. - pub registrations: Vec, -} - -/// Result of transitioning a fenced run into compensation execution. -#[derive(Clone, Debug, PartialEq)] -pub enum BeginCompensationOutcome { - /// The run entered compensation. - Applied(Box), - /// The run was already compensating under the exact pending terminal intent. - Replayed(Box), - /// No committed effect requires undo; finalize the held terminal intent directly. - NoCompensations(Box), - /// Forward tasks still require a definitive generation-fenced settlement. - ForwardTasksPending(Vec), - /// No visible run exists. - NotFound, - /// Revision, wake epoch, fence, or run state differed. - Conflict, -} - -/// Result of installing a fenced terminal intent without executing undo work. -#[derive(Clone, Debug, PartialEq)] -pub enum FencedTerminalFinalizationOutcome { - /// The held terminal intent was installed and its fence cleared. - Finalized(ExecutionRunRecord), - /// The exact terminal state was already installed. - Replayed(ExecutionRunRecord), - /// A forward ambiguity finalized as compensation failure with manual repair required. - ManualRepairRequired(ExecutionRunRecord), - /// Forward tasks still require generation-fenced settlement. - ForwardTasksPending(Vec), - /// No visible run exists. - NotFound, - /// Revision, wake epoch, or pending terminal intent differed. - Conflict, -} - -/// Complete repository projection used to drive compensation workflows. -#[derive(Clone, Debug, PartialEq)] -pub struct ExecutionCompensationSnapshot { - /// Current durable run. - pub run: ExecutionRunRecord, - /// Registrations in descending commit sequence. - pub registrations: Vec, - /// Nonterminal forward tasks that must settle before compensation starts. - pub nonterminal_forward_tasks: Vec, - /// Whether automatic compensation progress is blocked on manual repair. - pub manual_repair_required: bool, -} - -/// Result of claiming the next strict reverse-order compensation. -#[derive(Clone, Debug, PartialEq)] -pub enum CompensationClaimOutcome { - /// The highest pending sequence entered the requested running generation. - Claimed(CompensationRegistrationProjection), - /// The same registration and generation were already claimed. - Replayed(CompensationRegistrationProjection), - /// Budget admission failed terminally and manual repair is required. - BudgetRejected(CompensationRegistrationProjection), - /// No visible registration exists. - NotFound, - /// Another higher sequence, generation, run status, or repair fence blocks this claim. - Conflict, -} - -/// Result of recording one generation-fenced compensation attempt outcome. -#[derive(Clone, Debug, PartialEq)] -pub enum CompensationOutcomeWrite { - /// The attempt completed and the registration settled successfully. - Completed(CompensationRegistrationProjection), - /// A typed retryable failure advanced attempt and generation and returned to pending. - Requeued(CompensationRegistrationProjection), - /// A terminal failure persisted and fenced automatic progress for manual repair. - Failed(CompensationRegistrationProjection), - /// An ambiguous effect persisted and fenced automatic progress for manual repair. - UnknownOutcome(CompensationRegistrationProjection), - /// The exact outcome was already accepted for this generation. - Replayed(CompensationRegistrationProjection), - /// No visible registration exists. - NotFound, - /// The generation, run state, or registration status rejected this outcome. - Conflict, -} - -/// Result of installing the pending terminal intent after compensation settles. -#[derive(Clone, Debug, PartialEq)] -pub enum CompensationFinalizationOutcome { - /// All undo work completed and the original terminal intent was installed. - Finalized(ExecutionRunRecord), - /// The same final state was already committed. - Replayed(ExecutionRunRecord), - /// Automatic finalization is blocked by failed or ambiguous undo work. - ManualRepairRequired(ExecutionRunRecord), - /// No visible run exists. - NotFound, - /// Wake epoch, active registrations, or terminal intent differed. - Conflict, -} - /// Compiler-validated amendment data persisted under a revision fence. #[derive(Clone, Debug)] pub struct ValidatedAmendment { @@ -1076,73 +1100,6 @@ pub struct ExecutionSchedulingSnapshot { pub projection: ExecutionProjection, } -/// Result of compare-and-set wake acknowledgement. -#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] -pub enum WakeAckOutcome { - /// The exact current epoch was acknowledged. - Acknowledged { - /// Newly persisted processed epoch. - processed_wake_epoch: u64, - }, - /// The same epoch was already acknowledged. - Replayed { - /// Persisted processed epoch. - processed_wake_epoch: u64, - }, - /// A later scheduling mutation occurred and remains unacknowledged. - Changed { - /// Current persisted wake epoch. - current_wake_epoch: u64, - }, - /// No visible run exists. - NotFound, -} - -/// Result of terminal run finalization. -#[derive(Clone, Debug, PartialEq)] -pub enum FinalizationOutcome { - /// Terminal state and completion evidence were persisted. - Finalized(ExecutionRunRecord), - /// The same terminal projection was already persisted. - Replayed(ExecutionRunRecord), - /// No visible run exists. - NotFound, - /// Revision, status, or completion evaluation did not match. - Conflict, -} - -/// Optimistically fenced request to atomically persist one terminal run projection. -#[derive(Clone, Debug, PartialEq)] -pub struct RunFinalizationRequest { - /// Run to finalize. - pub run_uid: Uuid, - /// Active plan revision used for completion evaluation. - pub expected_revision: u64, - /// Wake epoch of the structured projection used for completion evaluation. - pub expected_wake_epoch: u64, - /// Exact terminal projection selected by the scheduler. - pub terminal_projection: TerminalProjection, - /// Deterministic completion evaluation over the observed projection. - pub completion_evaluation: CompletionEvaluation, - /// Exact typed cause and requirement-count replay identity. - pub terminal_evidence: ExecutionTerminalEvidence, - /// Exact normalized terminal reason selected from typed evidence. - pub terminal_reason: ExecutionTerminalReason, -} - -/// Result of idempotently persisting one action-review resolution delivery. -#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] -pub enum ActionReviewResolutionWrite { - /// The review UID was accepted for the current running generation. - Applied, - /// The same review UID and generation were already recorded. - Replayed, - /// The review was audited but its task generation or status was stale. - AuditedStale, - /// No visible task exists. - NotFound, -} - /// Scoped repository for durable execution runs and logical tasks. #[derive(Clone, Debug)] pub struct ExecutionRepository { @@ -1205,15 +1162,18 @@ fn to_u32(value: i32, field: &str) -> Result { } fn storage_error(error: moa_core::error::MoaError) -> Error { - Error::Storage { - message: error.to_string(), + match error { + moa_core::error::MoaError::StorageUnavailable(message) => { + Error::StorageUnavailable { message } + } + terminal => Error::Storage { + message: terminal.to_string(), + }, } } fn sqlx_error(error: sqlx::Error) -> Error { - Error::Storage { - message: error.to_string(), - } + Error::Database { source: error } } fn row_error(error: sqlx::Error) -> Error { @@ -1222,5 +1182,34 @@ fn row_error(error: sqlx::Error) -> Error { } } +#[cfg(test)] +mod storage_error_tests { + use super::*; + + #[test] + fn repository_conversions_keep_retry_and_decode_boundaries_distinct() { + // Pins: SQL execution retains concrete SQLx provenance, shared transient + // failures remain retryable, and row decoding is deterministic corruption. + let direct = sqlx_error(sqlx::Error::PoolTimedOut); + assert!(direct.is_retryable_storage()); + assert!(matches!( + direct, + Error::Database { + source: sqlx::Error::PoolTimedOut + } + )); + + let scoped = storage_error(moa_core::error::MoaError::StorageUnavailable( + "database restarting".to_string(), + )); + assert!(scoped.is_retryable_storage()); + assert!(matches!(scoped, Error::StorageUnavailable { .. })); + + let corrupt_row = row_error(sqlx::Error::ColumnNotFound("status".to_string())); + assert!(!corrupt_row.is_retryable_storage()); + assert!(matches!(corrupt_row, Error::InvalidRepositoryData { .. })); + } +} + #[cfg(test)] mod tests; diff --git a/crates/moa-execution/src/repository/outbox.rs b/crates/moa-execution/src/repository/outbox.rs new file mode 100644 index 000000000..aecccd06c --- /dev/null +++ b/crates/moa-execution/src/repository/outbox.rs @@ -0,0 +1,1731 @@ +//! Transactional dispatch outbox persistence for bounded execution activations. + +use std::{collections::HashSet, str::FromStr, time::Duration}; + +use crate::wire::{ + ExecutionCompensationAttemptCancelRequest, ExecutionCompensationAttemptRequest, + ExecutionExternalJobCancelRequest, ExecutionTaskAttemptCancelRequest, + ExecutionTaskAttemptRequest, +}; +use chrono::{DateTime, Utc}; +use moa_core::types::identifiers::TenantId; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use sqlx::{PgConnection, Row}; +use uuid::Uuid; + +use super::{ + Error, ExecutionRepository, ExecutionScope, Result, sqlx_error, storage_error, + task::{TaskAttemptFence, TaskAttemptSettlementOutcome}, + to_optional_i64, to_u32, +}; + +const MAX_CLAIM_BATCH_SIZE: u32 = 1_000; +const MAX_HEALTH_SAMPLE_SIZE: u32 = 100_000; +const MAX_ERROR_CHARS: usize = 4_096; +const MAX_MAINTENANCE_ERROR_BYTES: usize = 4_096; + +/// Durable dispatch target selected by the execution controller. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ExecutionDispatchKind { + /// Wake one bounded run-controller activation. + RunActivation, + /// Start one bounded task-attempt generation. + TaskAttempt, + /// Start one bounded compensation-attempt generation. + CompensationAttempt, + /// Cancel one exact active task-attempt generation. + TaskAttemptCancel, + /// Cancel one exact active compensation-attempt generation. + CompensationAttemptCancel, + /// Deliver one immutable temporal trigger. + TriggerDelivery, + /// Request cancellation of one asynchronous provider job. + ExternalCancel, +} + +impl ExecutionDispatchKind { + /// Returns the canonical database label. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::RunActivation => "run_activation", + Self::TaskAttempt => "task_attempt", + Self::CompensationAttempt => "compensation_attempt", + Self::TaskAttemptCancel => "task_attempt_cancel", + Self::CompensationAttemptCancel => "compensation_attempt_cancel", + Self::TriggerDelivery => "trigger_delivery", + Self::ExternalCancel => "external_cancel", + } + } +} + +impl FromStr for ExecutionDispatchKind { + type Err = Error; + + fn from_str(value: &str) -> Result { + match value { + "run_activation" => Ok(Self::RunActivation), + "task_attempt" => Ok(Self::TaskAttempt), + "compensation_attempt" => Ok(Self::CompensationAttempt), + "task_attempt_cancel" => Ok(Self::TaskAttemptCancel), + "compensation_attempt_cancel" => Ok(Self::CompensationAttemptCancel), + "trigger_delivery" => Ok(Self::TriggerDelivery), + "external_cancel" => Ok(Self::ExternalCancel), + _ => Err(Error::InvalidRepositoryData { + message: format!("unknown execution dispatch kind `{value}`"), + }), + } + } +} + +/// Lifecycle state shared by execution trigger and dispatch queues. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ExecutionDeliveryState { + /// Durable work has not been claimed. + Pending, + /// One replica owns a time-bounded delivery claim. + Dispatching, + /// Delivery was acknowledged successfully. + Delivered, + /// A newer generation replaced this work. + Superseded, + /// Cancellation fenced this work before delivery. + Cancelled, + /// Delivery exhausted its bounded retry policy. + DeadLetter, +} + +impl ExecutionDeliveryState { + /// Returns the canonical database label. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Pending => "pending", + Self::Dispatching => "dispatching", + Self::Delivered => "delivered", + Self::Superseded => "superseded", + Self::Cancelled => "cancelled", + Self::DeadLetter => "dead_letter", + } + } +} + +impl FromStr for ExecutionDeliveryState { + type Err = Error; + + fn from_str(value: &str) -> Result { + match value { + "pending" => Ok(Self::Pending), + "dispatching" => Ok(Self::Dispatching), + "delivered" => Ok(Self::Delivered), + "superseded" => Ok(Self::Superseded), + "cancelled" => Ok(Self::Cancelled), + "dead_letter" => Ok(Self::DeadLetter), + _ => Err(Error::InvalidRepositoryData { + message: format!("unknown execution delivery state `{value}`"), + }), + } + } +} + +/// Immutable dispatch intent inserted in the same transaction as its state mutation. +#[derive(Clone, Debug, PartialEq)] +pub struct NewExecutionDispatch { + /// Stable dispatch identity used by replaying callers. + pub dispatch_uid: Uuid, + /// Tenant that owns every referenced row. + pub tenant_id: TenantId, + /// Owning execution run, when this target is run-scoped. + pub run_uid: Option, + /// Owning logical task, for task-attempt and external-cancel dispatches. + pub task_id: Option, + /// Owning compensation registration for compensation-attempt dispatches. + pub compensation_id: Option, + /// Immutable temporal trigger target. + pub trigger_uid: Option, + /// Asynchronous provider job target. + pub external_job_uid: Option, + /// Durable dispatch target kind. + pub kind: ExecutionDispatchKind, + /// Current run-controller generation fence. + pub controller_generation: Option, + /// Exact wake epoch for a run activation. + pub wake_epoch: Option, + /// Exact task-attempt generation fence. + pub attempt_generation: Option, + /// Exact compensation registration generation fence. + pub compensation_generation: Option, + /// Exact compensation-attempt generation fence. + pub compensation_attempt_generation: Option, + /// Earliest time at which delivery may be claimed. + pub not_before_at: DateTime, + /// Bounded structured delivery payload. + pub payload: Value, +} + +/// One persisted dispatch row. +#[derive(Clone, Debug, PartialEq)] +pub struct ExecutionDispatchRecord { + /// Stable dispatch identity. + pub dispatch_uid: Uuid, + /// Tenant that owns the row. + pub tenant_id: TenantId, + /// Owning execution run. + pub run_uid: Option, + /// Owning task. + pub task_id: Option, + /// Owning compensation registration. + pub compensation_id: Option, + /// Target temporal trigger. + pub trigger_uid: Option, + /// Target external job. + pub external_job_uid: Option, + /// Durable target kind. + pub kind: ExecutionDispatchKind, + /// Current delivery lifecycle state. + pub state: ExecutionDeliveryState, + /// Run-controller generation fence. + pub controller_generation: Option, + /// Exact run wake epoch. + pub wake_epoch: Option, + /// Exact task-attempt generation. + pub attempt_generation: Option, + /// Exact compensation registration generation. + pub compensation_generation: Option, + /// Exact compensation-attempt generation. + pub compensation_attempt_generation: Option, + /// Earliest delivery time. + pub not_before_at: DateTime, + /// Structured delivery payload. + pub payload: Value, + /// Current claim owner. + pub claim_owner: Option, + /// Claim acquisition time. + pub claimed_at: Option>, + /// Claim expiry time. + pub claim_expires_at: Option>, + /// Number of bounded delivery attempts. + pub delivery_attempts: u32, + /// Successful delivery time. + pub delivered_at: Option>, + /// Latest bounded delivery error. + pub last_error: Option, + /// Creation time. + pub created_at: DateTime, + /// Last mutation time. + pub updated_at: DateTime, +} + +/// Bounded retry and dead-letter policy for dispatch delivery. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ExecutionDispatchRetryPolicy { + /// Maximum claimed attempts before dead-lettering. + pub max_attempts: u32, + /// Delay after the first failed attempt. + pub base_delay: Duration, + /// Maximum retry delay. + pub maximum_delay: Duration, +} + +impl ExecutionDispatchRetryPolicy { + /// Returns the bounded delay after one failed claimed attempt. + #[must_use] + pub fn retry_delay(self, attempts: u32) -> Duration { + let shift = attempts.saturating_sub(1).min(16); + self.base_delay + .saturating_mul(1_u32 << shift) + .min(self.maximum_delay) + } + + fn validate(self) -> Result<()> { + if self.max_attempts == 0 + || self.base_delay.is_zero() + || self.maximum_delay < self.base_delay + { + return Err(Error::InvalidRepositoryInput { + message: "execution dispatch retry policy requires positive ordered bounds" + .to_string(), + }); + } + Ok(()) + } +} + +/// Result of recording a claimed dispatch failure. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum ExecutionDispatchFailureOutcome { + /// The exact claim was released behind this durable retry time. + RetryScheduled { not_before_at: DateTime }, + /// The exact claim exhausted its delivery budget. + DeadLettered, + /// The row was absent, already terminal, or owned by another claim. + StaleClaim, +} + +/// One count-capped queue sample suitable for low-frequency fleet metrics. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ExecutionQueueBacklogSample { + /// Oldest observed due or terminal timestamp. + pub oldest_at: Option>, + /// Number of observed rows, capped at the caller's sample limit. + pub observed_count: u32, + /// Whether at least one additional row existed beyond the reported count. + pub saturated: bool, +} + +/// Bounded trigger/outbox health observed in one scoped transaction. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ExecutionQueueHealthSnapshot { + /// Canonical database observation time. + pub observed_at: DateTime, + /// Pending triggers whose canonical deadline has arrived. + pub due_triggers: ExecutionQueueBacklogSample, + /// Pending or claim-expired dispatches eligible for delivery. + pub claimable_dispatches: ExecutionQueueBacklogSample, + /// Trigger deliveries that exhausted their retry policy. + pub dead_letter_triggers: ExecutionQueueBacklogSample, + /// Outbox deliveries that exhausted their retry policy. + pub dead_letter_dispatches: ExecutionQueueBacklogSample, +} + +/// Database clock and earliest indexed deadline for pending dispatch work. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct ExecutionDispatchWake { + /// Database time observed in the same statement as the pending deadline. + pub observed_at: DateTime, + /// Exact queue-head dispatch, if any pending work exists. + pub dispatch_uid: Option, + /// Earliest pending delivery deadline, if any pending work exists. + pub next_due_at: Option>, + /// Revision of the exact queue head, changed whenever delivery is rearmed or requeued. + pub head_updated_at: Option>, +} + +/// Closed, low-cardinality execution maintenance job identity. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ExecutionMaintenanceJobKind { + /// Repairs due trigger delivery and drains the transactional dispatch outbox. + DispatchReconciliation, +} + +impl ExecutionMaintenanceJobKind { + /// Returns the canonical database label. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::DispatchReconciliation => "execution_dispatch_reconciliation", + } + } +} + +impl FromStr for ExecutionMaintenanceJobKind { + type Err = Error; + + fn from_str(value: &str) -> Result { + match value { + "execution_dispatch_reconciliation" => Ok(Self::DispatchReconciliation), + _ => Err(Error::InvalidRepositoryData { + message: format!("unknown execution maintenance job kind `{value}`"), + }), + } + } +} + +/// Durable fleet-wide execution maintenance checkpoint. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ExecutionMaintenanceCheckpoint { + /// Closed maintenance job kind. + pub job_kind: ExecutionMaintenanceJobKind, + /// Monotonic invocation generation. + pub generation: u64, + /// Most recent invocation start. + pub last_started_at: Option>, + /// Most recent successful completion. + pub last_succeeded_at: Option>, + /// Most recent failed completion. + pub last_failure_at: Option>, + /// Bounded error from the most recent failure. + pub last_error: Option, + /// Last checkpoint mutation time. + pub updated_at: DateTime, +} + +/// Generation-fenced maintenance completion result. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum ExecutionMaintenanceSettlementOutcome { + /// The exact invocation generation recorded its result. + Applied(ExecutionMaintenanceCheckpoint), + /// The checkpoint was absent or a newer invocation generation already started. + StaleOrMissing, +} + +impl ExecutionRepository { + /// Starts one fleet maintenance invocation and returns its new generation. + pub async fn begin_execution_maintenance( + &self, + scope: ExecutionScope, + job_kind: ExecutionMaintenanceJobKind, + ) -> Result { + require_control_plane_scope(scope)?; + let mut conn = scope.begin(&self.pool).await?; + let row = sqlx::query( + r#" + INSERT INTO moa.execution_maintenance_checkpoint ( + job_kind, generation, last_started_at, updated_at + ) VALUES ($1, 1, now(), now()) + ON CONFLICT (job_kind) DO UPDATE + SET generation = moa.execution_maintenance_checkpoint.generation + 1, + last_started_at = now(), updated_at = now() + RETURNING * + "#, + ) + .bind(job_kind.as_str()) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let checkpoint = maintenance_checkpoint_from_row(&row)?; + conn.commit().await.map_err(storage_error)?; + Ok(checkpoint) + } + + /// Records successful completion only for the exact invocation generation. + pub async fn complete_execution_maintenance( + &self, + scope: ExecutionScope, + job_kind: ExecutionMaintenanceJobKind, + expected_generation: u64, + ) -> Result { + settle_execution_maintenance(self, scope, job_kind, expected_generation, None).await + } + + /// Records a bounded failure only for the exact invocation generation. + pub async fn fail_execution_maintenance( + &self, + scope: ExecutionScope, + job_kind: ExecutionMaintenanceJobKind, + expected_generation: u64, + error: &str, + ) -> Result { + let error = bounded_maintenance_error(error)?; + settle_execution_maintenance(self, scope, job_kind, expected_generation, Some(error)).await + } + + /// Loads the durable health receipt for one fleet maintenance job. + pub async fn load_execution_maintenance_checkpoint( + &self, + scope: ExecutionScope, + job_kind: ExecutionMaintenanceJobKind, + ) -> Result> { + require_control_plane_scope(scope)?; + let mut conn = scope.begin(&self.pool).await?; + let row = + sqlx::query("SELECT * FROM moa.execution_maintenance_checkpoint WHERE job_kind = $1") + .bind(job_kind.as_str()) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let checkpoint = row + .as_ref() + .map(maintenance_checkpoint_from_row) + .transpose()?; + conn.commit().await.map_err(storage_error)?; + Ok(checkpoint) + } + + /// Samples trigger/outbox health with a strict per-queue row-read bound. + pub async fn sample_execution_queue_health( + &self, + scope: ExecutionScope, + sample_limit: u32, + ) -> Result { + if sample_limit == 0 || sample_limit > MAX_HEALTH_SAMPLE_SIZE { + return Err(Error::InvalidRepositoryInput { + message: format!( + "execution queue health sample must be 1..={MAX_HEALTH_SAMPLE_SIZE}" + ), + }); + } + let fetch_limit = i64::from(sample_limit) + 1; + let mut conn = scope.begin(&self.pool).await?; + let observed_at = sqlx::query_scalar::<_, DateTime>("SELECT now()") + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let due_triggers = sqlx::query_scalar::<_, DateTime>( + r#" + SELECT due_at + FROM moa.execution_trigger + WHERE state = 'pending' AND due_at <= now() + ORDER BY due_at, tenant_id, trigger_uid + LIMIT $1 + "#, + ) + .bind(fetch_limit) + .fetch_all(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let claimable_dispatches = sqlx::query_scalar::<_, DateTime>( + r#" + SELECT claimable_at + FROM ( + SELECT not_before_at AS claimable_at, dispatch_uid + FROM moa.execution_dispatch_outbox + WHERE state = 'pending' AND not_before_at <= now() + UNION ALL + SELECT claim_expires_at AS claimable_at, dispatch_uid + FROM moa.execution_dispatch_outbox + WHERE state = 'dispatching' AND claim_expires_at <= now() + ) AS claimable + ORDER BY claimable_at, dispatch_uid + LIMIT $1 + "#, + ) + .bind(fetch_limit) + .fetch_all(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let dead_letter_triggers = sqlx::query_scalar::<_, DateTime>( + r#" + SELECT created_at + FROM moa.execution_trigger + WHERE state = 'dead_letter' + ORDER BY created_at, tenant_id, trigger_uid + LIMIT $1 + "#, + ) + .bind(fetch_limit) + .fetch_all(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let dead_letter_dispatches = sqlx::query_scalar::<_, DateTime>( + r#" + SELECT created_at + FROM moa.execution_dispatch_outbox + WHERE state = 'dead_letter' + ORDER BY created_at, tenant_id, dispatch_uid + LIMIT $1 + "#, + ) + .bind(fetch_limit) + .fetch_all(conn.as_mut()) + .await + .map_err(sqlx_error)?; + conn.commit().await.map_err(storage_error)?; + Ok(ExecutionQueueHealthSnapshot { + observed_at, + due_triggers: backlog_sample(due_triggers, sample_limit), + claimable_dispatches: backlog_sample(claimable_dispatches, sample_limit), + dead_letter_triggers: backlog_sample(dead_letter_triggers, sample_limit), + dead_letter_dispatches: backlog_sample(dead_letter_dispatches, sample_limit), + }) + } + + /// Enqueues one dispatch in its own scoped transaction. + pub async fn enqueue_dispatch( + &self, + scope: ExecutionScope, + request: NewExecutionDispatch, + ) -> Result { + let mut conn = scope.begin(&self.pool).await?; + let record = enqueue_dispatch_in_conn(conn.as_mut(), &request).await?; + conn.commit().await.map_err(storage_error)?; + Ok(record) + } + + /// Claims a bounded due batch, including claims abandoned past their expiry. + pub async fn claim_due_dispatches( + &self, + scope: ExecutionScope, + claim_owner: &str, + batch_size: u32, + claim_ttl: Duration, + ) -> Result> { + validate_claim_request(claim_owner, batch_size, claim_ttl)?; + let claim_ttl_seconds = duration_seconds_ceil(claim_ttl, "dispatch claim TTL")?; + let mut conn = scope.begin(&self.pool).await?; + let rows = sqlx::query( + r#" + WITH claimable AS ( + SELECT dispatch_uid + FROM moa.execution_dispatch_outbox + WHERE ( + state = 'pending' + AND not_before_at <= now() + ) OR ( + state = 'dispatching' + AND claim_expires_at <= now() + ) + ORDER BY not_before_at, created_at, dispatch_uid + LIMIT $1 + FOR UPDATE SKIP LOCKED + ) + UPDATE moa.execution_dispatch_outbox AS dispatch + SET state = 'dispatching', + claim_owner = $2, + claimed_at = now(), + claim_expires_at = now() + make_interval(secs => $3), + delivery_attempts = dispatch.delivery_attempts + 1, + updated_at = now() + FROM claimable + WHERE dispatch.dispatch_uid = claimable.dispatch_uid + RETURNING dispatch.* + "#, + ) + .bind(i64::from(batch_size)) + .bind(claim_owner) + .bind(claim_ttl_seconds) + .fetch_all(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let records = rows.iter().map(dispatch_from_row).collect::>()?; + conn.commit().await.map_err(storage_error)?; + Ok(records) + } + + /// Returns the earliest pending outbox deadline through the bounded queue index. + pub async fn next_pending_dispatch_wake( + &self, + scope: ExecutionScope, + ) -> Result { + let mut conn = scope.begin(&self.pool).await?; + let (observed_at, dispatch_uid, next_due_at, head_updated_at) = sqlx::query_as::< + _, + ( + DateTime, + Option, + Option>, + Option>, + ), + >( + r#" + SELECT now(), head.dispatch_uid, head.not_before_at, head.updated_at + FROM (SELECT 1) AS singleton + LEFT JOIN LATERAL ( + SELECT candidate.dispatch_uid, candidate.not_before_at, candidate.updated_at + FROM ( + (SELECT dispatch_uid, not_before_at, updated_at, created_at + FROM moa.execution_dispatch_outbox + WHERE state = 'pending' + ORDER BY not_before_at, created_at, dispatch_uid + LIMIT 1) + UNION ALL + (SELECT dispatch_uid, claim_expires_at AS not_before_at, updated_at, created_at + FROM moa.execution_dispatch_outbox + WHERE state = 'dispatching' AND claim_expires_at IS NOT NULL + ORDER BY claim_expires_at, created_at, dispatch_uid + LIMIT 1) + ) AS candidate + ORDER BY candidate.not_before_at, candidate.created_at, candidate.dispatch_uid + LIMIT 1 + ) AS head ON TRUE + "#, + ) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + conn.commit().await.map_err(storage_error)?; + Ok(ExecutionDispatchWake { + observed_at, + dispatch_uid, + next_due_at, + head_updated_at, + }) + } + + /// Acknowledges a bounded delivery batch only for the exact current claim owner. + /// + /// Returned identities retain request order. Missing identities were absent, terminal, or + /// changed claim owner before this acknowledgement committed. + pub async fn mark_dispatches_delivered( + &self, + scope: ExecutionScope, + dispatch_uids: &[Uuid], + claim_owner: &str, + ) -> Result> { + validate_claim_owner(claim_owner)?; + validate_ack_batch(dispatch_uids)?; + let mut conn = scope.begin(&self.pool).await?; + let applied = sqlx::query_scalar::<_, Uuid>( + r#" + UPDATE moa.execution_dispatch_outbox + SET state = 'delivered', delivered_at = now(), claim_owner = NULL, + claimed_at = NULL, claim_expires_at = NULL, last_error = NULL, + updated_at = now() + WHERE dispatch_uid = ANY($1::UUID[]) + AND state = 'dispatching' AND claim_owner = $2 + RETURNING dispatch_uid + "#, + ) + .bind(dispatch_uids) + .bind(claim_owner) + .fetch_all(conn.as_mut()) + .await + .map_err(sqlx_error)?; + conn.commit().await.map_err(storage_error)?; + let applied = applied.into_iter().collect::>(); + Ok(dispatch_uids + .iter() + .copied() + .filter(|dispatch_uid| applied.contains(dispatch_uid)) + .collect()) + } + + /// Releases one exact failed claim behind backoff or moves it to dead letter. + pub async fn record_dispatch_failure( + &self, + scope: ExecutionScope, + dispatch_uid: Uuid, + claim_owner: &str, + error: &str, + retry: ExecutionDispatchRetryPolicy, + ) -> Result { + validate_claim_owner(claim_owner)?; + retry.validate()?; + let mut conn = scope.begin(&self.pool).await?; + let dispatch = sqlx::query( + r#" + SELECT * + FROM moa.execution_dispatch_outbox + WHERE dispatch_uid = $1 AND state = 'dispatching' AND claim_owner = $2 + FOR UPDATE + "#, + ) + .bind(dispatch_uid) + .bind(claim_owner) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(dispatch) = dispatch else { + conn.commit().await.map_err(storage_error)?; + return Ok(ExecutionDispatchFailureOutcome::StaleClaim); + }; + let dispatch = dispatch_from_row(&dispatch)?; + let attempts = dispatch.delivery_attempts; + let dispatch_kind = dispatch.kind; + let last_error = error.chars().take(MAX_ERROR_CHARS).collect::(); + let requires_durable_retry = dispatch_requires_durable_retry(dispatch_kind); + let outcome = if attempts >= retry.max_attempts && !requires_durable_retry { + if dispatch_kind == ExecutionDispatchKind::TaskAttempt { + repair_dead_lettered_task_dispatch_in_conn(&mut conn, &dispatch).await?; + } else if dispatch_kind == ExecutionDispatchKind::CompensationAttempt { + repair_dead_lettered_compensation_dispatch_in_conn(&mut conn, &dispatch).await?; + } + sqlx::query( + r#" + UPDATE moa.execution_dispatch_outbox + SET state = 'dead_letter', claim_owner = NULL, claimed_at = NULL, + claim_expires_at = NULL, last_error = $3, updated_at = now() + WHERE dispatch_uid = $1 AND state = 'dispatching' AND claim_owner = $2 + "#, + ) + .bind(dispatch_uid) + .bind(claim_owner) + .bind(&last_error) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + ExecutionDispatchFailureOutcome::DeadLettered + } else { + let delay = if attempts >= retry.max_attempts { + retry.maximum_delay + } else { + retry.retry_delay(attempts) + }; + let delay_seconds = duration_seconds_ceil(delay, "dispatch retry delay")?; + let not_before_at = sqlx::query_scalar::<_, DateTime>( + r#" + UPDATE moa.execution_dispatch_outbox + SET state = 'pending', not_before_at = now() + make_interval(secs => $3), + claim_owner = NULL, claimed_at = NULL, claim_expires_at = NULL, + last_error = $4, updated_at = now() + WHERE dispatch_uid = $1 AND state = 'dispatching' AND claim_owner = $2 + RETURNING not_before_at + "#, + ) + .bind(dispatch_uid) + .bind(claim_owner) + .bind(delay_seconds) + .bind(last_error) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + ExecutionDispatchFailureOutcome::RetryScheduled { not_before_at } + }; + conn.commit().await.map_err(storage_error)?; + Ok(outcome) + } +} + +fn dispatch_requires_durable_retry(kind: ExecutionDispatchKind) -> bool { + match kind { + ExecutionDispatchKind::RunActivation + | ExecutionDispatchKind::TaskAttemptCancel + | ExecutionDispatchKind::CompensationAttemptCancel + | ExecutionDispatchKind::TriggerDelivery + | ExecutionDispatchKind::ExternalCancel => true, + ExecutionDispatchKind::TaskAttempt | ExecutionDispatchKind::CompensationAttempt => false, + } +} + +async fn repair_dead_lettered_task_dispatch_in_conn( + conn: &mut super::ScopedConn<'_>, + dispatch: &ExecutionDispatchRecord, +) -> Result<()> { + let request = serde_json::from_value::(dispatch.payload.clone()) + .map_err(|error| Error::InvalidRepositoryData { + message: format!("invalid dead-letter task-attempt payload: {error}"), + })?; + if request.dispatch_uid != dispatch.dispatch_uid + || request.tenant_id != dispatch.tenant_id + || Some(request.run_uid) != dispatch.run_uid + || Some(request.task_id.as_uuid()) != dispatch.task_id + || Some(request.controller_generation) != dispatch.controller_generation + || Some(request.attempt_generation) != dispatch.attempt_generation + { + return Err(Error::InvalidRepositoryData { + message: "dead-letter task-attempt payload lost immutable outbox fences".to_string(), + }); + } + let settled_at = sqlx::query_scalar::<_, DateTime>("SELECT now()") + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let outcome = super::task::settle_unstarted_task_attempt_in_conn( + conn, + TaskAttemptFence { + tenant_id: request.tenant_id, + run_uid: request.run_uid, + task_id: request.task_id, + controller_generation: request.controller_generation, + attempt_generation: request.attempt_generation, + dispatch_uid: request.dispatch_uid, + capacity_reservation_uid: request.capacity_reservation_uid, + watchdog_trigger_uid: request.watchdog_trigger_uid, + attempt_deadline_at: request.attempt_deadline_at, + }, + settled_at, + ) + .await?; + match outcome { + TaskAttemptSettlementOutcome::Applied { .. } + | TaskAttemptSettlementOutcome::Replayed { .. } => Ok(()), + TaskAttemptSettlementOutcome::NotFound + | TaskAttemptSettlementOutcome::Stale + | TaskAttemptSettlementOutcome::InvalidState => Err(Error::InvalidRepositoryData { + message: "task-attempt dead letter could not repair its exact Dispatching owner" + .to_string(), + }), + } +} + +async fn repair_dead_lettered_compensation_dispatch_in_conn( + conn: &mut super::ScopedConn<'_>, + dispatch: &ExecutionDispatchRecord, +) -> Result<()> { + let request = + serde_json::from_value::(dispatch.payload.clone()) + .map_err(|error| Error::InvalidRepositoryData { + message: format!("invalid dead-letter compensation-attempt payload: {error}"), + })?; + if request.dispatch_uid != dispatch.dispatch_uid + || request.tenant_id != dispatch.tenant_id + || Some(request.run_uid) != dispatch.run_uid + || Some(request.compensation_id.as_uuid()) != dispatch.compensation_id + || Some(request.controller_generation) != dispatch.controller_generation + || Some(request.compensation_generation) != dispatch.compensation_generation + || Some(request.compensation_attempt_generation) != dispatch.compensation_attempt_generation + { + return Err(Error::InvalidRepositoryData { + message: "dead-letter compensation-attempt payload lost immutable outbox fences" + .to_string(), + }); + } + let settled_at = sqlx::query_scalar::<_, DateTime>("SELECT now()") + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let outcome = super::compensation::settle_unstarted_compensation_attempt_in_conn( + conn, &request, settled_at, + ) + .await?; + match outcome { + super::compensation::CompensationAttemptWriteOutcome::Applied(_) + | super::compensation::CompensationAttemptWriteOutcome::Replayed(_) => Ok(()), + super::compensation::CompensationAttemptWriteOutcome::Conflict + | super::compensation::CompensationAttemptWriteOutcome::NotFound => { + Err(Error::InvalidRepositoryData { + message: + "compensation-attempt dead letter could not repair its exact Dispatching owner" + .to_string(), + }) + } + } +} + +async fn settle_execution_maintenance( + repository: &ExecutionRepository, + scope: ExecutionScope, + job_kind: ExecutionMaintenanceJobKind, + expected_generation: u64, + error: Option, +) -> Result { + require_control_plane_scope(scope)?; + if expected_generation == 0 { + return Err(Error::InvalidRepositoryInput { + message: "execution maintenance generation must be positive".to_string(), + }); + } + let expected_generation = + i64::try_from(expected_generation).map_err(|_| Error::InvalidRepositoryInput { + message: "execution maintenance generation exceeds PostgreSQL BIGINT".to_string(), + })?; + let mut conn = scope.begin(&repository.pool).await?; + let row = if let Some(error) = error { + sqlx::query( + r#" + UPDATE moa.execution_maintenance_checkpoint + SET last_failure_at = now(), last_error = $3, updated_at = now() + WHERE job_kind = $1 AND generation = $2 + RETURNING * + "#, + ) + .bind(job_kind.as_str()) + .bind(expected_generation) + .bind(error) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + } else { + sqlx::query( + r#" + UPDATE moa.execution_maintenance_checkpoint + SET last_succeeded_at = now(), updated_at = now() + WHERE job_kind = $1 AND generation = $2 + RETURNING * + "#, + ) + .bind(job_kind.as_str()) + .bind(expected_generation) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + }; + let outcome = row + .as_ref() + .map(maintenance_checkpoint_from_row) + .transpose()? + .map_or( + ExecutionMaintenanceSettlementOutcome::StaleOrMissing, + ExecutionMaintenanceSettlementOutcome::Applied, + ); + conn.commit().await.map_err(storage_error)?; + Ok(outcome) +} + +fn require_control_plane_scope(scope: ExecutionScope) -> Result<()> { + if scope != ExecutionScope::ControlPlane { + return Err(Error::InvalidRepositoryInput { + message: "execution maintenance checkpoints require control-plane scope".to_string(), + }); + } + Ok(()) +} + +fn bounded_maintenance_error(error: &str) -> Result { + let error = error.trim(); + if error.is_empty() { + return Err(Error::InvalidRepositoryInput { + message: "execution maintenance failure requires a non-empty error".to_string(), + }); + } + let mut end = error.len().min(MAX_MAINTENANCE_ERROR_BYTES); + while !error.is_char_boundary(end) { + end -= 1; + } + Ok(error[..end].to_string()) +} + +fn maintenance_checkpoint_from_row( + row: &sqlx::postgres::PgRow, +) -> Result { + let generation = row + .try_get::("generation") + .map_err(super::row_error)?; + Ok(ExecutionMaintenanceCheckpoint { + job_kind: row + .try_get::("job_kind") + .map_err(super::row_error)? + .parse()?, + generation: super::to_u64(generation, "execution maintenance generation")?, + last_started_at: row.try_get("last_started_at").map_err(super::row_error)?, + last_succeeded_at: row.try_get("last_succeeded_at").map_err(super::row_error)?, + last_failure_at: row.try_get("last_failure_at").map_err(super::row_error)?, + last_error: row.try_get("last_error").map_err(super::row_error)?, + updated_at: row.try_get("updated_at").map_err(super::row_error)?, + }) +} + +fn backlog_sample( + mut timestamps: Vec>, + sample_limit: u32, +) -> ExecutionQueueBacklogSample { + let saturated = timestamps.len() > sample_limit as usize; + timestamps.truncate(sample_limit as usize); + ExecutionQueueBacklogSample { + oldest_at: timestamps.first().copied(), + observed_count: u32::try_from(timestamps.len()).unwrap_or(sample_limit), + saturated, + } +} + +/// Inserts one durable dispatch without committing the caller-owned transaction. +pub async fn enqueue_dispatch_in_conn( + conn: &mut PgConnection, + request: &NewExecutionDispatch, +) -> Result { + validate_dispatch(request)?; + let controller_generation = + to_optional_i64(request.controller_generation, "controller generation")?; + let wake_epoch = to_optional_i64(request.wake_epoch, "wake epoch")?; + let attempt_generation = to_optional_i64(request.attempt_generation, "attempt generation")?; + let compensation_generation = + to_optional_i64(request.compensation_generation, "compensation generation")?; + let compensation_attempt_generation = to_optional_i64( + request.compensation_attempt_generation, + "compensation attempt generation", + )?; + let inserted = sqlx::query( + r#" + INSERT INTO moa.execution_dispatch_outbox ( + dispatch_uid, tenant_id, run_uid, task_id, compensation_id, + trigger_uid, external_job_uid, + dispatch_kind, controller_generation, wake_epoch, attempt_generation, + compensation_generation, compensation_attempt_generation, + not_before_at, payload + ) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15 + ) + ON CONFLICT (dispatch_uid) DO NOTHING + RETURNING * + "#, + ) + .bind(request.dispatch_uid) + .bind(request.tenant_id.0) + .bind(request.run_uid) + .bind(request.task_id) + .bind(request.compensation_id) + .bind(request.trigger_uid) + .bind(request.external_job_uid) + .bind(request.kind.as_str()) + .bind(controller_generation) + .bind(wake_epoch) + .bind(attempt_generation) + .bind(compensation_generation) + .bind(compensation_attempt_generation) + .bind(request.not_before_at) + .bind(&request.payload) + .fetch_optional(&mut *conn) + .await + .map_err(sqlx_error)?; + let row = match inserted { + Some(row) => row, + None => sqlx::query("SELECT * FROM moa.execution_dispatch_outbox WHERE dispatch_uid = $1") + .bind(request.dispatch_uid) + .fetch_optional(&mut *conn) + .await + .map_err(sqlx_error)? + .ok_or_else(|| Error::Storage { + message: "dispatch insert conflicted without a visible replay row".to_string(), + })?, + }; + let record = dispatch_from_row(&row)?; + if !dispatch_matches_request(&record, request) { + return Err(Error::InvalidRepositoryInput { + message: "dispatch UID is already bound to different immutable semantics".to_string(), + }); + } + Ok(record) +} + +/// Requeues one previously accepted dispatch without changing its immutable identity. +/// +/// The caller must first establish the authoritative generation fence while holding the +/// corresponding trigger row lock. A non-delivered replay is left untouched. +pub(super) async fn requeue_delivered_dispatch_in_conn( + conn: &mut PgConnection, + request: &NewExecutionDispatch, +) -> Result> { + validate_dispatch(request)?; + let row = sqlx::query( + r#" + UPDATE moa.execution_dispatch_outbox + SET state = 'pending', delivered_at = NULL, delivery_attempts = 0, + claim_owner = NULL, claimed_at = NULL, claim_expires_at = NULL, + last_error = NULL, updated_at = now() + WHERE dispatch_uid = $1 AND state = 'delivered' + RETURNING * + "#, + ) + .bind(request.dispatch_uid) + .fetch_optional(&mut *conn) + .await + .map_err(sqlx_error)?; + let Some(row) = row else { + return Ok(None); + }; + let record = dispatch_from_row(&row)?; + if !dispatch_matches_request(&record, request) { + return Err(Error::InvalidRepositoryData { + message: "delivered dispatch replay no longer matches its immutable intent".to_string(), + }); + } + Ok(Some(record)) +} + +/// Requeues a bounded page of accepted run activations whose authoritative wake remains queued. +pub(super) async fn requeue_current_run_activations_in_conn( + conn: &mut PgConnection, + batch_size: u32, + grace_seconds: i64, +) -> Result> { + if batch_size == 0 { + return Ok(Vec::new()); + } + let rows = sqlx::query( + r#" + WITH candidates AS ( + SELECT dispatch.dispatch_uid + FROM moa.execution_run AS run + JOIN moa.execution_dispatch_outbox AS dispatch + ON dispatch.tenant_id = run.tenant_id + AND dispatch.run_uid = run.run_uid + AND dispatch.controller_generation = run.controller_generation + AND dispatch.wake_epoch = run.wake_epoch + AND dispatch.dispatch_kind = 'run_activation' + WHERE run.activation_state = 'queued' + AND run.processed_wake_epoch < run.wake_epoch + AND run.status NOT IN ( + 'completed', 'partial', 'blocked', 'unsupported', 'failed', 'cancelled' + ) + AND dispatch.state = 'delivered' + AND dispatch.delivered_at <= now() - make_interval(secs => $2) + ORDER BY run.updated_at, run.run_uid + LIMIT $1 + FOR UPDATE OF dispatch SKIP LOCKED + ) + UPDATE moa.execution_dispatch_outbox AS dispatch + SET state = 'pending', delivered_at = NULL, delivery_attempts = 0, + claim_owner = NULL, claimed_at = NULL, claim_expires_at = NULL, + last_error = NULL, updated_at = now() + FROM candidates + WHERE dispatch.dispatch_uid = candidates.dispatch_uid + AND dispatch.state = 'delivered' + RETURNING dispatch.* + "#, + ) + .bind(i64::from(batch_size)) + .bind(grace_seconds) + .fetch_all(&mut *conn) + .await + .map_err(sqlx_error)?; + rows.iter().map(dispatch_from_row).collect() +} + +/// Requeues accepted deliveries only while their exact bounded work has not started. +/// +/// A running attempt is deliberately excluded: once effects may have begun, its +/// watchdog owns ambiguity resolution and the dispatcher must not replay it. +pub(super) async fn requeue_current_accepted_dispatches_in_conn( + conn: &mut PgConnection, + batch_size: u32, + grace_seconds: i64, +) -> Result> { + if batch_size == 0 { + return Ok(Vec::new()); + } + let rows = sqlx::query( + r#" + WITH candidates AS ( + SELECT dispatch.dispatch_uid + FROM moa.execution_dispatch_outbox AS dispatch + WHERE dispatch.state='delivered' + AND dispatch.delivered_at <= now() - make_interval(secs => $2) + AND ( + ( + dispatch.dispatch_kind='task_attempt' + AND EXISTS ( + SELECT 1 + FROM moa.execution_task AS task + JOIN moa.execution_run AS run + ON run.tenant_id=task.tenant_id AND run.run_uid=task.run_uid + WHERE task.tenant_id=dispatch.tenant_id + AND task.run_uid=dispatch.run_uid + AND task.task_id=dispatch.task_id + AND task.attempt_generation=dispatch.attempt_generation + AND task.active_dispatch_uid=dispatch.dispatch_uid + AND task.status='running' AND task.attempt_state='dispatching' + AND run.controller_generation=dispatch.controller_generation + ) + ) OR ( + dispatch.dispatch_kind='compensation_attempt' + AND EXISTS ( + SELECT 1 + FROM moa.execution_compensation AS compensation + JOIN moa.execution_run AS run + ON run.tenant_id=compensation.tenant_id + AND run.run_uid=compensation.run_uid + WHERE compensation.tenant_id=dispatch.tenant_id + AND compensation.run_uid=dispatch.run_uid + AND compensation.compensation_id=dispatch.compensation_id + AND compensation.generation=dispatch.compensation_generation + AND compensation.attempt_generation=dispatch.compensation_attempt_generation + AND compensation.active_dispatch_uid=dispatch.dispatch_uid + AND compensation.status='running' + AND compensation.attempt_state='dispatching' + AND run.controller_generation=dispatch.controller_generation + ) + ) OR ( + dispatch.dispatch_kind='task_attempt_cancel' + AND EXISTS ( + SELECT 1 + FROM moa.execution_task AS task + JOIN moa.execution_run AS run + ON run.tenant_id=task.tenant_id AND run.run_uid=task.run_uid + WHERE task.tenant_id=dispatch.tenant_id + AND task.run_uid=dispatch.run_uid + AND task.task_id=dispatch.task_id + AND task.attempt_generation=dispatch.attempt_generation + AND task.attempt_state='cancelling' + AND task.active_dispatch_uid::text=dispatch.payload->>'active_dispatch_uid' + AND task.generation::text=dispatch.payload->>'task_generation' + AND run.controller_generation=dispatch.controller_generation + ) + ) OR ( + dispatch.dispatch_kind='compensation_attempt_cancel' + AND EXISTS ( + SELECT 1 + FROM moa.execution_compensation AS compensation + JOIN moa.execution_run AS run + ON run.tenant_id=compensation.tenant_id + AND run.run_uid=compensation.run_uid + WHERE compensation.tenant_id=dispatch.tenant_id + AND compensation.run_uid=dispatch.run_uid + AND compensation.compensation_id=dispatch.compensation_id + AND compensation.generation=dispatch.compensation_generation + AND compensation.attempt_generation=dispatch.compensation_attempt_generation + AND compensation.attempt_state='cancelling' + AND compensation.active_dispatch_uid::text + =dispatch.payload->>'active_dispatch_uid' + AND run.controller_generation=dispatch.controller_generation + ) + ) OR ( + dispatch.dispatch_kind='external_cancel' + AND EXISTS ( + SELECT 1 + FROM moa.execution_external_job AS job + JOIN moa.execution_run AS run + ON run.tenant_id=job.tenant_id AND run.run_uid=job.run_uid + WHERE job.tenant_id=dispatch.tenant_id + AND job.run_uid=dispatch.run_uid + AND job.external_job_uid=dispatch.external_job_uid + AND job.task_id IS NOT DISTINCT FROM dispatch.task_id + AND job.attempt_generation IS NOT DISTINCT FROM dispatch.attempt_generation + AND job.compensation_id IS NOT DISTINCT FROM dispatch.compensation_id + AND job.compensation_generation IS NOT DISTINCT FROM dispatch.compensation_generation + AND job.compensation_attempt_generation + IS NOT DISTINCT FROM dispatch.compensation_attempt_generation + AND job.job_generation::text=dispatch.payload->>'job_generation' + AND job.provider=dispatch.payload->>'provider' + AND job.provider_job_id=dispatch.payload->>'provider_job_id' + AND job.idempotency_key=dispatch.payload->>'idempotency_key' + AND job.state='cancel_requested' + AND run.controller_generation=dispatch.controller_generation + ) + ) + ) + ORDER BY dispatch.delivered_at, dispatch.tenant_id, dispatch.dispatch_uid + LIMIT $1 + FOR UPDATE OF dispatch SKIP LOCKED + ) + UPDATE moa.execution_dispatch_outbox AS dispatch + SET state='pending', delivered_at=NULL, delivery_attempts=0, + claim_owner=NULL, claimed_at=NULL, claim_expires_at=NULL, + last_error=NULL, updated_at=now() + FROM candidates + WHERE dispatch.dispatch_uid=candidates.dispatch_uid + AND dispatch.state='delivered' + RETURNING dispatch.* + "#, + ) + .bind(i64::from(batch_size)) + .bind(grace_seconds) + .fetch_all(&mut *conn) + .await + .map_err(sqlx_error)?; + rows.iter().map(dispatch_from_row).collect() +} + +fn validate_dispatch(request: &NewExecutionDispatch) -> Result<()> { + let generation_is_zero = [ + request.controller_generation, + request.wake_epoch, + request.attempt_generation, + request.compensation_generation, + request.compensation_attempt_generation, + ] + .into_iter() + .flatten() + .any(|generation| generation == 0); + if request.dispatch_uid.is_nil() || !request.payload.is_object() || generation_is_zero { + return Err(Error::InvalidRepositoryInput { + message: "execution dispatch requires a non-nil UID, positive generations, and object payload" + .to_string(), + }); + } + let shape_is_valid = match request.kind { + ExecutionDispatchKind::RunActivation => { + request.run_uid.is_some() + && request.task_id.is_none() + && request.compensation_id.is_none() + && request.trigger_uid.is_none() + && request.external_job_uid.is_none() + && request.controller_generation.is_some() + && request.wake_epoch.is_some() + && request.attempt_generation.is_none() + && request.compensation_generation.is_none() + && request.compensation_attempt_generation.is_none() + } + ExecutionDispatchKind::TaskAttempt => { + request.run_uid.is_some() + && request.task_id.is_some() + && request.compensation_id.is_none() + && request.trigger_uid.is_none() + && request.external_job_uid.is_none() + && request.controller_generation.is_some() + && request.wake_epoch.is_none() + && request.attempt_generation.is_some() + && request.compensation_generation.is_none() + && request.compensation_attempt_generation.is_none() + } + ExecutionDispatchKind::TaskAttemptCancel => { + request.run_uid.is_some() + && request.task_id.is_some() + && request.compensation_id.is_none() + && request.trigger_uid.is_none() + && request.external_job_uid.is_none() + && request.controller_generation.is_some() + && request.wake_epoch.is_none() + && request.attempt_generation.is_some() + && request.compensation_generation.is_none() + && request.compensation_attempt_generation.is_none() + && task_cancel_payload_matches(request) + } + ExecutionDispatchKind::CompensationAttempt => { + request.run_uid.is_some() + && request.task_id.is_none() + && request.compensation_id.is_some() + && request.trigger_uid.is_none() + && request.external_job_uid.is_none() + && request.controller_generation.is_some() + && request.wake_epoch.is_none() + && request.attempt_generation.is_none() + && request.compensation_generation.is_some() + && request.compensation_attempt_generation.is_some() + } + ExecutionDispatchKind::CompensationAttemptCancel => { + request.run_uid.is_some() + && request.task_id.is_none() + && request.compensation_id.is_some() + && request.trigger_uid.is_none() + && request.external_job_uid.is_none() + && request.controller_generation.is_some() + && request.wake_epoch.is_none() + && request.attempt_generation.is_none() + && request.compensation_generation.is_some() + && request.compensation_attempt_generation.is_some() + && compensation_cancel_payload_matches(request) + } + ExecutionDispatchKind::TriggerDelivery => { + request.run_uid.is_none() + && request.task_id.is_none() + && request.compensation_id.is_none() + && request.trigger_uid.is_some() + && request.external_job_uid.is_none() + && request.wake_epoch.is_none() + && request.attempt_generation.is_none() + && request.controller_generation.is_none() + && request.compensation_generation.is_none() + && request.compensation_attempt_generation.is_none() + } + ExecutionDispatchKind::ExternalCancel => { + request.run_uid.is_some() + && request.trigger_uid.is_none() + && request.external_job_uid.is_some() + && request.controller_generation.is_some() + && request.wake_epoch.is_none() + && ((request.task_id.is_some() + && request.compensation_id.is_none() + && request.attempt_generation.is_some() + && request.compensation_generation.is_none() + && request.compensation_attempt_generation.is_none()) + || (request.task_id.is_none() + && request.compensation_id.is_some() + && request.attempt_generation.is_none() + && request.compensation_generation.is_some() + && request.compensation_attempt_generation.is_some())) + && external_cancel_payload_matches(request) + } + }; + if !shape_is_valid { + return Err(Error::InvalidRepositoryInput { + message: format!( + "execution dispatch target shape does not match {}", + request.kind.as_str() + ), + }); + } + Ok(()) +} + +fn external_cancel_payload_matches(request: &NewExecutionDispatch) -> bool { + serde_json::from_value::(request.payload.clone()).is_ok_and( + |payload| { + payload.tenant_id == request.tenant_id + && Some(payload.external_job_uid) == request.external_job_uid + && payload.job_generation > 0 + && !payload.provider.trim().is_empty() + && !payload.provider_job_id.trim().is_empty() + && !payload.idempotency_key.trim().is_empty() + }, + ) +} + +fn task_cancel_payload_matches(request: &NewExecutionDispatch) -> bool { + serde_json::from_value::(request.payload.clone()).is_ok_and( + |payload| { + payload.cancellation_dispatch_uid == request.dispatch_uid + && payload.tenant_id == request.tenant_id + && Some(payload.run_uid) == request.run_uid + && Some(payload.task_id.as_uuid()) == request.task_id + && Some(payload.controller_generation) == request.controller_generation + && Some(payload.attempt_generation) == request.attempt_generation + && payload.attempt_controller_generation > 0 + && payload.task_generation > 0 + && !payload.active_dispatch_uid.is_nil() + && !payload.capacity_reservation_uid.is_nil() + && !payload.watchdog_trigger_uid.is_nil() + }, + ) +} + +fn compensation_cancel_payload_matches(request: &NewExecutionDispatch) -> bool { + serde_json::from_value::(request.payload.clone()) + .is_ok_and(|payload| { + payload.cancellation_dispatch_uid == request.dispatch_uid + && payload.tenant_id == request.tenant_id + && Some(payload.run_uid) == request.run_uid + && Some(payload.compensation_id.as_uuid()) == request.compensation_id + && Some(payload.controller_generation) == request.controller_generation + && payload.attempt_controller_generation > 0 + && Some(payload.compensation_generation) == request.compensation_generation + && Some(payload.compensation_attempt_generation) + == request.compensation_attempt_generation + && !payload.active_dispatch_uid.is_nil() + && !payload.capacity_reservation_uid.is_nil() + && !payload.watchdog_trigger_uid.is_nil() + }) +} + +fn validate_claim_request(claim_owner: &str, batch_size: u32, claim_ttl: Duration) -> Result<()> { + validate_claim_owner(claim_owner)?; + if batch_size == 0 || batch_size > MAX_CLAIM_BATCH_SIZE || claim_ttl.is_zero() { + return Err(Error::InvalidRepositoryInput { + message: format!( + "execution dispatch claim requires batch size 1..={MAX_CLAIM_BATCH_SIZE} and positive TTL" + ), + }); + } + Ok(()) +} + +fn validate_claim_owner(claim_owner: &str) -> Result<()> { + if claim_owner.trim().is_empty() || claim_owner.len() > 256 { + return Err(Error::InvalidRepositoryInput { + message: "execution dispatch claim owner must contain 1..=256 bytes".to_string(), + }); + } + Ok(()) +} + +fn validate_ack_batch(dispatch_uids: &[Uuid]) -> Result<()> { + if dispatch_uids.is_empty() || dispatch_uids.len() > MAX_CLAIM_BATCH_SIZE as usize { + return Err(Error::InvalidRepositoryInput { + message: format!( + "execution dispatch acknowledgement requires 1..={MAX_CLAIM_BATCH_SIZE} identities" + ), + }); + } + if dispatch_uids.iter().copied().collect::>().len() != dispatch_uids.len() { + return Err(Error::InvalidRepositoryInput { + message: "execution dispatch acknowledgement identities must be unique".to_string(), + }); + } + Ok(()) +} + +fn duration_seconds_ceil(duration: Duration, field: &str) -> Result { + let seconds = duration + .as_secs() + .saturating_add(u64::from(duration.subsec_nanos() > 0)); + i64::try_from(seconds).map_err(|_| Error::InvalidRepositoryInput { + message: format!("{field} exceeds PostgreSQL interval bounds"), + }) +} + +fn dispatch_matches_request( + record: &ExecutionDispatchRecord, + request: &NewExecutionDispatch, +) -> bool { + record.dispatch_uid == request.dispatch_uid + && record.tenant_id == request.tenant_id + && record.run_uid == request.run_uid + && record.task_id == request.task_id + && record.compensation_id == request.compensation_id + && record.trigger_uid == request.trigger_uid + && record.external_job_uid == request.external_job_uid + && record.kind == request.kind + && record.controller_generation == request.controller_generation + && record.wake_epoch == request.wake_epoch + && record.attempt_generation == request.attempt_generation + && record.compensation_generation == request.compensation_generation + && record.compensation_attempt_generation == request.compensation_attempt_generation + && record.not_before_at == request.not_before_at + && record.payload == request.payload +} + +fn dispatch_from_row(row: &sqlx::postgres::PgRow) -> Result { + let controller_generation = row + .try_get::, _>("controller_generation") + .map_err(super::row_error)?; + let wake_epoch = row + .try_get::, _>("wake_epoch") + .map_err(super::row_error)?; + let attempt_generation = row + .try_get::, _>("attempt_generation") + .map_err(super::row_error)?; + let compensation_generation = row + .try_get::, _>("compensation_generation") + .map_err(super::row_error)?; + let compensation_attempt_generation = row + .try_get::, _>("compensation_attempt_generation") + .map_err(super::row_error)?; + Ok(ExecutionDispatchRecord { + dispatch_uid: row.try_get("dispatch_uid").map_err(super::row_error)?, + tenant_id: TenantId(row.try_get("tenant_id").map_err(super::row_error)?), + run_uid: row.try_get("run_uid").map_err(super::row_error)?, + task_id: row.try_get("task_id").map_err(super::row_error)?, + compensation_id: row.try_get("compensation_id").map_err(super::row_error)?, + trigger_uid: row.try_get("trigger_uid").map_err(super::row_error)?, + external_job_uid: row.try_get("external_job_uid").map_err(super::row_error)?, + kind: row + .try_get::("dispatch_kind") + .map_err(super::row_error)? + .parse()?, + state: row + .try_get::("state") + .map_err(super::row_error)? + .parse()?, + controller_generation: controller_generation + .map(|value| super::to_u64(value, "controller generation")) + .transpose()?, + wake_epoch: wake_epoch + .map(|value| super::to_u64(value, "wake epoch")) + .transpose()?, + attempt_generation: attempt_generation + .map(|value| super::to_u64(value, "attempt generation")) + .transpose()?, + compensation_generation: compensation_generation + .map(|value| super::to_u64(value, "compensation generation")) + .transpose()?, + compensation_attempt_generation: compensation_attempt_generation + .map(|value| super::to_u64(value, "compensation attempt generation")) + .transpose()?, + not_before_at: row.try_get("not_before_at").map_err(super::row_error)?, + payload: row.try_get("payload").map_err(super::row_error)?, + claim_owner: row.try_get("claim_owner").map_err(super::row_error)?, + claimed_at: row.try_get("claimed_at").map_err(super::row_error)?, + claim_expires_at: row.try_get("claim_expires_at").map_err(super::row_error)?, + delivery_attempts: to_u32( + row.try_get("delivery_attempts").map_err(super::row_error)?, + "delivery attempts", + )?, + delivered_at: row.try_get("delivered_at").map_err(super::row_error)?, + last_error: row.try_get("last_error").map_err(super::row_error)?, + created_at: row.try_get("created_at").map_err(super::row_error)?, + updated_at: row.try_get("updated_at").map_err(super::row_error)?, + }) +} + +/// Decodes an outbox row for sibling repository transactions. +pub(super) fn dispatch_from_row_for_repository( + row: &sqlx::postgres::PgRow, +) -> Result { + dispatch_from_row(row) +} + +#[cfg(test)] +mod tests { + use moa_core::types::identifiers::TenantId; + use serde_json::json; + use uuid::Uuid; + + use super::{ + ExecutionDispatchKind, NewExecutionDispatch, dispatch_requires_durable_retry, + validate_dispatch, + }; + use crate::{ + state::{CompensationId, ExecutionTaskId}, + wire::{ + ExecutionAttemptCancelReason, ExecutionCompensationAttemptCancelRequest, + ExecutionTaskAttemptCancelRequest, + }, + }; + + #[test] + fn cancel_dispatch_payload_must_duplicate_every_persisted_fence_offline() { + // Pins: an identity-free cancellation payload cannot redirect the outbox's exact + // tenant/run/owner/generation target or omit active ownership receipts. + let tenant_id = TenantId::new(); + let run_uid = Uuid::now_v7(); + let task_id = ExecutionTaskId::from_uuid(Uuid::now_v7()); + let dispatch_uid = Uuid::now_v7(); + let payload = ExecutionTaskAttemptCancelRequest { + cancellation_dispatch_uid: dispatch_uid, + tenant_id, + run_uid, + task_id, + controller_generation: 3, + attempt_controller_generation: 3, + task_generation: 4, + attempt_generation: 5, + active_dispatch_uid: Uuid::now_v7(), + capacity_reservation_uid: Uuid::now_v7(), + watchdog_trigger_uid: Uuid::now_v7(), + reason: ExecutionAttemptCancelReason::PauseRequested, + }; + let serialized_payload = + serde_json::to_value(payload).expect("serialize typed task cancel"); + assert_eq!(serialized_payload["dispatch_uid"], json!(dispatch_uid)); + assert!( + serialized_payload + .get("cancellation_dispatch_uid") + .is_none() + ); + let mut dispatch = NewExecutionDispatch { + dispatch_uid, + tenant_id, + run_uid: Some(run_uid), + task_id: Some(task_id.as_uuid()), + compensation_id: None, + trigger_uid: None, + external_job_uid: None, + kind: ExecutionDispatchKind::TaskAttemptCancel, + controller_generation: Some(3), + wake_epoch: None, + attempt_generation: Some(5), + compensation_generation: None, + compensation_attempt_generation: None, + not_before_at: chrono::Utc::now(), + payload: serialized_payload, + }; + validate_dispatch(&dispatch).expect("exact typed task cancellation validates"); + dispatch.attempt_generation = Some(6); + assert!(validate_dispatch(&dispatch).is_err()); + dispatch.attempt_generation = Some(5); + dispatch.payload["capacity_reservation_uid"] = json!(Uuid::nil()); + assert!(validate_dispatch(&dispatch).is_err()); + } + + #[test] + fn compensation_cancel_kind_and_payload_round_trip_exactly_offline() { + // Pins: compensation cancellation has a distinct durable label and requires both + // logical and bounded-attempt generations plus exact active ownership receipts. + assert_eq!( + "compensation_attempt_cancel" + .parse::() + .expect("parse compensation cancel kind"), + ExecutionDispatchKind::CompensationAttemptCancel + ); + assert_eq!( + ExecutionDispatchKind::TaskAttemptCancel.as_str(), + "task_attempt_cancel" + ); + let tenant_id = TenantId::new(); + let run_uid = Uuid::now_v7(); + let compensation_id = CompensationId::from_uuid(Uuid::now_v7()); + let dispatch_uid = Uuid::now_v7(); + let payload = ExecutionCompensationAttemptCancelRequest { + cancellation_dispatch_uid: dispatch_uid, + tenant_id, + run_uid, + compensation_id, + controller_generation: 7, + attempt_controller_generation: 7, + compensation_generation: 8, + compensation_attempt_generation: 9, + active_dispatch_uid: Uuid::now_v7(), + capacity_reservation_uid: Uuid::now_v7(), + watchdog_trigger_uid: Uuid::now_v7(), + intent: crate::wire::ExecutionCompensationReleaseIntent::RunTerminal, + }; + let serialized_payload = + serde_json::to_value(payload).expect("serialize typed compensation cancellation"); + assert_eq!(serialized_payload["dispatch_uid"], json!(dispatch_uid)); + assert!( + serialized_payload + .get("cancellation_dispatch_uid") + .is_none() + ); + let dispatch = NewExecutionDispatch { + dispatch_uid, + tenant_id, + run_uid: Some(run_uid), + task_id: None, + compensation_id: Some(compensation_id.as_uuid()), + trigger_uid: None, + external_job_uid: None, + kind: ExecutionDispatchKind::CompensationAttemptCancel, + controller_generation: Some(7), + wake_epoch: None, + attempt_generation: None, + compensation_generation: Some(8), + compensation_attempt_generation: Some(9), + not_before_at: chrono::Utc::now(), + payload: serialized_payload, + }; + validate_dispatch(&dispatch).expect("exact typed compensation cancellation validates"); + } + + #[test] + fn correctness_dispatches_never_terminally_dead_letter_offline() { + // Pins: correctness work remains behind capped sparse retries; only never-started + // task and compensation attempts may dead-letter after exact owner repair. + for kind in [ + ExecutionDispatchKind::RunActivation, + ExecutionDispatchKind::TaskAttemptCancel, + ExecutionDispatchKind::CompensationAttemptCancel, + ExecutionDispatchKind::TriggerDelivery, + ExecutionDispatchKind::ExternalCancel, + ] { + assert!(dispatch_requires_durable_retry(kind), "{kind:?}"); + } + assert!(!dispatch_requires_durable_retry( + ExecutionDispatchKind::TaskAttempt + )); + assert!(!dispatch_requires_durable_retry( + ExecutionDispatchKind::CompensationAttempt + )); + } +} diff --git a/crates/moa-execution/src/repository/outcome.rs b/crates/moa-execution/src/repository/outcome.rs index f972a2288..85e1c5360 100644 --- a/crates/moa-execution/src/repository/outcome.rs +++ b/crates/moa-execution/src/repository/outcome.rs @@ -2,8 +2,14 @@ use super::*; use super::{ - compensation::register_compensation_for_completed_task, materialize::reconcile_outcome_usage, - outcome_support::*, rows::*, sql::*, transition::task_outcome_is_exact_replay, + capacity::{ExecutionCapacityDimension, prelock_capacity_dimensions_in_tx}, + compensation::register_compensation_for_completed_task, + materialize::reconcile_outcome_usage, + outcome_support::*, + rows::*, + run::enqueue_run_activation_in_conn, + sql::*, + transition::task_outcome_is_exact_replay, }; impl ExecutionRepository { @@ -19,332 +25,350 @@ impl ExecutionRepository { generation: u64, outcome: ExecutionTaskOutcome, ) -> Result { - let validation = moa_artifacts::validation::validate_execution_task_outcome(&outcome); - if let Some(error) = validation.errors.first() { - return Err(Error::InvalidRepositoryInput { - message: format!("{}: {}", error.path, error.message), - }); - } let mut conn = scope.begin(&self.pool).await?; - let Some(run_row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) - .bind(run_uid) - .fetch_optional(conn.as_mut()) - .await - .map_err(sqlx_error)? - else { - conn.commit().await.map_err(storage_error)?; - return Ok(TaskOutcomeWrite::NotFound); - }; - let run = run_from_row(&run_row)?; - let Some(task_row) = sqlx::query(LOAD_TASK_FOR_UPDATE_SQL) - .bind(run_uid) - .bind(task_id.as_uuid()) - .fetch_optional(conn.as_mut()) - .await - .map_err(sqlx_error)? - else { - conn.commit().await.map_err(storage_error)?; - return Ok(TaskOutcomeWrite::NotFound); - }; - let task = task_from_row(&task_row)?; + let write = + record_task_outcome_in_conn(&mut conn, run_uid, task_id, generation, outcome).await?; + conn.commit().await.map_err(storage_error)?; + Ok(write) + } +} - if task_outcome_is_exact_replay(&task, generation, &outcome) { - register_compensation_for_completed_task(&mut conn, &run, &task, &outcome, true) - .await?; - let budget_overrun = run.budget_overrun; - conn.commit().await.map_err(storage_error)?; - return Ok(TaskOutcomeWrite::Replayed { - run, - task, - budget_overrun, - }); - } +/// Records one canonical task outcome without committing the caller-owned transaction. +pub(super) async fn record_task_outcome_in_conn( + conn: &mut ScopedConn<'_>, + run_uid: Uuid, + task_id: ExecutionTaskId, + generation: u64, + outcome: ExecutionTaskOutcome, +) -> Result { + record_task_outcome_for_source_in_conn( + conn, + run_uid, + task_id, + generation, + outcome, + TaskOutcomeSource::ActiveAttempt, + ) + .await +} - let terminal = task_outcome_is_terminal(&outcome); - let fenced_terminal_settlement = run.pending_terminal.is_some() && terminal; - let rejection = if outcome.schema_version != 1 { - Some(TaskOutcomeRejection::UnsupportedSchemaVersion) - } else if run.status.is_terminal() { - Some(TaskOutcomeRejection::TerminalRun) - } else if task.status.is_terminal() { - Some(TaskOutcomeRejection::TerminalTask) - } else if task.generation != generation { - Some(TaskOutcomeRejection::StaleGeneration) - } else if (task.status != ExecutionTaskStatus::Running && !fenced_terminal_settlement) - || (run.pending_terminal.is_some() && !terminal) - { - Some(TaskOutcomeRejection::InvalidTaskStatus) - } else if !usage_is_cumulative(&task.actual, &outcome.usage) { - Some(TaskOutcomeRejection::NonCumulativeUsage) - } else { - None - }; - if let Some(reason) = rejection { - let task = - append_outcome_audit(&mut conn, &task, generation, &outcome, false, Some(reason)) - .await?; - conn.commit().await.map_err(storage_error)?; - return Ok(TaskOutcomeWrite::Rejected { task, reason }); - } +/// Records a terminal provider outcome for a task parked on that provider job. +pub(super) async fn record_waiting_external_task_outcome_in_conn( + conn: &mut ScopedConn<'_>, + run_uid: Uuid, + task_id: ExecutionTaskId, + generation: u64, + outcome: ExecutionTaskOutcome, +) -> Result { + if !task_outcome_is_terminal(&outcome) { + return Err(Error::InvalidRepositoryInput { + message: "external-job settlement requires a terminal task outcome".to_string(), + }); + } + record_task_outcome_for_source_in_conn( + conn, + run_uid, + task_id, + generation, + outcome, + TaskOutcomeSource::ExternalJob, + ) + .await +} - let Some(reconciliation) = reconcile_outcome_usage(&run, &task, &outcome, terminal) else { - let reason = TaskOutcomeRejection::NonCumulativeUsage; - let task = - append_outcome_audit(&mut conn, &task, generation, &outcome, false, Some(reason)) - .await?; - conn.commit().await.map_err(storage_error)?; - return Ok(TaskOutcomeWrite::Rejected { task, reason }); - }; - let task_status = task_status_from_outcome(&outcome, true); - let run_status = run_status_after_task_outcome(run.status, &outcome); - let (output, error, citations) = outcome_projection_fields(&outcome)?; - let audit = outcome_audit_entry(&task, generation, &outcome, true, None); - let current_outcome = serde_json::to_value(&outcome)?; - let citations = serde_json::to_value(citations)?; - let completed_increment = u64::from(task_status == ExecutionTaskStatus::Completed); - let failed_increment = u64::from(task_status == ExecutionTaskStatus::Failed); - let cancelled_increment = u64::from(task_status == ExecutionTaskStatus::Cancelled); +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum TaskOutcomeSource { + ActiveAttempt, + ExternalJob, +} - let run_row = sqlx::query(RECONCILE_RUN_OUTCOME_SQL) - .bind(run_uid) - .bind(run_status.as_str()) - .bind(to_i64( - reconciliation.run_reserved.cost_microusd, - "run reserved cost", - )?) - .bind(to_i64( - reconciliation.run_reserved.tokens, - "run reserved tokens", - )?) - .bind(to_i64( - reconciliation.run_reserved.tasks, - "run reserved tasks", - )?) - .bind(to_i64( - reconciliation.run_reserved.tool_calls, - "run reserved tool calls", - )?) - .bind(to_i64( - reconciliation.run_reserved.retrieved_bytes, - "run reserved retrieved bytes", - )?) - .bind(to_i64( - reconciliation.run_consumed.cost_microusd, - "run consumed cost", - )?) - .bind(to_i64( - reconciliation.run_consumed.tokens, - "run consumed tokens", - )?) - .bind(to_i64( - reconciliation.run_consumed.tasks, - "run consumed tasks", - )?) - .bind(to_i64( - reconciliation.run_consumed.tool_calls, - "run consumed tool calls", - )?) - .bind(to_i64( - reconciliation.run_consumed.retrieved_bytes, - "run consumed retrieved bytes", - )?) - .bind(reconciliation.budget_overrun) - .bind(to_i64(completed_increment, "completed task increment")?) - .bind(to_i64(failed_increment, "failed task increment")?) - .bind(to_i64(cancelled_increment, "cancelled task increment")?) - .fetch_one(conn.as_mut()) - .await - .map_err(sqlx_error)?; - let run = run_from_row(&run_row)?; +async fn record_task_outcome_for_source_in_conn( + conn: &mut ScopedConn<'_>, + run_uid: Uuid, + task_id: ExecutionTaskId, + generation: u64, + outcome: ExecutionTaskOutcome, + source: TaskOutcomeSource, +) -> Result { + let validation = moa_artifacts::validation::validate_execution_task_outcome(&outcome); + if let Some(error) = validation.errors.first() { + return Err(Error::InvalidRepositoryInput { + message: format!("{}: {}", error.path, error.message), + }); + } + let Some(run_row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) + .bind(run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + return Ok(TaskOutcomeWrite::NotFound); + }; + let run = run_from_row(&run_row)?; + let Some(task_row) = sqlx::query(LOAD_TASK_FOR_UPDATE_SQL) + .bind(run_uid) + .bind(task_id.as_uuid()) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + return Ok(TaskOutcomeWrite::NotFound); + }; + let task = task_from_row(&task_row)?; - let row = sqlx::query(RECORD_TASK_OUTCOME_SQL) - .bind(run_uid) - .bind(task_id.as_uuid()) - .bind(to_i64(generation, "task generation")?) - .bind(task_status.as_str()) - .bind(to_i64( - reconciliation.remaining_task_reservation.cost_microusd, - "remaining task cost reservation", - )?) - .bind(to_i64( - reconciliation.remaining_task_reservation.tokens, - "remaining task token reservation", - )?) - .bind(to_i64( - reconciliation.remaining_task_reservation.tasks, - "remaining task logical reservation", - )?) - .bind(to_i64( - reconciliation.remaining_task_reservation.tool_calls, - "remaining task tool-call reservation", - )?) - .bind(to_i64( - reconciliation.remaining_task_reservation.retrieved_bytes, - "remaining task byte reservation", - )?) - .bind(to_i64(outcome.usage.cost_microusd, "actual task cost")?) - .bind(to_i64(outcome.usage.tokens, "actual task tokens")?) - .bind(to_i64( - u64::from(reconciliation.terminal), - "actual logical task", - )?) - .bind(to_i64(outcome.usage.tool_calls, "actual task tool calls")?) - .bind(to_i64( - outcome.usage.retrieved_bytes, - "actual task retrieved bytes", - )?) - .bind(current_outcome) - .bind(output) - .bind(error) - .bind(citations) - .bind(audit) - .bind(reconciliation.terminal) - .fetch_one(conn.as_mut()) - .await - .map_err(sqlx_error)?; - let task = task_from_row(&row)?; - register_compensation_for_completed_task(&mut conn, &run, &task, &outcome, false).await?; - if matches!(outcome.result, ExecutionTaskResult::UnknownOutcome { .. }) { - sqlx::query( - "UPDATE moa.execution_run SET manual_repair_required = TRUE, \ - wake_epoch = wake_epoch + 1, updated_at = NOW() WHERE run_uid = $1", - ) - .bind(run_uid) - .execute(conn.as_mut()) - .await - .map_err(sqlx_error)?; - } - let run_row = sqlx::query(LOAD_RUN_SQL) - .bind(run_uid) - .fetch_one(conn.as_mut()) - .await - .map_err(sqlx_error)?; - let run = run_from_row(&run_row)?; - conn.commit().await.map_err(storage_error)?; - Ok(TaskOutcomeWrite::Applied { + if task_outcome_is_exact_replay(&task, generation, &outcome) { + register_compensation_for_completed_task(conn, &run, &task, &outcome, true).await?; + let budget_overrun = run.budget_overrun; + return Ok(TaskOutcomeWrite::Replayed { run, task, - budget_overrun: reconciliation.budget_overrun, - }) + budget_overrun, + }); + } + + let terminal = task_outcome_is_terminal(&outcome); + let fenced_terminal_settlement = run.pending_terminal.is_some() && terminal; + let unstarted_cancellation = task.status == ExecutionTaskStatus::Dispatching + && task.attempt_state == ExecutionAttemptState::Cancelling + && matches!(&outcome.result, ExecutionTaskResult::Cancelled { .. }); + let rejection = if outcome.schema_version != 1 { + Some(TaskOutcomeRejection::UnsupportedSchemaVersion) + } else if run.status.is_terminal() { + Some(TaskOutcomeRejection::TerminalRun) + } else if task.status.is_terminal() { + Some(TaskOutcomeRejection::TerminalTask) + } else if task.generation != generation { + Some(TaskOutcomeRejection::StaleGeneration) + } else if (task.status != ExecutionTaskStatus::Running + && !(source == TaskOutcomeSource::ExternalJob + && task.status == ExecutionTaskStatus::WaitingExternal + && terminal) + && !fenced_terminal_settlement + && !unstarted_cancellation) + || (run.pending_terminal.is_some() && !terminal) + { + Some(TaskOutcomeRejection::InvalidTaskStatus) + } else if !usage_is_cumulative(&task.actual, &outcome.usage) { + Some(TaskOutcomeRejection::NonCumulativeUsage) + } else { + None + }; + if let Some(reason) = rejection { + let task = + append_outcome_audit(conn, &task, generation, &outcome, false, Some(reason)).await?; + return Ok(TaskOutcomeWrite::Rejected { task, reason }); + } + + let Some(reconciliation) = reconcile_outcome_usage(&run, &task, &outcome, terminal) else { + let reason = TaskOutcomeRejection::NonCumulativeUsage; + let task = + append_outcome_audit(conn, &task, generation, &outcome, false, Some(reason)).await?; + return Ok(TaskOutcomeWrite::Rejected { task, reason }); + }; + let task_status = task_status_from_outcome(&outcome, true); + let run_status = run_status_after_task_outcome(run.status, &outcome); + let (output, error, citations) = outcome_projection_fields(&outcome)?; + let failure_fingerprint = task_failure_fingerprint_input_for_outcome(&task, &outcome) + .map(|input| failure_fingerprint(&input).map(|hash| hash.to_string())) + .transpose()?; + let audit = outcome_audit_entry(&task, generation, &outcome, true, None); + let current_outcome = serde_json::to_value(&outcome)?; + let citations = serde_json::to_value(citations)?; + let completed_increment = u64::from(task_status == ExecutionTaskStatus::Completed); + let failed_increment = u64::from(task_status == ExecutionTaskStatus::Failed); + let cancelled_increment = u64::from(task_status == ExecutionTaskStatus::Cancelled); + + let run_row = sqlx::query(RECONCILE_RUN_OUTCOME_SQL) + .bind(run_uid) + .bind(run_status.as_str()) + .bind(to_i64( + reconciliation.run_reserved.cost_microusd, + "run reserved cost", + )?) + .bind(to_i64( + reconciliation.run_reserved.tokens, + "run reserved tokens", + )?) + .bind(to_i64( + reconciliation.run_reserved.tasks, + "run reserved tasks", + )?) + .bind(to_i64( + reconciliation.run_reserved.tool_calls, + "run reserved tool calls", + )?) + .bind(to_i64( + reconciliation.run_reserved.retrieved_bytes, + "run reserved retrieved bytes", + )?) + .bind(to_i64( + reconciliation.run_consumed.cost_microusd, + "run consumed cost", + )?) + .bind(to_i64( + reconciliation.run_consumed.tokens, + "run consumed tokens", + )?) + .bind(to_i64( + reconciliation.run_consumed.tasks, + "run consumed tasks", + )?) + .bind(to_i64( + reconciliation.run_consumed.tool_calls, + "run consumed tool calls", + )?) + .bind(to_i64( + reconciliation.run_consumed.retrieved_bytes, + "run consumed retrieved bytes", + )?) + .bind(reconciliation.budget_overrun) + .bind(to_i64(completed_increment, "completed task increment")?) + .bind(to_i64(failed_increment, "failed task increment")?) + .bind(to_i64(cancelled_increment, "cancelled task increment")?) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let run = run_from_row(&run_row)?; + + let row = sqlx::query(RECORD_TASK_OUTCOME_SQL) + .bind(run_uid) + .bind(task_id.as_uuid()) + .bind(to_i64(generation, "task generation")?) + .bind(task_status.as_str()) + .bind(to_i64( + reconciliation.remaining_task_reservation.cost_microusd, + "remaining task cost reservation", + )?) + .bind(to_i64( + reconciliation.remaining_task_reservation.tokens, + "remaining task token reservation", + )?) + .bind(to_i64( + reconciliation.remaining_task_reservation.tasks, + "remaining task logical reservation", + )?) + .bind(to_i64( + reconciliation.remaining_task_reservation.tool_calls, + "remaining task tool-call reservation", + )?) + .bind(to_i64( + reconciliation.remaining_task_reservation.retrieved_bytes, + "remaining task byte reservation", + )?) + .bind(to_i64(outcome.usage.cost_microusd, "actual task cost")?) + .bind(to_i64(outcome.usage.tokens, "actual task tokens")?) + .bind(to_i64( + u64::from(reconciliation.terminal), + "actual logical task", + )?) + .bind(to_i64(outcome.usage.tool_calls, "actual task tool calls")?) + .bind(to_i64( + outcome.usage.retrieved_bytes, + "actual task retrieved bytes", + )?) + .bind(current_outcome) + .bind(output) + .bind(error) + .bind(citations) + .bind(audit) + .bind(reconciliation.terminal) + .bind(failure_fingerprint) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let task = task_from_row(&row)?; + register_compensation_for_completed_task(conn, &run, &task, &outcome, false).await?; + if matches!(outcome.result, ExecutionTaskResult::UnknownOutcome { .. }) { + sqlx::query( + "UPDATE moa.execution_run SET manual_repair_required = TRUE, \ + wake_epoch = wake_epoch + 1, updated_at = NOW() WHERE run_uid = $1", + ) + .bind(run_uid) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; } + let run_row = sqlx::query(LOAD_RUN_SQL) + .bind(run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let run = run_from_row(&run_row)?; + Ok(TaskOutcomeWrite::Applied { + run, + task, + budget_overrun: reconciliation.budget_overrun, + }) +} +impl ExecutionRepository { /// Recovers an exact committed amendment handoff before current-revision validation. pub async fn recover_amendment_handoff( &self, scope: ExecutionScope, run_uid: Uuid, + expected_session_id: SessionId, expected_revision: u64, amendment_hash: &ExecutionHash, ) -> Result { let mut conn = scope.begin(&self.pool).await?; - let Some(run_row) = sqlx::query(LOAD_RUN_SQL) - .bind(run_uid) - .fetch_optional(conn.as_mut()) - .await - .map_err(sqlx_error)? + let Some(run_row) = + sqlx::query("SELECT * FROM moa.execution_run WHERE run_uid=$1 AND session_id=$2") + .bind(run_uid) + .bind(expected_session_id.0) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? else { conn.commit().await.map_err(storage_error)?; return Ok(AmendmentReplayOutcome::NotFound); }; let run = run_from_row(&run_row)?; - let amendment_hash_text = amendment_hash.to_string(); - let exact_history = run.plan_history.iter().rev().find(|entry| { - entry.get("base_plan_revision").and_then(Value::as_u64) == Some(expected_revision) - && entry.get("amendment_hash").and_then(Value::as_str) - == Some(amendment_hash_text.as_str()) - }); - let task_rows = sqlx::query(LIST_ALL_TASKS_SQL) - .bind(run_uid) - .fetch_all(conn.as_mut()) - .await - .map_err(sqlx_error)?; - let tasks = task_rows - .iter() - .map(task_from_row) - .collect::>>()?; - let replan_stop_receipt_at_revision = tasks.iter().find_map(|task| { - let task_id = task.task_id.to_string(); - task.outcome_audit.iter().find(|entry| { - entry.get("kind").and_then(Value::as_str) == Some("replan_stop_fenced") - && entry.get("accepted").and_then(Value::as_bool) == Some(true) - && entry.get("task_id").and_then(Value::as_str) == Some(task_id.as_str()) - && entry.get("task_generation").and_then(Value::as_u64) == Some(task.generation) - && entry.get("base_plan_revision").and_then(Value::as_u64) - == Some(expected_revision) - }) - }); - if exact_history.is_none() - && let Some(receipt) = replan_stop_receipt_at_revision - { - let outcome = if receipt.get("amendment_hash").and_then(Value::as_str) - == Some(amendment_hash_text.as_str()) - { - AmendmentReplayOutcome::Replayed(Box::new(AmendmentCommit { - run, - task_ids_to_release: Vec::new(), - })) - } else { - AmendmentReplayOutcome::Conflict - }; - conn.commit().await.map_err(storage_error)?; - return Ok(outcome); - } - let audited_task_ids = tasks - .iter() - .filter(|task| { - task.outcome_audit.iter().any(|entry| { - entry.get("accepted").and_then(Value::as_bool) == Some(true) - && entry.get("amendment_hash").and_then(Value::as_str) - == Some(amendment_hash_text.as_str()) - && entry.get("base_plan_revision").and_then(Value::as_u64) - == Some(expected_revision) - }) - }) - .map(|task| task.task_id) - .collect::>(); - let task_ids_to_release = match exact_history { - Some(history) => history - .get("task_ids_to_release") - .cloned() - .and_then(|value| serde_json::from_value::>(value).ok()) - .filter(|task_ids| !task_ids.is_empty()) - .ok_or_else(|| Error::InvalidRepositoryData { - message: "committed amendment history is missing its handoff task IDs" - .to_string(), - })?, - None if run.plan_revision == expected_revision && !audited_task_ids.is_empty() => { - audited_task_ids - } - None => { - let outcome = if run.plan_revision == expected_revision && !run.status.is_terminal() - { - AmendmentReplayOutcome::NotApplied - } else { + let receipt = sqlx::query( + "SELECT amendment_hash,receipt_kind,task_ids_to_release \ + FROM moa.execution_amendment_receipt \ + WHERE tenant_id=$1 AND run_uid=$2 AND base_plan_revision=$3", + ) + .bind(run.tenant_id.0) + .bind(run_uid) + .bind(to_i64(expected_revision, "amendment base plan revision")?) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let outcome = match receipt { + Some(receipt) => { + let persisted_hash: String = + receipt.try_get("amendment_hash").map_err(row_error)?; + if persisted_hash != amendment_hash.to_string() { AmendmentReplayOutcome::Conflict - }; - conn.commit().await.map_err(storage_error)?; - return Ok(outcome); + } else { + let receipt_kind: String = + receipt.try_get("receipt_kind").map_err(row_error)?; + let task_ids_to_release = receipt + .try_get::, _>("task_ids_to_release") + .map_err(row_error)? + .into_iter() + .map(ExecutionTaskId::from_uuid) + .collect::>(); + if (receipt_kind == "applied" && task_ids_to_release.len() != 1) + || (receipt_kind == "replan_stop" && !task_ids_to_release.is_empty()) + || !matches!(receipt_kind.as_str(), "applied" | "replan_stop") + { + return Err(Error::InvalidRepositoryData { + message: "execution amendment receipt has an invalid release shape" + .to_string(), + }); + } + AmendmentReplayOutcome::Replayed(Box::new(AmendmentCommit { + run, + task_ids_to_release, + })) + } } - }; - let audit_matches = task_ids_to_release.iter().all(|task_id| { - tasks.iter().any(|task| { - task.task_id == *task_id - && task.outcome_audit.iter().any(|entry| { - entry.get("accepted").and_then(Value::as_bool) == Some(true) - && entry.get("amendment_hash").and_then(Value::as_str) - == Some(amendment_hash_text.as_str()) - && entry.get("base_plan_revision").and_then(Value::as_u64) - == Some(expected_revision) - }) - }) - }); - let outcome = if audit_matches { - AmendmentReplayOutcome::Replayed(Box::new(AmendmentCommit { - run, - task_ids_to_release, - })) - } else { - AmendmentReplayOutcome::Conflict + None if run.plan_revision == expected_revision && !run.status.is_terminal() => { + AmendmentReplayOutcome::NotApplied + } + None => AmendmentReplayOutcome::Conflict, }; conn.commit().await.map_err(storage_error)?; Ok(outcome) @@ -354,6 +378,7 @@ impl ExecutionRepository { pub async fn append_amendment( &self, scope: ExecutionScope, + config: &moa_config::ExecutionConfig, run_uid: Uuid, expected_revision: u64, validated: ValidatedAmendment, @@ -367,6 +392,27 @@ impl ExecutionRepository { return Ok(AmendmentWrite::Conflict); } let mut conn = scope.begin(&self.pool).await?; + let tenant_id = sqlx::query_scalar::<_, Uuid>( + "SELECT tenant_id FROM moa.execution_run WHERE run_uid=$1", + ) + .bind(run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(tenant_id) = tenant_id else { + conn.commit().await.map_err(storage_error)?; + return Ok(AmendmentWrite::NotFound); + }; + prelock_capacity_dimensions_in_tx( + conn.as_mut(), + config, + TenantId(tenant_id), + &[ + ExecutionCapacityDimension::ActiveRuns, + ExecutionCapacityDimension::ParkedRuns, + ], + ) + .await?; let Some(run_row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) .bind(run_uid) .fetch_optional(conn.as_mut()) @@ -381,37 +427,22 @@ impl ExecutionRepository { || run.status != ExecutionRunStatus::WaitingReplan || run.active_plan_hash == validated.active_plan.plan_hash { - let amendment_hash_text = validated.amendment_hash.to_string(); - let exact_history = run.plan_history.iter().rev().any(|entry| { - entry.get("base_plan_revision").and_then(Value::as_u64) == Some(expected_revision) - && entry.get("amendment_hash").and_then(Value::as_str) - == Some(amendment_hash_text.as_str()) - && entry.get("task_ids_to_release") - == Some(&json!([validated.superseded_task_id])) - }); - let exact_task = if exact_history { - sqlx::query(LOAD_TASK_FOR_UPDATE_SQL) - .bind(run_uid) - .bind(validated.superseded_task_id.as_uuid()) - .fetch_optional(conn.as_mut()) - .await - .map_err(sqlx_error)? - .map(|row| task_from_row(&row)) - .transpose()? - .is_some_and(|task| { - task.outcome_audit.iter().any(|entry| { - entry.get("accepted").and_then(Value::as_bool) == Some(true) - && entry.get("base_plan_revision").and_then(Value::as_u64) - == Some(expected_revision) - && entry.get("amendment_hash").and_then(Value::as_str) - == Some(amendment_hash_text.as_str()) - }) - }) - } else { - false - }; + let exact_receipt = sqlx::query_scalar::<_, bool>( + "SELECT EXISTS (SELECT 1 FROM moa.execution_amendment_receipt \ + WHERE tenant_id=$1 AND run_uid=$2 AND base_plan_revision=$3 \ + AND amendment_hash=$4 AND receipt_kind='applied' \ + AND superseded_task_id=$5 AND task_ids_to_release=ARRAY[$5]::UUID[])", + ) + .bind(run.tenant_id.0) + .bind(run_uid) + .bind(to_i64(expected_revision, "amendment base plan revision")?) + .bind(validated.amendment_hash.to_string()) + .bind(validated.superseded_task_id.as_uuid()) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; conn.commit().await.map_err(storage_error)?; - return Ok(if exact_task { + return Ok(if exact_receipt { AmendmentWrite::Replayed(Box::new(AmendmentCommit { run, task_ids_to_release: vec![validated.superseded_task_id], @@ -562,6 +593,40 @@ impl ExecutionRepository { .fetch_one(conn.as_mut()) .await .map_err(sqlx_error)?; + sqlx::query( + "INSERT INTO moa.execution_amendment_receipt (tenant_id,run_uid, \ + base_plan_revision,amendment_hash,receipt_kind,superseded_task_id, \ + task_generation,task_ids_to_release) \ + VALUES ($1,$2,$3,$4,'applied',$5,$6,ARRAY[$5]::UUID[])", + ) + .bind(run.tenant_id.0) + .bind(run_uid) + .bind(to_i64(expected_revision, "amendment base plan revision")?) + .bind(validated.amendment_hash.to_string()) + .bind(task.task_id.as_uuid()) + .bind(to_i64(task.generation, "amendment task generation")?) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + enqueue_run_activation_in_conn( + conn.as_mut(), + run.tenant_id, + run.run_uid, + run.controller_generation, + run.updated_at, + json!({ + "reason": "plan_amended", + "plan_revision": run.plan_revision, + "superseded_task_id": task.task_id, + }), + ) + .await?; + let row = sqlx::query(LOAD_RUN_SQL) + .bind(run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let run = run_from_row(&row)?; conn.commit().await.map_err(storage_error)?; Ok(AmendmentWrite::Applied(Box::new(AmendmentCommit { run, diff --git a/crates/moa-execution/src/repository/outcome_support.rs b/crates/moa-execution/src/repository/outcome_support.rs index 170012882..3a2e0c638 100644 --- a/crates/moa-execution/src/repository/outcome_support.rs +++ b/crates/moa-execution/src/repository/outcome_support.rs @@ -14,6 +14,13 @@ pub(super) fn task_failure_fingerprint_input( task: &ExecutionTaskRecord, ) -> Option { let outcome = task.current_outcome.as_ref()?; + task_failure_fingerprint_input_for_outcome(task, outcome) +} + +pub(super) fn task_failure_fingerprint_input_for_outcome( + task: &ExecutionTaskRecord, + outcome: &ExecutionTaskOutcome, +) -> Option { let (class, message) = match &outcome.result { ExecutionTaskResult::Failed { class, message } => (class.clone(), message.clone()), ExecutionTaskResult::NeedsReplan { reason, .. } => ( @@ -274,6 +281,9 @@ pub(super) async fn terminalize_reservation_rejection( .map_err(sqlx_error)?; let (_, error, citations) = outcome_projection_fields(&outcome)?; + let failure_fingerprint = task_failure_fingerprint_input_for_outcome(task, &outcome) + .map(|input| failure_fingerprint(&input).map(|hash| hash.to_string())) + .transpose()?; let audit = json!({ "kind": "reservation_admission_rejected", "attempt": task.attempt, @@ -298,6 +308,7 @@ pub(super) async fn terminalize_reservation_rejection( .bind(error) .bind(serde_json::to_value(citations)?) .bind(audit) + .bind(failure_fingerprint) .fetch_optional(conn.as_mut()) .await .map_err(sqlx_error)?; diff --git a/crates/moa-execution/src/repository/projection.rs b/crates/moa-execution/src/repository/projection.rs index d278c7705..bc360b5ba 100644 --- a/crates/moa-execution/src/repository/projection.rs +++ b/crates/moa-execution/src/repository/projection.rs @@ -22,93 +22,3 @@ pub(super) fn terminal_projection_output(projection: &TerminalProjection) -> Opt | TerminalProjection::Cancelled { .. } => None, } } - -pub(super) fn scheduling_projection( - run: &ExecutionRunRecord, - tasks: &[ExecutionTaskRecord], -) -> ExecutionProjection { - let task_projections = tasks - .iter() - .map(|task| ExecutionTaskProjection { - task_id: task.task_id, - node_id: task.node_id.clone(), - item_key: task.item_key.clone(), - status: task.status, - attempt: task.attempt, - generation: task.generation, - input: task.input.clone(), - outcome: task.current_outcome.clone(), - }) - .collect::>(); - let mut node_statuses = BTreeMap::new(); - for node in &run.active_plan.definition.nodes { - let node_tasks = tasks - .iter() - .filter(|task| task.node_id == node.id) - .collect::>(); - let status = persisted_node_status(&node.operation, &node_tasks); - node_statuses.insert(node.id.clone(), status); - } - ExecutionProjection { - plan_revision: run.plan_revision, - node_statuses, - tasks: task_projections, - } -} - -pub(super) fn persisted_node_status( - operation: &ExecutionOperation, - tasks: &[&ExecutionTaskRecord], -) -> ExecutionNodeStatus { - if tasks.is_empty() { - return ExecutionNodeStatus::Pending; - } - if tasks.iter().any(|task| { - matches!( - task.status, - ExecutionTaskStatus::WaitingInput | ExecutionTaskStatus::WaitingReplan - ) || (task.status == ExecutionTaskStatus::Running - && matches!( - task.kind, - LogicalTaskKind::Review { .. } | LogicalTaskKind::WaitSignal { .. } - )) - }) { - return ExecutionNodeStatus::Waiting; - } - if tasks.iter().any(|task| { - matches!( - task.status, - ExecutionTaskStatus::Pending - | ExecutionTaskStatus::Reserved - | ExecutionTaskStatus::Running - ) - }) { - return ExecutionNodeStatus::Running; - } - if tasks - .iter() - .any(|task| task.status == ExecutionTaskStatus::Failed) - { - return ExecutionNodeStatus::Failed; - } - if tasks - .iter() - .any(|task| task.status == ExecutionTaskStatus::Cancelled) - { - return ExecutionNodeStatus::Cancelled; - } - if matches!( - operation, - ExecutionOperation::Map { .. } | ExecutionOperation::Reduce { .. } - ) { - return ExecutionNodeStatus::Pending; - } - if tasks - .iter() - .all(|task| task.status == ExecutionTaskStatus::Skipped) - { - ExecutionNodeStatus::Skipped - } else { - ExecutionNodeStatus::Completed - } -} diff --git a/crates/moa-execution/src/repository/ready.rs b/crates/moa-execution/src/repository/ready.rs new file mode 100644 index 000000000..62d2b71fc --- /dev/null +++ b/crates/moa-execution/src/repository/ready.rs @@ -0,0 +1,2678 @@ +//! Bounded ready-queue materialization and persisted per-node scheduler aggregates. + +use std::collections::{BTreeMap, BTreeSet}; + +use moa_artifacts::execution_plan::{ExecutionTemporalTarget, ExecutionWaitPolicy, InputAudience}; +use moa_config::ExecutionConfig; +use sqlx::{Row, postgres::PgRow}; + +use crate::capability::node_output_hash; +use crate::schema::validate_instance; + +use super::*; +use super::{ + materialize::{ensure_materialization_replay_matches, prepare_task_materialization_batch}, + rows::*, + sql::*, + trigger::{ExecutionTriggerKind, NewExecutionTrigger, create_trigger_with_dispatch_in_conn}, +}; + +const MAX_READY_PAGE_SIZE: u32 = 1_000; +const MAX_READY_PAGE_SIZE_USIZE: usize = 1_000; +const MAX_MAP_AGGREGATE_PAGE_SIZE: i64 = 16; +const MAX_ACTIVATION_OUTPUT_BYTES: i64 = 1_048_576; +const MAX_WAITING_REASON_SAMPLES: usize = 64; +const MAX_WAITING_REASON_SAMPLE_BYTES: usize = 65_536; + +/// Durable scheduler lifecycle for one plan node. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ExecutionNodeQueueStatus { + /// Dependencies or conditions are not ready. + Pending, + /// Bounded logical work is available for fleet admission. + Ready, + /// At least one attempt owns active capacity. + Running, + /// Work is parked in storage awaiting an external event. + Waiting, + /// The deterministic node aggregate completed. + Completed, + /// The node condition evaluated false. + Skipped, + /// Node work failed terminally. + Failed, + /// Node work was cancelled. + Cancelled, +} + +/// Persisted bounded scheduler aggregate for one plan node. +#[derive(Clone, Debug, PartialEq)] +pub struct ExecutionNodeStateRecord { + /// Owning run. + pub run_uid: Uuid, + /// Stable plan node identifier. + pub node_id: String, + /// Deterministic plan order. + pub node_order: u64, + /// Current aggregate lifecycle. + pub status: ExecutionNodeQueueStatus, + /// Next source item not yet materialized. + pub materialization_cursor: u64, + /// Whether the deterministic source has no further logical tasks to materialize. + pub materialization_complete: bool, + /// One-based reduce round, including the initial plan-input round. + pub reduce_round: u64, + /// Number of batches already materialized in the current reduce round. + pub reduce_batch_cursor: u64, + /// Input item count for the current reduce round, once known. + pub reduce_round_input_count: Option, + /// Tasks materialized in the current reduce round. + pub reduce_round_task_count: u64, + /// Terminal tasks in the current reduce round. + pub reduce_round_terminal_task_count: u64, + /// Whether every source batch in the current reduce round has been materialized. + pub reduce_ready: bool, + /// Dependencies still blocking the node. + pub remaining_dependency_count: u64, + /// Total materialized logical tasks. + pub total_task_count: u64, + /// Tasks available to fleet admission. + pub ready_task_count: u64, + /// Attempts consuming active capacity. + pub active_task_count: u64, + /// Tasks parked without active compute. + pub waiting_task_count: u64, + /// Terminal logical tasks. + pub terminal_task_count: u64, + /// Deterministic aggregate output persisted when the node completes. + pub aggregate_output: Option, + /// Canonical hash verified whenever an aggregate output is loaded. + pub aggregate_output_hash: Option, + /// Last map item key durably appended to the bounded aggregate. + pub aggregate_cursor_item_key: Option, + /// Whether the aggregate is final and safe for dependent-node resolution. + pub aggregate_complete: bool, +} + +/// One lightweight pending map-aggregation page owned by a claimed controller wake. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct MapAggregatePageRequest { + /// Owning execution run. + pub run_uid: Uuid, + /// Exact active plan revision. + pub plan_revision: u64, + /// Exact controller generation. + pub controller_generation: u64, + /// Exact claimed wake. + pub wake_epoch: u64, + /// Map node being aggregated. + pub node_id: String, + /// Exact persisted item-key cursor observed by the controller. + pub expected_cursor_item_key: Option, +} + +/// Lightweight map-aggregation work discovered without loading partial aggregate JSON. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct MapAggregateCandidate { + /// Map node identifier. + pub node_id: String, + /// Deterministic plan order. + pub node_order: u64, + /// Exact persisted aggregation cursor. + pub cursor_item_key: Option, +} + +/// Result of one bounded map-aggregation page. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum MapAggregatePageOutcome { + /// One page committed, possibly including the final completion CAS. + Applied { + /// Cursor after this page. + next_cursor_item_key: Option, + /// Number of task outputs appended by this page. + aggregated_tasks: u32, + /// Whether this page completed the node and released its dependents. + aggregate_complete: bool, + }, + /// The exact page was already committed before its response was observed. + Replayed { + /// Current persisted cursor. + next_cursor_item_key: Option, + /// Whether the aggregate is already complete. + aggregate_complete: bool, + }, + /// The cumulative inline output exceeded the one-MiB aggregate ceiling. + Overflow, + /// Run, generation, wake, plan, node, or cursor is no longer current. + Conflict, + /// No run exists under the supplied tenant/contact scope. + NotFound, +} + +/// Exact persisted reduce-round fence supplied with one ready materialization page. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ExecutionReduceMaterializationCursor { + /// One-based round being materialized. + pub round: u64, + /// Number of batches already committed in this round. + pub batch_cursor: u64, + /// Total input values consumed by this round. + pub round_input_count: u64, +} + +/// Bounded prior-round output slice requested for one reduce materialization page. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ReduceRoundInputPageRequest { + /// Owning execution run. + pub run_uid: Uuid, + /// Reduce plan node identifier. + pub node_id: String, + /// Immediately preceding round whose outputs feed this page. + pub source_round: u64, + /// Persisted current-round cursor and input count. + pub cursor: ExecutionReduceMaterializationCursor, + /// Fixed reducer batch size. + pub batch_size: u32, + /// Maximum target batches materialized by this page. + pub target_batch_limit: u32, +} + +/// Atomic input for one bounded ready-page materialization transaction. +#[derive(Clone, Debug, PartialEq)] +pub struct ReadyMaterializationRequest { + /// Owning execution run. + pub run_uid: Uuid, + /// Immutable plan revision fenced by the page. + pub plan_revision: u64, + /// Single plan node materialized by this page. + pub node_id: String, + /// Total committed task count before this page. + pub expected_cursor: u64, + /// Exact reduce-round source position, only for reduce nodes. + pub reduce_cursor: Option, + /// Whether this page reached the end of its deterministic node source. + pub source_exhausted: bool, + /// Aggregate output when the source completes without creating a logical task. + pub terminal_output: Option, + /// Bounded deterministic logical tasks in source order. + pub tasks: Vec, +} + +/// One bounded controller input page reconstructed without loading every task row. +#[derive(Clone, Debug, PartialEq)] +pub struct ExecutionActivationProjection { + /// Canonical run snapshot containing the plan and generation fences. + pub run: ExecutionRunRecord, + /// Bounded node aggregate page in deterministic plan order. + pub nodes: Vec, + /// Exact completed outputs referenced by nodes in this page. + pub referenced_outputs: BTreeMap, + /// Whether another actionable node exists beyond this bounded page. + pub has_more_actionable: bool, +} + +/// Constant-size scheduler readiness summary used after each bounded activation. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ExecutionActivationReadiness { + /// At least one dependency-ready node still has source work to materialize. + pub has_actionable_nodes: bool, + /// At least one plan node has not reached a terminal aggregate status. + pub has_unfinished_nodes: bool, + /// At least one logical task remains outside a terminal task status. + pub has_nonterminal_tasks: bool, +} + +impl ExecutionActivationReadiness { + /// Returns whether every node and task has reached an ordinary terminal boundary. + #[must_use] + pub const fn terminal_ready(self) -> bool { + !self.has_unfinished_nodes && !self.has_nonterminal_tasks + } +} + +/// Result of atomically materializing one bounded page directly into the ready queue. +#[derive(Clone, Debug, PartialEq)] +pub enum ReadyMaterializationOutcome { + /// New logical tasks were inserted and made ready. + Applied { + /// Exact persisted task records in deterministic request order. + tasks: Vec, + /// Cursor to supply when materializing the next page for this node. + next_cursor: u64, + /// Exact delayed trigger deliveries committed for storage-only waits. + triggers: Vec, + }, + /// The exact page had already committed before the caller retried. + Replayed { + /// Exact persisted task records in deterministic request order. + tasks: Vec, + /// Current cursor after the replayed page. + next_cursor: u64, + /// Exact delayed trigger deliveries reconstructed for replay. + triggers: Vec, + }, + /// The node cursor, plan revision, or immutable task semantics changed. + Conflict, +} + +/// Exact delayed delivery a controller must schedule after the transaction commits. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ExecutionScheduledTrigger { + /// Durable trigger-delivery dispatch identity. + pub dispatch_uid: Uuid, + /// Owning tenant. + pub tenant_id: TenantId, + /// Immutable trigger identity. + pub trigger_uid: Uuid, + /// Exact absolute delivery time. + pub due_at: DateTime, +} + +/// Bounded page used to verify that no nonterminal task remains before finalization. +#[derive(Clone, Debug, PartialEq)] +pub struct ExecutionTerminalVerificationPage { + /// Nonterminal tasks found in this page. + pub nonterminal_tasks: Vec, + /// Stable cursor for the next page, or `None` when verification is complete. + pub next_cursor: Option, +} + +impl ExecutionRepository { + /// Loads one bounded controller projection from node aggregates and exact dependencies. + pub async fn load_activation_projection( + &self, + scope: ExecutionScope, + run_uid: Uuid, + limit: u32, + ) -> Result> { + let limit = limit.clamp(1, MAX_READY_PAGE_SIZE); + let fetch_limit = i64::from(limit) + 1; + let mut conn = scope.begin(&self.pool).await?; + let Some(run_row) = sqlx::query(LOAD_RUN_SQL) + .bind(run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + conn.commit().await.map_err(storage_error)?; + return Ok(None); + }; + let run = run_from_row(&run_row)?; + let rows = sqlx::query( + "SELECT run_uid, node_id, node_order, node_status, materialization_cursor, \ + materialization_complete, \ + remaining_dependency_count, total_task_count, ready_task_count, \ + active_task_count, waiting_task_count, terminal_task_count, \ + aggregate_output, aggregate_output_hash, aggregate_cursor_item_key, \ + aggregate_complete \ + , reduce_round, reduce_batch_cursor, reduce_round_input_count, \ + reduce_round_task_count, reduce_round_terminal_task_count, reduce_ready \ + FROM moa.execution_node_state WHERE run_uid = $1 \ + AND remaining_dependency_count = 0 AND NOT materialization_complete \ + AND node_status NOT IN ('completed', 'skipped', 'failed', 'cancelled') \ + ORDER BY updated_at, node_order, node_state_uid LIMIT $2", + ) + .bind(run_uid) + .bind(fetch_limit) + .fetch_all(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let has_more = rows.len() + > usize::try_from(limit).map_err(|_| Error::InvalidRepositoryInput { + message: "activation node page does not fit in memory".to_string(), + })?; + let nodes = rows + .iter() + .take( + usize::try_from(limit).map_err(|_| Error::InvalidRepositoryInput { + message: "activation node page does not fit in memory".to_string(), + })?, + ) + .map(node_state_from_row) + .collect::>>()?; + let page_node_ids = nodes + .iter() + .map(|node| node.node_id.as_str()) + .collect::>(); + let referenced_node_ids = run + .active_plan + .definition + .nodes + .iter() + .filter(|node| page_node_ids.contains(node.id.as_str())) + .flat_map(|node| node.depends_on.iter().cloned()) + .collect::>(); + let referenced_outputs = if referenced_node_ids.is_empty() { + BTreeMap::new() + } else { + let ids = referenced_node_ids.into_iter().collect::>(); + let referenced_bytes = sqlx::query_scalar::<_, i64>( + "SELECT COALESCE(SUM(pg_column_size(aggregate_output)), 0)::BIGINT \ + FROM moa.execution_node_state WHERE run_uid = $1 \ + AND node_id = ANY($2::TEXT[]) AND aggregate_output IS NOT NULL", + ) + .bind(run_uid) + .bind(&ids) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if referenced_bytes > MAX_ACTIVATION_OUTPUT_BYTES { + return Err(Error::InvalidRepositoryData { + message: format!( + "activation dependency outputs exceed {MAX_ACTIVATION_OUTPUT_BYTES} bytes" + ), + }); + } + sqlx::query( + "SELECT node_id, aggregate_output, aggregate_output_hash \ + FROM moa.execution_node_state \ + WHERE run_uid = $1 AND node_id = ANY($2::TEXT[]) \ + AND node_status IN ('completed', 'skipped') ORDER BY node_order", + ) + .bind(run_uid) + .bind(&ids) + .fetch_all(conn.as_mut()) + .await + .map_err(sqlx_error)? + .into_iter() + .map(|row| { + let node_id: String = row.try_get("node_id").map_err(row_error)?; + let output: Option = row.try_get("aggregate_output").map_err(row_error)?; + let hash: Option = + row.try_get("aggregate_output_hash").map_err(row_error)?; + let output = output.unwrap_or(Value::Null); + let hash = hash.ok_or_else(|| Error::InvalidRepositoryData { + message: format!("node `{node_id}` aggregate output is missing its hash"), + })?; + if node_output_hash(&output)?.to_string() != hash { + return Err(Error::InvalidRepositoryData { + message: format!("node `{node_id}` aggregate output hash mismatch"), + }); + } + Ok((node_id, output)) + }) + .collect::>()? + }; + conn.commit().await.map_err(storage_error)?; + Ok(Some(ExecutionActivationProjection { + run, + nodes, + referenced_outputs, + has_more_actionable: has_more, + })) + } + + /// Loads the oldest pending map aggregate without loading its partial JSON value. + pub async fn load_map_aggregate_candidate( + &self, + scope: ExecutionScope, + run_uid: Uuid, + controller_generation: u64, + wake_epoch: u64, + ) -> Result> { + let mut conn = scope.begin(&self.pool).await?; + let Some(run_row) = sqlx::query(LOAD_RUN_SQL) + .bind(run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + conn.commit().await.map_err(storage_error)?; + return Ok(None); + }; + let run = run_from_row(&run_row)?; + if run.controller_generation != controller_generation + || run.wake_epoch != wake_epoch + || run.activation_state != ExecutionActivationState::Advancing + { + conn.commit().await.map_err(storage_error)?; + return Ok(None); + } + let row = sqlx::query( + "SELECT node_id,node_order,aggregate_cursor_item_key \ + FROM moa.execution_node_state WHERE run_uid=$1 AND materialization_complete \ + AND NOT aggregate_complete AND node_status='pending' \ + AND terminal_task_count=total_task_count AND total_task_count>0 \ + ORDER BY updated_at,node_order,node_state_uid LIMIT 1", + ) + .bind(run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let candidate = row + .map(|row| { + let node_id: String = row.try_get("node_id").map_err(row_error)?; + let is_map = run + .active_plan + .definition + .nodes + .iter() + .find(|node| node.id == node_id) + .is_some_and(|node| matches!(node.operation, ExecutionOperation::Map { .. })); + if !is_map { + return Err(Error::InvalidRepositoryData { + message: format!( + "non-map node `{node_id}` entered the pending aggregate queue" + ), + }); + } + Ok(MapAggregateCandidate { + node_id, + node_order: required_u64(&row, "node_order")?, + cursor_item_key: row + .try_get("aggregate_cursor_item_key") + .map_err(row_error)?, + }) + }) + .transpose()?; + conn.commit().await.map_err(storage_error)?; + Ok(candidate) + } + + /// Appends at most sixteen inline map outputs and completes the node only at source exhaustion. + pub async fn advance_map_aggregate_page( + &self, + scope: ExecutionScope, + request: MapAggregatePageRequest, + ) -> Result { + let mut conn = scope.begin(&self.pool).await?; + let Some(run_row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) + .bind(request.run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + conn.commit().await.map_err(storage_error)?; + return Ok(MapAggregatePageOutcome::NotFound); + }; + let run = run_from_row(&run_row)?; + if run.plan_revision != request.plan_revision + || run.controller_generation != request.controller_generation + || run.wake_epoch != request.wake_epoch + || run.activation_state != ExecutionActivationState::Advancing + || run.status.is_terminal() + || run.pending_terminal.is_some() + || !run + .active_plan + .definition + .nodes + .iter() + .find(|node| node.id == request.node_id) + .is_some_and(|node| matches!(node.operation, ExecutionOperation::Map { .. })) + { + conn.commit().await.map_err(storage_error)?; + return Ok(MapAggregatePageOutcome::Conflict); + } + let Some(node) = sqlx::query( + "SELECT node_status,materialization_complete,total_task_count,terminal_task_count, \ + succeeded_task_count, \ + aggregate_output,aggregate_output_hash,aggregate_cursor_item_key,aggregate_complete \ + FROM moa.execution_node_state WHERE run_uid=$1 AND node_id=$2 FOR UPDATE", + ) + .bind(request.run_uid) + .bind(&request.node_id) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + conn.commit().await.map_err(storage_error)?; + return Ok(MapAggregatePageOutcome::Conflict); + }; + let current_cursor: Option = node + .try_get("aggregate_cursor_item_key") + .map_err(row_error)?; + let aggregate_complete: bool = node.try_get("aggregate_complete").map_err(row_error)?; + if current_cursor != request.expected_cursor_item_key { + let stale_replay = match (¤t_cursor, &request.expected_cursor_item_key) { + (Some(current), Some(expected)) => current > expected, + (Some(_), None) => true, + (None, Some(_)) | (None, None) => false, + }; + conn.commit().await.map_err(storage_error)?; + return Ok(if stale_replay { + MapAggregatePageOutcome::Replayed { + next_cursor_item_key: current_cursor, + aggregate_complete, + } + } else { + MapAggregatePageOutcome::Conflict + }); + } + if aggregate_complete { + conn.commit().await.map_err(storage_error)?; + return Ok(MapAggregatePageOutcome::Replayed { + next_cursor_item_key: current_cursor, + aggregate_complete: true, + }); + } + let node_status: String = node.try_get("node_status").map_err(row_error)?; + let materialization_complete: bool = node + .try_get("materialization_complete") + .map_err(row_error)?; + let total_task_count = required_u64(&node, "total_task_count")?; + if node_status != "pending" + || !materialization_complete + || total_task_count == 0 + || required_u64(&node, "terminal_task_count")? != total_task_count + { + conn.commit().await.map_err(storage_error)?; + return Ok(MapAggregatePageOutcome::Conflict); + } + + let rows = sqlx::query( + "SELECT item_key,COALESCE(output,'null'::JSONB) AS output \ + FROM moa.execution_task WHERE run_uid=$1 AND node_id=$2 \ + AND status IN ('completed','skipped') \ + AND ($3::TEXT IS NULL OR item_key>$3) \ + ORDER BY item_key,task_id LIMIT $4", + ) + .bind(request.run_uid) + .bind(&request.node_id) + .bind(¤t_cursor) + .bind(MAX_MAP_AGGREGATE_PAGE_SIZE) + .fetch_all(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let next_cursor = rows + .last() + .map(|row| row.try_get::("item_key").map_err(row_error)) + .transpose()? + .or_else(|| current_cursor.clone()); + let has_more = sqlx::query_scalar::<_, bool>( + "SELECT EXISTS (SELECT 1 FROM moa.execution_task WHERE run_uid=$1 AND node_id=$2 \ + AND status IN ('completed','skipped') \ + AND ($3::TEXT IS NULL OR item_key>$3))", + ) + .bind(request.run_uid) + .bind(&request.node_id) + .bind(&next_cursor) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let mut aggregate = match node + .try_get::, _>("aggregate_output") + .map_err(row_error)? + { + Some(Value::Array(values)) => values, + None => Vec::new(), + Some(_) => { + return Err(Error::InvalidRepositoryData { + message: "partial map aggregate is not a JSON array".to_string(), + }); + } + }; + let persisted_hash: Option = + node.try_get("aggregate_output_hash").map_err(row_error)?; + if !aggregate.is_empty() { + let current = Value::Array(aggregate.clone()); + let current_hash = node_output_hash(¤t)?.to_string(); + if persisted_hash.as_deref() != Some(current_hash.as_str()) { + return Err(Error::InvalidRepositoryData { + message: "partial map aggregate hash mismatch".to_string(), + }); + } + } else if persisted_hash.is_some() { + return Err(Error::InvalidRepositoryData { + message: "empty map aggregate unexpectedly has a persisted hash".to_string(), + }); + } + for row in &rows { + aggregate.push(row.try_get("output").map_err(row_error)?); + } + let aggregate = Value::Array(aggregate); + let aggregate_bytes = moa_core::canonical_json::canonical_json_bytes(&aggregate)?; + if aggregate_bytes.len() + > usize::try_from(MAX_ACTIVATION_OUTPUT_BYTES).map_err(|_| { + Error::InvalidRepositoryData { + message: "activation output byte ceiling is invalid".to_string(), + } + })? + { + let updated = sqlx::query( + "UPDATE moa.execution_node_state SET node_status='failed',aggregate_output=NULL, \ + aggregate_output_hash=NULL,aggregate_complete=TRUE,updated_at=NOW() \ + WHERE run_uid=$1 AND node_id=$2 \ + AND aggregate_cursor_item_key IS NOT DISTINCT FROM $3 AND NOT aggregate_complete", + ) + .bind(request.run_uid) + .bind(&request.node_id) + .bind(¤t_cursor) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if updated.rows_affected() != 1 { + conn.rollback().await.map_err(storage_error)?; + return Ok(MapAggregatePageOutcome::Conflict); + } + conn.commit().await.map_err(storage_error)?; + return Ok(MapAggregatePageOutcome::Overflow); + } + let complete = !has_more; + let completed_status = if required_u64(&node, "succeeded_task_count")? == 0 { + "skipped" + } else { + "completed" + }; + let persisted_output = if complete && completed_status == "skipped" { + Value::Null + } else { + aggregate + }; + let output_hash = node_output_hash(&persisted_output)?.to_string(); + let updated = sqlx::query( + "UPDATE moa.execution_node_state SET aggregate_cursor_item_key=$4,aggregate_output=$5, \ + aggregate_output_hash=$6,aggregate_complete=$7, \ + node_status=CASE WHEN $7 THEN $8 ELSE 'pending' END,updated_at=NOW() \ + WHERE run_uid=$1 AND node_id=$2 \ + AND aggregate_cursor_item_key IS NOT DISTINCT FROM $3 AND NOT aggregate_complete", + ) + .bind(request.run_uid) + .bind(&request.node_id) + .bind(¤t_cursor) + .bind(&next_cursor) + .bind(&persisted_output) + .bind(output_hash) + .bind(complete) + .bind(completed_status) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if updated.rows_affected() != 1 { + conn.rollback().await.map_err(storage_error)?; + return Ok(MapAggregatePageOutcome::Conflict); + } + if complete { + release_node_dependencies_in_tx(conn.as_mut(), &run, &request.node_id).await?; + } + let aggregated_tasks = + u32::try_from(rows.len()).map_err(|_| Error::ArithmeticOverflow { + context: "map aggregate page task count".to_string(), + })?; + conn.commit().await.map_err(storage_error)?; + Ok(MapAggregatePageOutcome::Applied { + next_cursor_item_key: next_cursor, + aggregated_tasks, + aggregate_complete: complete, + }) + } + + /// Loads a constant-size ordinary-progress and terminal-readiness summary. + pub async fn load_activation_readiness( + &self, + scope: ExecutionScope, + run_uid: Uuid, + ) -> Result> { + let mut conn = scope.begin(&self.pool).await?; + let exists: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM moa.execution_run WHERE run_uid = $1)", + ) + .bind(run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if !exists { + conn.commit().await.map_err(storage_error)?; + return Ok(None); + } + let row = sqlx::query( + "SELECT \ + EXISTS (SELECT 1 FROM moa.execution_node_state \ + WHERE run_uid = $1 AND remaining_dependency_count = 0 AND ( \ + NOT materialization_complete OR (materialization_complete \ + AND NOT aggregate_complete AND node_status='pending' \ + AND terminal_task_count=total_task_count AND total_task_count>0)) \ + AND node_status NOT IN ('completed','skipped','failed','cancelled')) \ + AS has_actionable_nodes, \ + EXISTS (SELECT 1 FROM moa.execution_node_state \ + WHERE run_uid = $1 \ + AND node_status NOT IN ('completed','skipped','failed','cancelled')) \ + AS has_unfinished_nodes, \ + EXISTS (SELECT 1 FROM moa.execution_task WHERE run_uid = $1 \ + AND status NOT IN \ + ('completed','skipped','failed','cancelled','unknown_outcome')) \ + AS has_nonterminal_tasks", + ) + .bind(run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + conn.commit().await.map_err(storage_error)?; + Ok(Some(ExecutionActivationReadiness { + has_actionable_nodes: row.try_get("has_actionable_nodes").map_err(row_error)?, + has_unfinished_nodes: row.try_get("has_unfinished_nodes").map_err(row_error)?, + has_nonterminal_tasks: row.try_get("has_nonterminal_tasks").map_err(row_error)?, + })) + } + + /// Loads one contiguous, byte-bounded prior-round output slice for reduce paging. + pub async fn load_reduce_round_inputs( + &self, + scope: ExecutionScope, + request: ReduceRoundInputPageRequest, + ) -> Result> { + let ReduceRoundInputPageRequest { + run_uid, + node_id, + source_round, + cursor, + batch_size, + target_batch_limit, + } = request; + if source_round == 0 + || cursor.round != source_round + 1 + || batch_size < 2 + || target_batch_limit == 0 + || target_batch_limit > MAX_READY_PAGE_SIZE + { + return Err(Error::InvalidRepositoryInput { + message: "reduce input page requires adjacent rounds and bounded positive limits" + .to_string(), + }); + } + let input_offset = cursor + .batch_cursor + .checked_mul(u64::from(batch_size)) + .ok_or_else(|| Error::InvalidRepositoryInput { + message: "reduce input page offset overflow".to_string(), + })?; + let remaining = cursor.round_input_count.saturating_sub(input_offset); + let input_limit = remaining.min( + u64::from(target_batch_limit) + .checked_mul(u64::from(batch_size)) + .ok_or_else(|| Error::InvalidRepositoryInput { + message: "reduce input page length overflow".to_string(), + })?, + ); + let source_prefix = format!("r{source_round}:b%"); + let input_end = + input_offset + .checked_add(input_limit) + .ok_or_else(|| Error::InvalidRepositoryInput { + message: "reduce input page end overflow".to_string(), + })?; + let mut conn = scope.begin(&self.pool).await?; + let bytes = sqlx::query_scalar::<_, i64>( + "SELECT COALESCE(SUM(pg_column_size(output)), 0)::BIGINT \ + FROM moa.execution_task WHERE run_uid = $1 AND node_id = $2 \ + AND item_key LIKE $3 AND status = 'completed' \ + AND split_part(item_key, ':b', 2)::BIGINT >= $4 \ + AND split_part(item_key, ':b', 2)::BIGINT < $5", + ) + .bind(run_uid) + .bind(&node_id) + .bind(&source_prefix) + .bind(to_i64(input_offset, "reduce input offset")?) + .bind(to_i64(input_end, "reduce input end")?) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if bytes > MAX_ACTIVATION_OUTPUT_BYTES { + return Err(Error::InvalidRepositoryData { + message: format!( + "reduce round input page exceeds {MAX_ACTIVATION_OUTPUT_BYTES} bytes" + ), + }); + } + let rows = sqlx::query_scalar::<_, Value>( + "SELECT output FROM moa.execution_task \ + WHERE run_uid = $1 AND node_id = $2 AND item_key LIKE $3 \ + AND status = 'completed' \ + AND split_part(item_key, ':b', 2)::BIGINT >= $4 \ + AND split_part(item_key, ':b', 2)::BIGINT < $5 \ + ORDER BY split_part(item_key, ':b', 2)::BIGINT", + ) + .bind(run_uid) + .bind(&node_id) + .bind(source_prefix) + .bind(to_i64(input_offset, "reduce input offset")?) + .bind(to_i64(input_end, "reduce input end")?) + .fetch_all(conn.as_mut()) + .await + .map_err(sqlx_error)?; + conn.commit().await.map_err(storage_error)?; + if u64::try_from(rows.len()).map_err(|_| Error::InvalidRepositoryData { + message: "reduce input page row count does not fit in u64".to_string(), + })? != input_limit + { + return Err(Error::InvalidRepositoryData { + message: "reduce input page is incomplete or non-contiguous".to_string(), + }); + } + Ok(rows) + } + + /// Creates durable node aggregates and tenant fairness state for one accepted run. + pub async fn initialize_scheduler_state( + &self, + scope: ExecutionScope, + run_uid: Uuid, + ) -> Result { + let mut conn = scope.begin(&self.pool).await?; + let Some(run_row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) + .bind(run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + conn.commit().await.map_err(storage_error)?; + return Ok(false); + }; + let run = run_from_row(&run_row)?; + sqlx::query( + "INSERT INTO moa.execution_tenant_dispatch_state (tenant_id) VALUES ($1) \ + ON CONFLICT (tenant_id) DO NOTHING", + ) + .bind(run.tenant_id.0) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + + let mut inserted_any = false; + for (node_order, node) in run.active_plan.definition.nodes.iter().enumerate() { + let node_order = + i64::try_from(node_order).map_err(|_| Error::InvalidRepositoryInput { + message: "execution node order exceeds PostgreSQL BIGINT".to_string(), + })?; + let dependency_count = i64::try_from(node.depends_on.len()).map_err(|_| { + Error::InvalidRepositoryInput { + message: "execution dependency count exceeds PostgreSQL BIGINT".to_string(), + } + })?; + let inserted = sqlx::query( + "INSERT INTO moa.execution_node_state (\ + node_state_uid, tenant_id, run_uid, node_id, node_order, \ + dependency_count, remaining_dependency_count\ + ) VALUES ($1, $2, $3, $4, $5, $6, $6) \ + ON CONFLICT (run_uid, node_id) DO NOTHING", + ) + .bind(Uuid::now_v7()) + .bind(run.tenant_id.0) + .bind(run_uid) + .bind(&node.id) + .bind(node_order) + .bind(dependency_count) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + inserted_any |= inserted.rows_affected() == 1; + } + conn.commit().await.map_err(storage_error)?; + Ok(inserted_any) + } + + /// Materializes one cursor-fenced page and makes only that bounded page ready. + pub async fn materialize_ready_page( + &self, + scope: ExecutionScope, + config: &ExecutionConfig, + request: ReadyMaterializationRequest, + ) -> Result { + let ReadyMaterializationRequest { + run_uid, + plan_revision, + node_id, + expected_cursor, + reduce_cursor, + source_exhausted, + terminal_output, + tasks, + } = request; + if tasks.len() > MAX_READY_PAGE_SIZE_USIZE + || (tasks.is_empty() && (!source_exhausted || terminal_output.is_none())) + || (!tasks.is_empty() && terminal_output.is_some()) + { + return Err(Error::InvalidRepositoryInput { + message: format!( + "ready page must contain 1..={MAX_READY_PAGE_SIZE} tasks or one exhausted source output" + ), + }); + } + if tasks.iter().any(|task| task.node_id != node_id) { + return Err(Error::InvalidRepositoryInput { + message: "ready materialization page must contain exactly one node".to_string(), + }); + } + let page_count = u64::try_from(tasks.len()).map_err(|_| Error::InvalidRepositoryInput { + message: "ready materialization page does not fit in u64".to_string(), + })?; + let next_cursor = expected_cursor.checked_add(page_count).ok_or_else(|| { + Error::InvalidRepositoryInput { + message: "ready materialization cursor overflow".to_string(), + } + })?; + let mut conn = scope.begin(&self.pool).await?; + let Some(run_row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) + .bind(run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + conn.commit().await.map_err(storage_error)?; + return Ok(ReadyMaterializationOutcome::Conflict); + }; + let run = run_from_row(&run_row)?; + if run.plan_revision != plan_revision + || !matches!( + run.status, + ExecutionRunStatus::Queued + | ExecutionRunStatus::Running + | ExecutionRunStatus::WaitingInput + | ExecutionRunStatus::WaitingReview + | ExecutionRunStatus::WaitingSignal + | ExecutionRunStatus::WaitingTimer + | ExecutionRunStatus::WaitingExternal + | ExecutionRunStatus::WaitingReplan + ) + || run.pending_terminal.is_some() + { + conn.commit().await.map_err(storage_error)?; + return Ok(ReadyMaterializationOutcome::Conflict); + } + let wait_entered_at = + sqlx::query_scalar::<_, DateTime>("SELECT statement_timestamp()") + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let storage_wait = storage_wait_for_tasks(&tasks, &run, wait_entered_at)?; + let plan_node = run + .active_plan + .definition + .nodes + .iter() + .find(|node| node.id == node_id) + .ok_or_else(|| Error::InvalidRepositoryData { + message: format!("active plan is missing materialized node `{node_id}`"), + })?; + let reduce_batch_size = match (&plan_node.operation, reduce_cursor) { + (ExecutionOperation::Reduce { batch_size, .. }, Some(cursor)) => { + let minimum_inputs = if tasks.is_empty() { 1 } else { 2 }; + if cursor.round == 0 || cursor.round_input_count < minimum_inputs { + return Err(Error::InvalidRepositoryInput { + message: + "reduce cursor requires a one-based round with at least two inputs" + .to_string(), + }); + } + Some(u64::from(*batch_size)) + } + (ExecutionOperation::Reduce { .. }, None) => { + return Err(Error::InvalidRepositoryInput { + message: "reduce ready materialization requires its persisted round cursor" + .to_string(), + }); + } + (_, Some(_)) => { + return Err(Error::InvalidRepositoryInput { + message: "non-reduce materialization cannot advance a reduce cursor" + .to_string(), + }); + } + (_, None) => None, + }; + let cursor_row = sqlx::query( + "SELECT materialization_cursor, materialization_complete, \ + aggregate_output, aggregate_output_hash, reduce_round, reduce_batch_cursor, \ + reduce_round_input_count, reduce_round_task_count, reduce_ready \ + FROM moa.execution_node_state \ + WHERE run_uid = $1 AND node_id = $2 FOR UPDATE", + ) + .bind(run_uid) + .bind(&node_id) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(cursor_row) = cursor_row else { + conn.commit().await.map_err(storage_error)?; + return Ok(ReadyMaterializationOutcome::Conflict); + }; + let cursor = required_u64(&cursor_row, "materialization_cursor")?; + let materialization_complete: bool = cursor_row + .try_get("materialization_complete") + .map_err(row_error)?; + if let Some(output) = terminal_output { + if cursor != expected_cursor { + conn.commit().await.map_err(storage_error)?; + return Ok(ReadyMaterializationOutcome::Conflict); + } + validate_instance( + &plan_node.output_schema, + &output, + &format!("node.{}.output", plan_node.id), + )?; + let output_bytes = moa_core::canonical_json::canonical_json_bytes(&output)?; + if output_bytes.len() + > usize::try_from(MAX_ACTIVATION_OUTPUT_BYTES).map_err(|_| { + Error::InvalidRepositoryData { + message: "activation output byte ceiling is invalid".to_string(), + } + })? + { + return Err(Error::InvalidRepositoryInput { + message: format!( + "node `{node_id}` aggregate output exceeds {MAX_ACTIVATION_OUTPUT_BYTES} bytes" + ), + }); + } + let output_hash = node_output_hash(&output)?.to_string(); + if materialization_complete { + let persisted_output: Option = + cursor_row.try_get("aggregate_output").map_err(row_error)?; + let persisted_hash: Option = cursor_row + .try_get("aggregate_output_hash") + .map_err(row_error)?; + conn.commit().await.map_err(storage_error)?; + return Ok( + if persisted_output.as_ref() == Some(&output) + && persisted_hash.as_deref() == Some(output_hash.as_str()) + { + ReadyMaterializationOutcome::Replayed { + tasks: Vec::new(), + next_cursor: expected_cursor, + triggers: Vec::new(), + } + } else { + ReadyMaterializationOutcome::Conflict + }, + ); + } + let (reduce_round, reduce_input_count, reduce_ready) = match ( + &plan_node.operation, + reduce_cursor, + ) { + (ExecutionOperation::Map { .. }, None) => (None, None, false), + (ExecutionOperation::Reduce { .. }, Some(reduce)) + if reduce.round == 1 + && reduce.batch_cursor == 0 + && reduce.round_input_count == 1 => + { + let persisted_round = required_u64(&cursor_row, "reduce_round")?; + let persisted_batch = required_u64(&cursor_row, "reduce_batch_cursor")?; + let persisted_input = optional_u64(&cursor_row, "reduce_round_input_count")?; + if persisted_round != 1 + || persisted_batch != 0 + || persisted_input.is_some_and(|count| count != 1) + { + conn.commit().await.map_err(storage_error)?; + return Ok(ReadyMaterializationOutcome::Conflict); + } + (Some(1_u64), Some(1_u64), true) + } + _ => { + return Err(Error::InvalidRepositoryInput { + message: "only an empty map or one-item initial reduce may complete without tasks" + .to_string(), + }); + } + }; + let updated = sqlx::query( + "UPDATE moa.execution_node_state SET node_status = 'completed', \ + materialization_complete = TRUE, aggregate_output = $4, \ + aggregate_output_hash = $5, aggregate_complete = TRUE, \ + reduce_round = COALESCE($6, reduce_round), \ + reduce_round_input_count = COALESCE($7, reduce_round_input_count), \ + reduce_ready = $8, updated_at = NOW() \ + WHERE run_uid = $1 AND node_id = $2 AND materialization_cursor = $3 \ + AND NOT materialization_complete", + ) + .bind(run_uid) + .bind(&node_id) + .bind(to_i64( + expected_cursor, + "expected node materialization cursor", + )?) + .bind(&output) + .bind(&output_hash) + .bind( + reduce_round + .map(|value| to_i64(value, "reduce round")) + .transpose()?, + ) + .bind( + reduce_input_count + .map(|value| to_i64(value, "reduce input count")) + .transpose()?, + ) + .bind(reduce_ready) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if updated.rows_affected() != 1 { + conn.rollback().await.map_err(storage_error)?; + return Ok(ReadyMaterializationOutcome::Conflict); + } + release_node_dependents_in_tx(&mut conn, &run, &node_id).await?; + sqlx::query( + "UPDATE moa.execution_run SET last_progress_at = NOW(), updated_at = NOW() \ + WHERE run_uid = $1", + ) + .bind(run_uid) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + conn.commit().await.map_err(storage_error)?; + return Ok(ReadyMaterializationOutcome::Applied { + tasks: Vec::new(), + next_cursor: expected_cursor, + triggers: Vec::new(), + }); + } + let task_batch = prepare_task_materialization_batch(run_uid, plan_revision, &tasks)?; + let next_reduce_cursor = if let (Some(reduce), Some(batch_size)) = + (reduce_cursor, reduce_batch_size) + { + let persisted_round = required_u64(&cursor_row, "reduce_round")?; + let persisted_batch = required_u64(&cursor_row, "reduce_batch_cursor")?; + let persisted_input = optional_u64(&cursor_row, "reduce_round_input_count")?; + let persisted_ready: bool = cursor_row.try_get("reduce_ready").map_err(row_error)?; + let next_batch = reduce.batch_cursor.checked_add(page_count).ok_or_else(|| { + Error::InvalidRepositoryInput { + message: "reduce batch cursor overflow".to_string(), + } + })?; + let total_batches = reduce.round_input_count.div_ceil(batch_size); + if source_exhausted != (next_batch == total_batches) { + return Err(Error::InvalidRepositoryInput { + message: "reduce page exhaustion does not match its round cursor".to_string(), + }); + } + let expected_persisted_batch = if cursor == next_cursor { + next_batch + } else { + reduce.batch_cursor + }; + let keys_match = tasks.iter().enumerate().all(|(offset, task)| { + u64::try_from(offset) + .ok() + .and_then(|offset| reduce.batch_cursor.checked_add(offset)) + .is_some_and(|batch| task.item_key == format!("r{}:b{batch}", reduce.round)) + }); + if persisted_round != reduce.round + || persisted_batch != expected_persisted_batch + || persisted_input.is_some_and(|count| count != reduce.round_input_count) + || (cursor == next_cursor && persisted_ready != source_exhausted) + || (cursor != next_cursor && persisted_ready) + { + conn.commit().await.map_err(storage_error)?; + return Ok(ReadyMaterializationOutcome::Conflict); + } + if next_batch > total_batches || !keys_match { + return Err(Error::InvalidRepositoryInput { + message: "reduce page tasks do not match their round/batch cursor".to_string(), + }); + } + Some((reduce, next_batch)) + } else { + None + }; + if cursor == next_cursor { + let records = load_and_validate_page(&mut conn, run_uid, &task_batch, &tasks).await?; + let triggers = load_scheduled_triggers(&mut conn, run_uid, &tasks).await?; + conn.commit().await.map_err(storage_error)?; + return Ok(ReadyMaterializationOutcome::Replayed { + tasks: records, + next_cursor, + triggers, + }); + } + if cursor != expected_cursor { + conn.commit().await.map_err(storage_error)?; + return Ok(ReadyMaterializationOutcome::Conflict); + } + + let inserted = sqlx::query(INSERT_TASK_BATCH_SQL) + .bind(&task_batch) + .bind(run_uid) + .bind(run.tenant_id.0) + .bind(run.contact_id.map(|value| value.0)) + .bind(to_i64(plan_revision, "plan revision")?) + .fetch_all(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if inserted.len() != tasks.len() { + conn.rollback().await.map_err(storage_error)?; + return Ok(ReadyMaterializationOutcome::Conflict); + } + let task_ids = tasks + .iter() + .map(|task| task.task_id.as_uuid()) + .collect::>(); + let (task_status, attempt_state, waiting_since, ready_at) = storage_wait + .as_ref() + .map_or(("ready", "idle", None, Some(wait_entered_at)), |wait| { + (wait.task_status, "waiting", Some(wait_entered_at), None) + }); + let transitioned = sqlx::query( + "UPDATE moa.execution_task SET status = $3, attempt_state = $4, \ + waiting_since = $5, ready_at = $6, \ + last_progress_at = NOW(), updated_at = NOW() \ + WHERE run_uid = $1 AND task_id = ANY($2::UUID[]) AND status = 'pending'", + ) + .bind(run_uid) + .bind(&task_ids) + .bind(task_status) + .bind(attempt_state) + .bind(waiting_since) + .bind(ready_at) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if transitioned.rows_affected() != page_count { + return Err(Error::InvalidRepositoryData { + message: "inserted ready page did not transition every task".to_string(), + }); + } + let ready_delta = if storage_wait.is_some() { + 0 + } else { + page_count + }; + let waiting_delta = if storage_wait.is_some() { + page_count + } else { + 0 + }; + let node_status = if storage_wait.is_some() { + "waiting" + } else { + "ready" + }; + let (reduce_round, reduce_batch_cursor, reduce_input_count, reduce_task_delta) = + next_reduce_cursor.map_or((None, None, None, 0), |(cursor, next_batch)| { + ( + Some(cursor.round), + Some(next_batch), + Some(cursor.round_input_count), + page_count, + ) + }); + sqlx::query( + "UPDATE moa.execution_node_state \ + SET materialization_cursor = $3, node_status = $6, \ + materialization_complete = $13, \ + total_task_count = total_task_count + $4, \ + ready_task_count = ready_task_count + $7, \ + waiting_task_count = waiting_task_count + $8, \ + reduce_round = COALESCE($9, reduce_round), \ + reduce_batch_cursor = COALESCE($10, reduce_batch_cursor), \ + reduce_round_input_count = COALESCE($11, reduce_round_input_count), \ + reduce_round_task_count = reduce_round_task_count + $12, \ + reduce_ready = CASE WHEN $9::BIGINT IS NULL THEN reduce_ready ELSE $14 END, \ + updated_at = NOW() \ + WHERE run_uid = $1 AND node_id = $2 AND materialization_cursor = $5", + ) + .bind(run_uid) + .bind(&node_id) + .bind(to_i64(next_cursor, "next node materialization cursor")?) + .bind(to_i64(page_count, "ready materialization page count")?) + .bind(to_i64( + expected_cursor, + "expected node materialization cursor", + )?) + .bind(node_status) + .bind(to_i64(ready_delta, "ready materialization count")?) + .bind(to_i64(waiting_delta, "waiting materialization count")?) + .bind( + reduce_round + .map(|value| to_i64(value, "reduce round")) + .transpose()?, + ) + .bind( + reduce_batch_cursor + .map(|value| to_i64(value, "reduce batch cursor")) + .transpose()?, + ) + .bind( + reduce_input_count + .map(|value| to_i64(value, "reduce round input count")) + .transpose()?, + ) + .bind(to_i64(reduce_task_delta, "reduce round task count")?) + .bind(source_exhausted) + .bind(source_exhausted) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let next_wake_at = storage_wait.as_ref().map(|wait| wait.due_at); + let (waiting_status, waiting_reasons, waiting_since, waiting_truncated) = storage_wait + .as_ref() + .map(|wait| { + let sample_limit = config + .maximum_activation_steps + .min(MAX_WAITING_REASON_SAMPLES); + let reasons = bounded_waiting_reason_sample( + &run.waiting_reasons, + &wait.reason, + sample_limit, + )?; + let waiting_count = run.waiting_task_count.checked_add(1).ok_or_else(|| { + Error::ArithmeticOverflow { + context: "run waiting task count".to_string(), + } + })?; + let sample_count = + u64::try_from(reasons.len()).map_err(|_| Error::ArithmeticOverflow { + context: "run waiting reason sample count".to_string(), + })?; + Ok::<_, Error>(( + Some(waiting_run_status_after(&run, wait.task_status).as_str()), + Some(serde_json::to_value(reasons)?), + Some(run.waiting_since.unwrap_or(wait_entered_at)), + Some(waiting_count > sample_count), + )) + }) + .transpose()? + .unwrap_or((None, None, None, None)); + let run_waiting = storage_wait.as_ref().map_or_else( + || Ok(RunWaitingCounterDelta::default()), + |wait| { + run_waiting_counter_delta( + ExecutionTaskStatus::Pending, + storage_wait_task_status(wait.task_status)?, + None, + ) + }, + )?; + sqlx::query( + "UPDATE moa.execution_run \ + SET progress_total_tasks = progress_total_tasks + $2, \ + ready_task_count = ready_task_count + $3, \ + next_wake_at = CASE WHEN $4::TIMESTAMPTZ IS NULL THEN next_wake_at \ + WHEN next_wake_at IS NULL THEN $4 ELSE LEAST(next_wake_at, $4) END, \ + status = COALESCE($5, status), \ + waiting_reasons = COALESCE($6, waiting_reasons), \ + waiting_since = CASE WHEN $6::JSONB IS NULL THEN waiting_since \ + ELSE COALESCE(waiting_since, $7) END, \ + waiting_reasons_truncated = COALESCE($8, waiting_reasons_truncated), \ + waiting_task_count = waiting_task_count + $9, \ + waiting_review_task_count = waiting_review_task_count + $10, \ + waiting_signal_task_count = waiting_signal_task_count + $11, \ + waiting_timer_task_count = waiting_timer_task_count + $12, \ + last_progress_at = NOW(), updated_at = NOW() \ + WHERE run_uid = $1", + ) + .bind(run_uid) + .bind(to_i64(page_count, "ready materialization page count")?) + .bind(to_i64(ready_delta, "ready materialization count")?) + .bind(next_wake_at) + .bind(waiting_status) + .bind(waiting_reasons) + .bind(waiting_since) + .bind(waiting_truncated) + .bind(run_waiting.total) + .bind(run_waiting.review) + .bind(run_waiting.signal) + .bind(run_waiting.timer) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let mut triggers = Vec::new(); + if let Some(wait) = storage_wait { + let task = &tasks[0]; + let write = create_trigger_with_dispatch_in_conn( + conn.as_mut(), + config, + &NewExecutionTrigger { + trigger_uid: storage_wait_trigger_uid( + task.task_id, + task.generation, + wait.trigger_kind, + ), + tenant_id: run.tenant_id, + run_uid: Some(run_uid), + task_id: Some(task.task_id.as_uuid()), + compensation_id: None, + schedule_uid: None, + kind: wait.trigger_kind, + controller_generation: Some(run.controller_generation), + attempt_generation: Some(task.generation), + compensation_generation: None, + compensation_attempt_generation: None, + schedule_incarnation: None, + occurrence_sequence: None, + due_at: wait.due_at, + payload: json!({ "task_id": task.task_id }), + }, + ) + .await?; + triggers.push(ExecutionScheduledTrigger { + dispatch_uid: write.dispatch.dispatch_uid, + tenant_id: run.tenant_id, + trigger_uid: write.trigger.trigger_uid, + due_at: wait.due_at, + }); + } + let records = load_and_validate_page(&mut conn, run_uid, &task_batch, &tasks).await?; + conn.commit().await.map_err(storage_error)?; + Ok(ReadyMaterializationOutcome::Applied { + tasks: records, + next_cursor, + triggers, + }) + } + + /// Loads only exact referenced terminal outputs, with a hard request-size bound. + pub async fn load_referenced_task_outputs( + &self, + scope: ExecutionScope, + run_uid: Uuid, + task_ids: &[ExecutionTaskId], + ) -> Result> { + if task_ids.len() > MAX_READY_PAGE_SIZE_USIZE { + return Err(Error::InvalidRepositoryInput { + message: format!("referenced output load exceeds {MAX_READY_PAGE_SIZE} tasks"), + }); + } + let ids = task_ids.iter().map(|id| id.as_uuid()).collect::>(); + let mut conn = scope.begin(&self.pool).await?; + let rows = sqlx::query( + "SELECT task_id, output FROM moa.execution_task \ + WHERE run_uid = $1 AND task_id = ANY($2::UUID[]) \ + AND status IN ('completed', 'skipped') ORDER BY task_id", + ) + .bind(run_uid) + .bind(ids) + .fetch_all(conn.as_mut()) + .await + .map_err(sqlx_error)?; + conn.commit().await.map_err(storage_error)?; + rows.into_iter() + .map(|row| { + let task_id: Uuid = row.try_get("task_id").map_err(row_error)?; + let output: Value = row.try_get("output").map_err(row_error)?; + Ok((ExecutionTaskId::from_uuid(task_id), output)) + }) + .collect() + } + + /// Scans one bounded task page while proving terminality without an unbounded load. + pub async fn load_terminal_verification_page( + &self, + scope: ExecutionScope, + run_uid: Uuid, + after: Option, + limit: u32, + ) -> Result { + let limit = limit.clamp(1, MAX_READY_PAGE_SIZE); + let mut conn = scope.begin(&self.pool).await?; + let rows = sqlx::query( + "SELECT * FROM moa.execution_task WHERE run_uid = $1 \ + AND ($2::UUID IS NULL OR task_id > $2) \ + ORDER BY task_id LIMIT $3", + ) + .bind(run_uid) + .bind(after.map(ExecutionTaskId::as_uuid)) + .bind(i64::from(limit)) + .fetch_all(conn.as_mut()) + .await + .map_err(sqlx_error)?; + conn.commit().await.map_err(storage_error)?; + let tasks = rows.iter().map(task_from_row).collect::>>()?; + let limit = usize::try_from(limit).map_err(|_| Error::InvalidRepositoryInput { + message: "terminal verification page does not fit in memory".to_string(), + })?; + let next_cursor = (tasks.len() == limit) + .then(|| tasks.last().map(|task| task.task_id)) + .flatten(); + let nonterminal_tasks = tasks + .into_iter() + .filter(|task| !task.status.is_terminal()) + .collect(); + Ok(ExecutionTerminalVerificationPage { + nonterminal_tasks, + next_cursor, + }) + } + + /// Builds compact terminal Session delivery evidence with SQL-enforced row limits. + pub async fn load_bounded_terminal_delivery( + &self, + scope: ExecutionScope, + run_uid: Uuid, + ) -> Result> { + let mut conn = scope.begin(&self.pool).await?; + let Some(run_row) = sqlx::query(LOAD_RUN_SQL) + .bind(run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + conn.commit().await.map_err(storage_error)?; + return Ok(None); + }; + let run = run_from_row(&run_row)?; + if !run.status.is_terminal() { + return Err(Error::InvalidRepositoryData { + message: format!("execution run `{run_uid}` is not terminal"), + }); + } + let citation_limit = i64::try_from(crate::wire::EXECUTION_TERMINAL_MAX_CITATION_IDS) + .map_err(|_| Error::InvalidRepositoryData { + message: "terminal citation limit exceeds PostgreSQL BIGINT".to_string(), + })?; + let failure_limit = + i64::try_from(crate::wire::EXECUTION_TERMINAL_MAX_FAILURES).map_err(|_| { + Error::InvalidRepositoryData { + message: "terminal failure limit exceeds PostgreSQL BIGINT".to_string(), + } + })?; + let citation_ids = sqlx::query_scalar::<_, String>( + "SELECT citation.value ->> 'source_id' \ + FROM moa.execution_task task \ + CROSS JOIN LATERAL jsonb_array_elements(task.citations) \ + WITH ORDINALITY AS citation(value, position) \ + WHERE task.run_uid = $1 \ + AND NULLIF(btrim(citation.value ->> 'source_id'), '') IS NOT NULL \ + ORDER BY task.node_id, task.item_key, task.task_id, citation.position \ + LIMIT $2", + ) + .bind(run_uid) + .bind(citation_limit) + .fetch_all(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let failures = sqlx::query_scalar::<_, String>( + "SELECT COALESCE(error, current_outcome #>> '{result,message}') \ + FROM moa.execution_task WHERE run_uid = $1 \ + AND status IN ('failed', 'unknown_outcome') \ + AND COALESCE(error, current_outcome #>> '{result,message}') IS NOT NULL \ + ORDER BY node_id, item_key, task_id LIMIT $2", + ) + .bind(run_uid) + .bind(failure_limit) + .fetch_all(conn.as_mut()) + .await + .map_err(sqlx_error)?; + conn.commit().await.map_err(storage_error)?; + let summary = crate::wire::build_execution_terminal_summary( + run.run_uid, + run.originating_user_sequence_num, + run.output.as_ref(), + citation_ids, + failures, + run.terminal_gaps.clone(), + )?; + Ok(Some(ExecutionTerminalDelivery { + status: run.status, + summary, + })) + } +} + +fn storage_wait_trigger_uid( + task_id: ExecutionTaskId, + generation: u64, + kind: ExecutionTriggerKind, +) -> Uuid { + Uuid::new_v5( + &task_id.as_uuid(), + format!( + "execution-storage-wait-trigger-v1:{generation}:{}", + kind.as_str() + ) + .as_bytes(), + ) +} + +fn node_state_from_row(row: &PgRow) -> Result { + let status: String = row.try_get("node_status").map_err(row_error)?; + let status = match status.as_str() { + "pending" => ExecutionNodeQueueStatus::Pending, + "ready" => ExecutionNodeQueueStatus::Ready, + "running" => ExecutionNodeQueueStatus::Running, + "waiting" => ExecutionNodeQueueStatus::Waiting, + "completed" => ExecutionNodeQueueStatus::Completed, + "skipped" => ExecutionNodeQueueStatus::Skipped, + "failed" => ExecutionNodeQueueStatus::Failed, + "cancelled" => ExecutionNodeQueueStatus::Cancelled, + other => { + return Err(Error::InvalidRepositoryData { + message: format!("unknown execution node queue status `{other}`"), + }); + } + }; + let aggregate_output: Option = row.try_get("aggregate_output").map_err(row_error)?; + let aggregate_output_hash: Option = + row.try_get("aggregate_output_hash").map_err(row_error)?; + let aggregate_output_hash = aggregate_output_hash + .map(|hash| ExecutionHash::from_str(&hash)) + .transpose()?; + if aggregate_output.is_some() != aggregate_output_hash.is_some() { + return Err(Error::InvalidRepositoryData { + message: "node aggregate output/hash pair is incomplete".to_string(), + }); + } + if let (Some(output), Some(expected_hash)) = (&aggregate_output, aggregate_output_hash) + && node_output_hash(output)? != expected_hash + { + return Err(Error::InvalidRepositoryData { + message: "node aggregate output hash does not match canonical bytes".to_string(), + }); + } + Ok(ExecutionNodeStateRecord { + run_uid: row.try_get("run_uid").map_err(row_error)?, + node_id: row.try_get("node_id").map_err(row_error)?, + node_order: required_u64(row, "node_order")?, + status, + materialization_cursor: required_u64(row, "materialization_cursor")?, + materialization_complete: row.try_get("materialization_complete").map_err(row_error)?, + reduce_round: required_u64(row, "reduce_round")?, + reduce_batch_cursor: required_u64(row, "reduce_batch_cursor")?, + reduce_round_input_count: optional_u64(row, "reduce_round_input_count")?, + reduce_round_task_count: required_u64(row, "reduce_round_task_count")?, + reduce_round_terminal_task_count: required_u64(row, "reduce_round_terminal_task_count")?, + reduce_ready: row.try_get("reduce_ready").map_err(row_error)?, + remaining_dependency_count: required_u64(row, "remaining_dependency_count")?, + total_task_count: required_u64(row, "total_task_count")?, + ready_task_count: required_u64(row, "ready_task_count")?, + active_task_count: required_u64(row, "active_task_count")?, + waiting_task_count: required_u64(row, "waiting_task_count")?, + terminal_task_count: required_u64(row, "terminal_task_count")?, + aggregate_output, + aggregate_output_hash, + aggregate_cursor_item_key: row + .try_get("aggregate_cursor_item_key") + .map_err(row_error)?, + aggregate_complete: row.try_get("aggregate_complete").map_err(row_error)?, + }) +} + +struct StorageWaitMaterialization { + task_status: &'static str, + trigger_kind: ExecutionTriggerKind, + due_at: DateTime, + reason: WaitingReason, +} + +fn storage_wait_for_tasks( + tasks: &[LogicalTask], + run: &ExecutionRunRecord, + wait_entered_at: DateTime, +) -> Result> { + let Some(first) = tasks.first() else { + return Ok(None); + }; + let (task_status, trigger_kind, target) = match &first.kind { + LogicalTaskKind::Review { wait_policy, .. } => ( + "waiting_review", + ExecutionTriggerKind::WaitExpiry, + &wait_policy.expiry, + ), + LogicalTaskKind::WaitSignal { wait_policy, .. } => ( + "waiting_signal", + ExecutionTriggerKind::WaitExpiry, + &wait_policy.expiry, + ), + LogicalTaskKind::WaitUntil { wake, .. } => { + ("waiting_timer", ExecutionTriggerKind::TaskTimer, wake) + } + _ => return Ok(None), + }; + if tasks.len() != 1 { + return Err(Error::InvalidRepositoryInput { + message: "storage-only wait nodes must materialize exactly one logical task" + .to_string(), + }); + } + let run_deadline_at = + run.approved_budget + .deadline_at + .ok_or_else(|| Error::InvalidRepositoryInput { + message: "storage-only waits require an absolute run deadline".to_string(), + })?; + let due_at = + crate::interpreter::resolve_temporal_target(target, wait_entered_at, run_deadline_at)?; + let exact_target = ExecutionTemporalTarget::At { at: due_at }; + let reason = match &first.kind { + LogicalTaskKind::Review { + prompt, + wait_policy, + } => WaitingReason::Review { + task_id: first.task_id, + prompt: prompt.clone(), + wait_policy: ExecutionWaitPolicy { + expiry: exact_target, + on_expiry: wait_policy.on_expiry.clone(), + }, + }, + LogicalTaskKind::WaitSignal { + signal_name, + wait_policy, + } => WaitingReason::Signal { + task_id: first.task_id, + signal_name: signal_name.clone(), + wait_policy: ExecutionWaitPolicy { + expiry: exact_target, + on_expiry: wait_policy.on_expiry.clone(), + }, + }, + LogicalTaskKind::WaitUntil { .. } => WaitingReason::Timer { + task_id: first.task_id, + wake: exact_target, + }, + LogicalTaskKind::Capability { .. } + | LogicalTaskKind::Agent { .. } + | LogicalTaskKind::Output { .. } + | LogicalTaskKind::CompletionVerifier { .. } => { + return Err(Error::InvalidRepositoryData { + message: "compute task reached storage-wait materialization".to_string(), + }); + } + }; + Ok(Some(StorageWaitMaterialization { + task_status, + trigger_kind, + due_at, + reason, + })) +} + +fn waiting_reason_task_id(reason: &WaitingReason) -> Option { + match reason { + WaitingReason::Input { task_id, .. } + | WaitingReason::Review { task_id, .. } + | WaitingReason::Signal { task_id, .. } + | WaitingReason::Timer { task_id, .. } + | WaitingReason::External { task_id } => Some(*task_id), + WaitingReason::RunningTasks | WaitingReason::Dependencies { .. } => None, + } +} + +fn bounded_waiting_reason_sample( + existing: &[WaitingReason], + inserted: &WaitingReason, + limit: usize, +) -> Result> { + let mut candidates = existing + .iter() + .filter(|reason| waiting_reason_task_id(reason).is_some()) + .cloned() + .collect::>(); + if let Some(inserted_task_id) = waiting_reason_task_id(inserted) + && !candidates + .iter() + .any(|reason| waiting_reason_task_id(reason) == Some(inserted_task_id)) + { + candidates.push(inserted.clone()); + } + candidates.sort_by_key(waiting_reason_task_id); + let mut sample = Vec::new(); + for reason in candidates { + if sample.len() >= limit { + break; + } + sample.push(reason); + if serde_json::to_vec(&sample)?.len() > MAX_WAITING_REASON_SAMPLE_BYTES { + sample.pop(); + } + } + Ok(sample) +} + +pub(super) async fn append_run_wait_reason_in_tx( + conn: &mut ScopedConn<'_>, + run_uid: Uuid, + reason: &WaitingReason, + entered_at: DateTime, +) -> Result<()> { + let row = sqlx::query( + "SELECT waiting_reasons, waiting_task_count, waiting_since \ + FROM moa.execution_run WHERE run_uid=$1 FOR UPDATE", + ) + .bind(run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + .ok_or_else(|| Error::InvalidRepositoryData { + message: "waiting task references a missing execution run".to_string(), + })?; + let existing: Vec = + serde_json::from_value(row.try_get("waiting_reasons").map_err(row_error)?)?; + let waiting_task_count = required_u64(&row, "waiting_task_count")?; + let waiting_since: Option> = row.try_get("waiting_since").map_err(row_error)?; + let reasons = bounded_waiting_reason_sample(&existing, reason, MAX_WAITING_REASON_SAMPLES)?; + let sample_count = u64::try_from(reasons.len()).map_err(|_| Error::ArithmeticOverflow { + context: "run waiting reason sample count".to_string(), + })?; + sqlx::query( + "UPDATE moa.execution_run SET waiting_reasons=$2, \ + waiting_reasons_truncated=$3, waiting_since=$4, \ + status=CASE \ + WHEN status IN ('pause_requested','pausing','paused') THEN status \ + WHEN waiting_input_task_count > 0 THEN 'waiting_input' \ + WHEN waiting_review_task_count > 0 THEN 'waiting_review' \ + WHEN waiting_signal_task_count > 0 THEN 'waiting_signal' \ + WHEN waiting_timer_task_count > 0 THEN 'waiting_timer' \ + WHEN waiting_external_task_count > 0 THEN 'waiting_external' \ + WHEN waiting_replan_task_count > 0 THEN 'waiting_replan' \ + ELSE 'running' END, \ + last_progress_at=GREATEST(last_progress_at,$5), updated_at=NOW() \ + WHERE run_uid=$1", + ) + .bind(run_uid) + .bind(serde_json::to_value(reasons)?) + .bind(sample_count < waiting_task_count) + .bind(waiting_since.unwrap_or(entered_at)) + .bind(entered_at) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + Ok(()) +} + +fn storage_wait_task_status(status: &str) -> Result { + match status { + "waiting_review" => Ok(ExecutionTaskStatus::WaitingReview), + "waiting_signal" => Ok(ExecutionTaskStatus::WaitingSignal), + "waiting_timer" => Ok(ExecutionTaskStatus::WaitingTimer), + other => Err(Error::InvalidRepositoryData { + message: format!("unknown storage wait task status `{other}`"), + }), + } +} + +fn waiting_run_status_after( + run: &ExecutionRunRecord, + inserted_task_status: &str, +) -> ExecutionRunStatus { + let inserted_review = u64::from(inserted_task_status == "waiting_review"); + let inserted_signal = u64::from(inserted_task_status == "waiting_signal"); + let inserted_timer = u64::from(inserted_task_status == "waiting_timer"); + if run.waiting_input_task_count > 0 { + ExecutionRunStatus::WaitingInput + } else if run + .waiting_review_task_count + .saturating_add(inserted_review) + > 0 + { + ExecutionRunStatus::WaitingReview + } else if run + .waiting_signal_task_count + .saturating_add(inserted_signal) + > 0 + { + ExecutionRunStatus::WaitingSignal + } else if run.waiting_timer_task_count.saturating_add(inserted_timer) > 0 { + ExecutionRunStatus::WaitingTimer + } else if run.waiting_external_task_count > 0 { + ExecutionRunStatus::WaitingExternal + } else if run.waiting_replan_task_count > 0 { + ExecutionRunStatus::WaitingReplan + } else { + ExecutionRunStatus::Running + } +} + +async fn release_node_dependents_in_tx( + conn: &mut ScopedConn<'_>, + run: &ExecutionRunRecord, + node_id: &str, +) -> Result<()> { + for dependent in run.active_plan.definition.nodes.iter().filter(|node| { + node.depends_on + .iter() + .any(|dependency| dependency == node_id) + }) { + let released = sqlx::query( + "UPDATE moa.execution_node_state \ + SET remaining_dependency_count = remaining_dependency_count - 1, \ + updated_at = NOW() \ + WHERE run_uid = $1 AND node_id = $2 AND remaining_dependency_count > 0", + ) + .bind(run.run_uid) + .bind(&dependent.id) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if released.rows_affected() != 1 { + return Err(Error::InvalidRepositoryData { + message: format!( + "dependent node `{}` lost its dependency counter", + dependent.id + ), + }); + } + } + Ok(()) +} + +async fn load_scheduled_triggers( + conn: &mut ScopedConn<'_>, + run_uid: Uuid, + tasks: &[LogicalTask], +) -> Result> { + let task_ids = tasks + .iter() + .map(|task| task.task_id.as_uuid()) + .collect::>(); + let rows = sqlx::query( + "SELECT dispatch.dispatch_uid, trigger.tenant_id, trigger.trigger_uid, trigger.due_at \ + FROM moa.execution_trigger AS trigger \ + JOIN moa.execution_dispatch_outbox AS dispatch \ + ON dispatch.tenant_id = trigger.tenant_id \ + AND dispatch.trigger_uid = trigger.trigger_uid \ + AND dispatch.dispatch_kind = 'trigger_delivery' \ + WHERE trigger.run_uid = $1 AND trigger.task_id = ANY($2::UUID[]) \ + AND trigger.trigger_kind IN ('task_timer', 'wait_expiry') \ + ORDER BY trigger.task_id, trigger.trigger_uid", + ) + .bind(run_uid) + .bind(task_ids) + .fetch_all(conn.as_mut()) + .await + .map_err(sqlx_error)?; + rows.into_iter() + .map(|row| { + Ok(ExecutionScheduledTrigger { + dispatch_uid: row.try_get("dispatch_uid").map_err(row_error)?, + tenant_id: TenantId::from(row.try_get::("tenant_id").map_err(row_error)?), + trigger_uid: row.try_get("trigger_uid").map_err(row_error)?, + due_at: row.try_get("due_at").map_err(row_error)?, + }) + }) + .collect() +} + +async fn load_and_validate_page( + conn: &mut ScopedConn<'_>, + run_uid: Uuid, + task_batch: &Value, + requested: &[LogicalTask], +) -> Result> { + let rows = sqlx::query(LOAD_TASK_BATCH_SQL) + .bind(task_batch) + .bind(run_uid) + .fetch_all(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if rows.len() != requested.len() { + return Err(Error::InvalidRepositoryData { + message: "ready materialization replay did not reload the exact page".to_string(), + }); + } + rows.iter() + .zip(requested) + .map(|(row, requested)| { + let record = task_from_row(row)?; + ensure_materialization_replay_matches(&record, requested)?; + if record.status != ExecutionTaskStatus::Ready + && record.status != ExecutionTaskStatus::Dispatching + && record.status != ExecutionTaskStatus::Running + && record.status != ExecutionTaskStatus::WaitingReview + && record.status != ExecutionTaskStatus::WaitingSignal + && record.status != ExecutionTaskStatus::WaitingTimer + && !record.status.is_terminal() + { + return Err(Error::InvalidRepositoryData { + message: "replayed ready page contains a task outside its lifecycle" + .to_string(), + }); + } + Ok(record) + }) + .collect() +} + +/// Updates task-derived node/run counters in the caller's canonical mutation transaction. +pub(super) async fn transition_node_counters_in_tx( + conn: &mut ScopedConn<'_>, + run_uid: Uuid, + node_id: &str, + item_key: &str, + from: ExecutionTaskStatus, + to: ExecutionTaskStatus, +) -> Result<()> { + transition_node_counters_inner(conn, run_uid, node_id, item_key, from, to, None).await +} + +/// Updates counters for one transition into or out of an audience-qualified input wait. +pub(super) async fn transition_node_counters_with_input_audience_in_tx( + conn: &mut ScopedConn<'_>, + run_uid: Uuid, + node_id: &str, + item_key: &str, + from: ExecutionTaskStatus, + to: ExecutionTaskStatus, + input_audience: &InputAudience, +) -> Result<()> { + transition_node_counters_inner( + conn, + run_uid, + node_id, + item_key, + from, + to, + Some(input_audience), + ) + .await +} + +async fn transition_node_counters_inner( + conn: &mut ScopedConn<'_>, + run_uid: Uuid, + node_id: &str, + item_key: &str, + from: ExecutionTaskStatus, + to: ExecutionTaskStatus, + input_audience: Option<&InputAudience>, +) -> Result<()> { + if from == to { + return Ok(()); + } + let touches_input = + from == ExecutionTaskStatus::WaitingInput || to == ExecutionTaskStatus::WaitingInput; + if touches_input != input_audience.is_some() { + return Err(Error::InvalidRepositoryInput { + message: "WaitingInput counter transitions require exactly one typed input audience" + .to_string(), + }); + } + let run_row = sqlx::query(LOAD_RUN_SQL) + .bind(run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let run = run_from_row(&run_row)?; + let node = run + .active_plan + .definition + .nodes + .iter() + .find(|node| node.id == node_id); + let is_verifier = node_id.starts_with("@check/"); + if node.is_none() && !is_verifier { + return Err(Error::InvalidRepositoryData { + message: format!("active plan is missing transitioned node `{node_id}`"), + }); + } + let is_reduce = + node.is_some_and(|node| matches!(node.operation, ExecutionOperation::Reduce { .. })); + let is_map = node.is_some_and(|node| matches!(node.operation, ExecutionOperation::Map { .. })); + let previous_node = sqlx::query( + "SELECT node_status, reduce_round, reduce_round_task_count, \ + reduce_round_terminal_task_count, reduce_ready \ + FROM moa.execution_node_state \ + WHERE run_uid = $1 AND node_id = $2 FOR UPDATE", + ) + .bind(run_uid) + .bind(node_id) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + .ok_or_else(|| Error::InvalidRepositoryData { + message: format!("missing node counters for `{node_id}`"), + })?; + let previous_node_status: String = previous_node.try_get("node_status").map_err(row_error)?; + let from_counts = task_counter_class(from); + let to_counts = task_counter_class(to); + let ready_delta = to_counts.ready - from_counts.ready; + let active_delta = to_counts.active - from_counts.active; + let waiting_delta = to_counts.waiting - from_counts.waiting; + let terminal_delta = to_counts.terminal - from_counts.terminal; + let succeeded_delta = to_counts.succeeded - from_counts.succeeded; + let failed_delta = to_counts.failed - from_counts.failed; + let cancelled_delta = to_counts.cancelled - from_counts.cancelled; + let run_waiting = run_waiting_counter_delta(from, to, input_audience)?; + let reduce_terminal_delta = if is_reduce && terminal_delta != 0 { + let task_round = reduce_round_from_item_key(item_key)?; + let current_round = required_u64(&previous_node, "reduce_round")?; + if task_round != current_round { + return Err(Error::InvalidRepositoryData { + message: format!( + "reduce task round {task_round} does not match current round {current_round}" + ), + }); + } + terminal_delta + } else { + 0 + }; + let updated = sqlx::query( + "UPDATE moa.execution_node_state SET \ + ready_task_count = ready_task_count + $3, \ + active_task_count = active_task_count + $4, \ + waiting_task_count = waiting_task_count + $5, \ + terminal_task_count = terminal_task_count + $6, \ + succeeded_task_count = succeeded_task_count + $7, \ + failed_task_count = failed_task_count + $8, \ + cancelled_task_count = cancelled_task_count + $9, \ + reduce_round_terminal_task_count = \ + reduce_round_terminal_task_count + $10, \ + node_status = CASE \ + WHEN failed_task_count + $8 > 0 THEN 'failed' \ + WHEN cancelled_task_count + $9 > 0 THEN 'cancelled' \ + WHEN $11 AND reduce_ready \ + AND reduce_round_terminal_task_count + $10 = reduce_round_task_count \ + AND reduce_round_task_count = 1 THEN 'completed' \ + WHEN $11 AND reduce_ready \ + AND reduce_round_terminal_task_count + $10 = reduce_round_task_count \ + AND reduce_round_task_count > 1 THEN 'pending' \ + WHEN $12 AND materialization_complete \ + AND terminal_task_count + $6 = total_task_count \ + AND total_task_count > 0 THEN 'pending' \ + WHEN NOT $11 AND materialization_complete \ + AND terminal_task_count + $6 = total_task_count \ + AND total_task_count > 0 THEN \ + CASE WHEN succeeded_task_count + $7 = 0 THEN 'skipped' ELSE 'completed' END \ + WHEN waiting_task_count + $5 > 0 THEN 'waiting' \ + WHEN active_task_count + $4 > 0 THEN 'running' \ + WHEN ready_task_count + $3 > 0 THEN 'ready' \ + ELSE 'pending' END, \ + updated_at = NOW() \ + WHERE run_uid = $1 AND node_id = $2 \ + AND ready_task_count + $3 >= 0 AND active_task_count + $4 >= 0 \ + AND waiting_task_count + $5 >= 0 AND terminal_task_count + $6 >= 0 \ + AND reduce_round_terminal_task_count + $10 >= 0 \ + RETURNING node_status, reduce_round, reduce_round_task_count, \ + reduce_round_terminal_task_count, reduce_ready", + ) + .bind(run_uid) + .bind(node_id) + .bind(ready_delta) + .bind(active_delta) + .bind(waiting_delta) + .bind(terminal_delta) + .bind(succeeded_delta) + .bind(failed_delta) + .bind(cancelled_delta) + .bind(reduce_terminal_delta) + .bind(is_reduce) + .bind(is_map) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(updated) = updated else { + return Err(Error::InvalidRepositoryData { + message: format!("missing or inconsistent node counters for `{node_id}`"), + }); + }; + let mut updated_node_status: String = updated.try_get("node_status").map_err(row_error)?; + if is_reduce { + let round_task_count = required_u64(&updated, "reduce_round_task_count")?; + let round_terminal_count = required_u64(&updated, "reduce_round_terminal_task_count")?; + let round_ready: bool = updated.try_get("reduce_ready").map_err(row_error)?; + if round_ready && round_task_count > 1 && round_terminal_count == round_task_count { + let advanced = sqlx::query( + "UPDATE moa.execution_node_state \ + SET reduce_round = reduce_round + 1, reduce_batch_cursor = 0, \ + reduce_round_input_count = $3, reduce_round_task_count = 0, \ + reduce_round_terminal_task_count = 0, reduce_ready = FALSE, \ + materialization_complete = FALSE, node_status = 'pending', \ + updated_at = NOW() \ + WHERE run_uid = $1 AND node_id = $2 AND reduce_round = $4", + ) + .bind(run_uid) + .bind(node_id) + .bind(to_i64(round_task_count, "next reduce round input count")?) + .bind(to_i64( + required_u64(&updated, "reduce_round")?, + "reduce round", + )?) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if advanced.rows_affected() != 1 { + return Err(Error::InvalidRepositoryData { + message: "completed reduce round lost its cursor fence".to_string(), + }); + } + updated_node_status = "pending".to_string(); + } + } + let run_updated = sqlx::query( + "UPDATE moa.execution_run SET ready_task_count = ready_task_count + $2, \ + active_task_count = active_task_count + $3, \ + waiting_task_count = waiting_task_count + $4, \ + waiting_input_task_count = waiting_input_task_count + $5, \ + waiting_review_task_count = waiting_review_task_count + $6, \ + waiting_signal_task_count = waiting_signal_task_count + $7, \ + waiting_timer_task_count = waiting_timer_task_count + $8, \ + waiting_external_task_count = waiting_external_task_count + $9, \ + waiting_replan_task_count = waiting_replan_task_count + $10, \ + waiting_input_user_task_count = waiting_input_user_task_count + $11, \ + waiting_input_tenant_admin_task_count = \ + waiting_input_tenant_admin_task_count + $12, \ + waiting_input_external_task_count = waiting_input_external_task_count + $13, \ + last_progress_at = GREATEST(last_progress_at, NOW()), \ + wake_epoch = wake_epoch + 1, updated_at = NOW() \ + WHERE run_uid = $1 AND ready_task_count + $2 >= 0 AND active_task_count + $3 >= 0 \ + AND waiting_task_count + $4 >= 0 AND waiting_input_task_count + $5 >= 0 \ + AND waiting_review_task_count + $6 >= 0 \ + AND waiting_signal_task_count + $7 >= 0 \ + AND waiting_timer_task_count + $8 >= 0 \ + AND waiting_external_task_count + $9 >= 0 \ + AND waiting_replan_task_count + $10 >= 0 \ + AND waiting_input_user_task_count + $11 >= 0 \ + AND waiting_input_tenant_admin_task_count + $12 >= 0 \ + AND waiting_input_external_task_count + $13 >= 0", + ) + .bind(run_uid) + .bind(ready_delta) + .bind(active_delta) + .bind(run_waiting.total) + .bind(run_waiting.input) + .bind(run_waiting.review) + .bind(run_waiting.signal) + .bind(run_waiting.timer) + .bind(run_waiting.external) + .bind(run_waiting.replan) + .bind(run_waiting.input_user) + .bind(run_waiting.input_tenant_admin) + .bind(run_waiting.input_external) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if run_updated.rows_affected() != 1 { + return Err(Error::InvalidRepositoryData { + message: "run ready/active/waiting counters would underflow".to_string(), + }); + } + if !matches!(previous_node_status.as_str(), "failed" | "cancelled") + && matches!(updated_node_status.as_str(), "failed" | "cancelled") + { + cancel_unmaterialized_dependents_in_tx(conn.as_mut(), &run, node_id).await?; + } + if !matches!(previous_node_status.as_str(), "completed" | "skipped") + && matches!(updated_node_status.as_str(), "completed" | "skipped") + { + let aggregate_output = if updated_node_status == "skipped" { + Value::Null + } else { + match node.map(|node| &node.operation) { + Some(ExecutionOperation::Map { .. }) => sqlx::query_scalar::<_, Value>( + "SELECT COALESCE(jsonb_agg(output ORDER BY item_key), '[]'::JSONB) \ + FROM moa.execution_task WHERE run_uid = $1 AND node_id = $2 \ + AND status IN ('completed', 'skipped')", + ) + .bind(run_uid) + .bind(node_id) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?, + Some(ExecutionOperation::Reduce { .. }) => sqlx::query_scalar::<_, Value>( + "SELECT output FROM moa.execution_task \ + WHERE run_uid = $1 AND node_id = $2 AND status = 'completed' \ + ORDER BY created_at DESC, task_id DESC LIMIT 1", + ) + .bind(run_uid) + .bind(node_id) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?, + Some(_) | None => sqlx::query_scalar::<_, Value>( + "SELECT output FROM moa.execution_task \ + WHERE run_uid = $1 AND node_id = $2 \ + AND status IN ('completed', 'skipped') ORDER BY task_id LIMIT 1", + ) + .bind(run_uid) + .bind(node_id) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?, + } + }; + let aggregate_bytes = moa_core::canonical_json::canonical_json_bytes(&aggregate_output)?; + if aggregate_bytes.len() + > usize::try_from(MAX_ACTIVATION_OUTPUT_BYTES).map_err(|_| { + Error::InvalidRepositoryData { + message: "activation output byte ceiling is invalid".to_string(), + } + })? + { + return Err(Error::InvalidRepositoryData { + message: format!( + "node `{node_id}` aggregate output exceeds {MAX_ACTIVATION_OUTPUT_BYTES} bytes" + ), + }); + } + sqlx::query( + "UPDATE moa.execution_node_state SET aggregate_output = $3, \ + aggregate_output_hash = $4, aggregate_complete = TRUE, updated_at = NOW() \ + WHERE run_uid = $1 AND node_id = $2", + ) + .bind(run_uid) + .bind(node_id) + .bind(&aggregate_output) + .bind(node_output_hash(&aggregate_output)?.to_string()) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + release_node_dependencies_in_tx(conn.as_mut(), &run, node_id).await?; + } + Ok(()) +} + +async fn cancel_unmaterialized_dependents_in_tx( + conn: &mut PgConnection, + run: &ExecutionRunRecord, + failed_node_id: &str, +) -> Result<()> { + let mut blocked = BTreeSet::from([failed_node_id]); + loop { + let prior_len = blocked.len(); + for node in &run.active_plan.definition.nodes { + if node + .depends_on + .iter() + .any(|dependency| blocked.contains(dependency.as_str())) + { + blocked.insert(node.id.as_str()); + } + } + if blocked.len() == prior_len { + break; + } + } + blocked.remove(failed_node_id); + if blocked.is_empty() { + return Ok(()); + } + let dependent_ids = blocked.into_iter().collect::>(); + let (pending_count, cancelled_count) = sqlx::query_as::<_, (i64, i64)>( + "SELECT COUNT(*) FILTER (WHERE node_status='pending' AND total_task_count=0), \ + COUNT(*) FILTER (WHERE node_status='cancelled' AND total_task_count=0 \ + AND materialization_complete AND aggregate_complete \ + AND remaining_dependency_count=0) \ + FROM moa.execution_node_state WHERE run_uid=$1 AND node_id=ANY($2::TEXT[])", + ) + .bind(run.run_uid) + .bind(&dependent_ids) + .fetch_one(&mut *conn) + .await + .map_err(sqlx_error)?; + let expected = i64::try_from(dependent_ids.len()).map_err(|_| Error::ArithmeticOverflow { + context: "failed dependency node count".to_string(), + })?; + if pending_count + cancelled_count != expected { + return Err(Error::InvalidRepositoryData { + message: "failed dependency cascade found a materialized or non-pending dependent" + .to_string(), + }); + } + let cancelled = sqlx::query( + "UPDATE moa.execution_node_state SET node_status='cancelled', \ + materialization_complete=TRUE, aggregate_complete=TRUE, \ + remaining_dependency_count=0, updated_at=NOW() \ + WHERE run_uid=$1 AND node_id=ANY($2::TEXT[]) \ + AND node_status='pending' AND total_task_count=0", + ) + .bind(run.run_uid) + .bind(&dependent_ids) + .execute(&mut *conn) + .await + .map_err(sqlx_error)?; + let applied_count = + i64::try_from(cancelled.rows_affected()).map_err(|_| Error::ArithmeticOverflow { + context: "newly cancelled dependency node count".to_string(), + })?; + if applied_count != pending_count { + return Err(Error::InvalidRepositoryData { + message: "failed dependency cascade lost its exact pending-node set".to_string(), + }); + } + Ok(()) +} + +async fn release_node_dependencies_in_tx( + conn: &mut PgConnection, + run: &ExecutionRunRecord, + completed_node_id: &str, +) -> Result<()> { + for dependent in run.active_plan.definition.nodes.iter().filter(|node| { + node.depends_on + .iter() + .any(|dependency| dependency == completed_node_id) + }) { + let released = sqlx::query( + "UPDATE moa.execution_node_state \ + SET remaining_dependency_count = remaining_dependency_count - 1,updated_at=NOW() \ + WHERE run_uid=$1 AND node_id=$2 AND remaining_dependency_count>0", + ) + .bind(run.run_uid) + .bind(&dependent.id) + .execute(&mut *conn) + .await + .map_err(sqlx_error)?; + if released.rows_affected() != 1 { + let already_cancelled = sqlx::query_scalar::<_, bool>( + "SELECT EXISTS (SELECT 1 FROM moa.execution_node_state \ + WHERE run_uid=$1 AND node_id=$2 AND node_status='cancelled' \ + AND total_task_count=0 AND remaining_dependency_count=0 \ + AND materialization_complete AND aggregate_complete)", + ) + .bind(run.run_uid) + .bind(&dependent.id) + .fetch_one(&mut *conn) + .await + .map_err(sqlx_error)?; + if already_cancelled { + continue; + } + return Err(Error::InvalidRepositoryData { + message: format!( + "dependent node `{}` lost its dependency counter", + dependent.id + ), + }); + } + } + Ok(()) +} + +#[derive(Clone, Copy)] +struct TaskCounterClass { + ready: i64, + active: i64, + waiting: i64, + terminal: i64, + succeeded: i64, + failed: i64, + cancelled: i64, +} + +#[derive(Clone, Copy, Debug, Default)] +struct RunWaitingCounterDelta { + total: i64, + input: i64, + review: i64, + signal: i64, + timer: i64, + external: i64, + replan: i64, + input_user: i64, + input_tenant_admin: i64, + input_external: i64, +} + +fn run_waiting_counter_delta( + from: ExecutionTaskStatus, + to: ExecutionTaskStatus, + input_audience: Option<&InputAudience>, +) -> Result { + let mut delta = RunWaitingCounterDelta::default(); + apply_run_waiting_counter(&mut delta, from, input_audience, -1)?; + apply_run_waiting_counter(&mut delta, to, input_audience, 1)?; + Ok(delta) +} + +fn apply_run_waiting_counter( + delta: &mut RunWaitingCounterDelta, + status: ExecutionTaskStatus, + input_audience: Option<&InputAudience>, + direction: i64, +) -> Result<()> { + match status { + ExecutionTaskStatus::WaitingInput => { + delta.total += direction; + delta.input += direction; + match input_audience.ok_or_else(|| Error::InvalidRepositoryInput { + message: "WaitingInput counter transition is missing its audience".to_string(), + })? { + InputAudience::User => delta.input_user += direction, + InputAudience::TenantAdmin => delta.input_tenant_admin += direction, + InputAudience::ExternalSystem => delta.input_external += direction, + } + } + ExecutionTaskStatus::WaitingReview => { + delta.total += direction; + delta.review += direction; + } + ExecutionTaskStatus::WaitingSignal => { + delta.total += direction; + delta.signal += direction; + } + ExecutionTaskStatus::WaitingTimer => { + delta.total += direction; + delta.timer += direction; + } + ExecutionTaskStatus::WaitingExternal => { + delta.total += direction; + delta.external += direction; + } + ExecutionTaskStatus::WaitingReplan => { + delta.total += direction; + delta.replan += direction; + } + ExecutionTaskStatus::Pending + | ExecutionTaskStatus::Ready + | ExecutionTaskStatus::Reserved + | ExecutionTaskStatus::Dispatching + | ExecutionTaskStatus::Running + | ExecutionTaskStatus::Completed + | ExecutionTaskStatus::Skipped + | ExecutionTaskStatus::Failed + | ExecutionTaskStatus::Cancelled + | ExecutionTaskStatus::UnknownOutcome => {} + } + Ok(()) +} + +const fn task_counter_class(status: ExecutionTaskStatus) -> TaskCounterClass { + let mut counts = TaskCounterClass { + ready: 0, + active: 0, + waiting: 0, + terminal: 0, + succeeded: 0, + failed: 0, + cancelled: 0, + }; + match status { + ExecutionTaskStatus::Ready => counts.ready = 1, + ExecutionTaskStatus::Reserved + | ExecutionTaskStatus::Dispatching + | ExecutionTaskStatus::Running => counts.active = 1, + ExecutionTaskStatus::WaitingInput + | ExecutionTaskStatus::WaitingReview + | ExecutionTaskStatus::WaitingSignal + | ExecutionTaskStatus::WaitingTimer + | ExecutionTaskStatus::WaitingExternal + | ExecutionTaskStatus::WaitingReplan => counts.waiting = 1, + ExecutionTaskStatus::Completed => { + counts.terminal = 1; + counts.succeeded = 1; + } + ExecutionTaskStatus::Skipped => counts.terminal = 1, + ExecutionTaskStatus::Failed | ExecutionTaskStatus::UnknownOutcome => { + counts.terminal = 1; + counts.failed = 1; + } + ExecutionTaskStatus::Cancelled => { + counts.terminal = 1; + counts.cancelled = 1; + } + ExecutionTaskStatus::Pending => {} + } + counts +} + +fn reduce_round_from_item_key(item_key: &str) -> Result { + let (round, batch) = item_key + .strip_prefix('r') + .and_then(|value| value.split_once(":b")) + .ok_or_else(|| Error::InvalidRepositoryData { + message: format!("reduce task item key `{item_key}` is malformed"), + })?; + if batch.parse::().is_err() { + return Err(Error::InvalidRepositoryData { + message: format!("reduce task item key `{item_key}` has an invalid batch"), + }); + } + round + .parse::() + .ok() + .filter(|round| *round > 0) + .ok_or_else(|| Error::InvalidRepositoryData { + message: format!("reduce task item key `{item_key}` has an invalid round"), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn storage_wait_trigger_identity_replays_and_fences_generation_and_kind_offline() { + // Pins: replaying one materialization page addresses the same timer/expiry trigger, + // while a new logical generation or a different trigger contract cannot alias it. + let task_id = ExecutionTaskId::from_uuid(Uuid::from_u128(41)); + let timer = storage_wait_trigger_uid(task_id, 7, ExecutionTriggerKind::TaskTimer); + + assert_eq!( + timer, + Uuid::parse_str("42cf2d13-1c01-5eb4-baf5-193b682fb523") + .expect("pinned storage-wait trigger UUID") + ); + assert_eq!( + timer, + storage_wait_trigger_uid(task_id, 7, ExecutionTriggerKind::TaskTimer) + ); + assert_ne!( + timer, + storage_wait_trigger_uid(task_id, 8, ExecutionTriggerKind::TaskTimer) + ); + assert_ne!( + timer, + storage_wait_trigger_uid(task_id, 7, ExecutionTriggerKind::WaitExpiry) + ); + } + + #[test] + fn thousand_wait_reasons_keep_the_same_bounded_canonical_sample_offline() { + // Pins: high-fanout timer admission never grows the hot run-row blocker sample beyond + // 64 entries/64KiB, and insertion order cannot change the canonical retained task IDs. + let ordered = (1_u128..=1_000).collect::>(); + let mut reversed = ordered.clone(); + reversed.reverse(); + let wake_at = + DateTime::::from_timestamp(1_700_000_000, 0).expect("fixed timer wake timestamp"); + let forward = accumulate_timer_sample(&ordered, wake_at); + let reverse = accumulate_timer_sample(&reversed, wake_at); + assert_eq!(forward, reverse); + assert_eq!(forward.len(), MAX_WAITING_REASON_SAMPLES); + assert!( + serde_json::to_vec(&forward) + .expect("serialize bounded wait sample") + .len() + <= MAX_WAITING_REASON_SAMPLE_BYTES + ); + let ids = forward + .iter() + .filter_map(waiting_reason_task_id) + .map(ExecutionTaskId::as_uuid) + .collect::>(); + assert_eq!(ids, (1_u128..=64).map(Uuid::from_u128).collect::>()); + } + + fn accumulate_timer_sample(ids: &[u128], wake_at: DateTime) -> Vec { + let mut sample = Vec::new(); + for id in ids { + let reason = WaitingReason::Timer { + task_id: ExecutionTaskId::from_uuid(Uuid::from_u128(*id)), + wake: ExecutionTemporalTarget::At { at: wake_at }, + }; + sample = bounded_waiting_reason_sample(&sample, &reason, MAX_WAITING_REASON_SAMPLES) + .expect("bound waiting reason sample"); + } + sample + } +} diff --git a/crates/moa-execution/src/repository/replan_stop.rs b/crates/moa-execution/src/repository/replan_stop.rs new file mode 100644 index 000000000..10c965b82 --- /dev/null +++ b/crates/moa-execution/src/repository/replan_stop.rs @@ -0,0 +1,323 @@ +//! Durable bounded replan-stop intent handoff. + +use moa_core::types::identifiers::SessionId; + +use super::*; +use super::{ + capacity::{ExecutionCapacityDimension, prelock_capacity_dimensions_in_tx}, + rows::{required_u64, run_from_row}, + run::enqueue_run_activation_in_conn, + sql::LOAD_RUN_SQL, + terminal::ReplanStopReceipt, +}; +use crate::ReplanStopReason; + +const MAX_REPLAN_STOP_DETAIL_CHARS: usize = 512; + +/// Exact task- and revision-fenced intent requested by the public amendment boundary. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct NewExecutionReplanStopIntent { + /// Owning run. + pub run_uid: Uuid, + /// Parent session that owns the request. + pub session_id: SessionId, + /// Expected active plan revision. + pub base_plan_revision: u64, + /// WaitingReplan task that triggered the stop. + pub origin_task_id: ExecutionTaskId, + /// Exact logical task generation. + pub task_generation: u64, + /// Deterministic amendment hash associated with the stop. + pub amendment_hash: ExecutionHash, + /// Typed deterministic stop reason. + pub stop_reason: ReplanStopReason, + /// Optional bounded diagnostic detail. + pub detail: Option, +} + +/// Persisted intent consumed only by its exact claimed controller wake. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ExecutionReplanStopIntentRecord { + /// Owning tenant. + pub tenant_id: TenantId, + /// Owning run. + pub run_uid: Uuid, + /// Exact controller generation. + pub controller_generation: u64, + /// Exact controller wake. + pub wake_epoch: u64, + /// WaitingReplan origin. + pub origin_task_id: ExecutionTaskId, + /// Exact logical task generation. + pub task_generation: u64, + /// Plan revision that stopped. + pub base_plan_revision: u64, + /// Typed stop reason. + pub stop_reason: ReplanStopReason, + /// Bounded human-readable detail. + pub detail: String, + /// Exact amendment hash. + pub amendment_hash: ExecutionHash, +} + +impl ExecutionReplanStopIntentRecord { + /// Reconstructs the exact receipt eventually written with the terminal fence. + #[must_use] + pub const fn receipt(&self) -> ReplanStopReceipt { + ReplanStopReceipt { + task_id: self.origin_task_id, + task_generation: self.task_generation, + base_plan_revision: self.base_plan_revision, + amendment_hash: self.amendment_hash, + } + } +} + +/// Result of persisting one exact replan-stop intent and activation. +#[derive(Clone, Debug, PartialEq)] +pub enum ReplanStopIntentWriteOutcome { + /// New intent and exact activation committed atomically. + Applied(Box), + /// The identical intent was already persisted. + Replayed(Box), + /// Run/session does not exist under the supplied scope. + NotFound, + /// Revision, task, generation, or an existing intent differs. + Conflict, +} + +impl ExecutionRepository { + /// Persists one exact replan-stop command and queues its owning controller wake atomically. + pub async fn request_replan_stop( + &self, + scope: ExecutionScope, + config: &moa_config::ExecutionConfig, + request: NewExecutionReplanStopIntent, + ) -> Result { + let detail = request + .detail + .as_deref() + .filter(|detail| !detail.trim().is_empty()) + .unwrap_or(request.stop_reason.as_str()) + .chars() + .take(MAX_REPLAN_STOP_DETAIL_CHARS) + .collect::(); + let mut conn = scope.begin(&self.pool).await?; + let tenant_id = sqlx::query_scalar::<_, Uuid>( + "SELECT tenant_id FROM moa.execution_run WHERE run_uid=$1 AND session_id=$2", + ) + .bind(request.run_uid) + .bind(request.session_id.0) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(tenant_id) = tenant_id else { + conn.commit().await.map_err(storage_error)?; + return Ok(ReplanStopIntentWriteOutcome::NotFound); + }; + prelock_capacity_dimensions_in_tx( + conn.as_mut(), + config, + TenantId(tenant_id), + &[ + ExecutionCapacityDimension::ActiveRuns, + ExecutionCapacityDimension::ParkedRuns, + ], + ) + .await?; + let Some(run_row) = sqlx::query( + "SELECT * FROM moa.execution_run WHERE run_uid=$1 AND session_id=$2 FOR UPDATE", + ) + .bind(request.run_uid) + .bind(request.session_id.0) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + conn.commit().await.map_err(storage_error)?; + return Ok(ReplanStopIntentWriteOutcome::NotFound); + }; + let run = run_from_row(&run_row)?; + let existing = sqlx::query( + "SELECT tenant_id,run_uid,controller_generation,wake_epoch,origin_task_id, \ + task_generation,base_plan_revision,stop_reason,detail,amendment_hash \ + FROM moa.execution_replan_stop_intent WHERE run_uid=$1", + ) + .bind(run.run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if let Some(existing) = existing { + let existing = replan_stop_intent_from_row(&existing)?; + let exact = existing.origin_task_id == request.origin_task_id + && existing.task_generation == request.task_generation + && existing.base_plan_revision == request.base_plan_revision + && existing.amendment_hash == request.amendment_hash + && existing.stop_reason == request.stop_reason + && existing.detail == detail; + conn.commit().await.map_err(storage_error)?; + return Ok(if exact { + ReplanStopIntentWriteOutcome::Replayed(Box::new(run)) + } else { + ReplanStopIntentWriteOutcome::Conflict + }); + } + if run.plan_revision != request.base_plan_revision + || run.status != ExecutionRunStatus::WaitingReplan + || run.pending_terminal.is_some() + { + conn.commit().await.map_err(storage_error)?; + return Ok(ReplanStopIntentWriteOutcome::Conflict); + } + let task = sqlx::query( + "SELECT generation,status,current_outcome FROM moa.execution_task \ + WHERE run_uid=$1 AND task_id=$2 FOR UPDATE", + ) + .bind(run.run_uid) + .bind(request.origin_task_id.as_uuid()) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(task) = task else { + conn.commit().await.map_err(storage_error)?; + return Ok(ReplanStopIntentWriteOutcome::Conflict); + }; + let task_generation = required_u64(&task, "generation")?; + let task_status: String = task.try_get("status").map_err(row_error)?; + let current_outcome: Option = task + .try_get::, _>("current_outcome") + .map_err(row_error)? + .map(serde_json::from_value) + .transpose()?; + if task_generation != request.task_generation + || task_status != "waiting_replan" + || !matches!( + current_outcome.as_ref().map(|outcome| &outcome.result), + Some(ExecutionTaskResult::NeedsReplan { .. }) + ) + { + conn.commit().await.map_err(storage_error)?; + return Ok(ReplanStopIntentWriteOutcome::Conflict); + } + + let dispatch = enqueue_run_activation_in_conn( + conn.as_mut(), + run.tenant_id, + run.run_uid, + run.controller_generation, + Utc::now(), + json!({ + "reason": "replan_stop_completion", + "base_plan_revision": request.base_plan_revision, + "origin_task_id": request.origin_task_id, + }), + ) + .await?; + let wake_epoch = dispatch + .wake_epoch + .ok_or_else(|| Error::InvalidRepositoryData { + message: "replan-stop activation is missing its wake epoch".to_string(), + })?; + sqlx::query( + "INSERT INTO moa.execution_replan_stop_intent (tenant_id,run_uid, \ + controller_generation,wake_epoch,origin_task_id,task_generation, \ + base_plan_revision,stop_reason,detail,amendment_hash) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)", + ) + .bind(run.tenant_id.0) + .bind(run.run_uid) + .bind(to_i64( + run.controller_generation, + "replan-stop controller generation", + )?) + .bind(to_i64(wake_epoch, "replan-stop wake epoch")?) + .bind(request.origin_task_id.as_uuid()) + .bind(to_i64( + request.task_generation, + "replan-stop task generation", + )?) + .bind(to_i64( + request.base_plan_revision, + "replan-stop plan revision", + )?) + .bind(request.stop_reason.as_str()) + .bind(detail) + .bind(request.amendment_hash.to_string()) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let run = sqlx::query(LOAD_RUN_SQL) + .bind(run.run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error) + .and_then(|row| run_from_row(&row))?; + conn.commit().await.map_err(storage_error)?; + Ok(ReplanStopIntentWriteOutcome::Applied(Box::new(run))) + } + + /// Loads an intent only for the exact claimed controller generation and wake. + pub async fn load_replan_stop_intent( + &self, + scope: ExecutionScope, + run_uid: Uuid, + controller_generation: u64, + wake_epoch: u64, + ) -> Result> { + let mut conn = scope.begin(&self.pool).await?; + let row = sqlx::query( + "SELECT tenant_id,run_uid,controller_generation,wake_epoch,origin_task_id, \ + task_generation,base_plan_revision,stop_reason,detail,amendment_hash \ + FROM moa.execution_replan_stop_intent WHERE run_uid=$1 \ + AND controller_generation=$2 AND wake_epoch=$3", + ) + .bind(run_uid) + .bind(to_i64( + controller_generation, + "replan-stop controller generation", + )?) + .bind(to_i64(wake_epoch, "replan-stop wake epoch")?) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let intent = row.as_ref().map(replan_stop_intent_from_row).transpose()?; + conn.commit().await.map_err(storage_error)?; + Ok(intent) + } +} + +fn replan_stop_intent_from_row(row: &PgRow) -> Result { + Ok(ExecutionReplanStopIntentRecord { + tenant_id: TenantId(row.try_get("tenant_id").map_err(row_error)?), + run_uid: row.try_get("run_uid").map_err(row_error)?, + controller_generation: required_u64(row, "controller_generation")?, + wake_epoch: required_u64(row, "wake_epoch")?, + origin_task_id: ExecutionTaskId::from_uuid( + row.try_get("origin_task_id").map_err(row_error)?, + ), + task_generation: required_u64(row, "task_generation")?, + base_plan_revision: required_u64(row, "base_plan_revision")?, + stop_reason: parse_replan_stop_reason( + &row.try_get::("stop_reason").map_err(row_error)?, + )?, + detail: row.try_get("detail").map_err(row_error)?, + amendment_hash: row + .try_get::("amendment_hash") + .map_err(row_error)? + .parse()?, + }) +} + +fn parse_replan_stop_reason(value: &str) -> Result { + match value { + "duplicate_plan" => Ok(ReplanStopReason::DuplicatePlan), + "duplicate_amendment" => Ok(ReplanStopReason::DuplicateAmendment), + "repeated_failure" => Ok(ReplanStopReason::RepeatedFailure), + "no_progress" => Ok(ReplanStopReason::NoProgress), + "deadline_exceeded" => Ok(ReplanStopReason::DeadlineExceeded), + "budget_exhausted" => Ok(ReplanStopReason::BudgetExhausted), + other => Err(Error::InvalidRepositoryData { + message: format!("unknown replan-stop reason `{other}`"), + }), + } +} diff --git a/crates/moa-execution/src/repository/retention.rs b/crates/moa-execution/src/repository/retention.rs new file mode 100644 index 000000000..5a9bd2592 --- /dev/null +++ b/crates/moa-execution/src/repository/retention.rs @@ -0,0 +1,1375 @@ +//! Bounded, archive-first retention for terminal execution detail. + +use chrono::{DateTime, Utc}; +use moa_core::canonical_json::canonical_json_bytes; +use moa_core::types::identifiers::TenantId; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use sqlx::Row; +use uuid::Uuid; + +use super::{ + Error, ExecutionRepository, ExecutionScope, Result, row_error, sqlx_error, storage_error, + to_i64, to_u64, +}; + +const ARCHIVE_FORMAT_VERSION: i64 = 1; +const MAX_SEGMENT_BYTES: usize = 4 * 1024 * 1024; +const MAX_MAINTENANCE_ERROR_BYTES: usize = 4_096; +const RETENTION_CLAIM_TTL_SECONDS: i64 = 5 * 60; +const TERMINAL_ARCHIVE_NAMESPACE: Uuid = Uuid::from_u128(0x3bc3_2231_6df2_5dc0_8b93_47da_7e68_8229); + +/// One bounded terminal-detail retention disposition. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(tag = "outcome", rename_all = "snake_case")] +pub enum ExecutionRetentionPageOutcome { + /// No eligible terminal run remains at the captured boundary. + Idle, + /// One immutable archive segment was durably appended. + SegmentArchived { + /// Archived run. + run_uid: Uuid, + /// Stable source-table label. + segment_kind: String, + /// Rows captured in the segment. + records: u32, + }, + /// All segments were verified and the root receipt was bound to the run. + ArchiveFinalized { + /// Finalized run. + run_uid: Uuid, + /// Canonical archive root digest. + root_digest: String, + }, + /// One dependency-ordered live-detail page was deleted. + DetailDeleted { + /// Run whose archived detail was deleted. + run_uid: Uuid, + /// Source table deleted in this page. + segment_kind: String, + /// Rows deleted in this page. + records: u32, + }, + /// A finalized archive has no remaining live detail. + Complete { + /// Fully retained run. + run_uid: Uuid, + }, +} + +/// Durable self-schedule claim for the singleton retention service. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(tag = "outcome", rename_all = "snake_case")] +pub enum ExecutionRetentionClaimOutcome { + /// This invocation owns the returned generation. + Claimed { + /// Exact generation fence. + generation: u64, + /// Delay used by the preceding idle schedule, when one exists. + previous_delay_seconds: Option, + }, + /// A newer or not-yet-due invocation already owns the schedule. + NotDue { + /// Persisted next eligible time. + next_run_at: Option>, + /// Persisted generation carried by the accepted delayed invocation. + scheduled_generation: Option, + }, +} + +/// Persisted schedule receipt returned after a completed or failed pass. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct ExecutionRetentionScheduleReceipt { + /// Generation the delayed self-invocation must present. + pub scheduled_generation: u64, + /// Database time at which that invocation becomes eligible. + pub next_run_at: DateTime, +} + +/// Durable health and self-schedule receipt for terminal-detail retention. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ExecutionRetentionCheckpoint { + /// Monotonic claimed pass generation. + pub generation: u64, + /// Most recent claimed pass start. + pub last_started_at: Option>, + /// Most recent successfully completed page. + pub last_succeeded_at: Option>, + /// Most recent failed page. + pub last_failure_at: Option>, + /// Persisted eligibility time for the delayed self-call. + pub next_run_at: Option>, + /// Exact generation carried by the delayed self-call. + pub scheduled_generation: Option, + /// Bounded diagnostic for the most recent failure. + pub last_error: Option, + /// Database time of the latest checkpoint mutation. + pub updated_at: DateTime, +} + +#[derive(Clone, Copy)] +struct ArchiveSource { + kind: &'static str, + select_page_sql: &'static str, +} + +const ARCHIVE_SOURCES: &[ArchiveSource] = &[ + ArchiveSource { + kind: "execution_task_checkpoint", + select_page_sql: "SELECT to_jsonb(source) AS record, to_jsonb(checkpoint_uid::TEXT) AS cursor FROM moa.execution_task_checkpoint AS source WHERE tenant_id = $1 AND run_uid = $2 AND ($4::JSONB IS NULL OR checkpoint_uid > (($4 #>> '{}')::UUID)) ORDER BY checkpoint_uid LIMIT $3", + }, + ArchiveSource { + kind: "sandbox_execution_hand_release_receipts", + select_page_sql: "SELECT to_jsonb(source) AS record, to_jsonb(receipt_id::TEXT) AS cursor FROM moa.sandbox_execution_hand_release_receipts AS source WHERE tenant_id = $1 AND run_uid = $2 AND ($4::JSONB IS NULL OR receipt_id > (($4 #>> '{}')::UUID)) ORDER BY receipt_id LIMIT $3", + }, + ArchiveSource { + kind: "execution_external_job_callback_receipt", + select_page_sql: "SELECT to_jsonb(source) AS record, jsonb_build_array(external_job_uid::TEXT, job_generation, provider, provider_event_id) AS cursor FROM moa.execution_external_job_callback_receipt AS source WHERE tenant_id = $1 AND external_job_uid IN (SELECT external_job_uid FROM moa.execution_external_job WHERE tenant_id = $1 AND run_uid = $2) AND ($4::JSONB IS NULL OR (external_job_uid, job_generation, provider, provider_event_id) > (($4->>0)::UUID, ($4->>1)::BIGINT, $4->>2, $4->>3)) ORDER BY external_job_uid, job_generation, provider, provider_event_id LIMIT $3", + }, + ArchiveSource { + kind: "execution_trigger", + select_page_sql: "SELECT to_jsonb(source) AS record, to_jsonb(trigger_uid::TEXT) AS cursor FROM moa.execution_trigger AS source WHERE tenant_id = $1 AND run_uid = $2 AND ($4::JSONB IS NULL OR trigger_uid > (($4 #>> '{}')::UUID)) ORDER BY trigger_uid LIMIT $3", + }, + ArchiveSource { + kind: "execution_dispatch_outbox", + select_page_sql: "SELECT to_jsonb(source) AS record, to_jsonb(dispatch_uid::TEXT) AS cursor FROM moa.execution_dispatch_outbox AS source WHERE tenant_id = $1 AND run_uid = $2 AND ($4::JSONB IS NULL OR dispatch_uid > (($4 #>> '{}')::UUID)) ORDER BY dispatch_uid LIMIT $3", + }, + ArchiveSource { + kind: "execution_external_job", + select_page_sql: "SELECT to_jsonb(source) AS record, to_jsonb(external_job_uid::TEXT) AS cursor FROM moa.execution_external_job AS source WHERE tenant_id = $1 AND run_uid = $2 AND ($4::JSONB IS NULL OR external_job_uid > (($4 #>> '{}')::UUID)) ORDER BY external_job_uid LIMIT $3", + }, + ArchiveSource { + kind: "execution_action_review_outbox", + select_page_sql: "SELECT to_jsonb(source) AS record, to_jsonb(review_uid::TEXT) AS cursor FROM moa.execution_action_review_outbox AS source WHERE tenant_id = $1 AND run_uid = $2 AND ($4::JSONB IS NULL OR review_uid > (($4 #>> '{}')::UUID)) ORDER BY review_uid LIMIT $3", + }, + ArchiveSource { + kind: "execution_capacity_reservation", + select_page_sql: "SELECT to_jsonb(source) AS record, to_jsonb(reservation_uid::TEXT) AS cursor FROM moa.execution_capacity_reservation AS source WHERE tenant_id = $1 AND run_uid = $2 AND ($4::JSONB IS NULL OR reservation_uid > (($4 #>> '{}')::UUID)) ORDER BY reservation_uid LIMIT $3", + }, + ArchiveSource { + kind: "execution_compensation", + select_page_sql: "SELECT to_jsonb(source) AS record, to_jsonb(compensation_id::TEXT) AS cursor FROM moa.execution_compensation AS source WHERE tenant_id = $1 AND run_uid = $2 AND ($4::JSONB IS NULL OR compensation_id > (($4 #>> '{}')::UUID)) ORDER BY compensation_id LIMIT $3", + }, + ArchiveSource { + kind: "execution_task", + select_page_sql: "SELECT to_jsonb(source) AS record, to_jsonb(task_id::TEXT) AS cursor FROM moa.execution_task AS source WHERE tenant_id = $1 AND run_uid = $2 AND ($4::JSONB IS NULL OR task_id > (($4 #>> '{}')::UUID)) ORDER BY task_id LIMIT $3", + }, + ArchiveSource { + kind: "execution_node_state", + select_page_sql: "SELECT to_jsonb(source) AS record, to_jsonb(node_state_uid::TEXT) AS cursor FROM moa.execution_node_state AS source WHERE tenant_id = $1 AND run_uid = $2 AND ($4::JSONB IS NULL OR node_state_uid > (($4 #>> '{}')::UUID)) ORDER BY node_state_uid LIMIT $3", + }, + ArchiveSource { + kind: "execution_completion_scan", + select_page_sql: "SELECT to_jsonb(source) AS record, to_jsonb(run_uid::TEXT) AS cursor FROM moa.execution_completion_scan AS source WHERE tenant_id = $1 AND run_uid = $2 AND ($4::JSONB IS NULL OR run_uid > (($4 #>> '{}')::UUID)) ORDER BY run_uid LIMIT $3", + }, + ArchiveSource { + kind: "execution_replan_stop_intent", + select_page_sql: "SELECT to_jsonb(source) AS record, to_jsonb(run_uid::TEXT) AS cursor FROM moa.execution_replan_stop_intent AS source WHERE tenant_id = $1 AND run_uid = $2 AND ($4::JSONB IS NULL OR run_uid > (($4 #>> '{}')::UUID)) ORDER BY run_uid LIMIT $3", + }, + ArchiveSource { + kind: "execution_amendment_receipt", + select_page_sql: "SELECT to_jsonb(source) AS record, to_jsonb(base_plan_revision) AS cursor FROM moa.execution_amendment_receipt AS source WHERE tenant_id = $1 AND run_uid = $2 AND ($4::JSONB IS NULL OR base_plan_revision > (($4 #>> '{}')::BIGINT)) ORDER BY base_plan_revision LIMIT $3", + }, + ArchiveSource { + kind: "execution_node_materialization", + select_page_sql: "SELECT to_jsonb(source) AS record, jsonb_build_array(plan_revision, node_id) AS cursor FROM moa.execution_node_materialization AS source WHERE tenant_id = $1 AND run_uid = $2 AND ($4::JSONB IS NULL OR (plan_revision, node_id) > (($4->>0)::BIGINT, $4->>1)) ORDER BY plan_revision, node_id LIMIT $3", + }, + ArchiveSource { + kind: "execution_planner_call_audit", + select_page_sql: "SELECT to_jsonb(source) AS record, to_jsonb(audit_uid::TEXT) AS cursor FROM moa.execution_planner_call_audit AS source WHERE tenant_id = $1 AND run_uid = $2 AND ($4::JSONB IS NULL OR audit_uid > (($4 #>> '{}')::UUID)) ORDER BY audit_uid LIMIT $3", + }, + ArchiveSource { + kind: "execution_compile_audit", + select_page_sql: "SELECT to_jsonb(source) AS record, to_jsonb(audit_uid::TEXT) AS cursor FROM moa.execution_compile_audit AS source WHERE tenant_id = $1 AND run_uid = $2 AND ($4::JSONB IS NULL OR audit_uid > (($4 #>> '{}')::UUID)) ORDER BY audit_uid LIMIT $3", + }, + ArchiveSource { + kind: "execution_template_admission", + select_page_sql: "SELECT to_jsonb(source) AS record, to_jsonb(operation_uid::TEXT) AS cursor FROM moa.execution_template_admission AS source WHERE tenant_id = $1 AND execution_run_uid = $2 AND ($4::JSONB IS NULL OR operation_uid > (($4 #>> '{}')::UUID)) ORDER BY operation_uid LIMIT $3", + }, +]; + +#[derive(Serialize)] +struct ArchiveSegmentBody<'a> { + format_version: i64, + segment_kind: &'a str, + records: Vec<&'a Value>, +} + +struct RetentionCandidate { + tenant_id: TenantId, + run_uid: Uuid, + contact_id: Option, + status: String, + completed_at: DateTime, + goal_contract: Value, + initial_plan_hash: String, + active_plan_hash: String, + terminal_summary: Value, +} + +struct ArchiveManifest { + archive_uid: Uuid, + finalized_at: Option>, + details_deleted_at: Option>, + source_cursor: ArchiveCursor, + rolling_chain_digest: Option, + source_record_count: u64, + source_logical_bytes: u64, + segment_count: u64, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +struct ArchiveCursor { + kind: String, + #[serde(skip_serializing_if = "Option::is_none")] + after: Option, +} + +struct ArchivePageRow { + record: Value, + cursor: Value, +} + +impl ExecutionRepository { + /// Loads the durable terminal-detail retention health and schedule receipt. + pub async fn load_execution_retention_checkpoint( + &self, + scope: ExecutionScope, + ) -> Result> { + require_control_plane(scope)?; + let mut conn = scope.begin(&self.pool).await?; + let row = sqlx::query( + "SELECT generation, last_started_at, last_succeeded_at, last_failure_at, \ + next_run_at, scheduled_generation, last_error, updated_at \ + FROM moa.execution_maintenance_checkpoint \ + WHERE job_kind = 'execution_terminal_retention'", + ) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let checkpoint = row + .as_ref() + .map(execution_retention_checkpoint_from_row) + .transpose()?; + conn.commit().await.map_err(storage_error)?; + Ok(checkpoint) + } + + /// Claims the due singleton retention generation, or reports the existing schedule. + pub async fn claim_execution_retention( + &self, + scope: ExecutionScope, + expected_generation: Option, + ) -> Result { + require_control_plane(scope)?; + let expected_generation = expected_generation + .map(|generation| { + i64::try_from(generation).map_err(|_| Error::InvalidRepositoryInput { + message: "execution retention generation exceeds PostgreSQL BIGINT".to_string(), + }) + }) + .transpose()?; + let mut conn = scope.begin(&self.pool).await?; + let current = sqlx::query( + "SELECT generation, last_started_at, last_succeeded_at, next_run_at, scheduled_generation, now() AS observed_at FROM moa.execution_maintenance_checkpoint WHERE job_kind = 'execution_terminal_retention' FOR UPDATE", + ) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let outcome = if let Some(row) = current { + let next_run_at: Option> = + row.try_get("next_run_at").map_err(row_error)?; + let persisted_scheduled_generation: Option = + row.try_get("scheduled_generation").map_err(row_error)?; + let observed_at: DateTime = row.try_get("observed_at").map_err(row_error)?; + let last_started_at: Option> = + row.try_get("last_started_at").map_err(row_error)?; + let stale_claim = next_run_at.is_none() + && persisted_scheduled_generation.is_none() + && last_started_at.is_some_and(|started| { + observed_at.signed_duration_since(started).num_seconds() + >= RETENTION_CLAIM_TTL_SECONDS + }); + let due = next_run_at.is_some_and(|next| next <= observed_at) || stale_claim; + let generation_matches = expected_generation.is_none() + || expected_generation == persisted_scheduled_generation; + if !due || !generation_matches { + let scheduled_generation = persisted_scheduled_generation + .map(|value| to_u64(value, "scheduled execution retention generation")) + .transpose()?; + ExecutionRetentionClaimOutcome::NotDue { + next_run_at, + scheduled_generation, + } + } else { + let previous_delay_seconds = row + .try_get::>, _>("last_succeeded_at") + .map_err(row_error)? + .zip(next_run_at) + .and_then(|(succeeded, next)| { + next.signed_duration_since(succeeded).to_std().ok() + }) + .map(|delay| delay.as_secs()); + let generation: i64 = sqlx::query_scalar( + r#" + UPDATE moa.execution_maintenance_checkpoint + SET generation = generation + 1, last_started_at = now(), + next_run_at = NULL, scheduled_generation = NULL, updated_at = now() + WHERE job_kind = 'execution_terminal_retention' + RETURNING generation + "#, + ) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let generation = to_u64(generation, "execution retention generation")?; + ExecutionRetentionClaimOutcome::Claimed { + generation, + previous_delay_seconds, + } + } + } else { + let generation: i64 = sqlx::query_scalar( + r#" + INSERT INTO moa.execution_maintenance_checkpoint ( + job_kind, generation, last_started_at, updated_at + ) VALUES ('execution_terminal_retention', 1, now(), now()) + RETURNING generation + "#, + ) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + ExecutionRetentionClaimOutcome::Claimed { + generation: to_u64(generation, "execution retention generation")?, + previous_delay_seconds: None, + } + }; + conn.commit().await.map_err(storage_error)?; + Ok(outcome) + } + + /// Advances at most one archive or dependency-ordered deletion page. + pub async fn advance_execution_retention_page( + &self, + scope: ExecutionScope, + retention_days: u64, + page_size: u32, + ) -> Result { + require_control_plane(scope)?; + if retention_days == 0 || page_size == 0 || page_size > 1_000 { + return Err(Error::InvalidRepositoryInput { + message: "execution retention requires positive days and a page size of 1..=1000" + .to_string(), + }); + } + let retention_days = + i64::try_from(retention_days).map_err(|_| Error::InvalidRepositoryInput { + message: "execution retention days exceed PostgreSQL BIGINT".to_string(), + })?; + let mut conn = scope.begin(&self.pool).await?; + let Some(candidate) = load_retention_candidate(conn.as_mut(), retention_days).await? else { + conn.commit().await.map_err(storage_error)?; + return Ok(ExecutionRetentionPageOutcome::Idle); + }; + + lock_and_recheck_retention_candidate(conn.as_mut(), &candidate, retention_days).await?; + let mut manifest = ensure_archive_manifest(conn.as_mut(), &candidate).await?; + let outcome = if manifest.finalized_at.is_none() { + advance_archive(conn.as_mut(), &candidate, &mut manifest, page_size).await? + } else if manifest.details_deleted_at.is_none() { + advance_deletion(conn.as_mut(), &candidate, &manifest, page_size).await? + } else { + ExecutionRetentionPageOutcome::Complete { + run_uid: candidate.run_uid, + } + }; + conn.commit().await.map_err(storage_error)?; + Ok(outcome) + } + + /// Persists the next delayed invocation only for the exact claimed generation. + pub async fn schedule_execution_retention( + &self, + scope: ExecutionScope, + generation: u64, + delay_seconds: u64, + failure: Option<&str>, + ) -> Result { + require_control_plane(scope)?; + if generation == 0 || delay_seconds == 0 { + return Err(Error::InvalidRepositoryInput { + message: "execution retention scheduling requires positive generation and delay" + .to_string(), + }); + } + let generation = i64::try_from(generation).map_err(|_| Error::InvalidRepositoryInput { + message: "execution retention generation exceeds PostgreSQL BIGINT".to_string(), + })?; + let delay_seconds = + i64::try_from(delay_seconds).map_err(|_| Error::InvalidRepositoryInput { + message: "execution retention delay exceeds PostgreSQL BIGINT".to_string(), + })?; + let failure = failure + .map(str::trim) + .filter(|message| !message.is_empty()) + .map(bounded_maintenance_error); + let mut conn = scope.begin(&self.pool).await?; + let row = sqlx::query( + r#" + UPDATE moa.execution_maintenance_checkpoint + SET last_succeeded_at = CASE WHEN $3::TEXT IS NULL THEN now() ELSE last_succeeded_at END, + last_failure_at = CASE WHEN $3::TEXT IS NULL THEN last_failure_at ELSE now() END, + last_error = CASE WHEN $3::TEXT IS NULL THEN last_error ELSE $3 END, + next_run_at = now() + make_interval(secs => $2), + scheduled_generation = generation + 1, + updated_at = now() + WHERE job_kind = 'execution_terminal_retention' AND generation = $1 + AND next_run_at IS NULL AND scheduled_generation IS NULL + RETURNING next_run_at, scheduled_generation + "#, + ) + .bind(generation) + .bind(delay_seconds) + .bind(failure) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let row = match row { + Some(row) => row, + None => { + let existing = sqlx::query( + "SELECT generation, next_run_at, scheduled_generation FROM moa.execution_maintenance_checkpoint WHERE job_kind = 'execution_terminal_retention'", + ) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(existing) = existing else { + return Err(Error::InvalidRepositoryInput { + message: "execution retention generation was superseded before scheduling" + .to_string(), + }); + }; + let persisted_generation: i64 = + existing.try_get("generation").map_err(row_error)?; + let scheduled_generation: Option = existing + .try_get("scheduled_generation") + .map_err(row_error)?; + if persisted_generation != generation + || scheduled_generation != generation.checked_add(1) + { + return Err(Error::InvalidRepositoryInput { + message: "execution retention generation was superseded before scheduling" + .to_string(), + }); + } + existing + } + }; + let receipt = ExecutionRetentionScheduleReceipt { + scheduled_generation: to_u64( + row.try_get::("scheduled_generation") + .map_err(row_error)?, + "scheduled execution retention generation", + )?, + next_run_at: row.try_get("next_run_at").map_err(row_error)?, + }; + conn.commit().await.map_err(storage_error)?; + Ok(receipt) + } +} + +async fn load_retention_candidate( + conn: &mut sqlx::PgConnection, + retention_days: i64, +) -> Result> { + let row = sqlx::query( + r#" + SELECT run.tenant_id, run.run_uid, run.contact_id, run.status, + run.completed_at, run.goal_contract, run.initial_plan_hash, + run.active_plan_hash, + jsonb_build_object( + 'schema_version', 1, + 'terminal_cause', run.terminal_cause, + 'terminal_reason', run.terminal_reason, + 'satisfied_requirement_count', run.terminal_satisfied_requirement_count, + 'requirement_count', run.terminal_requirement_count + ) AS terminal_summary + FROM moa.execution_run AS run + LEFT JOIN moa.execution_terminal_archive AS archive + ON archive.tenant_id = run.tenant_id AND archive.run_uid = run.run_uid + WHERE run.status IN ('completed', 'partial', 'blocked', 'unsupported', 'failed', 'cancelled') + AND run.completed_at <= now() - make_interval(days => $1) + AND (archive.archive_uid IS NULL OR archive.details_deleted_at IS NULL) + AND NOT EXISTS ( + SELECT 1 FROM moa.execution_task AS task + WHERE task.tenant_id = run.tenant_id AND task.run_uid = run.run_uid + AND task.status = 'unknown_outcome' + ) + AND NOT EXISTS ( + SELECT 1 FROM moa.execution_compensation AS compensation + WHERE compensation.tenant_id = run.tenant_id + AND compensation.run_uid = run.run_uid + AND compensation.status = 'unknown_outcome' + ) + AND NOT EXISTS ( + SELECT 1 FROM moa.execution_external_job AS job + WHERE job.tenant_id = run.tenant_id AND job.run_uid = run.run_uid + AND job.state = 'unknown_outcome' + ) + AND NOT EXISTS ( + SELECT 1 FROM moa.legal_hold AS hold + WHERE hold.tenant_id = run.tenant_id AND hold.released_at IS NULL + AND (hold.subject_id IS NULL OR hold.subject_id = run.contact_id) + ) + AND NOT EXISTS ( + SELECT 1 FROM moa.destruction_operation_fence AS fence + WHERE fence.tenant_id = run.tenant_id + AND (fence.subject_id IS NULL OR fence.subject_id = run.contact_id) + ) + ORDER BY run.completed_at, run.tenant_id, run.run_uid + FOR UPDATE OF run SKIP LOCKED + LIMIT 1 + "#, + ) + .bind(retention_days) + .fetch_optional(&mut *conn) + .await + .map_err(sqlx_error)?; + row.map(candidate_from_row).transpose() +} + +async fn lock_and_recheck_retention_candidate( + conn: &mut sqlx::PgConnection, + candidate: &RetentionCandidate, + retention_days: i64, +) -> Result<()> { + sqlx::query( + "SELECT pg_advisory_xact_lock_shared(hashtextextended('moa:destruction:tenant:' || $1::text, 0))", + ) + .bind(candidate.tenant_id.0) + .execute(&mut *conn) + .await + .map_err(sqlx_error)?; + let eligible: bool = sqlx::query_scalar( + r#" + SELECT EXISTS ( + SELECT 1 FROM moa.execution_run AS run + WHERE run.tenant_id = $1 AND run.run_uid = $2 + AND run.status IN ('completed', 'partial', 'blocked', 'unsupported', 'failed', 'cancelled') + AND run.completed_at <= now() - make_interval(days => $3) + AND NOT EXISTS ( + SELECT 1 FROM moa.legal_hold AS hold + WHERE hold.tenant_id = run.tenant_id AND hold.released_at IS NULL + AND (hold.subject_id IS NULL OR hold.subject_id = run.contact_id) + ) + AND NOT EXISTS ( + SELECT 1 FROM moa.destruction_operation_fence AS fence + WHERE fence.tenant_id = run.tenant_id + AND (fence.subject_id IS NULL OR fence.subject_id = run.contact_id) + ) + AND NOT EXISTS ( + SELECT 1 FROM moa.execution_task AS task + WHERE task.tenant_id = run.tenant_id AND task.run_uid = run.run_uid + AND task.status = 'unknown_outcome' + ) + AND NOT EXISTS ( + SELECT 1 FROM moa.execution_compensation AS compensation + WHERE compensation.tenant_id = run.tenant_id + AND compensation.run_uid = run.run_uid + AND compensation.status = 'unknown_outcome' + ) + AND NOT EXISTS ( + SELECT 1 FROM moa.execution_external_job AS job + WHERE job.tenant_id = run.tenant_id AND job.run_uid = run.run_uid + AND job.state = 'unknown_outcome' + ) + ) + "#, + ) + .bind(candidate.tenant_id.0) + .bind(candidate.run_uid) + .bind(retention_days) + .fetch_one(&mut *conn) + .await + .map_err(sqlx_error)?; + if !eligible { + return Err(Error::InvalidRepositoryInput { + message: "execution retention candidate lost its terminal/legal-hold fence".to_string(), + }); + } + Ok(()) +} + +async fn ensure_archive_manifest( + conn: &mut sqlx::PgConnection, + candidate: &RetentionCandidate, +) -> Result { + let archive_uid = Uuid::new_v5(&TERMINAL_ARCHIVE_NAMESPACE, candidate.run_uid.as_bytes()); + let goal_hash = + canonical_value_hash("moa.execution-retention.goal.v1", &candidate.goal_contract)?; + let row = sqlx::query( + r#" + INSERT INTO moa.execution_terminal_archive ( + archive_uid, tenant_id, run_uid, contact_id, format_version, + terminal_status, terminal_completed_at, goal_hash, + initial_plan_hash, active_plan_hash, source_cursor + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) + ON CONFLICT (tenant_id, run_uid) DO NOTHING + RETURNING archive_uid, finalized_at, details_deleted_at, source_cursor, + rolling_chain_digest, source_record_count, source_logical_bytes, + segment_count + "#, + ) + .bind(archive_uid) + .bind(candidate.tenant_id.0) + .bind(candidate.run_uid) + .bind(candidate.contact_id) + .bind(ARCHIVE_FORMAT_VERSION) + .bind(&candidate.status) + .bind(candidate.completed_at) + .bind(goal_hash) + .bind(&candidate.initial_plan_hash) + .bind(&candidate.active_plan_hash) + .bind(serde_json::to_value(ArchiveCursor { + kind: "terminal_summary".to_string(), + after: None, + })?) + .fetch_optional(&mut *conn) + .await + .map_err(sqlx_error)?; + let row = match row { + Some(row) => row, + None => sqlx::query( + "SELECT archive_uid, finalized_at, details_deleted_at, source_cursor, \ + rolling_chain_digest, source_record_count, source_logical_bytes, \ + segment_count \ + FROM moa.execution_terminal_archive \ + WHERE tenant_id = $1 AND run_uid = $2 FOR UPDATE", + ) + .bind(candidate.tenant_id.0) + .bind(candidate.run_uid) + .fetch_one(&mut *conn) + .await + .map_err(sqlx_error)?, + }; + let mut source_cursor: Value = row.try_get("source_cursor").map_err(row_error)?; + if source_cursor + .as_object() + .is_some_and(serde_json::Map::is_empty) + { + source_cursor = serde_json::to_value(ArchiveCursor { + kind: "terminal_summary".to_string(), + after: None, + })?; + sqlx::query( + "UPDATE moa.execution_terminal_archive SET source_cursor = $2 \ + WHERE archive_uid = $1 AND finalized_at IS NULL AND source_cursor = '{}'::JSONB", + ) + .bind(row.try_get::("archive_uid").map_err(row_error)?) + .bind(&source_cursor) + .execute(&mut *conn) + .await + .map_err(sqlx_error)?; + } + let source_cursor = + serde_json::from_value(source_cursor).map_err(|error| Error::InvalidRepositoryData { + message: format!("decode execution archive source cursor: {error}"), + })?; + Ok(ArchiveManifest { + archive_uid: row.try_get("archive_uid").map_err(row_error)?, + finalized_at: row.try_get("finalized_at").map_err(row_error)?, + details_deleted_at: row.try_get("details_deleted_at").map_err(row_error)?, + source_cursor, + rolling_chain_digest: row.try_get("rolling_chain_digest").map_err(row_error)?, + source_record_count: to_u64( + row.try_get("source_record_count").map_err(row_error)?, + "execution archive source record count", + )?, + source_logical_bytes: to_u64( + row.try_get("source_logical_bytes").map_err(row_error)?, + "execution archive source logical bytes", + )?, + segment_count: to_u64( + row.try_get("segment_count").map_err(row_error)?, + "execution archive segment count", + )?, + }) +} + +async fn advance_archive( + conn: &mut sqlx::PgConnection, + candidate: &RetentionCandidate, + manifest: &mut ArchiveManifest, + page_size: u32, +) -> Result { + if manifest.source_cursor.kind == "terminal_summary" { + return insert_archive_segment( + conn, + candidate, + manifest, + "terminal_summary", + &[ArchivePageRow { + record: candidate.terminal_summary.clone(), + cursor: Value::Null, + }], + Some(ARCHIVE_SOURCES[0].kind), + ) + .await; + } + if manifest.source_cursor.kind == "complete" { + return finalize_archive(conn, candidate, manifest).await; + } + let start = ARCHIVE_SOURCES + .iter() + .position(|source| source.kind == manifest.source_cursor.kind) + .ok_or_else(|| Error::InvalidRepositoryData { + message: format!( + "unknown execution archive source cursor `{}`", + manifest.source_cursor.kind + ), + })?; + for (index, source) in ARCHIVE_SOURCES.iter().copied().enumerate().skip(start) { + let after = if source.kind == manifest.source_cursor.kind { + manifest.source_cursor.after.clone() + } else { + None + }; + let rows = sqlx::query(source.select_page_sql) + .bind(candidate.tenant_id.0) + .bind(candidate.run_uid) + .bind(i64::from(page_size)) + .bind(after) + .fetch_all(&mut *conn) + .await + .map_err(sqlx_error)?; + if rows.is_empty() { + let next_kind = ARCHIVE_SOURCES + .get(index + 1) + .map_or("complete", |next| next.kind); + advance_archive_cursor( + conn, + manifest, + ArchiveCursor { + kind: next_kind.to_string(), + after: None, + }, + ) + .await?; + continue; + } + let records = rows + .into_iter() + .map(|row| { + Ok(ArchivePageRow { + record: row.try_get("record").map_err(row_error)?, + cursor: row.try_get("cursor").map_err(row_error)?, + }) + }) + .collect::>>()?; + return insert_archive_segment(conn, candidate, manifest, source.kind, &records, None) + .await; + } + finalize_archive(conn, candidate, manifest).await +} + +async fn insert_archive_segment( + conn: &mut sqlx::PgConnection, + candidate: &RetentionCandidate, + manifest: &mut ArchiveManifest, + kind: &str, + rows: &[ArchivePageRow], + next_kind: Option<&str>, +) -> Result { + let mut accepted = rows.len(); + let bytes = loop { + let records = rows[..accepted] + .iter() + .map(|row| &row.record) + .collect::>(); + let encoded = serde_json::to_vec(&ArchiveSegmentBody { + format_version: ARCHIVE_FORMAT_VERSION, + segment_kind: kind, + records, + }) + .map_err(|error| Error::InvalidRepositoryData { + message: format!("encode execution terminal archive segment: {error}"), + })?; + if encoded.len() <= MAX_SEGMENT_BYTES { + break encoded; + } + if accepted <= 1 { + return Err(Error::InvalidRepositoryData { + message: format!( + "one {kind} archive record exceeds the {MAX_SEGMENT_BYTES}-byte segment bound" + ), + }); + } + accepted = accepted.div_ceil(2); + }; + let digest = blake3::hash(&bytes); + let sequence = + manifest + .segment_count + .checked_add(1) + .ok_or_else(|| Error::InvalidRepositoryData { + message: "execution archive segment sequence overflow".to_string(), + })?; + let sequence = i64::try_from(sequence).map_err(|_| Error::InvalidRepositoryData { + message: "execution archive segment sequence exceeds PostgreSQL BIGINT".to_string(), + })?; + let record_count = i64::try_from(accepted).map_err(|_| Error::InvalidRepositoryData { + message: "execution archive segment record count exceeds PostgreSQL BIGINT".to_string(), + })?; + let stored = sqlx::query( + r#" + INSERT INTO moa.execution_terminal_archive_segment ( + archive_uid, tenant_id, run_uid, segment_kind, segment_sequence, + format_version, record_count, payload, content_digest + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + RETURNING payload, content_digest + "#, + ) + .bind(manifest.archive_uid) + .bind(candidate.tenant_id.0) + .bind(candidate.run_uid) + .bind(kind) + .bind(sequence) + .bind(ARCHIVE_FORMAT_VERSION) + .bind(record_count) + .bind(&bytes) + .bind(digest.as_bytes().as_slice()) + .fetch_one(&mut *conn) + .await + .map_err(sqlx_error)?; + let stored_bytes: Vec = stored.try_get("payload").map_err(row_error)?; + let stored_digest: Vec = stored.try_get("content_digest").map_err(row_error)?; + if blake3::hash(&stored_bytes).as_bytes().as_slice() != stored_digest.as_slice() + || stored_digest.as_slice() != digest.as_bytes().as_slice() + { + return Err(Error::InvalidRepositoryData { + message: "execution terminal archive segment failed digest read-back".to_string(), + }); + } + let source_cursor = ArchiveCursor { + kind: next_kind.unwrap_or(kind).to_string(), + after: next_kind + .is_none() + .then(|| rows[accepted - 1].cursor.clone()), + }; + let new_record_count = manifest + .source_record_count + .checked_add( + u64::try_from(accepted).map_err(|_| Error::InvalidRepositoryData { + message: "execution archive page count exceeds u64".to_string(), + })?, + ) + .ok_or_else(|| Error::InvalidRepositoryData { + message: "execution archive source record count overflow".to_string(), + })?; + let new_logical_bytes = manifest + .source_logical_bytes + .checked_add( + u64::try_from(bytes.len()).map_err(|_| Error::InvalidRepositoryData { + message: "execution archive payload length exceeds u64".to_string(), + })?, + ) + .ok_or_else(|| Error::InvalidRepositoryData { + message: "execution archive source logical byte count overflow".to_string(), + })?; + let new_segment_count = + manifest + .segment_count + .checked_add(1) + .ok_or_else(|| Error::InvalidRepositoryData { + message: "execution archive segment count overflow".to_string(), + })?; + let rolling_chain_digest = rolling_chain_digest( + manifest.rolling_chain_digest.as_deref(), + kind, + sequence, + record_count, + i64::try_from(bytes.len()).map_err(|_| Error::InvalidRepositoryData { + message: "execution archive payload length exceeds PostgreSQL BIGINT".to_string(), + })?, + digest.as_bytes(), + ); + let old_cursor = serde_json::to_value(&manifest.source_cursor)?; + let new_cursor = serde_json::to_value(&source_cursor)?; + let advanced = sqlx::query( + "UPDATE moa.execution_terminal_archive \ + SET source_record_count = $2, source_logical_bytes = $3, segment_count = $4, \ + source_cursor = $5, rolling_chain_digest = $6 \ + WHERE archive_uid = $1 AND finalized_at IS NULL \ + AND source_record_count = $7 AND source_logical_bytes = $8 \ + AND segment_count = $9 AND source_cursor = $10 \ + AND rolling_chain_digest IS NOT DISTINCT FROM $11 \ + RETURNING archive_uid", + ) + .bind(manifest.archive_uid) + .bind(to_i64( + new_record_count, + "execution archive source record count", + )?) + .bind(to_i64( + new_logical_bytes, + "execution archive source logical bytes", + )?) + .bind(to_i64( + new_segment_count, + "execution archive segment count", + )?) + .bind(&new_cursor) + .bind(&rolling_chain_digest) + .bind(to_i64( + manifest.source_record_count, + "prior execution archive source record count", + )?) + .bind(to_i64( + manifest.source_logical_bytes, + "prior execution archive source logical bytes", + )?) + .bind(to_i64( + manifest.segment_count, + "prior execution archive segment count", + )?) + .bind(&old_cursor) + .bind(&manifest.rolling_chain_digest) + .fetch_optional(&mut *conn) + .await + .map_err(sqlx_error)?; + if advanced.is_none() { + return Err(Error::InvalidRepositoryData { + message: "execution archive accumulator lost its exact progress fence".to_string(), + }); + } + manifest.source_record_count = new_record_count; + manifest.source_logical_bytes = new_logical_bytes; + manifest.segment_count = new_segment_count; + manifest.source_cursor = source_cursor; + manifest.rolling_chain_digest = Some(rolling_chain_digest); + Ok(ExecutionRetentionPageOutcome::SegmentArchived { + run_uid: candidate.run_uid, + segment_kind: kind.to_string(), + records: u32::try_from(accepted).map_err(|_| Error::InvalidRepositoryData { + message: "execution archive page count exceeds u32".to_string(), + })?, + }) +} + +async fn finalize_archive( + conn: &mut sqlx::PgConnection, + candidate: &RetentionCandidate, + manifest: &ArchiveManifest, +) -> Result { + if manifest.source_cursor.kind != "complete" || manifest.segment_count == 0 { + return Err(Error::InvalidRepositoryData { + message: "cannot finalize an incomplete execution terminal archive".to_string(), + }); + } + let digest = + manifest + .rolling_chain_digest + .clone() + .ok_or_else(|| Error::InvalidRepositoryData { + message: "execution archive has segments without a rolling chain digest" + .to_string(), + })?; + let source_cursor = serde_json::to_value(&manifest.source_cursor)?; + let finalized = sqlx::query( + r#" + UPDATE moa.execution_terminal_archive + SET root_digest = rolling_chain_digest, finalized_at = now() + WHERE archive_uid = $1 AND finalized_at IS NULL + AND source_record_count = $2 AND source_logical_bytes = $3 + AND segment_count = $4 AND source_cursor = $5 + AND rolling_chain_digest = $6 + RETURNING root_digest + "#, + ) + .bind(manifest.archive_uid) + .bind(to_i64( + manifest.source_record_count, + "execution archive source record count", + )?) + .bind(to_i64( + manifest.source_logical_bytes, + "execution archive source logical bytes", + )?) + .bind(to_i64( + manifest.segment_count, + "execution archive segment count", + )?) + .bind(source_cursor) + .bind(&digest) + .fetch_one(&mut *conn) + .await + .map_err(sqlx_error)?; + let stored_digest: String = finalized.try_get("root_digest").map_err(row_error)?; + if stored_digest != digest { + return Err(Error::InvalidRepositoryData { + message: "execution terminal archive root digest failed read-back".to_string(), + }); + } + let bound = sqlx::query( + r#" + UPDATE moa.execution_run + SET terminal_archive_uid = $3, terminal_archive_hash = $4, + terminal_details_archived_at = now(), updated_at = now() + WHERE tenant_id = $1 AND run_uid = $2 AND terminal_archive_uid IS NULL + RETURNING run_uid + "#, + ) + .bind(candidate.tenant_id.0) + .bind(candidate.run_uid) + .bind(manifest.archive_uid) + .bind(&digest) + .fetch_optional(&mut *conn) + .await + .map_err(sqlx_error)?; + if bound.is_none() { + return Err(Error::InvalidRepositoryData { + message: "execution archive finalized without binding its run receipt".to_string(), + }); + } + Ok(ExecutionRetentionPageOutcome::ArchiveFinalized { + run_uid: candidate.run_uid, + root_digest: digest, + }) +} + +async fn advance_archive_cursor( + conn: &mut sqlx::PgConnection, + manifest: &mut ArchiveManifest, + next: ArchiveCursor, +) -> Result<()> { + let previous = serde_json::to_value(&manifest.source_cursor)?; + let next_value = serde_json::to_value(&next)?; + let updated = sqlx::query( + "UPDATE moa.execution_terminal_archive SET source_cursor = $2 \ + WHERE archive_uid = $1 AND finalized_at IS NULL AND source_cursor = $3 \ + AND source_record_count = $4 AND source_logical_bytes = $5 \ + AND segment_count = $6 AND rolling_chain_digest IS NOT DISTINCT FROM $7 \ + RETURNING archive_uid", + ) + .bind(manifest.archive_uid) + .bind(&next_value) + .bind(&previous) + .bind(to_i64( + manifest.source_record_count, + "execution archive source record count", + )?) + .bind(to_i64( + manifest.source_logical_bytes, + "execution archive source logical bytes", + )?) + .bind(to_i64( + manifest.segment_count, + "execution archive segment count", + )?) + .bind(&manifest.rolling_chain_digest) + .fetch_optional(&mut *conn) + .await + .map_err(sqlx_error)?; + if updated.is_none() { + return Err(Error::InvalidRepositoryData { + message: "execution archive cursor lost its exact progress fence".to_string(), + }); + } + manifest.source_cursor = next; + Ok(()) +} + +fn rolling_chain_digest( + previous: Option<&str>, + kind: &str, + sequence: i64, + record_count: i64, + logical_bytes: i64, + content_digest: &[u8; 32], +) -> String { + let mut chain = blake3::Hasher::new(); + chain.update(b"moa.execution-terminal-archive.chain.v1\0"); + match previous { + Some(previous) => chain.update(previous.as_bytes()), + None => chain.update(b"genesis"), + }; + chain.update(&(kind.len() as u64).to_be_bytes()); + chain.update(kind.as_bytes()); + chain.update(&sequence.to_be_bytes()); + chain.update(&record_count.to_be_bytes()); + chain.update(&logical_bytes.to_be_bytes()); + chain.update(content_digest); + chain.finalize().to_hex().to_string() +} + +async fn advance_deletion( + conn: &mut sqlx::PgConnection, + candidate: &RetentionCandidate, + manifest: &ArchiveManifest, + page_size: u32, +) -> Result { + let stages = [ + ( + "execution_task_checkpoint", + "DELETE FROM moa.execution_task_checkpoint WHERE checkpoint_uid IN (SELECT checkpoint_uid FROM moa.execution_task_checkpoint WHERE tenant_id = $1 AND run_uid = $2 ORDER BY checkpoint_uid LIMIT $3)", + ), + ( + "sandbox_execution_hand_release_receipts", + "DELETE FROM moa.sandbox_execution_hand_release_receipts WHERE receipt_id IN (SELECT receipt_id FROM moa.sandbox_execution_hand_release_receipts WHERE tenant_id = $1 AND run_uid = $2 ORDER BY receipt_id LIMIT $3)", + ), + ( + "execution_external_job_callback_receipt", + "DELETE FROM moa.execution_external_job_callback_receipt WHERE ctid IN (SELECT receipt.ctid FROM moa.execution_external_job_callback_receipt AS receipt JOIN moa.execution_external_job AS job ON job.tenant_id = receipt.tenant_id AND job.external_job_uid = receipt.external_job_uid WHERE job.tenant_id = $1 AND job.run_uid = $2 ORDER BY receipt.external_job_uid, receipt.job_generation, receipt.provider, receipt.provider_event_id LIMIT $3)", + ), + ( + "execution_trigger", + "DELETE FROM moa.execution_trigger WHERE trigger_uid IN (SELECT trigger_uid FROM moa.execution_trigger WHERE tenant_id = $1 AND run_uid = $2 ORDER BY trigger_uid LIMIT $3)", + ), + ( + "execution_dispatch_outbox", + "DELETE FROM moa.execution_dispatch_outbox WHERE dispatch_uid IN (SELECT dispatch_uid FROM moa.execution_dispatch_outbox WHERE tenant_id = $1 AND run_uid = $2 ORDER BY dispatch_uid LIMIT $3)", + ), + ( + "execution_external_job", + "DELETE FROM moa.execution_external_job WHERE external_job_uid IN (SELECT external_job_uid FROM moa.execution_external_job WHERE tenant_id = $1 AND run_uid = $2 ORDER BY external_job_uid LIMIT $3)", + ), + ( + "execution_action_review_outbox", + "DELETE FROM moa.execution_action_review_outbox WHERE review_uid IN (SELECT review_uid FROM moa.execution_action_review_outbox WHERE tenant_id = $1 AND run_uid = $2 ORDER BY review_uid LIMIT $3)", + ), + ( + "execution_capacity_reservation", + "DELETE FROM moa.execution_capacity_reservation WHERE reservation_uid IN (SELECT reservation_uid FROM moa.execution_capacity_reservation WHERE tenant_id = $1 AND run_uid = $2 ORDER BY reservation_uid LIMIT $3)", + ), + ( + "execution_node_materialization", + "DELETE FROM moa.execution_node_materialization WHERE ctid IN (SELECT ctid FROM moa.execution_node_materialization WHERE tenant_id = $1 AND run_uid = $2 ORDER BY plan_revision, node_id LIMIT $3)", + ), + ( + "execution_node_state", + "DELETE FROM moa.execution_node_state WHERE node_state_uid IN (SELECT node_state_uid FROM moa.execution_node_state WHERE tenant_id = $1 AND run_uid = $2 ORDER BY node_state_uid LIMIT $3)", + ), + ( + "execution_completion_scan", + "DELETE FROM moa.execution_completion_scan WHERE tenant_id = $1 AND run_uid IN (SELECT run_uid FROM moa.execution_completion_scan WHERE tenant_id = $1 AND run_uid = $2 ORDER BY run_uid LIMIT $3)", + ), + ( + "execution_replan_stop_intent", + "DELETE FROM moa.execution_replan_stop_intent WHERE tenant_id = $1 AND run_uid IN (SELECT run_uid FROM moa.execution_replan_stop_intent WHERE tenant_id = $1 AND run_uid = $2 ORDER BY run_uid LIMIT $3)", + ), + ( + "execution_amendment_receipt", + "DELETE FROM moa.execution_amendment_receipt WHERE ctid IN (SELECT ctid FROM moa.execution_amendment_receipt WHERE tenant_id = $1 AND run_uid = $2 ORDER BY base_plan_revision LIMIT $3)", + ), + ( + "execution_compensation", + "DELETE FROM moa.execution_compensation WHERE compensation_id IN (SELECT compensation_id FROM moa.execution_compensation WHERE tenant_id = $1 AND run_uid = $2 ORDER BY compensation_id LIMIT $3)", + ), + ( + "execution_task", + "DELETE FROM moa.execution_task WHERE task_id IN (SELECT task_id FROM moa.execution_task WHERE tenant_id = $1 AND run_uid = $2 ORDER BY task_id LIMIT $3)", + ), + ( + "execution_planner_call_audit", + "DELETE FROM moa.execution_planner_call_audit WHERE audit_uid IN (SELECT audit_uid FROM moa.execution_planner_call_audit WHERE tenant_id = $1 AND run_uid = $2 ORDER BY audit_uid LIMIT $3)", + ), + ( + "execution_compile_audit", + "DELETE FROM moa.execution_compile_audit WHERE audit_uid IN (SELECT audit_uid FROM moa.execution_compile_audit WHERE tenant_id = $1 AND run_uid = $2 ORDER BY audit_uid LIMIT $3)", + ), + ( + "execution_template_admission", + "DELETE FROM moa.execution_template_admission WHERE operation_uid IN (SELECT operation_uid FROM moa.execution_template_admission WHERE tenant_id = $1 AND execution_run_uid = $2 ORDER BY operation_uid LIMIT $3)", + ), + ]; + for (kind, sql) in stages { + let affected = sqlx::query(sql) + .bind(candidate.tenant_id.0) + .bind(candidate.run_uid) + .bind(i64::from(page_size)) + .execute(&mut *conn) + .await + .map_err(sqlx_error)? + .rows_affected(); + if affected > 0 { + return Ok(ExecutionRetentionPageOutcome::DetailDeleted { + run_uid: candidate.run_uid, + segment_kind: kind.to_string(), + records: u32::try_from(affected).map_err(|_| Error::InvalidRepositoryData { + message: "execution retention deleted row count exceeds u32".to_string(), + })?, + }); + } + } + sqlx::query( + "UPDATE moa.execution_terminal_archive SET details_deleted_at = now() WHERE archive_uid = $1 AND finalized_at IS NOT NULL AND details_deleted_at IS NULL", + ) + .bind(manifest.archive_uid) + .execute(&mut *conn) + .await + .map_err(sqlx_error)?; + Ok(ExecutionRetentionPageOutcome::Complete { + run_uid: candidate.run_uid, + }) +} + +fn candidate_from_row(row: sqlx::postgres::PgRow) -> Result { + Ok(RetentionCandidate { + tenant_id: TenantId::from(row.try_get::("tenant_id").map_err(row_error)?), + run_uid: row.try_get("run_uid").map_err(row_error)?, + contact_id: row.try_get("contact_id").map_err(row_error)?, + status: row.try_get("status").map_err(row_error)?, + completed_at: row + .try_get::>, _>("completed_at") + .map_err(row_error)? + .ok_or_else(|| Error::InvalidRepositoryData { + message: "terminal execution retention candidate lacks completed_at".to_string(), + })?, + goal_contract: row.try_get("goal_contract").map_err(row_error)?, + initial_plan_hash: row.try_get("initial_plan_hash").map_err(row_error)?, + active_plan_hash: row.try_get("active_plan_hash").map_err(row_error)?, + terminal_summary: row.try_get("terminal_summary").map_err(row_error)?, + }) +} + +fn execution_retention_checkpoint_from_row( + row: &sqlx::postgres::PgRow, +) -> Result { + Ok(ExecutionRetentionCheckpoint { + generation: to_u64( + row.try_get("generation").map_err(row_error)?, + "execution retention checkpoint generation", + )?, + last_started_at: row.try_get("last_started_at").map_err(row_error)?, + last_succeeded_at: row.try_get("last_succeeded_at").map_err(row_error)?, + last_failure_at: row.try_get("last_failure_at").map_err(row_error)?, + next_run_at: row.try_get("next_run_at").map_err(row_error)?, + scheduled_generation: row + .try_get::, _>("scheduled_generation") + .map_err(row_error)? + .map(|generation| to_u64(generation, "scheduled execution retention generation")) + .transpose()?, + last_error: row.try_get("last_error").map_err(row_error)?, + updated_at: row.try_get("updated_at").map_err(row_error)?, + }) +} + +fn canonical_value_hash(domain: &str, value: &Value) -> Result { + let bytes = canonical_json_bytes(value).map_err(|error| Error::InvalidRepositoryData { + message: format!("canonicalize execution retention value: {error}"), + })?; + let mut hasher = blake3::Hasher::new(); + hasher.update(domain.as_bytes()); + hasher.update(&[0]); + hasher.update(&bytes); + Ok(hasher.finalize().to_hex().to_string()) +} + +fn require_control_plane(scope: ExecutionScope) -> Result<()> { + if scope != ExecutionScope::ControlPlane { + return Err(Error::InvalidRepositoryInput { + message: "execution terminal retention requires control-plane scope".to_string(), + }); + } + Ok(()) +} + +fn bounded_maintenance_error(error: &str) -> String { + let mut end = error.len().min(MAX_MAINTENANCE_ERROR_BYTES); + while !error.is_char_boundary(end) { + end -= 1; + } + error[..end].to_string() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn canonical_archive_hash_is_domain_separated_and_stable() { + // Pins: archive manifests cannot alias another hash domain or depend on object key order. + let left = serde_json::json!({"b": 2, "a": 1}); + let right = serde_json::json!({"a": 1, "b": 2}); + assert_eq!( + canonical_value_hash("retention", &left).expect("hash left"), + canonical_value_hash("retention", &right).expect("hash right") + ); + assert_ne!( + canonical_value_hash("retention", &left).expect("retention hash"), + canonical_value_hash("another-domain", &left).expect("other hash") + ); + } + + #[test] + fn maintenance_error_bound_preserves_utf8_and_postgres_octet_limit() { + // Pins: multibyte failures cannot violate the maintenance checkpoint's + // octet-length constraint while recording the self-scheduled retry. + let bounded = bounded_maintenance_error(&"é".repeat(3_000)); + assert_eq!(bounded.len(), MAX_MAINTENANCE_ERROR_BYTES); + assert!(bounded.is_char_boundary(bounded.len())); + assert_eq!(bounded.chars().count(), MAX_MAINTENANCE_ERROR_BYTES / 2); + } + + #[test] + fn archive_sources_use_persisted_keyset_cursors_without_offsets() { + // Pins: every archive source resumes strictly after its last committed key; restoring + // OFFSET paging would make later pages increasingly expensive and replay-fragile. + assert_eq!(ARCHIVE_SOURCES.len(), 18); + for source in ARCHIVE_SOURCES { + assert!(source.select_page_sql.contains("$4"), "{}", source.kind); + assert!( + source.select_page_sql.contains(" AS cursor"), + "{}", + source.kind + ); + assert!( + !source.select_page_sql.contains("OFFSET"), + "{}", + source.kind + ); + } + } + + #[test] + fn rolling_archive_chain_binds_order_counts_and_prior_root() { + // Pins: O(1) finalization relies on the manifest accumulator binding every verified + // segment in insertion order; changing any prior root or segment fact changes the root. + let digest = *blake3::hash(b"segment").as_bytes(); + let first = rolling_chain_digest(None, "execution_task", 1, 2, 128, &digest); + assert_eq!( + first, + rolling_chain_digest(None, "execution_task", 1, 2, 128, &digest) + ); + assert_ne!( + first, + rolling_chain_digest(None, "execution_task", 1, 3, 128, &digest) + ); + assert_ne!( + first, + rolling_chain_digest(None, "execution_task", 1, 2, 129, &digest) + ); + assert_ne!( + rolling_chain_digest(Some(&first), "execution_task", 2, 1, 64, &digest), + rolling_chain_digest(None, "execution_task", 2, 1, 64, &digest) + ); + } + + #[test] + fn archive_finalization_never_rescans_segment_payloads() { + // Pins: finalization is a constant-space manifest CAS; segment verification and rolling + // accumulation happen at insertion, never through a terminal fetch_all. + let source = include_str!("retention.rs"); + let body = source + .split_once("async fn finalize_archive") + .expect("finalizer") + .1 + .split_once("async fn advance_archive_cursor") + .expect("cursor helper") + .0; + assert!(!body.contains("execution_terminal_archive_segment")); + assert!(!body.contains("fetch_all")); + assert!(body.contains("root_digest = rolling_chain_digest")); + } +} diff --git a/crates/moa-execution/src/repository/rows.rs b/crates/moa-execution/src/repository/rows.rs index 2f7295708..cb0cbe0ae 100644 --- a/crates/moa-execution/src/repository/rows.rs +++ b/crates/moa-execution/src/repository/rows.rs @@ -1,5 +1,6 @@ //! PostgreSQL row decoding for execution repository projections. +use super::audit::ExecutionPlanningContextRecord; use super::*; pub(super) fn planning_context_from_row(row: &PgRow) -> Result { @@ -52,7 +53,52 @@ pub(super) fn run_from_row(row: &PgRow) -> Result { }); } }; - let waiting_reasons: Value = row.try_get("waiting_reasons").map_err(row_error)?; + let waiting_reasons: Vec = + serde_json::from_value(row.try_get("waiting_reasons").map_err(row_error)?)?; + let waiting_task_count = required_u64(row, "waiting_task_count")?; + let waiting_input_task_count = required_u64(row, "waiting_input_task_count")?; + let waiting_review_task_count = required_u64(row, "waiting_review_task_count")?; + let waiting_signal_task_count = required_u64(row, "waiting_signal_task_count")?; + let waiting_timer_task_count = required_u64(row, "waiting_timer_task_count")?; + let waiting_external_task_count = required_u64(row, "waiting_external_task_count")?; + let waiting_replan_task_count = required_u64(row, "waiting_replan_task_count")?; + let waiting_input_user_task_count = required_u64(row, "waiting_input_user_task_count")?; + let waiting_input_tenant_admin_task_count = + required_u64(row, "waiting_input_tenant_admin_task_count")?; + let waiting_input_external_task_count = required_u64(row, "waiting_input_external_task_count")?; + let waiting_reasons_truncated: bool = row + .try_get("waiting_reasons_truncated") + .map_err(row_error)?; + let classified_waiting_count = waiting_input_task_count + .checked_add(waiting_review_task_count) + .and_then(|count| count.checked_add(waiting_signal_task_count)) + .and_then(|count| count.checked_add(waiting_timer_task_count)) + .and_then(|count| count.checked_add(waiting_external_task_count)) + .and_then(|count| count.checked_add(waiting_replan_task_count)) + .ok_or_else(|| Error::ArithmeticOverflow { + context: "execution run waiting task count".to_string(), + })?; + let classified_input_count = waiting_input_user_task_count + .checked_add(waiting_input_tenant_admin_task_count) + .and_then(|count| count.checked_add(waiting_input_external_task_count)) + .ok_or_else(|| Error::ArithmeticOverflow { + context: "execution run waiting input task count".to_string(), + })?; + let sample_count = + u64::try_from(waiting_reasons.len()).map_err(|_| Error::ArithmeticOverflow { + context: "execution run waiting reason sample".to_string(), + })?; + if classified_waiting_count != waiting_task_count + || classified_input_count != waiting_input_task_count + || sample_count > waiting_task_count + || waiting_reasons.len() > 64 + || waiting_reasons_truncated != (sample_count < waiting_task_count) + { + return Err(Error::InvalidRepositoryData { + message: "execution run waiting counters disagree with the bounded reason sample" + .to_string(), + }); + } let source_provenance: ExecutionSourceProvenance = serde_json::from_value(row.try_get("source_provenance").map_err(row_error)?)?; let source_kind = ExecutionSourceKind::from_str( @@ -116,9 +162,17 @@ pub(super) fn run_from_row(row: &PgRow) -> Result { let contact_id: Option = row.try_get("contact_id").map_err(row_error)?; let session_id: Uuid = row.try_get("session_id").map_err(row_error)?; let owner_user_id: String = row.try_get("owner_user_id").map_err(row_error)?; + let tenant_id = TenantId(row.try_get("tenant_id").map_err(row_error)?); + let admitted_identity: Identity = + serde_json::from_value(row.try_get("admitted_identity").map_err(row_error)?)?; + if admitted_identity.tenant_id != tenant_id { + return Err(Error::InvalidRepositoryData { + message: "execution admitted identity tenant disagrees with run tenant".to_string(), + }); + } Ok(ExecutionRunRecord { run_uid, - tenant_id: TenantId(row.try_get("tenant_id").map_err(row_error)?), + tenant_id, contact_id: contact_id.map(ContactId), session_id: SessionId(session_id), originating_user_sequence_num: required_u64(row, "originating_user_sequence_num")?, @@ -128,6 +182,7 @@ pub(super) fn run_from_row(row: &PgRow) -> Result { .map_err(row_error)? .parse()?, owner_user_id: UserId::new(owner_user_id), + admitted_identity, goal: serde_json::from_value(goal_value)?, initial_plan: serde_json::from_value(initial_plan_value)?, active_plan: serde_json::from_value(active_plan_value)?, @@ -165,6 +220,28 @@ pub(super) fn run_from_row(row: &PgRow) -> Result { terminal_evidence, terminal_reason, status, + controller_generation: required_u64(row, "controller_generation")?, + activation_state: ExecutionActivationState::from_str( + &row.try_get::("activation_state") + .map_err(row_error)?, + )?, + next_wake_at: row.try_get("next_wake_at").map_err(row_error)?, + waiting_since: row.try_get("waiting_since").map_err(row_error)?, + last_progress_at: row.try_get("last_progress_at").map_err(row_error)?, + pause_requested_at: row.try_get("pause_requested_at").map_err(row_error)?, + paused_at: row.try_get("paused_at").map_err(row_error)?, + ready_task_count: required_u64(row, "ready_task_count")?, + active_task_count: required_u64(row, "active_task_count")?, + waiting_task_count, + waiting_input_task_count, + waiting_review_task_count, + waiting_signal_task_count, + waiting_timer_task_count, + waiting_external_task_count, + waiting_replan_task_count, + waiting_input_user_task_count, + waiting_input_tenant_admin_task_count, + waiting_input_external_task_count, approved_budget: ExecutionBudgetLimit { max_cost_microusd: optional_u64(row, "budget_max_cost_microusd")?, max_tokens: optional_u64(row, "budget_max_tokens")?, @@ -180,7 +257,8 @@ pub(super) fn run_from_row(row: &PgRow) -> Result { progress_completed_tasks: required_u64(row, "progress_completed_tasks")?, progress_failed_tasks: required_u64(row, "progress_failed_tasks")?, progress_cancelled_tasks: required_u64(row, "progress_cancelled_tasks")?, - waiting_reasons: serde_json::from_value(waiting_reasons)?, + waiting_reasons, + waiting_reasons_truncated, wake_epoch: required_u64(row, "wake_epoch")?, processed_wake_epoch: required_u64(row, "processed_wake_epoch")?, next_compensation_sequence: required_u64(row, "next_compensation_sequence")?, @@ -224,6 +302,19 @@ pub(super) fn task_from_row(row: &PgRow) -> Result { )?, attempt: to_u32(row.try_get("attempt").map_err(row_error)?, "attempt")?, generation: required_u64(row, "generation")?, + attempt_generation: required_u64(row, "attempt_generation")?, + attempt_state: ExecutionAttemptState::from_str( + &row.try_get::("attempt_state") + .map_err(row_error)?, + )?, + attempt_started_at: row.try_get("attempt_started_at").map_err(row_error)?, + last_progress_at: row.try_get("last_progress_at").map_err(row_error)?, + attempt_deadline_at: row.try_get("attempt_deadline_at").map_err(row_error)?, + waiting_since: row.try_get("waiting_since").map_err(row_error)?, + ready_at: row.try_get("ready_at").map_err(row_error)?, + external_job_uid: row.try_get("external_job_uid").map_err(row_error)?, + active_dispatch_uid: row.try_get("active_dispatch_uid").map_err(row_error)?, + dispatch_sequence: required_u64(row, "dispatch_sequence")?, input: row.try_get("input").map_err(row_error)?, resume_input_history: serde_json::from_value(resume_input_history)?, kind: serde_json::from_value(kind)?, diff --git a/crates/moa-execution/src/repository/run.rs b/crates/moa-execution/src/repository/run.rs index 2be2b939e..1beb16e28 100644 --- a/crates/moa-execution/src/repository/run.rs +++ b/crates/moa-execution/src/repository/run.rs @@ -2,28 +2,88 @@ use super::*; use super::{ - projection::{budget_ledger, scheduling_projection}, + capacity::{ + ActiveRunCapacityReserveOutcome, CapacityReserveOutcome, ExecutionCapacityDimension, + ExecutionCapacityOwner, ExecutionCapacityRequest, execution_capacity_reservation_uid, + prelock_capacity_dimensions_in_tx, reserve_active_run_capacity_in_tx, + transfer_active_run_to_parked_in_tx, transfer_parked_run_to_active_in_tx, + }, + outbox::{ + ExecutionDispatchKind, ExecutionDispatchRecord, NewExecutionDispatch, + enqueue_dispatch_in_conn, + }, rows::*, sql::*, + trigger::{ + ExecutionTriggerKind, ExecutionTriggerSupersedeOutcome, NewExecutionTrigger, + create_trigger_with_dispatch_in_conn, supersede_trigger_in_conn, + }, }; +use moa_config::ExecutionConfig; + +const RUN_ACTIVATION_DISPATCH_NAMESPACE: Uuid = + Uuid::from_u128(0x83f0_a3b7_6f50_5c12_99d0_48a0_3b09_2cd4); +const RUN_DEADLINE_TRIGGER_NAMESPACE: Uuid = + Uuid::from_u128(0xb14e_032e_1b46_5e32_82b0_999d_1d45_9cb2); +const RUN_LIFETIME_CAPACITY_GENERATION: u64 = 1; +const LOAD_RUN_FOR_SESSION_SQL: &str = r#" + SELECT * + FROM moa.execution_run + WHERE run_uid = $1 + AND session_id = $2 +"#; + +/// Bounded cancellation evidence loaded under the caller's exact session fence. +#[derive(Clone, Debug, PartialEq)] +pub struct ExecutionCancellationProjection { + /// Current durable run projection. + pub run: ExecutionRunRecord, + /// Completed plan-node identities, bounded by the compiler-capped active plan. + pub completed_node_ids: Vec, +} +const LOAD_RUN_BY_IDEMPOTENCY_FOR_SESSION_SQL: &str = r#" + SELECT * + FROM moa.execution_run + WHERE tenant_id = $1 + AND contact_id IS NOT DISTINCT FROM $2 + AND idempotency_key = $3 + AND session_id = $4 +"#; + +/// Atomic run-admission result, including durable idempotency replay and capacity deferral. +#[derive(Clone, Debug, PartialEq)] +pub enum RunAdmissionOutcome { + /// A new run, its scheduler state, and its lifetime capacity receipt committed together. + Admitted(Box), + /// The exact scoped idempotency key already owns this admitted run. + Replayed(Box), + /// Fleet or tenant admission capacity is exhausted; no run row committed. + CapacitySaturated { + /// The lifetime capacity dimension that rejected admission. + dimension: ExecutionCapacityDimension, + }, +} impl ExecutionRepository { - /// Creates a run or returns the existing row for the same scoped idempotency key. + /// Admits a run, scheduler state, and lifetime capacity in one transaction. pub async fn create_run( &self, scope: ExecutionScope, + config: &ExecutionConfig, new_run: NewExecutionRun, - ) -> Result { + ) -> Result { validate_new_run(scope, &new_run)?; let budget = DbBudgetLimit::try_from(&new_run.approved_budget)?; let run_uid = Uuid::now_v7(); let plan_value = serde_json::to_value(&new_run.plan)?; let goal_value = serde_json::to_value(&new_run.goal)?; + let admitted_identity_value = serde_json::to_value(&new_run.admitted_identity)?; let catalog_value = serde_json::to_value(&new_run.catalog)?; let authorization_value = serde_json::to_value(&new_run.authorization)?; let pinned_skills_value = serde_json::to_value(&new_run.pinned_instruction_skills)?; let source_provenance_value = serde_json::to_value(&new_run.source_provenance)?; let source_fields = normalized_source_fields(&new_run.source_provenance); + let activation_state = initial_activation_state(new_run.status)?; let originating_user_sequence_num = to_i64( new_run.originating_user_sequence_num, "originating user sequence", @@ -38,6 +98,7 @@ impl ExecutionRepository { .bind(new_run.planning_context_uid) .bind(new_run.planning_context_hash.to_string()) .bind(new_run.owner_user_id.as_str()) + .bind(admitted_identity_value) .bind(goal_value) .bind(&plan_value) .bind(&plan_value) @@ -60,12 +121,16 @@ impl ExecutionRepository { .bind(budget.deadline_at) .bind(0_i64) .bind(new_run.idempotency_key.as_deref()) + .bind(activation_state.as_str()) + .bind(Option::::None) + .bind(Option::::None) + .bind(Option::::None) .fetch_optional(conn.as_mut()) .await .map_err(sqlx_error)?; - let record = if let Some(row) = row { - run_from_row(&row)? + let (record, admitted) = if let Some(row) = row { + (run_from_row(&row)?, true) } else if let Some(idempotency_key) = new_run.idempotency_key.as_deref() { let row = sqlx::query(LOAD_RUN_BY_IDEMPOTENCY_SQL) .bind(new_run.tenant_id.0) @@ -78,14 +143,117 @@ impl ExecutionRepository { message: "idempotent run insert conflicted without a visible existing row" .to_string(), })?; - run_from_row(&row)? + (run_from_row(&row)?, false) } else { return Err(Error::Storage { message: "execution run insert conflicted without an idempotency key".to_string(), }); }; + if record.admitted_identity != new_run.admitted_identity { + conn.rollback().await.map_err(storage_error)?; + return Err(Error::InvalidRepositoryInput { + message: + "execution idempotency key is already bound to a different admitted identity" + .to_string(), + }); + } + if !admitted { + conn.commit().await.map_err(storage_error)?; + return Ok(RunAdmissionOutcome::Replayed(Box::new(record))); + } + + seed_run_scheduler_state_in_tx( + conn.as_mut(), + record.tenant_id, + record.run_uid, + &record.active_plan, + ) + .await?; + let mut capacity_dimensions = vec![ + ExecutionCapacityDimension::ActiveRuns, + ExecutionCapacityDimension::ParkedRuns, + ]; + if record.approved_budget.deadline_at.is_some() { + capacity_dimensions.push(ExecutionCapacityDimension::ScheduledTriggers); + } + prelock_capacity_dimensions_in_tx( + conn.as_mut(), + config, + record.tenant_id, + &capacity_dimensions, + ) + .await?; + let capacity = reserve_active_run_capacity_in_tx( + conn.as_mut(), + config, + active_run_capacity_request(record.tenant_id, record.run_uid), + ) + .await?; + match capacity { + ActiveRunCapacityReserveOutcome::Reserved + | ActiveRunCapacityReserveOutcome::Replayed => {} + ActiveRunCapacityReserveOutcome::Saturated(dimension) => { + conn.rollback().await.map_err(storage_error)?; + return Ok(RunAdmissionOutcome::CapacitySaturated { dimension }); + } + } + match arm_run_deadline_in_conn(conn.as_mut(), config, &record).await { + Ok( + RunDeadlineArmOutcome::Armed(_) + | RunDeadlineArmOutcome::NoDeadline + | RunDeadlineArmOutcome::Terminal, + ) => {} + Ok(RunDeadlineArmOutcome::NotFound | RunDeadlineArmOutcome::StaleGeneration { .. }) => { + conn.rollback().await.map_err(storage_error)?; + return Err(Error::InvalidRepositoryData { + message: "newly inserted execution run lost its deadline arm fence".to_string(), + }); + } + Err(Error::CapacitySaturated { dimension }) + if dimension == ExecutionCapacityDimension::ScheduledTriggers.as_str() => + { + conn.rollback().await.map_err(storage_error)?; + return Ok(RunAdmissionOutcome::CapacitySaturated { + dimension: ExecutionCapacityDimension::ScheduledTriggers, + }); + } + Err(error) => return Err(error), + } + if record.status == ExecutionRunStatus::Queued { + enqueue_dispatch_in_conn( + conn.as_mut(), + &NewExecutionDispatch { + dispatch_uid: run_activation_dispatch_uid( + record.run_uid, + record.controller_generation, + record.wake_epoch, + ), + tenant_id: record.tenant_id, + run_uid: Some(record.run_uid), + task_id: None, + compensation_id: None, + trigger_uid: None, + external_job_uid: None, + kind: ExecutionDispatchKind::RunActivation, + controller_generation: Some(record.controller_generation), + wake_epoch: Some(record.wake_epoch), + attempt_generation: None, + compensation_generation: None, + compensation_attempt_generation: None, + not_before_at: record.created_at, + payload: json!({"reason": "run_admitted"}), + }, + ) + .await?; + } + let row = sqlx::query(LOAD_RUN_SQL) + .bind(record.run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let record = run_from_row(&row)?; conn.commit().await.map_err(storage_error)?; - Ok(record) + Ok(RunAdmissionOutcome::Admitted(Box::new(record))) } /// Loads one visible execution run. @@ -104,6 +272,81 @@ impl ExecutionRepository { row.as_ref().map(run_from_row).transpose() } + /// Loads one visible execution run only when it belongs to the expected parent session. + pub async fn load_run_for_session( + &self, + scope: ExecutionScope, + run_uid: Uuid, + expected_session_id: SessionId, + ) -> Result> { + let mut conn = scope.begin(&self.pool).await?; + let row = sqlx::query(LOAD_RUN_FOR_SESSION_SQL) + .bind(run_uid) + .bind(expected_session_id.0) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + conn.commit().await.map_err(storage_error)?; + row.as_ref().map(run_from_row).transpose() + } + + /// Loads one session-fenced run plus bounded completed-node cancellation evidence. + pub async fn load_cancellation_projection_for_session( + &self, + scope: ExecutionScope, + run_uid: Uuid, + expected_session_id: SessionId, + ) -> Result> { + let mut conn = scope.begin(&self.pool).await?; + let row = sqlx::query(LOAD_RUN_FOR_SESSION_SQL) + .bind(run_uid) + .bind(expected_session_id.0) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(row) = row else { + conn.commit().await.map_err(storage_error)?; + return Ok(None); + }; + let run = run_from_row(&row)?; + let plan_node_ids = run + .active_plan + .definition + .nodes + .iter() + .map(|node| node.id.clone()) + .collect::>(); + let fetch_limit = i64::try_from(plan_node_ids.len()) + .map_err(|_| Error::InvalidRepositoryData { + message: "active plan node count exceeds PostgreSQL BIGINT".to_string(), + })? + .checked_add(1) + .ok_or_else(|| Error::ArithmeticOverflow { + context: "cancellation projection node limit".to_string(), + })?; + let completed_node_ids = sqlx::query_scalar::<_, String>( + "SELECT node_id FROM moa.execution_node_state \ + WHERE run_uid=$1 AND node_id=ANY($2::TEXT[]) AND node_status='completed' \ + ORDER BY node_order, node_id LIMIT $3", + ) + .bind(run_uid) + .bind(&plan_node_ids) + .bind(fetch_limit) + .fetch_all(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if completed_node_ids.len() > plan_node_ids.len() { + return Err(Error::InvalidRepositoryData { + message: "cancellation projection exceeded its active-plan bound".to_string(), + }); + } + conn.commit().await.map_err(storage_error)?; + Ok(Some(ExecutionCancellationProjection { + run, + completed_node_ids, + })) + } + /// Loads one visible task under its owning run and stable task ID. pub async fn load_task( &self, @@ -147,6 +390,33 @@ impl ExecutionRepository { row.as_ref().map(run_from_row).transpose() } + /// Loads a scope-local idempotent run only when it belongs to the expected session. + pub async fn load_run_by_idempotency_key_for_session( + &self, + scope: ExecutionScope, + tenant_id: TenantId, + contact_id: Option, + expected_session_id: SessionId, + idempotency_key: &str, + ) -> Result> { + if !scope.permits_owner(tenant_id, contact_id) { + return Err(Error::InvalidRepositoryInput { + message: "idempotency lookup owner does not match repository scope".to_string(), + }); + } + let mut conn = scope.begin(&self.pool).await?; + let row = sqlx::query(LOAD_RUN_BY_IDEMPOTENCY_FOR_SESSION_SQL) + .bind(tenant_id.0) + .bind(contact_id.map(|value| value.0)) + .bind(idempotency_key) + .bind(expected_session_id.0) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + conn.commit().await.map_err(storage_error)?; + row.as_ref().map(run_from_row).transpose() + } + /// Lists one bounded, stable page of visible execution runs. pub async fn list_runs( &self, @@ -183,73 +453,118 @@ impl ExecutionRepository { Ok(ExecutionRunPage { runs, next_cursor }) } - /// Loads one repeatable-read scheduling snapshot with its complete ordered task projection. - pub async fn load_scheduling_snapshot( + /// Claims one exact queued controller generation for a bounded activation. + pub async fn claim_run_activation( &self, scope: ExecutionScope, run_uid: Uuid, - ) -> Result> { - let mut conn = self.pool.begin().await.map_err(sqlx_error)?; - sqlx::query("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ") - .execute(&mut *conn) - .await - .map_err(sqlx_error)?; - install_execution_scope(&mut conn, scope).await?; - sqlx::query("SET LOCAL ROLE moa_app") - .execute(&mut *conn) - .await - .map_err(sqlx_error)?; - let Some(row) = sqlx::query(LOAD_RUN_SQL) + controller_generation: u64, + ) -> Result { + let generation = to_i64(controller_generation, "controller generation")?; + let mut conn = scope.begin(&self.pool).await?; + let Some(locked_row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) .bind(run_uid) - .fetch_optional(&mut *conn) + .fetch_optional(conn.as_mut()) .await .map_err(sqlx_error)? else { - conn.commit().await.map_err(sqlx_error)?; - return Ok(None); + conn.commit().await.map_err(storage_error)?; + return Ok(RunActivationWriteOutcome::NotFound); }; - let run = run_from_row(&row)?; - let task_rows = sqlx::query(LIST_ALL_TASKS_SQL) + let locked = run_from_row(&locked_row)?; + if locked.controller_generation != controller_generation { + conn.commit().await.map_err(storage_error)?; + return Ok(RunActivationWriteOutcome::GenerationMismatch); + } + if locked.activation_state == ExecutionActivationState::Advancing { + conn.commit().await.map_err(storage_error)?; + return Ok(RunActivationWriteOutcome::AlreadyApplied(locked)); + } + if locked.activation_state != ExecutionActivationState::Queued + || locked.status.is_terminal() + { + conn.commit().await.map_err(storage_error)?; + return Ok(RunActivationWriteOutcome::InvalidState); + } + let row = sqlx::query(CLAIM_RUN_ACTIVATION_SQL) .bind(run_uid) - .fetch_all(&mut *conn) + .bind(generation) + .fetch_optional(conn.as_mut()) .await - .map_err(sqlx_error)?; - conn.commit().await.map_err(sqlx_error)?; - let tasks = task_rows - .iter() - .map(task_from_row) - .collect::>>()?; - let projection = scheduling_projection(&run, &tasks); - Ok(Some(ExecutionSchedulingSnapshot { - catalog: run.catalog.clone(), - authorization: run.authorization.clone(), - pinned_instruction_skills: run.pinned_instruction_skills.clone(), - budget_ledger: budget_ledger(&run), - run, - projection, - })) + .map_err(sqlx_error)? + .ok_or_else(|| Error::Storage { + message: "run activation claim lost its locked generation fence".to_string(), + })?; + let record = run_from_row(&row)?; + conn.commit().await.map_err(storage_error)?; + Ok(RunActivationWriteOutcome::Applied(record)) } - /// Loads one terminal run and derives its compact session delivery from the same snapshot. - pub async fn load_terminal_delivery( + /// Persists the exact terminal or parked checkpoint produced by one activation. + pub async fn checkpoint_run_activation( &self, scope: ExecutionScope, run_uid: Uuid, - ) -> Result> { - let Some(snapshot) = self.load_scheduling_snapshot(scope, run_uid).await? else { - return Ok(None); + controller_generation: u64, + checkpoint: ExecutionRunActivationCheckpoint, + ) -> Result { + validate_activation_checkpoint(&checkpoint)?; + let generation = to_i64(controller_generation, "controller generation")?; + let ready_task_count = to_i64(checkpoint.ready_task_count, "ready task count")?; + let active_task_count = to_i64(checkpoint.active_task_count, "active task count")?; + let mut conn = scope.begin(&self.pool).await?; + let Some(locked_row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) + .bind(run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + conn.commit().await.map_err(storage_error)?; + return Ok(RunActivationWriteOutcome::NotFound); }; - execution_terminal_delivery_from_state(&snapshot.run, &snapshot.projection).map(Some) + let locked = run_from_row(&locked_row)?; + if locked.controller_generation != controller_generation { + conn.commit().await.map_err(storage_error)?; + return Ok(RunActivationWriteOutcome::GenerationMismatch); + } + if activation_checkpoint_matches(&locked, &checkpoint) { + conn.commit().await.map_err(storage_error)?; + return Ok(RunActivationWriteOutcome::AlreadyApplied(locked)); + } + if locked.activation_state != ExecutionActivationState::Advancing { + conn.commit().await.map_err(storage_error)?; + return Ok(RunActivationWriteOutcome::InvalidState); + } + let row = sqlx::query(CHECKPOINT_RUN_ACTIVATION_SQL) + .bind(run_uid) + .bind(generation) + .bind(checkpoint.status.as_str()) + .bind(checkpoint.activation_state.as_str()) + .bind(checkpoint.next_wake_at) + .bind(checkpoint.waiting_since) + .bind(ready_task_count) + .bind(active_task_count) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + .ok_or_else(|| Error::Storage { + message: "run activation checkpoint lost its locked generation fence".to_string(), + })?; + let record = run_from_row(&row)?; + conn.commit().await.map_err(storage_error)?; + Ok(RunActivationWriteOutcome::Applied(record)) } - /// Acknowledges only the exact current wake epoch, preserving any later wake. - pub async fn ack_run_wake( + /// Claims one exact controller generation and unprocessed wake epoch atomically. + pub async fn claim_controller_wake( &self, scope: ExecutionScope, run_uid: Uuid, - expected_wake_epoch: u64, - ) -> Result { - let expected = to_i64(expected_wake_epoch, "wake epoch")?; + controller_generation: u64, + wake_epoch: u64, + ) -> Result { + let generation = to_i64(controller_generation, "controller generation")?; + let wake = to_i64(wake_epoch, "wake epoch")?; let mut conn = scope.begin(&self.pool).await?; let Some(row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) .bind(run_uid) @@ -258,46 +573,660 @@ impl ExecutionRepository { .map_err(sqlx_error)? else { conn.commit().await.map_err(storage_error)?; - return Ok(WakeAckOutcome::NotFound); + return Ok(RunControllerClaimOutcome::NotFound); }; let run = run_from_row(&row)?; - let outcome = if run.wake_epoch != expected_wake_epoch { - if expected_wake_epoch <= run.processed_wake_epoch { - WakeAckOutcome::Replayed { - processed_wake_epoch: run.processed_wake_epoch, - } - } else { - WakeAckOutcome::Changed { - current_wake_epoch: run.wake_epoch, - } + let outcome = if run.controller_generation != controller_generation { + RunControllerClaimOutcome::StaleGeneration { + current_generation: run.controller_generation, } - } else if run.processed_wake_epoch >= expected_wake_epoch { - WakeAckOutcome::Replayed { + } else if run.status.is_terminal() + || run.activation_state == ExecutionActivationState::Terminal + { + RunControllerClaimOutcome::Terminal(run) + } else if wake_epoch <= run.processed_wake_epoch { + RunControllerClaimOutcome::Replayed(run) + } else if run.wake_epoch != wake_epoch { + RunControllerClaimOutcome::StaleWake { + current_wake_epoch: run.wake_epoch, processed_wake_epoch: run.processed_wake_epoch, } + } else if run.activation_state == ExecutionActivationState::Advancing { + RunControllerClaimOutcome::Resumed(run) + } else if run.activation_state != ExecutionActivationState::Queued { + RunControllerClaimOutcome::InvalidState } else { let updated = sqlx::query( - "UPDATE moa.execution_run SET processed_wake_epoch = $2, updated_at = NOW() \ - WHERE run_uid = $1 AND wake_epoch = $2 AND processed_wake_epoch < $2", + "UPDATE moa.execution_run \ + SET activation_state = 'advancing', updated_at = NOW() \ + WHERE run_uid = $1 AND controller_generation = $2 AND wake_epoch = $3 \ + AND processed_wake_epoch < $3 AND activation_state = 'queued' \ + RETURNING *", ) .bind(run_uid) - .bind(expected) - .execute(conn.as_mut()) + .bind(generation) + .bind(wake) + .fetch_optional(conn.as_mut()) .await - .map_err(sqlx_error)?; - if updated.rows_affected() == 1 { - WakeAckOutcome::Acknowledged { - processed_wake_epoch: expected_wake_epoch, + .map_err(sqlx_error)? + .ok_or_else(|| Error::Storage { + message: "controller wake claim lost its locked compare-and-set".to_string(), + })?; + RunControllerClaimOutcome::Claimed(run_from_row(&updated)?) + }; + conn.commit().await.map_err(storage_error)?; + Ok(outcome) + } + + /// Checkpoints one exact wake, acknowledges it, and optionally enqueues one continuation. + pub async fn complete_controller_wake( + &self, + scope: ExecutionScope, + config: &ExecutionConfig, + run_uid: Uuid, + request: RunControllerCompletionRequest, + ) -> Result { + validate_activation_checkpoint(&request.checkpoint)?; + validate_controller_completion(&request)?; + let mut conn = scope.begin(&self.pool).await?; + let tenant_id = sqlx::query_scalar::<_, Uuid>( + "SELECT tenant_id FROM moa.execution_run WHERE run_uid=$1", + ) + .bind(run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(tenant_id) = tenant_id else { + conn.commit().await.map_err(storage_error)?; + return Ok(RunControllerCompletionOutcome::NotFound); + }; + prelock_capacity_dimensions_in_tx( + conn.as_mut(), + config, + TenantId(tenant_id), + &[ + ExecutionCapacityDimension::ActiveRuns, + ExecutionCapacityDimension::ParkedRuns, + ], + ) + .await?; + let checkpoint = if request.continuation_payload.is_some() { + ExecutionRunActivationCheckpoint { + activation_state: ExecutionActivationState::Idle, + ..request.checkpoint.clone() + } + } else { + request.checkpoint.clone() + }; + let outcome = complete_controller_wake_in_conn( + &mut conn, + run_uid, + request.controller_generation, + request.wake_epoch, + checkpoint, + ) + .await?; + let outcome = match (outcome, request.continuation_payload) { + (RunControllerCompletionOutcome::Applied { run, .. }, Some(payload)) => { + let continuation = enqueue_run_activation_in_conn( + conn.as_mut(), + run.tenant_id, + run_uid, + request.controller_generation, + request.continuation_not_before_at, + payload, + ) + .await?; + let row = sqlx::query(LOAD_RUN_SQL) + .bind(run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + RunControllerCompletionOutcome::Applied { + run: Box::new(run_from_row(&row)?), + continuation: Some(Box::new(continuation)), + } + } + (RunControllerCompletionOutcome::Applied { run, .. }, None) + if requires_parked_run_capacity(&run) => + { + if transfer_active_run_to_parked_in_tx( + conn.as_mut(), + config, + &run, + request.wake_epoch, + ) + .await? + == CapacityReserveOutcome::Saturated + { + conn.rollback().await.map_err(storage_error)?; + return Ok(RunControllerCompletionOutcome::CapacitySaturated { + dimension: ExecutionCapacityDimension::ParkedRuns, + }); + } + RunControllerCompletionOutcome::Applied { + run, + continuation: None, } - } else { - return Err(Error::Storage { - message: "wake acknowledgement lost its locked compare-and-set".to_string(), - }); } + (outcome, _) => outcome, }; conn.commit().await.map_err(storage_error)?; Ok(outcome) } + + /// Arms or idempotently replaces the exact deadline trigger for a run generation. + pub async fn arm_run_deadline( + &self, + scope: ExecutionScope, + run_uid: Uuid, + controller_generation: u64, + config: &ExecutionConfig, + ) -> Result { + let mut conn = scope.begin(&self.pool).await?; + let tenant_id = sqlx::query_scalar::<_, Uuid>( + "SELECT tenant_id FROM moa.execution_run WHERE run_uid=$1", + ) + .bind(run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(tenant_id) = tenant_id else { + conn.commit().await.map_err(storage_error)?; + return Ok(RunDeadlineArmOutcome::NotFound); + }; + prelock_capacity_dimensions_in_tx( + conn.as_mut(), + config, + TenantId(tenant_id), + &[ExecutionCapacityDimension::ScheduledTriggers], + ) + .await?; + let Some(row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) + .bind(run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + conn.commit().await.map_err(storage_error)?; + return Ok(RunDeadlineArmOutcome::NotFound); + }; + let run = run_from_row(&row)?; + if run.tenant_id.0 != tenant_id { + conn.rollback().await.map_err(storage_error)?; + return Err(Error::InvalidRepositoryData { + message: "run deadline tenant changed between capacity prelock and row lock" + .to_string(), + }); + } + if run.controller_generation != controller_generation { + conn.commit().await.map_err(storage_error)?; + return Ok(RunDeadlineArmOutcome::StaleGeneration { + current_generation: run.controller_generation, + }); + } + if run.status.is_terminal() { + conn.commit().await.map_err(storage_error)?; + return Ok(RunDeadlineArmOutcome::Terminal); + } + let outcome = arm_run_deadline_in_conn(conn.as_mut(), config, &run).await?; + conn.commit().await.map_err(storage_error)?; + Ok(outcome) + } +} + +/// Arms the exact immutable run-deadline trigger inside the caller's transaction. +/// +/// Multi-resource admission callers must prelock `ActiveRuns` then `ScheduledTriggers` before +/// invoking this helper. The helper never commits, so run admission, capacity, delayed delivery, +/// and the run wake projection share one crash-safe boundary. +pub(super) async fn arm_run_deadline_in_conn( + conn: &mut PgConnection, + config: &ExecutionConfig, + run: &ExecutionRunRecord, +) -> Result { + if run.status.is_terminal() { + return Ok(RunDeadlineArmOutcome::Terminal); + } + let Some(deadline_at) = run.approved_budget.deadline_at else { + return Ok(RunDeadlineArmOutcome::NoDeadline); + }; + let trigger_uid = run_deadline_trigger_uid(run.run_uid, run.controller_generation, deadline_at); + let stale_deadlines = sqlx::query( + "SELECT trigger_uid, controller_generation \ + FROM moa.execution_trigger \ + WHERE run_uid = $1 AND trigger_kind = 'run_deadline' \ + AND state IN ('pending', 'dispatching') AND trigger_uid <> $2 \ + ORDER BY controller_generation, trigger_uid \ + LIMIT 2 FOR UPDATE", + ) + .bind(run.run_uid) + .bind(trigger_uid) + .fetch_all(&mut *conn) + .await + .map_err(sqlx_error)?; + if stale_deadlines.len() > 1 { + return Err(Error::InvalidRepositoryData { + message: "execution run owns multiple stale active deadline triggers".to_string(), + }); + } + for stale in stale_deadlines { + let stale_trigger_uid = stale.try_get::("trigger_uid").map_err(row_error)?; + let stale_generation = required_u64(&stale, "controller_generation")?; + match supersede_trigger_in_conn( + conn, + stale_trigger_uid, + ExecutionTriggerKind::RunDeadline, + Some(stale_generation), + None, + None, + None, + ) + .await? + { + ExecutionTriggerSupersedeOutcome::Superseded + | ExecutionTriggerSupersedeOutcome::AlreadySuperseded + | ExecutionTriggerSupersedeOutcome::AlreadyInactive => {} + ExecutionTriggerSupersedeOutcome::StaleOrMissing => { + return Err(Error::InvalidRepositoryData { + message: "locked run deadline disappeared before capacity release".to_string(), + }); + } + } + } + let write = create_trigger_with_dispatch_in_conn( + conn, + config, + &NewExecutionTrigger { + trigger_uid, + tenant_id: run.tenant_id, + run_uid: Some(run.run_uid), + task_id: None, + compensation_id: None, + schedule_uid: None, + kind: ExecutionTriggerKind::RunDeadline, + controller_generation: Some(run.controller_generation), + attempt_generation: None, + compensation_generation: None, + compensation_attempt_generation: None, + schedule_incarnation: None, + occurrence_sequence: None, + due_at: deadline_at, + payload: json!({ "run_uid": run.run_uid, "deadline_at": deadline_at }), + }, + ) + .await?; + sqlx::query( + "UPDATE moa.execution_run SET next_wake_at = CASE \ + WHEN next_wake_at IS NULL THEN $3 ELSE LEAST(next_wake_at, $3) END, \ + updated_at = NOW() WHERE run_uid = $1 AND controller_generation = $2", + ) + .bind(run.run_uid) + .bind(to_i64(run.controller_generation, "controller generation")?) + .bind(deadline_at) + .execute(&mut *conn) + .await + .map_err(sqlx_error)?; + Ok(RunDeadlineArmOutcome::Armed(Box::new(write))) +} + +/// Increments one current run's wake epoch and enqueues its exact activation atomically. +/// +/// The caller must already own a transaction that makes the cause idempotent. This helper +/// deliberately does not commit, so trigger delivery, task settlement, and controller +/// continuation can share one persist-before-dispatch boundary. +pub async fn enqueue_run_activation_in_conn( + conn: &mut PgConnection, + tenant_id: TenantId, + run_uid: Uuid, + controller_generation: u64, + not_before_at: DateTime, + payload: Value, +) -> Result { + if !payload.is_object() { + return Err(Error::InvalidRepositoryInput { + message: "run activation payload must be a JSON object".to_string(), + }); + } + if transfer_parked_run_to_active_in_tx(conn, tenant_id, run_uid, controller_generation).await? + == CapacityReserveOutcome::Saturated + { + return Err(Error::CapacitySaturated { + dimension: ExecutionCapacityDimension::ActiveRuns.as_str(), + }); + } + let generation = to_i64(controller_generation, "controller generation")?; + let wake_epoch = sqlx::query_scalar::<_, i64>( + "UPDATE moa.execution_run \ + SET wake_epoch = wake_epoch + 1, activation_state = 'queued', updated_at = NOW() \ + WHERE run_uid = $1 AND tenant_id = $2 AND controller_generation = $3 \ + AND status NOT IN ('completed', 'partial', 'blocked', 'unsupported', 'failed', 'cancelled') \ + RETURNING wake_epoch", + ) + .bind(run_uid) + .bind(tenant_id.0) + .bind(generation) + .fetch_optional(&mut *conn) + .await + .map_err(sqlx_error)? + .ok_or_else(|| Error::InvalidRepositoryInput { + message: "run activation target is missing, terminal, or generation-stale".to_string(), + })?; + let wake_epoch = to_u64(wake_epoch, "wake epoch")?; + let dispatch_uid = run_activation_dispatch_uid(run_uid, controller_generation, wake_epoch); + enqueue_dispatch_in_conn( + conn, + &NewExecutionDispatch { + dispatch_uid, + tenant_id, + run_uid: Some(run_uid), + task_id: None, + compensation_id: None, + trigger_uid: None, + external_job_uid: None, + kind: ExecutionDispatchKind::RunActivation, + controller_generation: Some(controller_generation), + wake_epoch: Some(wake_epoch), + attempt_generation: None, + compensation_generation: None, + compensation_attempt_generation: None, + not_before_at, + payload, + }, + ) + .await +} + +/// Checkpoints and acknowledges one exact claimed wake inside the caller's transaction. +/// +/// This is the shared boundary for deadline/cancellation drain transactions. Callers that need a +/// continuation must invoke [`enqueue_run_activation_in_conn`] before committing, after this +/// helper returns `Applied`; the intermediate checkpoint must therefore be non-queued. +pub async fn complete_controller_wake_in_conn( + conn: &mut ScopedConn<'_>, + run_uid: Uuid, + controller_generation: u64, + wake_epoch: u64, + checkpoint: ExecutionRunActivationCheckpoint, +) -> Result { + validate_activation_checkpoint(&checkpoint)?; + if checkpoint.activation_state == ExecutionActivationState::Queued { + return Err(Error::InvalidRepositoryInput { + message: "transactional controller checkpoint must enqueue after acknowledging" + .to_string(), + }); + } + let generation = to_i64(controller_generation, "controller generation")?; + let wake = to_i64(wake_epoch, "wake epoch")?; + let ready = to_i64(checkpoint.ready_task_count, "ready task count")?; + let active = to_i64(checkpoint.active_task_count, "active task count")?; + let Some(row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) + .bind(run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + return Ok(RunControllerCompletionOutcome::NotFound); + }; + let run = run_from_row(&row)?; + if run.controller_generation != controller_generation { + return Ok(RunControllerCompletionOutcome::StaleGeneration { + current_generation: run.controller_generation, + }); + } + if wake_epoch <= run.processed_wake_epoch { + return Ok(RunControllerCompletionOutcome::Replayed(Box::new(run))); + } + if run.wake_epoch != wake_epoch { + return Ok(RunControllerCompletionOutcome::StaleWake { + current_wake_epoch: run.wake_epoch, + processed_wake_epoch: run.processed_wake_epoch, + }); + } + if !matches!( + run.activation_state, + ExecutionActivationState::Advancing | ExecutionActivationState::Queued + ) { + return Ok(RunControllerCompletionOutcome::InvalidState); + } + let updated = sqlx::query( + "UPDATE moa.execution_run SET status = $4, activation_state = $5, \ + next_wake_at = $6, waiting_since = $7, ready_task_count = $8, \ + active_task_count = $9, processed_wake_epoch = $3, updated_at = NOW() \ + WHERE run_uid = $1 AND controller_generation = $2 \ + AND wake_epoch = $3 AND processed_wake_epoch < $3 RETURNING *", + ) + .bind(run_uid) + .bind(generation) + .bind(wake) + .bind(checkpoint.status.as_str()) + .bind(checkpoint.activation_state.as_str()) + .bind(checkpoint.next_wake_at) + .bind(checkpoint.waiting_since) + .bind(ready) + .bind(active) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + .ok_or_else(|| Error::Storage { + message: "transactional controller completion lost its wake fence".to_string(), + })?; + Ok(RunControllerCompletionOutcome::Applied { + run: Box::new(run_from_row(&updated)?), + continuation: None, + }) +} + +fn run_activation_dispatch_uid(run_uid: Uuid, controller_generation: u64, wake_epoch: u64) -> Uuid { + let name = format!("{run_uid}:{controller_generation}:{wake_epoch}"); + Uuid::new_v5(&RUN_ACTIVATION_DISPATCH_NAMESPACE, name.as_bytes()) +} + +fn run_deadline_trigger_uid( + run_uid: Uuid, + controller_generation: u64, + deadline_at: DateTime, +) -> Uuid { + let name = format!( + "{run_uid}:{controller_generation}:{}", + deadline_at.timestamp_micros() + ); + Uuid::new_v5(&RUN_DEADLINE_TRIGGER_NAMESPACE, name.as_bytes()) +} + +fn initial_activation_state(status: ExecutionRunStatus) -> Result { + match status { + ExecutionRunStatus::AwaitingConfirmation => Ok(ExecutionActivationState::Idle), + ExecutionRunStatus::Queued => Ok(ExecutionActivationState::Queued), + _ => Err(Error::InvalidRepositoryInput { + message: "new execution run must be awaiting confirmation or queued".to_string(), + }), + } +} + +fn validate_activation_checkpoint(checkpoint: &ExecutionRunActivationCheckpoint) -> Result<()> { + if checkpoint.activation_state == ExecutionActivationState::Advancing { + return Err(Error::InvalidRepositoryInput { + message: "a completed controller activation cannot checkpoint as advancing".to_string(), + }); + } + if checkpoint.status.is_terminal() + != (checkpoint.activation_state == ExecutionActivationState::Terminal) + { + return Err(Error::InvalidRepositoryInput { + message: "terminal run status and terminal activation state must agree".to_string(), + }); + } + if checkpoint.activation_state == ExecutionActivationState::Terminal + && (checkpoint.next_wake_at.is_some() + || checkpoint.waiting_since.is_some() + || checkpoint.ready_task_count != 0 + || checkpoint.active_task_count != 0) + { + return Err(Error::InvalidRepositoryInput { + message: "terminal activation checkpoints cannot retain wakes, waits, or task counts" + .to_string(), + }); + } + if checkpoint.status == ExecutionRunStatus::Paused + && (checkpoint.activation_state != ExecutionActivationState::Paused + || checkpoint.active_task_count != 0) + { + return Err(Error::InvalidRepositoryInput { + message: "paused runs must use paused activation state with zero active tasks" + .to_string(), + }); + } + Ok(()) +} + +fn validate_controller_completion(request: &RunControllerCompletionRequest) -> Result<()> { + let continuation = request.continuation_payload.is_some(); + if continuation != (request.checkpoint.activation_state == ExecutionActivationState::Queued) { + return Err(Error::InvalidRepositoryInput { + message: "queued controller checkpoint requires exactly one continuation payload" + .to_string(), + }); + } + if request + .continuation_payload + .as_ref() + .is_some_and(|payload| !payload.is_object()) + { + return Err(Error::InvalidRepositoryInput { + message: "controller continuation payload must be a JSON object".to_string(), + }); + } + Ok(()) +} + +fn activation_checkpoint_matches( + run: &ExecutionRunRecord, + checkpoint: &ExecutionRunActivationCheckpoint, +) -> bool { + run.status == checkpoint.status + && run.activation_state == checkpoint.activation_state + && run.next_wake_at == checkpoint.next_wake_at + && run.waiting_since == checkpoint.waiting_since + && run.ready_task_count == checkpoint.ready_task_count + && run.active_task_count == checkpoint.active_task_count +} + +fn requires_parked_run_capacity(run: &ExecutionRunRecord) -> bool { + requires_parked_capacity_checkpoint( + run.status, + run.activation_state, + run.ready_task_count, + run.active_task_count, + ) +} + +fn requires_parked_capacity_checkpoint( + status: ExecutionRunStatus, + activation_state: ExecutionActivationState, + ready_task_count: u64, + active_task_count: u64, +) -> bool { + activation_state == ExecutionActivationState::Idle + && ready_task_count == 0 + && active_task_count == 0 + && matches!( + status, + ExecutionRunStatus::WaitingInput + | ExecutionRunStatus::WaitingReview + | ExecutionRunStatus::WaitingSignal + | ExecutionRunStatus::WaitingTimer + | ExecutionRunStatus::WaitingExternal + | ExecutionRunStatus::WaitingReplan + ) +} + +pub(super) fn active_run_capacity_request( + tenant_id: TenantId, + run_uid: Uuid, +) -> ExecutionCapacityRequest { + ExecutionCapacityRequest { + reservation_uid: execution_capacity_reservation_uid( + ExecutionCapacityDimension::ActiveRuns, + run_uid, + None, + ), + tenant_id, + run_uid: Some(run_uid), + controller_generation: Some(RUN_LIFETIME_CAPACITY_GENERATION), + dimension: ExecutionCapacityDimension::ActiveRuns, + owner: ExecutionCapacityOwner::Run, + expires_at: None, + } +} + +/// Set-seeds tenant dispatch state and every canonical node inside run admission. +pub(super) async fn seed_run_scheduler_state_in_tx( + conn: &mut PgConnection, + tenant_id: TenantId, + run_uid: Uuid, + plan: &CanonicalExecutionPlan, +) -> Result<()> { + sqlx::query( + "INSERT INTO moa.execution_tenant_dispatch_state (tenant_id) VALUES ($1) \ + ON CONFLICT (tenant_id) DO NOTHING", + ) + .bind(tenant_id.0) + .execute(&mut *conn) + .await + .map_err(sqlx_error)?; + + if plan.definition.nodes.is_empty() { + return Ok(()); + } + let mut node_state_uids = Vec::with_capacity(plan.definition.nodes.len()); + let mut node_ids = Vec::with_capacity(plan.definition.nodes.len()); + let mut node_orders = Vec::with_capacity(plan.definition.nodes.len()); + let mut dependency_counts = Vec::with_capacity(plan.definition.nodes.len()); + for (node_order, node) in plan.definition.nodes.iter().enumerate() { + node_state_uids.push(Uuid::new_v5(&run_uid, node.id.as_bytes())); + node_ids.push(node.id.clone()); + node_orders.push( + i64::try_from(node_order).map_err(|_| Error::InvalidRepositoryInput { + message: "execution node order exceeds PostgreSQL BIGINT".to_string(), + })?, + ); + dependency_counts.push(i64::try_from(node.depends_on.len()).map_err(|_| { + Error::InvalidRepositoryInput { + message: "execution dependency count exceeds PostgreSQL BIGINT".to_string(), + } + })?); + } + let inserted = sqlx::query( + "INSERT INTO moa.execution_node_state (\ + node_state_uid, tenant_id, run_uid, node_id, node_order, \ + dependency_count, remaining_dependency_count\ + ) \ + SELECT seed.node_state_uid, $1, $2, seed.node_id, seed.node_order, \ + seed.dependency_count, seed.dependency_count \ + FROM UNNEST($3::UUID[], $4::TEXT[], $5::BIGINT[], $6::BIGINT[]) AS seed(\ + node_state_uid, node_id, node_order, dependency_count\ + )", + ) + .bind(tenant_id.0) + .bind(run_uid) + .bind(node_state_uids) + .bind(node_ids) + .bind(node_orders) + .bind(dependency_counts) + .execute(&mut *conn) + .await + .map_err(sqlx_error)?; + let expected = + u64::try_from(plan.definition.nodes.len()).map_err(|_| Error::InvalidRepositoryInput { + message: "execution node count exceeds PostgreSQL BIGINT".to_string(), + })?; + if inserted.rows_affected() != expected { + return Err(Error::InvalidRepositoryData { + message: "execution run admission did not seed every canonical plan node".to_string(), + }); + } + Ok(()) } pub(super) fn validate_new_run(scope: ExecutionScope, new_run: &NewExecutionRun) -> Result<()> { @@ -314,6 +1243,11 @@ pub(super) fn validate_new_run(scope: ExecutionScope, new_run: &NewExecutionRun) message: "run owner does not match the repository scope".to_string(), }); } + if new_run.admitted_identity.tenant_id != new_run.tenant_id { + return Err(Error::InvalidRepositoryInput { + message: "admitted identity tenant does not match the execution run tenant".to_string(), + }); + } if !matches!( new_run.status, ExecutionRunStatus::AwaitingConfirmation | ExecutionRunStatus::Queued @@ -399,3 +1333,97 @@ pub(super) fn normalized_source_fields( }, } } + +#[cfg(test)] +mod tests { + use chrono::TimeZone; + + use super::*; + + #[test] + fn continuation_dispatch_identity_is_fenced_by_generation_and_wake() { + // Pins: retrying the same completion addresses one outbox row, while a later wake or + // controller generation can never alias that durable continuation. + let run_uid = Uuid::from_u128(0x11); + let first = run_activation_dispatch_uid(run_uid, 7, 9); + + assert_eq!(first, run_activation_dispatch_uid(run_uid, 7, 9)); + assert_ne!(first, run_activation_dispatch_uid(run_uid, 7, 10)); + assert_ne!(first, run_activation_dispatch_uid(run_uid, 8, 9)); + } + + #[test] + fn run_deadline_trigger_identity_changes_only_with_its_generation_or_deadline() { + // Pins: an amended deadline replaces its old immutable trigger while an exact replay + // resolves to the same trigger tombstone. + let run_uid = Uuid::from_u128(0x22); + let deadline = Utc + .timestamp_opt(1_800_000_000, 0) + .single() + .expect("test timestamp is representable"); + let first = run_deadline_trigger_uid(run_uid, 4, deadline); + + assert_eq!(first, run_deadline_trigger_uid(run_uid, 4, deadline)); + assert_ne!( + first, + run_deadline_trigger_uid(run_uid, 4, deadline + chrono::TimeDelta::seconds(1)) + ); + assert_ne!(first, run_deadline_trigger_uid(run_uid, 5, deadline)); + } + + #[test] + fn queued_checkpoint_requires_exactly_one_continuation_payload() { + // Pins: a queued checkpoint cannot commit without its same-transaction outbox row. + let request = RunControllerCompletionRequest { + controller_generation: 1, + wake_epoch: 2, + checkpoint: ExecutionRunActivationCheckpoint { + status: ExecutionRunStatus::Running, + activation_state: ExecutionActivationState::Queued, + next_wake_at: None, + waiting_since: None, + ready_task_count: 0, + active_task_count: 0, + }, + continuation_payload: None, + continuation_not_before_at: Utc::now(), + }; + + let error = validate_controller_completion(&request) + .expect_err("queued checkpoint without outbox payload must fail"); + assert_eq!( + error.to_string(), + "invalid execution repository request: queued controller checkpoint requires exactly one continuation payload" + ); + } + + #[test] + fn only_zero_compute_storage_waits_reserve_parked_run_capacity() { + // Pins: parked capacity is acquired exactly when the controller can return with no ready + // or active work; queued/running work cannot double-count as a parked run. + assert!(requires_parked_capacity_checkpoint( + ExecutionRunStatus::WaitingTimer, + ExecutionActivationState::Idle, + 0, + 0, + )); + assert!(!requires_parked_capacity_checkpoint( + ExecutionRunStatus::WaitingTimer, + ExecutionActivationState::Idle, + 0, + 1, + )); + assert!(!requires_parked_capacity_checkpoint( + ExecutionRunStatus::Running, + ExecutionActivationState::Idle, + 0, + 0, + )); + assert!(!requires_parked_capacity_checkpoint( + ExecutionRunStatus::WaitingTimer, + ExecutionActivationState::Queued, + 0, + 0, + )); + } +} diff --git a/crates/moa-execution/src/repository/schedule.rs b/crates/moa-execution/src/repository/schedule.rs new file mode 100644 index 000000000..7a080ddd2 --- /dev/null +++ b/crates/moa-execution/src/repository/schedule.rs @@ -0,0 +1,1463 @@ +//! Tenant-scoped recurring execution schedule persistence. + +use chrono::{DateTime, NaiveDateTime, TimeDelta, Utc}; +use moa_artifacts::execution_plan::{ExecutionBudgetLimit, ExecutionGoalContract}; +use moa_config::ExecutionConfig; +use moa_core::types::{ + execution_planning::{ + ExecutionScheduleCreateRequest, ExecutionScheduleDstPolicy, + ExecutionScheduleMissedFirePolicy, ExecutionScheduleOverlapPolicy, ExecutionSchedulePage, + ExecutionSchedulePolicy, ExecutionScheduleRecord, ExecutionScheduleStatus, + ExecutionScheduleTemplate, ExecutionScheduleUpdateRequest, + execution_schedule_occurrence_ids, + }, + identifiers::{SessionId, TenantId, UserId}, +}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use sqlx::{PgConnection, Row, postgres::PgRow}; +use uuid::Uuid; + +use super::{ + DbBudgetLimit, Error, ExecutionRepository, ExecutionScope, NewExecutionRun, Result, + RunDeadlineArmOutcome, + capacity::{ + ActiveRunCapacityReserveOutcome, CapacityReleaseOutcome, ExecutionCapacityDimension, + ExecutionCapacityOwner, ExecutionCapacityRequest, execution_capacity_reservation_uid, + prelock_capacity_dimensions_in_tx, release_capacity_in_tx, + reserve_active_run_capacity_in_tx, + }, + outbox::{ + ExecutionDispatchKind, ExecutionDispatchRecord, NewExecutionDispatch, + enqueue_dispatch_in_conn, + }, + rows::run_from_row, + run::{arm_run_deadline_in_conn, seed_run_scheduler_state_in_tx, validate_new_run}, + sql::CREATE_RUN_SQL, + sqlx_error, storage_error, to_i64, + trigger::{ExecutionTriggerWrite, NewExecutionTrigger, create_trigger_with_dispatch_in_conn}, +}; +use crate::{ + capability::{ExecutionAuthorizationEnvelope, ExecutionCapabilityCatalog, ExecutionHash}, + compiler::CanonicalExecutionPlan, + state::{ExecutionRunStatus, ExecutionSourceKind}, + wire::PinnedInstructionSkill, +}; + +const DEFAULT_PAGE_LIMIT: u32 = 100; +const MAX_PAGE_LIMIT: u32 = 1_000; + +/// Exact UTC/local occurrence pair computed by the wall-clock policy owner. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ExecutionScheduleOccurrence { + /// Absolute delivery instant. + pub at: DateTime, + /// Calendar-local value before timezone resolution. + pub local: NaiveDateTime, +} + +/// Fully compiled and admitted non-occurrence inputs pinned at schedule creation. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ExecutionScheduleRunBlueprint { + /// Parent session that receives occurrence progress and terminal events. + pub session_id: SessionId, + /// Exact persisted user event that originally authorized this recurring objective. + pub originating_user_sequence_num: u64, + /// Immutable planning context admitted when the schedule was created. + pub planning_context_uid: Uuid, + /// Canonical hash of the immutable planning context. + pub planning_context_hash: ExecutionHash, + /// Immutable recurring goal contract. + pub goal: ExecutionGoalContract, + /// Fully compiled canonical execution plan. + pub plan: CanonicalExecutionPlan, + /// Exact immutable capability catalog used by compilation. + pub catalog: ExecutionCapabilityCatalog, + /// Exact immutable authorization envelope. + pub authorization: ExecutionAuthorizationEnvelope, + /// Sorted pinned instruction-skill revisions. + pub pinned_instruction_skills: Vec, + /// Exact pinned-template provenance. + pub source_provenance: moa_core::types::execution_planning::ExecutionSourceProvenance, + /// Structured input copied into every fresh occurrence run. + pub input: Value, + /// Approved resource envelope copied into every fresh occurrence run. + pub approved_budget: ExecutionBudgetLimit, + /// Optional per-occurrence deadline offset, capped by deployment maximum horizon. + pub deadline_offset_seconds: Option, +} + +impl ExecutionScheduleRunBlueprint { + /// Builds one fresh queued run from this immutable blueprint and occurrence tuple. + pub fn instantiate( + &self, + schedule: &ExecutionScheduleRecord, + occurrence: ExecutionScheduleOccurrence, + occurrence_sequence: u64, + maximum_horizon_seconds: u64, + ) -> Result { + let owner_id = schedule + .run_as_identity + .acting_on_behalf_of + .unwrap_or(schedule.run_as_identity.id); + if maximum_horizon_seconds == 0 { + return Err(Error::InvalidRepositoryInput { + message: "schedule maximum horizon must be positive".to_string(), + }); + } + let deadline_seconds = self + .deadline_offset_seconds + .unwrap_or(maximum_horizon_seconds) + .min(maximum_horizon_seconds); + if deadline_seconds == 0 { + return Err(Error::InvalidRepositoryInput { + message: "schedule deadline offset must be positive".to_string(), + }); + } + let mut approved_budget = self.approved_budget.clone(); + let deadline_delta = i64::try_from(deadline_seconds) + .ok() + .and_then(TimeDelta::try_seconds) + .ok_or_else(|| Error::InvalidRepositoryInput { + message: "schedule deadline offset exceeds chrono bounds".to_string(), + })?; + approved_budget.deadline_at = Some( + occurrence + .at + .checked_add_signed(deadline_delta) + .ok_or_else(|| Error::InvalidRepositoryInput { + message: "schedule occurrence deadline exceeds timestamp bounds".to_string(), + })?, + ); + Ok(NewExecutionRun { + tenant_id: schedule.tenant_id, + contact_id: None, + session_id: self.session_id, + originating_user_sequence_num: self.originating_user_sequence_num, + planning_context_uid: self.planning_context_uid, + planning_context_hash: self.planning_context_hash, + owner_user_id: UserId::new(owner_id.to_string()), + admitted_identity: schedule.run_as_identity.clone(), + goal: self.goal.clone(), + plan: self.plan.clone(), + catalog: self.catalog.clone(), + authorization: self.authorization.clone(), + pinned_instruction_skills: self.pinned_instruction_skills.clone(), + source_provenance: self.source_provenance.clone(), + input: self.input.clone(), + status: ExecutionRunStatus::Queued, + approved_budget, + idempotency_key: Some(format!( + "schedule:{}:{}:{occurrence_sequence}", + schedule.schedule_uid, schedule.schedule_incarnation + )), + }) + } +} + +/// Decodes and revalidates the fully admitted run blueprint pinned on a schedule row. +pub fn execution_schedule_run_blueprint( + schedule: &ExecutionScheduleRecord, +) -> Result { + let blueprint: ExecutionScheduleRunBlueprint = + serde_json::from_value(schedule.template.snapshot.clone()).map_err(|error| { + Error::InvalidRepositoryData { + message: format!("invalid persisted scheduled run blueprint: {error}"), + } + })?; + if serde_json::to_value(&blueprint.approved_budget)? != schedule.policy.occurrence_budget + || blueprint.approved_budget.deadline_at.is_some() + || blueprint.deadline_offset_seconds == Some(0) + || scheduled_blueprint_revision(&blueprint) != Some(schedule.template.revision_uid) + { + return Err(Error::InvalidRepositoryData { + message: "persisted scheduled blueprint drifted from its budget or template revision" + .to_string(), + }); + } + Ok(blueprint) +} + +/// Result of a replay-safe schedule creation. +#[derive(Clone, Debug, PartialEq)] +pub enum ExecutionScheduleCreateOutcome { + /// A new schedule and its first trigger were committed. + Created { + /// Persisted schedule. + schedule: Box, + /// First delayed trigger, if the schedule has an occurrence in bounds. + trigger: Option>, + }, + /// The exact immutable creation was already committed. + Replayed(Box), + /// The schedule ID is bound to different immutable bytes. + Conflict, +} + +/// Result of a fenced schedule control mutation. +#[derive(Clone, Debug, PartialEq)] +pub enum ExecutionScheduleMutationOutcome { + /// The exact requested state was committed. + Updated { + /// Current persisted schedule. + schedule: Box, + /// Newly armed trigger, when the resulting state is active. + trigger: Option>, + }, + /// The target schedule does not exist in the tenant scope. + NotFound, + /// The expected incarnation or source lifecycle state was stale. + Stale, +} + +/// Complete fresh-run admission for one due immutable occurrence. +pub struct ExecutionScheduleRunAdmission { + /// Tenant-owned schedule. + pub tenant_id: TenantId, + /// Target schedule. + pub schedule_uid: Uuid, + /// Exact armed schedule incarnation. + pub schedule_incarnation: u64, + /// Exact occurrence sequence within the incarnation. + pub occurrence_sequence: u64, + /// Exact delayed trigger being consumed. + pub trigger_uid: Uuid, + /// Exact trigger-delivery outbox identity. + pub trigger_dispatch_uid: Uuid, + /// Due occurrence being consumed. + pub occurrence: ExecutionScheduleOccurrence, + /// Fresh fully admitted run snapshot built from the pinned template. + pub run: NewExecutionRun, + /// Next occurrence, or none when the schedule completed. + pub next_occurrence: Option, +} + +/// Transactional result of consuming one schedule occurrence. +#[derive(Clone, Debug, PartialEq)] +pub enum ExecutionScheduleRunAdmissionOutcome { + /// A fresh run and its initial controller activation were committed. + Admitted { + /// Deterministic fresh run. + run: Box, + /// Initial bounded controller activation. + activation: Box, + /// Next delayed occurrence, when one remains. + next_trigger: Option>, + }, + /// Overlap/concurrency policy deliberately omitted this occurrence. + Skipped { + /// Updated schedule with the occurrence consumed. + schedule: Box, + /// Next delayed occurrence, when one remains. + next_trigger: Option>, + }, + /// The same deterministic occurrence already committed. + Replayed { + /// Fresh run identity when the original occurrence was admitted. + run_uid: Option, + /// Initial controller activation when the original occurrence was admitted. + activation_dispatch_uid: Option, + }, + /// The schedule, incarnation, sequence, or due instant was stale. + Stale, +} + +impl ExecutionRepository { + /// Creates one immutable tenant schedule and atomically arms its first occurrence. + pub async fn create_schedule( + &self, + scope: ExecutionScope, + config: &ExecutionConfig, + request: ExecutionScheduleCreateRequest, + first_occurrence: Option, + ) -> Result { + request + .validate() + .map_err(|error| Error::InvalidRepositoryInput { + message: error.to_string(), + })?; + require_tenant_scope(scope, request.tenant_id)?; + validate_occurrence_in_policy(first_occurrence, &request.policy)?; + validate_blueprint(&request)?; + let owner_user_id = request + .run_as_identity + .acting_on_behalf_of + .unwrap_or(request.run_as_identity.id) + .to_string(); + let template = serde_json::to_value(&request.template.snapshot)?; + let run_as = serde_json::to_value(&request.run_as_identity)?; + let origin = serde_json::to_value(&request.origin)?; + let status = if first_occurrence.is_some() { + ExecutionScheduleStatus::Active + } else { + ExecutionScheduleStatus::Completed + }; + let mut conn = scope.begin(&self.pool).await?; + prelock_capacity_dimensions_in_tx( + conn.as_mut(), + config, + request.tenant_id, + &[ExecutionCapacityDimension::ScheduledTriggers], + ) + .await?; + let inserted = sqlx::query( + r#" + INSERT INTO moa.execution_schedule ( + schedule_uid, tenant_id, owner_user_id, name, timezone, + calendar_expression, template_revision_uid, template_snapshot, + template_hash, run_as_identity, creation_origin, status, + missed_fire_policy, overlap_policy, dst_policy, + maximum_concurrent_runs, occurrence_budget, schedule_incarnation, + start_at, next_occurrence_at, next_occurrence_local, end_at + ) VALUES ( + $1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17, + 1,$18,$19,$20,$21 + ) + ON CONFLICT (schedule_uid) DO NOTHING + RETURNING * + "#, + ) + .bind(request.schedule_uid) + .bind(request.tenant_id.0) + .bind(owner_user_id) + .bind(&request.name) + .bind(&request.policy.timezone) + .bind(&request.policy.calendar_expression) + .bind(request.template.revision_uid) + .bind(template) + .bind(&request.template.template_hash) + .bind(run_as) + .bind(origin) + .bind(status.as_str()) + .bind(request.policy.missed_fire_policy.as_str()) + .bind(request.policy.overlap_policy.as_str()) + .bind(request.policy.dst_policy.as_str()) + .bind(to_i64( + request.policy.maximum_concurrent_runs, + "maximum concurrent runs", + )?) + .bind(&request.policy.occurrence_budget) + .bind(request.policy.start_at) + .bind(first_occurrence.map(|occurrence| occurrence.at)) + .bind(first_occurrence.map(|occurrence| occurrence.local)) + .bind(request.policy.end_at) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(row) = inserted else { + let existing = load_schedule_in_conn(conn.as_mut(), request.schedule_uid).await?; + conn.commit().await.map_err(storage_error)?; + return Ok(match existing { + Some(existing) if schedule_matches_create(&existing, &request) => { + ExecutionScheduleCreateOutcome::Replayed(Box::new(existing)) + } + Some(_) | None => ExecutionScheduleCreateOutcome::Conflict, + }); + }; + let schedule = schedule_from_row(&row)?; + let trigger = + arm_occurrence_in_conn(conn.as_mut(), config, &schedule, first_occurrence, 1).await?; + conn.commit().await.map_err(storage_error)?; + Ok(ExecutionScheduleCreateOutcome::Created { + schedule: Box::new(schedule), + trigger: trigger.map(Box::new), + }) + } + + /// Loads one visible tenant schedule. + pub async fn load_schedule( + &self, + scope: ExecutionScope, + tenant_id: TenantId, + schedule_uid: Uuid, + ) -> Result> { + require_tenant_scope(scope, tenant_id)?; + let mut conn = scope.begin(&self.pool).await?; + let record = load_schedule_in_conn(conn.as_mut(), schedule_uid).await?; + conn.commit().await.map_err(storage_error)?; + Ok(record) + } + + /// Lists one stable bounded page of visible tenant schedules. + pub async fn list_schedules( + &self, + scope: ExecutionScope, + tenant_id: TenantId, + limit: u32, + cursor: Option, + ) -> Result { + require_tenant_scope(scope, tenant_id)?; + let limit = if limit == 0 { + DEFAULT_PAGE_LIMIT + } else { + limit.min(MAX_PAGE_LIMIT) + }; + let mut conn = scope.begin(&self.pool).await?; + let rows = sqlx::query( + "SELECT * FROM moa.execution_schedule WHERE ($1::UUID IS NULL OR schedule_uid > $1) \ + ORDER BY schedule_uid LIMIT $2", + ) + .bind(cursor) + .bind(i64::from(limit) + 1) + .fetch_all(conn.as_mut()) + .await + .map_err(sqlx_error)?; + conn.commit().await.map_err(storage_error)?; + let mut schedules = rows + .iter() + .map(schedule_from_row) + .collect::>>()?; + let has_more = schedules.len() > limit as usize; + if has_more { + let _ = schedules.pop(); + } + let next_cursor = has_more + .then(|| schedules.last().map(|schedule| schedule.schedule_uid)) + .flatten(); + Ok(ExecutionSchedulePage { + schedules, + next_cursor, + }) + } + + /// Pauses an active schedule, cancels its armed occurrence, and advances its fence. + pub async fn pause_schedule( + &self, + scope: ExecutionScope, + config: &ExecutionConfig, + tenant_id: TenantId, + schedule_uid: Uuid, + ) -> Result { + mutate_lifecycle( + self, + scope, + config, + ScheduleLifecycleMutation { + tenant_id, + schedule_uid, + expected_status: "active", + new_status: "paused", + next_occurrence: None, + }, + ) + .await + } + + /// Resumes a paused schedule and atomically arms its next occurrence. + pub async fn resume_schedule( + &self, + scope: ExecutionScope, + config: &ExecutionConfig, + tenant_id: TenantId, + schedule_uid: Uuid, + next_occurrence: Option, + ) -> Result { + mutate_lifecycle( + self, + scope, + config, + ScheduleLifecycleMutation { + tenant_id, + schedule_uid, + expected_status: "paused", + new_status: if next_occurrence.is_some() { + "active" + } else { + "completed" + }, + next_occurrence, + }, + ) + .await + } + + /// Permanently fences future occurrences while retaining schedule audit state. + pub async fn cancel_schedule( + &self, + scope: ExecutionScope, + config: &ExecutionConfig, + tenant_id: TenantId, + schedule_uid: Uuid, + ) -> Result { + require_tenant_scope(scope, tenant_id)?; + let mut conn = scope.begin(&self.pool).await?; + prelock_capacity_dimensions_in_tx( + conn.as_mut(), + config, + tenant_id, + &[ExecutionCapacityDimension::ScheduledTriggers], + ) + .await?; + let Some(current) = lock_schedule_in_conn(conn.as_mut(), schedule_uid).await? else { + conn.commit().await.map_err(storage_error)?; + return Ok(ExecutionScheduleMutationOutcome::NotFound); + }; + if matches!( + current.status, + ExecutionScheduleStatus::Cancelled | ExecutionScheduleStatus::Completed + ) { + conn.commit().await.map_err(storage_error)?; + return Ok(ExecutionScheduleMutationOutcome::Stale); + } + cancel_armed_occurrences(conn.as_mut(), ¤t).await?; + let row = sqlx::query( + "UPDATE moa.execution_schedule SET status='cancelled', \ + schedule_incarnation=schedule_incarnation+1, last_occurrence_sequence=0, \ + next_occurrence_at=NULL, next_occurrence_local=NULL, paused_at=NULL, updated_at=now() \ + WHERE schedule_uid=$1 RETURNING *", + ) + .bind(schedule_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let schedule = schedule_from_row(&row)?; + conn.commit().await.map_err(storage_error)?; + Ok(ExecutionScheduleMutationOutcome::Updated { + schedule: Box::new(schedule), + trigger: None, + }) + } + + /// Replaces mutable schedule policy behind an exact incarnation fence. + pub async fn update_schedule( + &self, + scope: ExecutionScope, + config: &ExecutionConfig, + request: ExecutionScheduleUpdateRequest, + next_occurrence: Option, + ) -> Result { + request + .validate() + .map_err(|error| Error::InvalidRepositoryInput { + message: error.to_string(), + })?; + require_tenant_scope(scope, request.tenant_id)?; + validate_occurrence_in_policy(next_occurrence, &request.policy)?; + let mut conn = scope.begin(&self.pool).await?; + prelock_capacity_dimensions_in_tx( + conn.as_mut(), + config, + request.tenant_id, + &[ExecutionCapacityDimension::ScheduledTriggers], + ) + .await?; + let Some(current) = lock_schedule_in_conn(conn.as_mut(), request.schedule_uid).await? + else { + conn.commit().await.map_err(storage_error)?; + return Ok(ExecutionScheduleMutationOutcome::NotFound); + }; + if current.schedule_incarnation != request.expected_incarnation + || matches!( + current.status, + ExecutionScheduleStatus::Completed | ExecutionScheduleStatus::Cancelled + ) + { + conn.commit().await.map_err(storage_error)?; + return Ok(ExecutionScheduleMutationOutcome::Stale); + } + cancel_armed_occurrences(conn.as_mut(), ¤t).await?; + let new_status = if current.status == ExecutionScheduleStatus::Paused { + ExecutionScheduleStatus::Paused + } else if next_occurrence.is_some() { + ExecutionScheduleStatus::Active + } else { + ExecutionScheduleStatus::Completed + }; + let row = sqlx::query( + r#" + UPDATE moa.execution_schedule + SET name=$2, timezone=$3, calendar_expression=$4, + missed_fire_policy=$5, overlap_policy=$6, dst_policy=$7, + maximum_concurrent_runs=$8, occurrence_budget=$9, + start_at=$10, end_at=$11, status=$12, + schedule_incarnation=schedule_incarnation+1, + last_occurrence_sequence=0, next_occurrence_at=$13, + next_occurrence_local=$14, updated_at=now() + WHERE schedule_uid=$1 + RETURNING * + "#, + ) + .bind(request.schedule_uid) + .bind(&request.name) + .bind(&request.policy.timezone) + .bind(&request.policy.calendar_expression) + .bind(request.policy.missed_fire_policy.as_str()) + .bind(request.policy.overlap_policy.as_str()) + .bind(request.policy.dst_policy.as_str()) + .bind(to_i64( + request.policy.maximum_concurrent_runs, + "maximum concurrent runs", + )?) + .bind(&request.policy.occurrence_budget) + .bind(request.policy.start_at) + .bind(request.policy.end_at) + .bind(new_status.as_str()) + .bind(next_occurrence.map(|occurrence| occurrence.at)) + .bind(next_occurrence.map(|occurrence| occurrence.local)) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let schedule = schedule_from_row(&row)?; + let trigger = if schedule.status == ExecutionScheduleStatus::Active { + arm_occurrence_in_conn(conn.as_mut(), config, &schedule, next_occurrence, 1).await? + } else { + None + }; + conn.commit().await.map_err(storage_error)?; + Ok(ExecutionScheduleMutationOutcome::Updated { + schedule: Box::new(schedule), + trigger: trigger.map(Box::new), + }) + } + + /// Atomically consumes one occurrence, creates its fresh run/activation, and arms the next. + pub async fn admit_schedule_occurrence( + &self, + scope: ExecutionScope, + config: &ExecutionConfig, + request: ExecutionScheduleRunAdmission, + ) -> Result { + require_tenant_scope(scope, request.tenant_id)?; + validate_new_run(scope, &request.run)?; + if request.run.status != ExecutionRunStatus::Queued + || request.run.tenant_id != request.tenant_id + || request.run.contact_id.is_some() + || request.occurrence_sequence == 0 + { + return Err(Error::InvalidRepositoryInput { + message: "schedule occurrences require a fresh queued tenant-owned run".to_string(), + }); + } + let mut conn = scope.begin(&self.pool).await?; + prelock_capacity_dimensions_in_tx( + conn.as_mut(), + config, + request.tenant_id, + &[ + ExecutionCapacityDimension::ActiveRuns, + ExecutionCapacityDimension::ParkedRuns, + ExecutionCapacityDimension::ScheduledTriggers, + ], + ) + .await?; + let dispatch_matches = sqlx::query_scalar::<_, bool>( + "SELECT EXISTS (SELECT 1 FROM moa.execution_dispatch_outbox \ + WHERE dispatch_uid=$1 AND tenant_id=$2 AND trigger_uid=$3 \ + AND dispatch_kind='trigger_delivery')", + ) + .bind(request.trigger_dispatch_uid) + .bind(request.tenant_id.0) + .bind(request.trigger_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if !dispatch_matches { + conn.commit().await.map_err(storage_error)?; + return Ok(ExecutionScheduleRunAdmissionOutcome::Stale); + } + match super::trigger::fire_trigger_in_conn(conn.as_mut(), request.trigger_uid).await? { + super::trigger::ExecutionTriggerFireOutcome::Delivered { activation: None } => {} + super::trigger::ExecutionTriggerFireOutcome::Delivered { + activation: Some(_), + } => { + return Err(Error::InvalidRepositoryData { + message: "schedule occurrence trigger unexpectedly owned a run activation" + .to_string(), + }); + } + super::trigger::ExecutionTriggerFireOutcome::NoOp( + super::trigger::ExecutionTriggerNoOp::Duplicate, + ) => { + let replay = load_occurrence_replay_in_conn( + conn.as_mut(), + request.schedule_uid, + request.schedule_incarnation, + request.occurrence_sequence, + ) + .await?; + conn.commit().await.map_err(storage_error)?; + return Ok(replay); + } + super::trigger::ExecutionTriggerFireOutcome::NoOp(_) => { + conn.commit().await.map_err(storage_error)?; + return Ok(ExecutionScheduleRunAdmissionOutcome::Stale); + } + } + let Some(schedule) = lock_schedule_in_conn(conn.as_mut(), request.schedule_uid).await? + else { + conn.commit().await.map_err(storage_error)?; + return Ok(ExecutionScheduleRunAdmissionOutcome::Stale); + }; + let mut occurrence_budget = request.run.approved_budget.clone(); + occurrence_budget.deadline_at = None; + if schedule.status != ExecutionScheduleStatus::Active + || schedule.schedule_incarnation != request.schedule_incarnation + || schedule.last_occurrence_sequence + 1 != request.occurrence_sequence + || schedule.next_occurrence_at != Some(request.occurrence.at) + || schedule.next_occurrence_local != Some(request.occurrence.local) + || schedule.run_as_identity != request.run.admitted_identity + || serde_json::to_value(&occurrence_budget)? != schedule.policy.occurrence_budget + || scheduled_template_revision(&request.run) != Some(schedule.template.revision_uid) + { + return Err(Error::Storage { + message: "schedule occurrence changed after its trigger currentness check" + .to_string(), + }); + } + let trigger_dispatch_matches = sqlx::query_scalar::<_, bool>( + "SELECT EXISTS (SELECT 1 FROM moa.execution_dispatch_outbox \ + WHERE dispatch_uid=$1 AND tenant_id=$2 AND trigger_uid=$3 \ + AND dispatch_kind='trigger_delivery' AND state='delivered')", + ) + .bind(request.trigger_dispatch_uid) + .bind(request.tenant_id.0) + .bind(request.trigger_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if !trigger_dispatch_matches { + return Err(Error::Storage { + message: "schedule occurrence trigger dispatch was not settled atomically" + .to_string(), + }); + } + let skip = schedule_overlap_limit_reached_in_conn( + conn.as_mut(), + request.tenant_id, + request.schedule_uid, + schedule.policy.overlap_policy, + schedule.policy.maximum_concurrent_runs, + ) + .await?; + let next_trigger = advance_schedule_after_occurrence( + conn.as_mut(), + config, + &schedule, + request.occurrence_sequence, + request.next_occurrence, + ) + .await?; + if skip { + let updated = load_schedule_in_conn(conn.as_mut(), request.schedule_uid) + .await? + .ok_or_else(|| Error::Storage { + message: "schedule disappeared while consuming occurrence".to_string(), + })?; + conn.commit().await.map_err(storage_error)?; + return Ok(ExecutionScheduleRunAdmissionOutcome::Skipped { + schedule: Box::new(updated), + next_trigger: next_trigger.map(Box::new), + }); + } + let ids = execution_schedule_occurrence_ids( + request.schedule_uid, + request.schedule_incarnation, + request.occurrence_sequence, + ); + let run = insert_occurrence_run_in_conn( + conn.as_mut(), + ids.run_uid, + request.schedule_uid, + request.schedule_incarnation, + request.occurrence_sequence, + &request.run, + ) + .await?; + seed_run_scheduler_state_in_tx( + conn.as_mut(), + request.tenant_id, + ids.run_uid, + &request.run.plan, + ) + .await?; + let active_run_reservation_uid = execution_capacity_reservation_uid( + ExecutionCapacityDimension::ActiveRuns, + ids.run_uid, + None, + ); + let active_run_capacity = ExecutionCapacityRequest { + reservation_uid: active_run_reservation_uid, + tenant_id: request.tenant_id, + run_uid: Some(ids.run_uid), + controller_generation: Some(1), + dimension: ExecutionCapacityDimension::ActiveRuns, + owner: ExecutionCapacityOwner::Run, + expires_at: None, + }; + match reserve_active_run_capacity_in_tx(conn.as_mut(), config, active_run_capacity).await? { + ActiveRunCapacityReserveOutcome::Reserved + | ActiveRunCapacityReserveOutcome::Replayed => {} + ActiveRunCapacityReserveOutcome::Saturated(_) => { + sqlx::query("DELETE FROM moa.execution_run WHERE run_uid=$1") + .bind(ids.run_uid) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let updated = load_schedule_in_conn(conn.as_mut(), request.schedule_uid) + .await? + .ok_or_else(|| Error::Storage { + message: "schedule disappeared after resident-run saturation".to_string(), + })?; + conn.commit().await.map_err(storage_error)?; + return Ok(ExecutionScheduleRunAdmissionOutcome::Skipped { + schedule: Box::new(updated), + next_trigger: next_trigger.map(Box::new), + }); + } + } + match arm_run_deadline_in_conn(conn.as_mut(), config, &run).await { + Ok( + RunDeadlineArmOutcome::Armed(_) + | RunDeadlineArmOutcome::NoDeadline + | RunDeadlineArmOutcome::Terminal, + ) => {} + Ok(RunDeadlineArmOutcome::NotFound | RunDeadlineArmOutcome::StaleGeneration { .. }) => { + return Err(Error::InvalidRepositoryData { + message: "new schedule occurrence run lost its deadline arm fence".to_string(), + }); + } + Err(Error::CapacitySaturated { dimension }) + if dimension == ExecutionCapacityDimension::ScheduledTriggers.as_str() => + { + match release_capacity_in_tx(conn.as_mut(), active_run_capacity).await? { + CapacityReleaseOutcome::Released | CapacityReleaseOutcome::AlreadyReleased => {} + CapacityReleaseOutcome::NotFound | CapacityReleaseOutcome::Stale => { + return Err(Error::InvalidRepositoryData { + message: "schedule occurrence lost its active-run capacity receipt" + .to_string(), + }); + } + } + sqlx::query("DELETE FROM moa.execution_run WHERE run_uid=$1") + .bind(ids.run_uid) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let updated = load_schedule_in_conn(conn.as_mut(), request.schedule_uid) + .await? + .ok_or_else(|| Error::Storage { + message: "schedule disappeared after deadline saturation".to_string(), + })?; + conn.commit().await.map_err(storage_error)?; + return Ok(ExecutionScheduleRunAdmissionOutcome::Skipped { + schedule: Box::new(updated), + next_trigger: next_trigger.map(Box::new), + }); + } + Err(error) => return Err(error), + } + let activation = enqueue_dispatch_in_conn( + conn.as_mut(), + &NewExecutionDispatch { + dispatch_uid: ids.activation_dispatch_uid, + tenant_id: request.tenant_id, + run_uid: Some(ids.run_uid), + task_id: None, + compensation_id: None, + trigger_uid: None, + external_job_uid: None, + kind: ExecutionDispatchKind::RunActivation, + controller_generation: Some(1), + wake_epoch: Some(1), + attempt_generation: None, + compensation_generation: None, + compensation_attempt_generation: None, + not_before_at: Utc::now(), + payload: json!({"reason":"schedule_occurrence"}), + }, + ) + .await?; + conn.commit().await.map_err(storage_error)?; + Ok(ExecutionScheduleRunAdmissionOutcome::Admitted { + run: Box::new(run), + activation: Box::new(activation), + next_trigger: next_trigger.map(Box::new), + }) + } +} + +struct ScheduleLifecycleMutation<'a> { + tenant_id: TenantId, + schedule_uid: Uuid, + expected_status: &'a str, + new_status: &'a str, + next_occurrence: Option, +} + +async fn mutate_lifecycle( + repository: &ExecutionRepository, + scope: ExecutionScope, + config: &ExecutionConfig, + mutation: ScheduleLifecycleMutation<'_>, +) -> Result { + require_tenant_scope(scope, mutation.tenant_id)?; + let mut conn = scope.begin(&repository.pool).await?; + prelock_capacity_dimensions_in_tx( + conn.as_mut(), + config, + mutation.tenant_id, + &[ExecutionCapacityDimension::ScheduledTriggers], + ) + .await?; + let Some(current) = lock_schedule_in_conn(conn.as_mut(), mutation.schedule_uid).await? else { + conn.commit().await.map_err(storage_error)?; + return Ok(ExecutionScheduleMutationOutcome::NotFound); + }; + if current.status.as_str() != mutation.expected_status { + conn.commit().await.map_err(storage_error)?; + return Ok(ExecutionScheduleMutationOutcome::Stale); + } + validate_occurrence_in_policy(mutation.next_occurrence, ¤t.policy)?; + cancel_armed_occurrences(conn.as_mut(), ¤t).await?; + let row = sqlx::query( + "UPDATE moa.execution_schedule SET status=$2, \ + schedule_incarnation=schedule_incarnation+1, last_occurrence_sequence=0, \ + next_occurrence_at=$3, next_occurrence_local=$4, \ + paused_at=CASE WHEN $2='paused' THEN now() ELSE NULL END, updated_at=now() \ + WHERE schedule_uid=$1 RETURNING *", + ) + .bind(mutation.schedule_uid) + .bind(mutation.new_status) + .bind(mutation.next_occurrence.map(|occurrence| occurrence.at)) + .bind(mutation.next_occurrence.map(|occurrence| occurrence.local)) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let schedule = schedule_from_row(&row)?; + let trigger = arm_occurrence_in_conn( + conn.as_mut(), + config, + &schedule, + mutation.next_occurrence, + 1, + ) + .await?; + conn.commit().await.map_err(storage_error)?; + Ok(ExecutionScheduleMutationOutcome::Updated { + schedule: Box::new(schedule), + trigger: trigger.map(Box::new), + }) +} + +async fn arm_occurrence_in_conn( + conn: &mut PgConnection, + config: &ExecutionConfig, + schedule: &ExecutionScheduleRecord, + occurrence: Option, + sequence: u64, +) -> Result> { + let Some(occurrence) = occurrence else { + return Ok(None); + }; + let ids = execution_schedule_occurrence_ids( + schedule.schedule_uid, + schedule.schedule_incarnation, + sequence, + ); + create_trigger_with_dispatch_in_conn( + conn, + config, + &NewExecutionTrigger { + trigger_uid: ids.trigger_uid, + tenant_id: schedule.tenant_id, + run_uid: None, + task_id: None, + compensation_id: None, + schedule_uid: Some(schedule.schedule_uid), + kind: super::trigger::ExecutionTriggerKind::ScheduleOccurrence, + controller_generation: None, + attempt_generation: None, + compensation_generation: None, + compensation_attempt_generation: None, + schedule_incarnation: Some(schedule.schedule_incarnation), + occurrence_sequence: Some(sequence), + due_at: occurrence.at, + payload: json!({ + "schedule_incarnation": schedule.schedule_incarnation, + "occurrence_sequence": sequence, + "occurrence_local": occurrence.local, + "timezone": schedule.policy.timezone, + }), + }, + ) + .await + .map(Some) +} + +async fn advance_schedule_after_occurrence( + conn: &mut PgConnection, + config: &ExecutionConfig, + schedule: &ExecutionScheduleRecord, + sequence: u64, + next: Option, +) -> Result> { + validate_occurrence_in_policy(next, &schedule.policy)?; + let status = if next.is_some() { + "active" + } else { + "completed" + }; + let row = sqlx::query( + "UPDATE moa.execution_schedule SET last_occurrence_sequence=$2, status=$3, \ + next_occurrence_at=$4, next_occurrence_local=$5, updated_at=now() \ + WHERE schedule_uid=$1 AND schedule_incarnation=$6 RETURNING *", + ) + .bind(schedule.schedule_uid) + .bind(to_i64(sequence, "occurrence sequence")?) + .bind(status) + .bind(next.map(|occurrence| occurrence.at)) + .bind(next.map(|occurrence| occurrence.local)) + .bind(to_i64( + schedule.schedule_incarnation, + "schedule incarnation", + )?) + .fetch_one(&mut *conn) + .await + .map_err(sqlx_error)?; + let updated = schedule_from_row(&row)?; + arm_occurrence_in_conn(conn, config, &updated, next, sequence + 1).await +} + +async fn cancel_armed_occurrences( + conn: &mut PgConnection, + schedule: &ExecutionScheduleRecord, +) -> Result<()> { + let trigger_uids = sqlx::query_scalar::<_, Uuid>( + "SELECT trigger_uid FROM moa.execution_trigger \ + WHERE schedule_uid=$1 AND schedule_incarnation=$2 \ + AND trigger_kind='schedule_occurrence' AND state IN ('pending','dispatching') \ + ORDER BY trigger_uid LIMIT 2 FOR UPDATE", + ) + .bind(schedule.schedule_uid) + .bind(to_i64( + schedule.schedule_incarnation, + "schedule incarnation", + )?) + .fetch_all(&mut *conn) + .await + .map_err(sqlx_error)?; + if trigger_uids.len() > 1 { + return Err(Error::InvalidRepositoryData { + message: "schedule incarnation owns more than one armed occurrence".to_string(), + }); + } + for trigger_uid in trigger_uids { + super::trigger::supersede_trigger_in_conn( + conn, + trigger_uid, + super::trigger::ExecutionTriggerKind::ScheduleOccurrence, + None, + None, + None, + None, + ) + .await?; + } + Ok(()) +} + +async fn load_schedule_in_conn( + conn: &mut PgConnection, + schedule_uid: Uuid, +) -> Result> { + sqlx::query("SELECT * FROM moa.execution_schedule WHERE schedule_uid=$1") + .bind(schedule_uid) + .fetch_optional(conn) + .await + .map_err(sqlx_error)? + .as_ref() + .map(schedule_from_row) + .transpose() +} + +async fn lock_schedule_in_conn( + conn: &mut PgConnection, + schedule_uid: Uuid, +) -> Result> { + sqlx::query("SELECT * FROM moa.execution_schedule WHERE schedule_uid=$1 FOR UPDATE") + .bind(schedule_uid) + .fetch_optional(conn) + .await + .map_err(sqlx_error)? + .as_ref() + .map(schedule_from_row) + .transpose() +} + +fn schedule_from_row(row: &PgRow) -> Result { + macro_rules! get { + ($column:literal) => { + row.try_get($column).map_err(super::row_error)? + }; + } + let status_label: String = get!("status"); + let missed_fire_label: String = get!("missed_fire_policy"); + let overlap_label: String = get!("overlap_policy"); + let dst_label: String = get!("dst_policy"); + let tenant_uuid: Uuid = get!("tenant_id"); + let status = parse_schedule_status(&status_label)?; + Ok(ExecutionScheduleRecord { + schedule_uid: get!("schedule_uid"), + tenant_id: TenantId::from(tenant_uuid), + name: get!("name"), + template: ExecutionScheduleTemplate { + revision_uid: get!("template_revision_uid"), + template_hash: get!("template_hash"), + snapshot: get!("template_snapshot"), + }, + run_as_identity: serde_json::from_value(get!("run_as_identity"))?, + origin: serde_json::from_value(get!("creation_origin"))?, + policy: ExecutionSchedulePolicy { + timezone: get!("timezone"), + calendar_expression: get!("calendar_expression"), + start_at: get!("start_at"), + end_at: get!("end_at"), + missed_fire_policy: parse_missed_fire_policy(&missed_fire_label)?, + overlap_policy: parse_overlap_policy(&overlap_label)?, + dst_policy: parse_dst_policy(&dst_label)?, + maximum_concurrent_runs: super::to_u64( + get!("maximum_concurrent_runs"), + "maximum concurrent runs", + )?, + occurrence_budget: get!("occurrence_budget"), + }, + status, + schedule_incarnation: super::to_u64(get!("schedule_incarnation"), "schedule incarnation")?, + last_occurrence_sequence: super::to_u64( + get!("last_occurrence_sequence"), + "last occurrence sequence", + )?, + next_occurrence_at: get!("next_occurrence_at"), + next_occurrence_local: get!("next_occurrence_local"), + paused_at: get!("paused_at"), + created_at: get!("created_at"), + updated_at: get!("updated_at"), + }) +} + +async fn insert_occurrence_run_in_conn( + conn: &mut PgConnection, + run_uid: Uuid, + schedule_uid: Uuid, + schedule_incarnation: u64, + occurrence_sequence: u64, + new_run: &NewExecutionRun, +) -> Result { + let budget = DbBudgetLimit::try_from(&new_run.approved_budget)?; + let plan = serde_json::to_value(&new_run.plan)?; + let source = match &new_run.source_provenance { + moa_core::types::execution_planning::ExecutionSourceProvenance::GeneratedPlan { + .. + } => (ExecutionSourceKind::GeneratedPlan, None, None), + moa_core::types::execution_planning::ExecutionSourceProvenance::SkillTemplate { + skill_template_ref, + skill_template_revision_uid, + } => ( + ExecutionSourceKind::SkillTemplate, + Some(skill_template_ref.as_str()), + Some(*skill_template_revision_uid), + ), + moa_core::types::execution_planning::ExecutionSourceProvenance::ExperimentTemplate { + skill_template_ref, + skill_template_revision_uid, + .. + } => ( + ExecutionSourceKind::ExperimentTemplate, + Some(skill_template_ref.as_str()), + Some(*skill_template_revision_uid), + ), + }; + let row = sqlx::query(CREATE_RUN_SQL) + .bind(run_uid) + .bind(new_run.tenant_id.0) + .bind(Option::::None) + .bind(new_run.session_id.0) + .bind(to_i64( + new_run.originating_user_sequence_num, + "originating user sequence", + )?) + .bind(new_run.planning_context_uid) + .bind(new_run.planning_context_hash.to_string()) + .bind(new_run.owner_user_id.as_str()) + .bind(serde_json::to_value(&new_run.admitted_identity)?) + .bind(serde_json::to_value(&new_run.goal)?) + .bind(&plan) + .bind(&plan) + .bind(new_run.plan.plan_hash.to_string()) + .bind(new_run.plan.plan_hash.to_string()) + .bind(serde_json::to_value(&new_run.catalog)?) + .bind(serde_json::to_value(&new_run.authorization)?) + .bind(serde_json::to_value(&new_run.pinned_instruction_skills)?) + .bind(serde_json::to_value(&new_run.source_provenance)?) + .bind(source.0.as_str()) + .bind(source.1) + .bind(source.2) + .bind(&new_run.input) + .bind("queued") + .bind(budget.max_cost_microusd) + .bind(budget.max_tokens) + .bind(budget.max_tasks) + .bind(budget.max_tool_calls) + .bind(budget.max_retrieved_bytes) + .bind(budget.deadline_at) + .bind(0_i64) + .bind(new_run.idempotency_key.as_deref()) + .bind("queued") + .bind(schedule_uid) + .bind(to_i64(schedule_incarnation, "schedule incarnation")?) + .bind(to_i64(occurrence_sequence, "occurrence sequence")?) + .fetch_optional(&mut *conn) + .await + .map_err(sqlx_error)?; + let _inserted_or_replayed = match row { + Some(row) => row, + None => sqlx::query("SELECT * FROM moa.execution_run WHERE run_uid=$1") + .bind(run_uid) + .fetch_one(&mut *conn) + .await + .map_err(sqlx_error)?, + }; + let row = sqlx::query("SELECT * FROM moa.execution_run WHERE run_uid=$1") + .bind(run_uid) + .fetch_one(conn) + .await + .map_err(sqlx_error)?; + run_from_row(&row) +} + +fn require_tenant_scope(scope: ExecutionScope, tenant_id: TenantId) -> Result<()> { + if !scope.permits_owner(tenant_id, None) { + return Err(Error::InvalidRepositoryInput { + message: "schedule tenant does not match repository scope".to_string(), + }); + } + Ok(()) +} + +async fn schedule_overlap_limit_reached_in_conn( + conn: &mut PgConnection, + tenant_id: TenantId, + schedule_uid: Uuid, + overlap_policy: ExecutionScheduleOverlapPolicy, + maximum_concurrent_runs: u64, +) -> Result { + match overlap_policy { + ExecutionScheduleOverlapPolicy::Skip => sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM moa.execution_run \ + WHERE tenant_id=$1 AND schedule_uid=$2 \ + AND status NOT IN \ + ('completed','partial','blocked','unsupported','failed','cancelled'))", + ) + .bind(tenant_id.0) + .bind(schedule_uid) + .fetch_one(&mut *conn) + .await + .map_err(sqlx_error), + ExecutionScheduleOverlapPolicy::QueueOne => sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM moa.execution_run \ + WHERE tenant_id=$1 AND schedule_uid=$2 AND status='queued')", + ) + .bind(tenant_id.0) + .bind(schedule_uid) + .fetch_one(&mut *conn) + .await + .map_err(sqlx_error), + ExecutionScheduleOverlapPolicy::Allow => { + let limit = to_i64(maximum_concurrent_runs, "maximum concurrent runs")?; + let count: i64 = sqlx::query_scalar( + "SELECT count(*) FROM (SELECT 1 FROM moa.execution_run \ + WHERE tenant_id=$1 AND schedule_uid=$2 \ + AND status NOT IN \ + ('completed','partial','blocked','unsupported','failed','cancelled') \ + LIMIT $3) AS bounded_nonterminal_runs", + ) + .bind(tenant_id.0) + .bind(schedule_uid) + .bind(limit) + .fetch_one(&mut *conn) + .await + .map_err(sqlx_error)?; + Ok(count >= limit) + } + } +} + +fn validate_occurrence_in_policy( + occurrence: Option, + policy: &ExecutionSchedulePolicy, +) -> Result<()> { + if occurrence.is_some_and(|value| { + value.at < policy.start_at || policy.end_at.is_some_and(|end| value.at >= end) + }) { + return Err(Error::InvalidRepositoryInput { + message: "schedule occurrence is outside its start/end bounds".to_string(), + }); + } + Ok(()) +} + +fn schedule_matches_create( + record: &ExecutionScheduleRecord, + request: &ExecutionScheduleCreateRequest, +) -> bool { + record.tenant_id == request.tenant_id + && record.name == request.name + && record.template == request.template + && record.run_as_identity == request.run_as_identity + && record.origin == request.origin + && record.policy == request.policy +} + +fn validate_blueprint(request: &ExecutionScheduleCreateRequest) -> Result<()> { + let blueprint: ExecutionScheduleRunBlueprint = + serde_json::from_value(request.template.snapshot.clone()).map_err(|error| { + Error::InvalidRepositoryInput { + message: format!("invalid scheduled run blueprint: {error}"), + } + })?; + if serde_json::to_value(&blueprint.approved_budget)? != request.policy.occurrence_budget + || blueprint.approved_budget.deadline_at.is_some() + || blueprint.deadline_offset_seconds == Some(0) + || scheduled_blueprint_revision(&blueprint) != Some(request.template.revision_uid) + { + return Err(Error::InvalidRepositoryInput { + message: + "scheduled blueprint budget or pinned template revision does not match schedule" + .to_string(), + }); + } + let record = ExecutionScheduleRecord { + schedule_uid: request.schedule_uid, + tenant_id: request.tenant_id, + name: request.name.clone(), + template: request.template.clone(), + run_as_identity: request.run_as_identity.clone(), + origin: request.origin.clone(), + policy: request.policy.clone(), + status: ExecutionScheduleStatus::Active, + schedule_incarnation: 1, + last_occurrence_sequence: 0, + next_occurrence_at: None, + next_occurrence_local: None, + paused_at: None, + created_at: request.policy.start_at, + updated_at: request.policy.start_at, + }; + let occurrence = ExecutionScheduleOccurrence { + at: request.policy.start_at, + local: request.policy.start_at.naive_utc(), + }; + let run = blueprint.instantiate( + &record, + occurrence, + 1, + blueprint.deadline_offset_seconds.unwrap_or(1), + )?; + validate_new_run( + ExecutionScope::Tenant { + tenant_id: request.tenant_id, + }, + &run, + ) +} + +fn scheduled_blueprint_revision(blueprint: &ExecutionScheduleRunBlueprint) -> Option { + match &blueprint.source_provenance { + moa_core::types::execution_planning::ExecutionSourceProvenance::SkillTemplate { + skill_template_revision_uid, + .. + } + | moa_core::types::execution_planning::ExecutionSourceProvenance::ExperimentTemplate { + skill_template_revision_uid, + .. + } => Some(*skill_template_revision_uid), + moa_core::types::execution_planning::ExecutionSourceProvenance::GeneratedPlan { + .. + } => None, + } +} + +async fn load_occurrence_replay_in_conn( + conn: &mut PgConnection, + schedule_uid: Uuid, + schedule_incarnation: u64, + occurrence_sequence: u64, +) -> Result { + let run_uid = sqlx::query_scalar::<_, Uuid>( + "SELECT run_uid FROM moa.execution_run WHERE schedule_uid=$1 \ + AND schedule_incarnation=$2 AND schedule_occurrence_sequence=$3", + ) + .bind(schedule_uid) + .bind(to_i64(schedule_incarnation, "schedule incarnation")?) + .bind(to_i64(occurrence_sequence, "occurrence sequence")?) + .fetch_optional(conn) + .await + .map_err(sqlx_error)?; + let activation_dispatch_uid = run_uid.map(|_| { + execution_schedule_occurrence_ids(schedule_uid, schedule_incarnation, occurrence_sequence) + .activation_dispatch_uid + }); + Ok(ExecutionScheduleRunAdmissionOutcome::Replayed { + run_uid, + activation_dispatch_uid, + }) +} + +fn scheduled_template_revision(run: &NewExecutionRun) -> Option { + match &run.source_provenance { + moa_core::types::execution_planning::ExecutionSourceProvenance::SkillTemplate { + skill_template_revision_uid, + .. + } + | moa_core::types::execution_planning::ExecutionSourceProvenance::ExperimentTemplate { + skill_template_revision_uid, + .. + } => Some(*skill_template_revision_uid), + moa_core::types::execution_planning::ExecutionSourceProvenance::GeneratedPlan { + .. + } => None, + } +} + +fn invalid_enum(value: &str) -> Error { + Error::InvalidRepositoryData { + message: format!("unknown schedule enum `{value}`"), + } +} + +fn parse_schedule_status(value: &str) -> Result { + match value { + "active" => Ok(ExecutionScheduleStatus::Active), + "paused" => Ok(ExecutionScheduleStatus::Paused), + "completed" => Ok(ExecutionScheduleStatus::Completed), + "cancelled" => Ok(ExecutionScheduleStatus::Cancelled), + _ => Err(invalid_enum(value)), + } +} + +fn parse_missed_fire_policy(value: &str) -> Result { + match value { + "skip" => Ok(ExecutionScheduleMissedFirePolicy::Skip), + "fire_once" => Ok(ExecutionScheduleMissedFirePolicy::FireOnce), + _ => Err(invalid_enum(value)), + } +} + +fn parse_overlap_policy(value: &str) -> Result { + match value { + "skip" => Ok(ExecutionScheduleOverlapPolicy::Skip), + "queue_one" => Ok(ExecutionScheduleOverlapPolicy::QueueOne), + "allow" => Ok(ExecutionScheduleOverlapPolicy::Allow), + _ => Err(invalid_enum(value)), + } +} + +fn parse_dst_policy(value: &str) -> Result { + match value { + "earliest" => Ok(ExecutionScheduleDstPolicy::Earliest), + "latest" => Ok(ExecutionScheduleDstPolicy::Latest), + "skip" => Ok(ExecutionScheduleDstPolicy::Skip), + _ => Err(invalid_enum(value)), + } +} diff --git a/crates/moa-execution/src/repository/sql.rs b/crates/moa-execution/src/repository/sql.rs index faa93de56..b1f60b555 100644 --- a/crates/moa-execution/src/repository/sql.rs +++ b/crates/moa-execution/src/repository/sql.rs @@ -175,19 +175,20 @@ pub(super) const LOAD_PLANNING_CONTEXT_BY_ORIGIN_SQL: &str = r#" pub(super) const CREATE_RUN_SQL: &str = r#" INSERT INTO moa.execution_run ( run_uid, tenant_id, contact_id, session_id, originating_user_sequence_num, - planning_context_uid, planning_context_hash, owner_user_id, + planning_context_uid, planning_context_hash, owner_user_id, admitted_identity, goal_contract, initial_plan, active_plan, initial_plan_hash, active_plan_hash, capability_catalog, authorization_envelope, pinned_instruction_skills, source_provenance, source_kind, skill_template_ref, skill_template_revision_uid, input, status, budget_max_cost_microusd, budget_max_tokens, budget_max_tasks, budget_max_tool_calls, budget_max_retrieved_bytes, budget_deadline_at, - progress_total_tasks, idempotency_key + progress_total_tasks, idempotency_key, activation_state, + schedule_uid, schedule_incarnation, schedule_occurrence_sequence ) VALUES ( $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, - $24, $25, $26, $27, $28, $29, $30 + $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35 ) ON CONFLICT ( tenant_id, @@ -230,9 +231,38 @@ pub(super) const LOAD_RUN_FOR_UPDATE_SQL: &str = r#" FOR UPDATE "#; +pub(super) const CLAIM_RUN_ACTIVATION_SQL: &str = r#" + UPDATE moa.execution_run + SET activation_state = 'advancing', + last_progress_at = NOW(), + updated_at = NOW() + WHERE run_uid = $1 + AND controller_generation = $2 + AND activation_state = 'queued' + AND status NOT IN ('completed', 'partial', 'blocked', 'unsupported', 'failed', 'cancelled') + RETURNING * +"#; + +pub(super) const CHECKPOINT_RUN_ACTIVATION_SQL: &str = r#" + UPDATE moa.execution_run + SET status = $3, + activation_state = $4, + next_wake_at = $5, + waiting_since = $6, + ready_task_count = $7, + active_task_count = $8, + last_progress_at = NOW(), + updated_at = NOW() + WHERE run_uid = $1 + AND controller_generation = $2 + AND activation_state = 'advancing' + RETURNING * +"#; + pub(super) const CONFIRM_RUN_SQL: &str = r#" UPDATE moa.execution_run SET status = 'queued', + activation_state = 'queued', queued_at = COALESCE(queued_at, NOW()), budget_max_cost_microusd = $3, budget_max_tokens = $4, @@ -242,7 +272,7 @@ pub(super) const CONFIRM_RUN_SQL: &str = r#" budget_deadline_at = $8, confirmed_plan_hash = $2, confirmed_at = NOW(), - wake_epoch = wake_epoch + 1, + last_progress_at = NOW(), updated_at = NOW() WHERE run_uid = $1 AND status = 'awaiting_confirmation' @@ -291,6 +321,7 @@ pub(super) const INSERT_TASK_BATCH_SQL: &str = r#" INSERT INTO moa.execution_task ( task_id, run_uid, tenant_id, contact_id, node_id, item_key, requirement_ids, plan_revision, status, attempt, generation, + attempt_generation, attempt_state, input, task_kind, compensation_contract, retry_policy, estimate_cost_microusd, estimate_tokens, estimate_tasks, estimate_tool_calls, estimate_retrieved_bytes, generation_history @@ -298,6 +329,7 @@ pub(super) const INSERT_TASK_BATCH_SQL: &str = r#" SELECT input.task_id, $2, $3, $4, input.node_id, input.item_key, input.requirement_ids, $5, 'pending', 1, input.generation, + input.generation, 'idle', input.input, input.task_kind, input.compensation_contract, input.retry_policy, input.estimate_cost_microusd, input.estimate_tokens, input.estimate_tasks, input.estimate_tool_calls, @@ -330,8 +362,11 @@ pub(super) const LOAD_TASK_BATCH_SQL: &str = r#" task.actual_cost_microusd, task.actual_tokens, task.actual_tasks, task.actual_tool_calls, task.actual_retrieved_bytes, task.current_outcome, task.output, task.error, task.citations, - task.generation_history, task.outcome_audit, task.created_at, - task.updated_at, task.reserved_at, task.started_at, task.completed_at + task.generation_history, task.outcome_audit, task.attempt_generation, task.attempt_state, task.attempt_started_at, + task.last_progress_at, task.attempt_deadline_at, task.waiting_since, task.ready_at, + task.external_job_uid, task.active_dispatch_uid, task.dispatch_sequence, + task.created_at, task.updated_at, task.reserved_at, + task.started_at, task.completed_at FROM input JOIN moa.execution_task AS task ON task.run_uid = $2 @@ -353,6 +388,9 @@ pub(super) const LOAD_TASK_FOR_UPDATE_SQL: &str = r#" actual_cost_microusd, actual_tokens, actual_tasks, actual_tool_calls, actual_retrieved_bytes, current_outcome, output, error, citations, generation_history, outcome_audit, + attempt_generation, attempt_state, attempt_started_at, last_progress_at, + attempt_deadline_at, waiting_since, ready_at, external_job_uid, + active_dispatch_uid, dispatch_sequence, created_at, updated_at, reserved_at, started_at, completed_at FROM moa.execution_task WHERE run_uid = $1 AND task_id = $2 @@ -371,6 +409,9 @@ pub(super) const LOAD_TASK_SQL: &str = r#" actual_cost_microusd, actual_tokens, actual_tasks, actual_tool_calls, actual_retrieved_bytes, current_outcome, output, error, citations, generation_history, outcome_audit, + attempt_generation, attempt_state, attempt_started_at, last_progress_at, + attempt_deadline_at, waiting_since, ready_at, external_job_uid, + active_dispatch_uid, dispatch_sequence, created_at, updated_at, reserved_at, started_at, completed_at FROM moa.execution_task WHERE run_uid = $1 AND task_id = $2 @@ -431,7 +472,8 @@ pub(super) const RESERVE_TASK_SQL: &str = r#" reserved_retrieved_bytes = $8, reserved_at = NOW(), updated_at = NOW() - WHERE run_uid = $1 AND task_id = $2 AND generation = $3 AND status = 'pending' + WHERE run_uid = $1 AND task_id = $2 + AND generation = $3 AND attempt_generation = $3 AND status = 'pending' RETURNING task_id, run_uid, tenant_id, contact_id, node_id, item_key, requirement_ids, plan_revision, status, attempt, generation, @@ -443,13 +485,23 @@ pub(super) const RESERVE_TASK_SQL: &str = r#" actual_cost_microusd, actual_tokens, actual_tasks, actual_tool_calls, actual_retrieved_bytes, current_outcome, output, error, citations, generation_history, outcome_audit, + attempt_generation, attempt_state, attempt_started_at, last_progress_at, + attempt_deadline_at, waiting_since, ready_at, external_job_uid, + active_dispatch_uid, dispatch_sequence, created_at, updated_at, reserved_at, started_at, completed_at "#; pub(super) const MARK_TASK_RUNNING_SQL: &str = r#" UPDATE moa.execution_task - SET status = 'running', started_at = COALESCE(started_at, NOW()), updated_at = NOW() - WHERE run_uid = $1 AND task_id = $2 AND generation = $3 AND status = 'reserved' + SET status = 'running', + attempt_state = 'running', + attempt_started_at = NOW(), + last_progress_at = NOW(), + waiting_since = NULL, + started_at = COALESCE(started_at, NOW()), + updated_at = NOW() + WHERE run_uid = $1 AND task_id = $2 + AND generation = $3 AND attempt_generation = $3 AND status = 'reserved' RETURNING task_id, run_uid, tenant_id, contact_id, node_id, item_key, requirement_ids, plan_revision, status, attempt, generation, @@ -461,14 +513,25 @@ pub(super) const MARK_TASK_RUNNING_SQL: &str = r#" actual_cost_microusd, actual_tokens, actual_tasks, actual_tool_calls, actual_retrieved_bytes, current_outcome, output, error, citations, generation_history, outcome_audit, + attempt_generation, attempt_state, attempt_started_at, last_progress_at, + attempt_deadline_at, waiting_since, ready_at, external_job_uid, + active_dispatch_uid, dispatch_sequence, created_at, updated_at, reserved_at, started_at, completed_at "#; pub(super) const RESUME_TASK_SQL: &str = r#" UPDATE moa.execution_task - SET status = 'running', + SET status = 'ready', attempt = $5, generation = $6, + attempt_generation = $9, + attempt_state = 'idle', + attempt_started_at = NULL, + attempt_deadline_at = NULL, + active_dispatch_uid = NULL, + waiting_since = NULL, + ready_at = $10, + last_progress_at = $10, generation_history = generation_history || jsonb_build_array($7::JSONB), resume_input_history = CASE WHEN $8::JSONB IS NULL THEN resume_input_history @@ -478,8 +541,9 @@ pub(super) const RESUME_TASK_SQL: &str = r#" WHEN $8::JSONB IS NULL THEN current_outcome ELSE NULL END, - updated_at = NOW() - WHERE run_uid = $1 AND task_id = $2 AND status = $3 AND generation = $4 + updated_at = $10 + WHERE run_uid = $1 AND task_id = $2 AND status = $3 + AND generation = $4 AND attempt_generation = $4 RETURNING task_id, run_uid, tenant_id, contact_id, node_id, item_key, requirement_ids, plan_revision, status, attempt, generation, @@ -491,6 +555,9 @@ pub(super) const RESUME_TASK_SQL: &str = r#" actual_cost_microusd, actual_tokens, actual_tasks, actual_tool_calls, actual_retrieved_bytes, current_outcome, output, error, citations, generation_history, outcome_audit, + attempt_generation, attempt_state, attempt_started_at, last_progress_at, + attempt_deadline_at, waiting_since, ready_at, external_job_uid, + active_dispatch_uid, dispatch_sequence, created_at, updated_at, reserved_at, started_at, completed_at "#; @@ -506,6 +573,9 @@ pub(super) const LIST_TASKS_SQL: &str = r#" actual_cost_microusd, actual_tokens, actual_tasks, actual_tool_calls, actual_retrieved_bytes, current_outcome, output, error, citations, generation_history, outcome_audit, + attempt_generation, attempt_state, attempt_started_at, last_progress_at, + attempt_deadline_at, waiting_since, ready_at, external_job_uid, + active_dispatch_uid, dispatch_sequence, created_at, updated_at, reserved_at, started_at, completed_at FROM moa.execution_task WHERE run_uid = $1 @@ -517,24 +587,6 @@ pub(super) const LIST_TASKS_SQL: &str = r#" LIMIT $5 "#; -pub(super) const LIST_ALL_TASKS_SQL: &str = r#" - SELECT - task_id, run_uid, tenant_id, contact_id, node_id, item_key, - requirement_ids, plan_revision, status, attempt, generation, - input, resume_input_history, task_kind, compensation_contract, retry_policy, - estimate_cost_microusd, estimate_tokens, estimate_tasks, - estimate_tool_calls, estimate_retrieved_bytes, - reserved_cost_microusd, reserved_tokens, reserved_tasks, - reserved_tool_calls, reserved_retrieved_bytes, - actual_cost_microusd, actual_tokens, actual_tasks, - actual_tool_calls, actual_retrieved_bytes, - current_outcome, output, error, citations, generation_history, outcome_audit, - created_at, updated_at, reserved_at, started_at, completed_at - FROM moa.execution_task - WHERE run_uid = $1 - ORDER BY node_id, item_key, task_id -"#; - pub(super) const RECONCILE_RUN_OUTCOME_SQL: &str = r#" UPDATE moa.execution_run SET status = $2, @@ -561,6 +613,22 @@ pub(super) const RECONCILE_RUN_OUTCOME_SQL: &str = r#" pub(super) const RECORD_TASK_OUTCOME_SQL: &str = r#" UPDATE moa.execution_task SET status = $4, + attempt_state = CASE + WHEN $4 IN ('waiting_input', 'waiting_review', 'waiting_signal', + 'waiting_timer', 'waiting_external', 'waiting_replan') THEN 'waiting' + WHEN $4 = 'unknown_outcome' THEN 'unknown_outcome' + WHEN $4 IN ('completed', 'skipped', 'failed', 'cancelled') THEN 'terminal' + WHEN $4 = 'running' THEN attempt_state + ELSE attempt_state + END, + waiting_since = CASE + WHEN $4 IN ('waiting_input', 'waiting_review', 'waiting_signal', + 'waiting_timer', 'waiting_external', 'waiting_replan') + THEN COALESCE(waiting_since, NOW()) + ELSE NULL + END, + attempt_deadline_at = CASE WHEN $4 = 'running' THEN attempt_deadline_at ELSE NULL END, + last_progress_at = GREATEST(last_progress_at, NOW()), reserved_cost_microusd = $5, reserved_tokens = $6, reserved_tasks = $7, @@ -577,9 +645,11 @@ pub(super) const RECORD_TASK_OUTCOME_SQL: &str = r#" citations = $18, outcome_audit = outcome_audit || jsonb_build_array($19::JSONB), completed_at = CASE WHEN $20 THEN COALESCE(completed_at, NOW()) ELSE NULL END, + failure_fingerprint = $21, updated_at = NOW() - WHERE run_uid = $1 AND task_id = $2 AND generation = $3 - AND status NOT IN ('completed', 'skipped', 'failed', 'cancelled') + WHERE run_uid = $1 AND task_id = $2 + AND generation = $3 + AND status NOT IN ('completed', 'skipped', 'failed', 'cancelled', 'unknown_outcome') RETURNING task_id, run_uid, tenant_id, contact_id, node_id, item_key, requirement_ids, plan_revision, status, attempt, generation, @@ -591,12 +661,19 @@ pub(super) const RECORD_TASK_OUTCOME_SQL: &str = r#" actual_cost_microusd, actual_tokens, actual_tasks, actual_tool_calls, actual_retrieved_bytes, current_outcome, output, error, citations, generation_history, outcome_audit, + attempt_generation, attempt_state, attempt_started_at, last_progress_at, + attempt_deadline_at, waiting_since, ready_at, external_job_uid, + active_dispatch_uid, dispatch_sequence, created_at, updated_at, reserved_at, started_at, completed_at "#; pub(super) const RECORD_RESERVATION_REJECTION_SQL: &str = r#" UPDATE moa.execution_task SET status = 'failed', + attempt_state = 'terminal', + attempt_deadline_at = NULL, + waiting_since = NULL, + last_progress_at = NOW(), reserved_cost_microusd = 0, reserved_tokens = 0, reserved_tasks = 0, @@ -612,9 +689,11 @@ pub(super) const RECORD_RESERVATION_REJECTION_SQL: &str = r#" error = $9, citations = $10, outcome_audit = outcome_audit || jsonb_build_array($11::JSONB), + failure_fingerprint = $12, completed_at = NOW(), updated_at = NOW() - WHERE run_uid = $1 AND task_id = $2 AND generation = $3 AND status = 'running' + WHERE run_uid = $1 AND task_id = $2 + AND generation = $3 AND attempt_generation = $3 AND status = 'running' RETURNING task_id, run_uid, tenant_id, contact_id, node_id, item_key, requirement_ids, plan_revision, status, attempt, generation, @@ -626,6 +705,9 @@ pub(super) const RECORD_RESERVATION_REJECTION_SQL: &str = r#" actual_cost_microusd, actual_tokens, actual_tasks, actual_tool_calls, actual_retrieved_bytes, current_outcome, output, error, citations, generation_history, outcome_audit, + attempt_generation, attempt_state, attempt_started_at, last_progress_at, + attempt_deadline_at, waiting_since, ready_at, external_job_uid, + active_dispatch_uid, dispatch_sequence, created_at, updated_at, reserved_at, started_at, completed_at "#; @@ -645,12 +727,19 @@ pub(super) const APPEND_TASK_OUTCOME_AUDIT_SQL: &str = r#" actual_cost_microusd, actual_tokens, actual_tasks, actual_tool_calls, actual_retrieved_bytes, current_outcome, output, error, citations, generation_history, outcome_audit, + attempt_generation, attempt_state, attempt_started_at, last_progress_at, + attempt_deadline_at, waiting_since, ready_at, external_job_uid, + active_dispatch_uid, dispatch_sequence, created_at, updated_at, reserved_at, started_at, completed_at "#; pub(super) const SUPERSEDE_REPLAN_TASK_SQL: &str = r#" UPDATE moa.execution_task SET status = 'cancelled', + attempt_state = 'terminal', + attempt_deadline_at = NULL, + waiting_since = NULL, + last_progress_at = NOW(), current_outcome = $4, reserved_cost_microusd = 0, reserved_tokens = 0, @@ -677,6 +766,9 @@ pub(super) const SUPERSEDE_REPLAN_TASK_SQL: &str = r#" actual_cost_microusd, actual_tokens, actual_tasks, actual_tool_calls, actual_retrieved_bytes, current_outcome, output, error, citations, generation_history, outcome_audit, + attempt_generation, attempt_state, attempt_started_at, last_progress_at, + attempt_deadline_at, waiting_since, ready_at, external_job_uid, + active_dispatch_uid, dispatch_sequence, created_at, updated_at, reserved_at, started_at, completed_at "#; @@ -695,45 +787,18 @@ pub(super) const APPEND_AMENDMENT_SQL: &str = r#" consumed_tasks = $12, budget_overrun = $13, progress_cancelled_tasks = progress_cancelled_tasks + 1, - wake_epoch = wake_epoch + 1, updated_at = NOW() WHERE run_uid = $1 AND plan_revision = $2 AND status = 'waiting_replan' RETURNING * "#; -pub(super) const LOAD_NONTERMINAL_TASKS_FOR_UPDATE_SQL: &str = r#" - SELECT - task_id, run_uid, tenant_id, contact_id, node_id, item_key, - requirement_ids, plan_revision, status, attempt, generation, - input, resume_input_history, task_kind, compensation_contract, retry_policy, - estimate_cost_microusd, estimate_tokens, estimate_tasks, - estimate_tool_calls, estimate_retrieved_bytes, - reserved_cost_microusd, reserved_tokens, reserved_tasks, - reserved_tool_calls, reserved_retrieved_bytes, - actual_cost_microusd, actual_tokens, actual_tasks, - actual_tool_calls, actual_retrieved_bytes, - current_outcome, output, error, citations, generation_history, outcome_audit, - created_at, updated_at, reserved_at, started_at, completed_at - FROM moa.execution_task - WHERE run_uid = $1 - AND status IN ('pending', 'reserved', 'running', 'waiting_input', 'waiting_replan') - ORDER BY task_id - FOR UPDATE -"#; - -pub(super) const LIST_COMPENSATIONS_SQL: &str = r#" - SELECT compensation_id, run_uid, forward_task_id, registered_sequence, - forward_generation, compensator, mapped_input, status, attempt, - generation, outcome, error, created_at, updated_at, started_at, completed_at - FROM moa.execution_compensation - WHERE run_uid = $1 - ORDER BY registered_sequence DESC -"#; - pub(super) const LOAD_COMPENSATION_FOR_UPDATE_SQL: &str = r#" SELECT compensation_id, run_uid, forward_task_id, registered_sequence, forward_generation, compensator, mapped_input, status, attempt, - generation, outcome, error, created_at, updated_at, started_at, completed_at + generation, outcome, error, created_at, updated_at, started_at, completed_at, + attempt_generation, attempt_state, attempt_started_at, last_progress_at, + attempt_deadline_at, waiting_since, active_dispatch_uid, external_job_uid, + release_intent, dispatch_sequence FROM moa.execution_compensation WHERE run_uid = $1 AND compensation_id = $2 FOR UPDATE @@ -762,59 +827,3 @@ pub(super) const INSERT_COMPENSATION_SQL: &str = r#" ON CONFLICT (forward_task_id) DO NOTHING RETURNING compensation_id "#; - -pub(super) const FENCE_RUN_FOR_COMPENSATION_SQL: &str = r#" - UPDATE moa.execution_run - SET pending_terminal_status = $4, - pending_terminal_reason = $5, - pending_terminal_cause = $6, - pending_terminal_output = $7, - cancellation_reason = $8, - waiting_reasons = '[]'::JSONB, - wake_epoch = wake_epoch + 1, - updated_at = NOW() - WHERE run_uid = $1 - AND plan_revision = $2 - AND wake_epoch = $3 - AND pending_terminal_status IS NULL - AND status NOT IN ('completed', 'partial', 'blocked', 'unsupported', 'failed', 'cancelled', 'compensating') - RETURNING * -"#; - -pub(super) const BEGIN_COMPENSATION_SQL: &str = r#" - UPDATE moa.execution_run - SET status = 'compensating', - waiting_reasons = '[]'::JSONB, - wake_epoch = wake_epoch + 1, - updated_at = NOW() - WHERE run_uid = $1 - AND plan_revision = $2 - AND wake_epoch = $3 - AND pending_terminal_status IS NOT NULL - AND status NOT IN ('completed', 'partial', 'blocked', 'unsupported', 'failed', 'cancelled', 'compensating') - AND NOT EXISTS ( - SELECT 1 FROM moa.execution_task - WHERE run_uid = $1 - AND status NOT IN ('completed', 'skipped', 'failed', 'cancelled') - ) - RETURNING * -"#; - -pub(super) const CLAIM_COMPENSATION_SQL: &str = r#" - UPDATE moa.execution_compensation - SET status = 'running', - started_at = COALESCE(started_at, NOW()), - updated_at = NOW() - WHERE run_uid = $1 - AND compensation_id = $2 - AND generation = $3 - AND status = 'pending' - AND registered_sequence = ( - SELECT MAX(registered_sequence) - FROM moa.execution_compensation - WHERE run_uid = $1 AND status <> 'completed' - ) - RETURNING compensation_id, run_uid, forward_task_id, registered_sequence, - forward_generation, compensator, mapped_input, status, attempt, - generation, outcome, error, created_at, updated_at, started_at, completed_at -"#; diff --git a/crates/moa-execution/src/repository/task.rs b/crates/moa-execution/src/repository/task.rs index 64cd683c2..93d242a6c 100644 --- a/crates/moa-execution/src/repository/task.rs +++ b/crates/moa-execution/src/repository/task.rs @@ -1,7 +1,4284 @@ //! Logical-task reservation, redispatch, waiting, and review-resolution persistence. +use crate::state::{cancelled_task_outcome, completed_task_outcome, failed_task_outcome}; +use crate::wire::{ExecutionActionReviewResolution, ExecutionTaskAttemptRequest}; +use moa_config::ExecutionConfig; +use moa_core::{ + canonical_json::canonical_json_bytes, + types::{ + context::ContextMessage, + sandbox_workspace::{ExecutionHandReleaseOwner, ExecutionHandReleaseReceipt}, + }, +}; + use super::*; -use super::{materialize::DbEstimate, outcome_support::*, rows::*, sql::*}; +use super::{ + capacity::{ + CapacityReleaseOutcome, ExecutionCapacityDimension, prelock_capacity_dimensions_in_tx, + prelock_existing_capacity_dimensions_in_tx, release_task_capacity_in_tx, + }, + external_job::{ + ExecutionExternalJobOwner, ExecutionExternalJobRecord, ExecutionExternalJobState, + NewExecutionExternalJobIntent, load_external_job_for_update_in_conn, + }, + materialize::DbEstimate, + outcome::{record_task_outcome_in_conn, record_waiting_external_task_outcome_in_conn}, + outcome_support::*, + ready::{ + append_run_wait_reason_in_tx, transition_node_counters_in_tx, + transition_node_counters_with_input_audience_in_tx, + }, + rows::*, + run::enqueue_run_activation_in_conn, + sql::*, + transition::{refresh_run_after_wait_settlement_in_conn, task_outcome_is_exact_replay}, + trigger::{ + ExecutionTriggerKind, ExecutionTriggerSupersedeOutcome, NewExecutionTrigger, + create_trigger_with_dispatch_in_conn, supersede_trigger_in_conn, + }, +}; + +const TASK_INPUT_WAIT_TRIGGER_NAMESPACE: Uuid = + Uuid::from_u128(0x9a2e_18f4_1c5e_57c4_8bf5_ea73_52e9_4a11); + +/// Immutable identity of one admitted bounded task-attempt slice. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct TaskAttemptFence { + /// Tenant that owns every referenced row. + pub tenant_id: TenantId, + /// Owning execution run. + pub run_uid: Uuid, + /// Stable logical task. + pub task_id: ExecutionTaskId, + /// Run-controller generation that admitted the slice. + pub controller_generation: u64, + /// Exact bounded attempt generation. + pub attempt_generation: u64, + /// Immutable durable-dispatch identity. + pub dispatch_uid: Uuid, + /// Exact shared-capacity receipt. + pub capacity_reservation_uid: Uuid, + /// Exact active-attempt watchdog trigger. + pub watchdog_trigger_uid: Uuid, + /// Absolute deadline frozen by admission. + pub attempt_deadline_at: DateTime, +} + +/// Generation-fenced transition from durable dispatch to active execution. +#[derive(Clone, Debug, PartialEq)] +pub enum TaskAttemptStartOutcome { + /// The exact admitted dispatch became active. + Started(Box), + /// The same dispatch was already marked active. + AlreadyStarted(Box), + /// No exact run or task exists. + NotFound, + /// Dispatch, controller, attempt, deadline, or tenant identity is stale. + Stale, + /// The run or task is not currently dispatchable. + InvalidState, +} + +/// Authoritative active task slice loaded from its row-locked owning run. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct TaskAttemptRecord { + /// Immutable run projection, including admitted identity and parent session. + pub run: ExecutionRunRecord, + /// Exact bounded task projection. + pub task: ExecutionTaskRecord, +} + +/// Result of recording durable progress for one active attempt. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum TaskAttemptProgressOutcome { + /// The exact active attempt advanced its progress timestamp. + Applied, + /// The supplied timestamp was already durably covered. + Replayed, + /// No exact run or task exists. + NotFound, + /// The immutable attempt identity no longer matches canonical state. + Stale, + /// The task does not currently own active capacity. + InvalidState, +} + +/// Result of claiming exact teardown ownership before provider checkpoint/destroy I/O. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub enum TaskAttemptReleaseClaimOutcome { + /// The active attempt entered the non-admissible cancelling phase. + Applied(Box), + /// The same exact attempt already owns the cancelling phase. + Replayed(Box), + /// No exact run or task exists. + NotFound, + /// An immutable generation, dispatch, capacity, or watchdog coordinate is stale. + Stale, + /// The task is not currently eligible to relinquish active ownership. + InvalidState, +} + +/// Result of the short, receipt-fenced capacity release before logical outcome settlement. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub enum ReleasedTaskAttemptCapacityOutcome { + /// Capacity and watchdog ownership were released by this call. + Applied, + /// The exact capacity and watchdog release had already committed. + Replayed, + /// No exact run, task, or capacity receipt exists. + NotFound, + /// One immutable attempt or receipt coordinate no longer matches canonical state. + Stale, + /// The task is not in its resource-release phase. + InvalidState, +} + +/// Result of atomically settling one bounded task attempt. +#[derive(Clone, Debug, PartialEq)] +pub enum TaskAttemptSettlementOutcome { + /// The exact active attempt committed its outcome and released capacity. + Applied { + /// Run after its controller activation was enqueued. + run: ExecutionRunRecord, + /// Logical task after its bounded attempt yielded or settled. + task: ExecutionTaskRecord, + }, + /// The exact complete settlement had already committed. + Replayed { + /// Current owning run. + run: ExecutionRunRecord, + /// Current logical task. + task: ExecutionTaskRecord, + }, + /// No exact run, task, or capacity receipt exists. + NotFound, + /// One immutable attempt coordinate no longer matches canonical state. + Stale, + /// The task or run cannot accept this settlement. + InvalidState, +} + +/// Exact storage disposition for an admitted attempt whose receiver never started. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum UnstartedTaskAttemptDisposition { + /// A terminal cancellation won before the receiver committed its start fence. + Cancelled { + /// Bounded durable cancellation reason. + reason: String, + }, + /// A pause fenced the run before the receiver committed its start fence. + Paused { + /// Current run-controller generation that owns the pause drain. + controller_generation: u64, + }, + /// Durable dispatch was accepted but no receiver start transaction committed. + DispatchDeliveryLost, +} + +/// Result of yielding one attempt to an asynchronous provider job. +#[derive(Clone, Debug, PartialEq)] +pub enum TaskAttemptExternalOutcome { + /// The exact provider job and storage-only wait committed atomically. + Applied { + /// Run after its controller activation was enqueued. + run: ExecutionRunRecord, + /// Waiting logical task. + task: ExecutionTaskRecord, + /// Durable provider job. + external_job: ExecutionExternalJobRecord, + }, + /// The exact external-job yield had already committed. + Replayed { + /// Current owning run. + run: ExecutionRunRecord, + /// Current waiting task. + task: ExecutionTaskRecord, + /// Existing durable provider job. + external_job: ExecutionExternalJobRecord, + }, + /// No exact run, task, or capacity receipt exists. + NotFound, + /// One immutable attempt coordinate no longer matches canonical state. + Stale, + /// The task or run cannot yield to an external job. + InvalidState, +} + +/// Result of settling a terminal provider job into its exact waiting task. +#[derive(Clone, Debug, PartialEq)] +pub enum ExternalJobTaskSettlementOutcome { + /// The waiting task consumed the terminal provider outcome. + Applied(ExecutionTaskRecord), + /// The same terminal provider outcome was already consumed. + Replayed(ExecutionTaskRecord), + /// Provider terminal state is durable, but sandbox release still owns task settlement. + DeferredRelease(ExecutionTaskRecord), + /// The job no longer owns the current waiting attempt. + Stale, + /// No owning task exists. + NotFound, +} + +/// Durable kind of one bounded task continuation checkpoint. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub enum TaskAttemptCheckpointKind { + /// Task-local agent state at a bounded model/tool boundary. + AgentContinuation, + /// Direct capability invocation waiting on an exact action review. + CapabilityReview, + /// Direct async-capable invocation persisted before provider start. + CapabilityExternalStart, +} + +impl TaskAttemptCheckpointKind { + /// Returns the stable PostgreSQL label. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::AgentContinuation => "agent_continuation", + Self::CapabilityReview => "capability_review", + Self::CapabilityExternalStart => "capability_external_start", + } + } + + fn parse(value: &str) -> Result { + match value { + "agent_continuation" => Ok(Self::AgentContinuation), + "capability_review" => Ok(Self::CapabilityReview), + "capability_external_start" => Ok(Self::CapabilityExternalStart), + other => Err(Error::InvalidRepositoryData { + message: format!("unknown task-attempt checkpoint kind `{other}`"), + }), + } + } +} + +/// Immutable bounded continuation written before an attempt relinquishes ownership. +#[derive(Clone, Debug, PartialEq)] +pub struct NewTaskAttemptCheckpoint { + /// Exact active attempt identity. + pub fence: TaskAttemptFence, + /// Exact logical task generation. + pub task_generation: u64, + /// Closed continuation kind. + pub kind: TaskAttemptCheckpointKind, + /// Typed payload schema version. + pub schema_version: u32, + /// Canonical object payload, capped at one MiB. + pub payload: Value, + /// Verified sandbox release receipt, when the attempt owned sandbox compute. + pub workspace_release_receipt: Option, + /// Durable checkpoint time. + pub created_at: DateTime, +} + +/// One immutable persisted task continuation. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct TaskAttemptCheckpointRecord { + /// Deterministic checkpoint identity. + pub checkpoint_uid: Uuid, + /// Monotonic per-task checkpoint sequence. + pub checkpoint_sequence: u64, + /// Owning tenant. + pub tenant_id: TenantId, + /// Owning run. + pub run_uid: Uuid, + /// Stable logical task. + pub task_id: ExecutionTaskId, + /// Exact controller generation. + pub controller_generation: u64, + /// Exact attempt generation. + pub attempt_generation: u64, + /// Immutable active dispatch that produced this checkpoint. + pub dispatch_uid: Uuid, + /// Exact logical task generation. + pub task_generation: u64, + /// Closed continuation kind. + pub kind: TaskAttemptCheckpointKind, + /// Typed payload schema version. + pub schema_version: u32, + /// Canonical bounded payload. + pub payload: Value, + /// Canonical BLAKE3 digest of the payload. + pub payload_hash: String, + /// Verified sandbox release receipt. + pub workspace_release_receipt: Option, + /// Durable creation time. + pub created_at: DateTime, +} + +/// Result of generation-fenced checkpoint persistence. +#[derive(Clone, Debug, PartialEq)] +pub enum TaskAttemptCheckpointWriteOutcome { + /// A new immutable checkpoint superseded the prior current row. + Applied(Box), + /// The exact checkpoint was already current. + Replayed(Box), + /// No exact task exists. + NotFound, + /// The task generation, attempt, dispatch, or receipt is stale. + Stale, + /// The payload or task state cannot be checkpointed. + InvalidState, +} + +/// Recovery result for an async provider start that did not create provider-owned work. +#[derive(Clone, Debug, PartialEq)] +pub enum TaskExternalStartRetryOutcome { + /// Active ownership was released and the exact provisional continuation became ready. + Applied { + /// Ready task on the successor bounded-attempt generation. + task: ExecutionTaskRecord, + /// Current predecessor checkpoint consumed by that successor. + checkpoint: Box, + }, + /// The same recovery transition was already committed. + Replayed { + /// Current ready task. + task: ExecutionTaskRecord, + /// Exact preserved provisional checkpoint. + checkpoint: Box, + }, + /// No exact active task or capacity owner exists. + NotFound, + /// Task, attempt, dispatch, checkpoint, or intent identity is obsolete. + Stale, + /// The current owner is not a running provisional external start. + InvalidState, +} + +/// Result of checkpointing one bounded agent slice for immediate redispatch. +#[derive(Clone, Debug, PartialEq)] +pub enum TaskAttemptContinuationYieldOutcome { + /// The exact continuation was persisted and the task became ready. + Applied { + /// Ready logical task with an advanced attempt generation. + task: ExecutionTaskRecord, + /// Immutable checkpoint consumed by the next admitted slice. + checkpoint: Box, + }, + /// The same continuation yield already committed. + Replayed { + /// Current logical task. + task: ExecutionTaskRecord, + /// Current immutable checkpoint. + checkpoint: Box, + }, + /// No exact task or capacity receipt exists. + NotFound, + /// An immutable task, attempt, dispatch, or checkpoint coordinate is stale. + Stale, + /// The task cannot yield a continuation from its current state. + InvalidState, +} + +/// Result of atomically parking one reviewed effect in storage. +#[derive(Clone, Debug, PartialEq)] +pub enum TaskAttemptReviewParkOutcome { + /// Capacity/watchdog ownership moved to an immutable continuation. + Applied { + /// Waiting logical task. + task: ExecutionTaskRecord, + /// Current bounded continuation. + checkpoint: Box, + }, + /// The exact review park had already committed. + Replayed { + /// Current waiting task. + task: ExecutionTaskRecord, + /// Existing bounded continuation. + checkpoint: Box, + }, + /// No exact task, capacity receipt, or checkpoint exists. + NotFound, + /// An immutable generation, dispatch, watchdog, or review identity is stale. + Stale, + /// The task cannot enter a review wait from its current state. + InvalidState, +} + +/// Result of consuming one storage-only action-review resolution. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub enum TaskAttemptReviewResolutionOutcome { + /// The exact resolution was checkpointed and the task became Ready. + Applied { + /// Ready task projection. + task: ExecutionTaskRecord, + /// Immutable resolved continuation. + checkpoint: Box, + }, + /// The same resolution had already been consumed. + Replayed { + /// Current task projection. + task: ExecutionTaskRecord, + /// Current resolved continuation. + checkpoint: Box, + }, + /// The review decision arrived before the attempt completed its durable park. + NotReady, + /// No exact task or checkpoint exists. + NotFound, + /// The logical generation or review identity is obsolete. + Stale, +} + +/// Exact storage request for consuming one reviewed task-attempt effect. +#[derive(Clone, Debug)] +pub struct ResolveTaskAttemptReviewRequest { + /// Repository visibility boundary for the mutation. + pub scope: ExecutionScope, + /// Owning execution run. + pub run_uid: Uuid, + /// Stable logical task identity. + pub task_id: ExecutionTaskId, + /// Exact logical task generation that parked the review. + pub expected_task_generation: u64, + /// Stable review identity persisted in the current checkpoint. + pub review_uid: Uuid, + /// Durable reviewed-effect resolution. + pub resolution: ExecutionActionReviewResolution, + /// Timestamp at which the resolution was accepted. + pub resolved_at: DateTime, +} + +struct SettleTaskAttemptRequest<'a> { + config: &'a ExecutionConfig, + fence: TaskAttemptFence, + outcome: ExecutionTaskOutcome, + retry_at: Option>, + settled_at: DateTime, + expected_attempt_state: ExecutionAttemptState, + workspace_release_receipt: Option, + continuation_checkpoint: Option, +} + +struct ResumeTaskRequest<'a> { + scope: ExecutionScope, + config: Option<&'a ExecutionConfig>, + run_uid: Uuid, + task_id: ExecutionTaskId, + generation: u64, + kind: ResumeKind, + resume_input: Option, +} + +impl ExecutionRepository { + /// Loads the current bounded continuation for one visible logical task. + pub async fn load_task_attempt_checkpoint( + &self, + scope: ExecutionScope, + run_uid: Uuid, + task_id: ExecutionTaskId, + ) -> Result> { + let mut conn = scope.begin(&self.pool).await?; + let row = sqlx::query( + "SELECT * FROM moa.execution_task_checkpoint \ + WHERE run_uid = $1 AND task_id = $2 AND superseded_at IS NULL", + ) + .bind(run_uid) + .bind(task_id.as_uuid()) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let checkpoint = row.as_ref().map(task_checkpoint_from_row).transpose()?; + conn.commit().await.map_err(storage_error)?; + Ok(checkpoint) + } + + /// Persists the exact async-provider start continuation while its attempt remains active. + /// + /// This checkpoint is the replay authority if provider start recovery later proves that no + /// asynchronous work began. It never releases capacity, clears the watchdog, or changes task + /// lifecycle state. + pub async fn persist_running_task_external_start_checkpoint( + &self, + checkpoint: NewTaskAttemptCheckpoint, + ) -> Result { + if checkpoint.workspace_release_receipt.is_some() + || !external_start_checkpoint_payload_is_provisional( + checkpoint.kind, + &checkpoint.payload, + ) + { + return Ok(TaskAttemptCheckpointWriteOutcome::InvalidState); + } + let mut conn = ExecutionScope::ControlPlane.begin(&self.pool).await?; + let outcome = persist_task_attempt_checkpoint_for_state_in_conn( + &mut conn, + &checkpoint, + ExecutionAttemptState::Running, + ) + .await?; + conn.commit().await.map_err(storage_error)?; + Ok(outcome) + } + + /// Consumes one exact reviewed effect without resolving a workflow promise. + pub async fn resolve_task_attempt_review( + &self, + config: &ExecutionConfig, + request: ResolveTaskAttemptReviewRequest, + ) -> Result { + let ResolveTaskAttemptReviewRequest { + scope, + run_uid, + task_id, + expected_task_generation, + review_uid, + resolution, + resolved_at, + } = request; + if !matches!(scope, ExecutionScope::ControlPlane) + || expected_task_generation == 0 + || review_uid.is_nil() + { + return Err(Error::InvalidRepositoryInput { + message: "task review resolution requires control-plane scope and exact identity" + .to_string(), + }); + } + let mut conn = ExecutionScope::ControlPlane.begin(&self.pool).await?; + let tenant_id = sqlx::query_scalar::<_, Uuid>( + "SELECT tenant_id FROM moa.execution_run WHERE run_uid=$1", + ) + .bind(run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(tenant_id) = tenant_id else { + conn.commit().await.map_err(storage_error)?; + return Ok(TaskAttemptReviewResolutionOutcome::NotFound); + }; + prelock_capacity_dimensions_in_tx( + conn.as_mut(), + config, + TenantId(tenant_id), + &[ + ExecutionCapacityDimension::ActiveRuns, + ExecutionCapacityDimension::ParkedRuns, + ], + ) + .await?; + let Some(run_row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) + .bind(run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + conn.commit().await.map_err(storage_error)?; + return Ok(TaskAttemptReviewResolutionOutcome::NotFound); + }; + let run = run_from_row(&run_row)?; + let Some(task_row) = sqlx::query(LOAD_TASK_FOR_UPDATE_SQL) + .bind(run_uid) + .bind(task_id.as_uuid()) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + conn.commit().await.map_err(storage_error)?; + return Ok(TaskAttemptReviewResolutionOutcome::NotFound); + }; + let task = task_from_row(&task_row)?; + let checkpoint_row = sqlx::query( + "SELECT * FROM moa.execution_task_checkpoint WHERE tenant_id=$1 AND run_uid=$2 \ + AND task_id=$3 AND superseded_at IS NULL FOR UPDATE", + ) + .bind(run.tenant_id.0) + .bind(run_uid) + .bind(task_id.as_uuid()) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(checkpoint_row) = checkpoint_row else { + conn.commit().await.map_err(storage_error)?; + return Ok(if task.status == ExecutionTaskStatus::Running { + TaskAttemptReviewResolutionOutcome::NotReady + } else { + TaskAttemptReviewResolutionOutcome::NotFound + }); + }; + let checkpoint = task_checkpoint_from_row(&checkpoint_row)?; + if task.generation != expected_task_generation + || checkpoint.task_generation != expected_task_generation + || checkpoint_review_uid(&checkpoint.payload) != Some(review_uid) + { + conn.commit().await.map_err(storage_error)?; + return Ok(TaskAttemptReviewResolutionOutcome::Stale); + } + let resolution_value = serde_json::to_value(&resolution)?; + if task.status == ExecutionTaskStatus::Ready + && checkpoint_review_resolution(&checkpoint.payload) == Some(&resolution_value) + { + conn.commit().await.map_err(storage_error)?; + return Ok(TaskAttemptReviewResolutionOutcome::Replayed { + task, + checkpoint: Box::new(checkpoint), + }); + } + if task.status == ExecutionTaskStatus::Running { + conn.commit().await.map_err(storage_error)?; + return Ok(TaskAttemptReviewResolutionOutcome::NotReady); + } + if task.status != ExecutionTaskStatus::WaitingReview + || task.attempt_state != ExecutionAttemptState::Waiting + { + conn.commit().await.map_err(storage_error)?; + return Ok(TaskAttemptReviewResolutionOutcome::Stale); + } + let mut payload = checkpoint.payload.clone(); + let Some(object) = payload.as_object_mut() else { + return Err(Error::InvalidRepositoryData { + message: "task review checkpoint payload is not an object".to_string(), + }); + }; + object.insert("review_resolution".to_string(), resolution_value); + let resolved_checkpoint = insert_resolved_task_checkpoint_in_conn( + &mut conn, + &checkpoint, + run.controller_generation, + checkpoint.task_generation, + checkpoint.attempt_generation, + payload, + resolved_at, + ) + .await?; + let next_attempt_generation = task.attempt_generation.checked_add(1).ok_or_else(|| { + Error::InvalidRepositoryInput { + message: "task attempt generation overflow".to_string(), + } + })?; + let row = sqlx::query( + "UPDATE moa.execution_task SET status='ready', attempt_state='idle', \ + attempt_generation=$5, waiting_since=NULL, ready_at=$6, \ + last_progress_at=GREATEST(last_progress_at,$6), \ + generation_history=generation_history || jsonb_build_array(jsonb_build_object( \ + 'kind','action_review_resolved','review_uid',$4::TEXT,'recorded_at',$6)), \ + updated_at=NOW() WHERE run_uid=$1 AND task_id=$2 AND generation=$3 \ + AND status='waiting_review' RETURNING *", + ) + .bind(run_uid) + .bind(task_id.as_uuid()) + .bind(to_i64(expected_task_generation, "task generation")?) + .bind(review_uid) + .bind(to_i64(next_attempt_generation, "next attempt generation")?) + .bind(resolved_at) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(row) = row else { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptReviewResolutionOutcome::Stale); + }; + let task = task_from_row(&row)?; + transition_node_counters_in_tx( + &mut conn, + run_uid, + &task.node_id, + &task.item_key, + ExecutionTaskStatus::WaitingReview, + ExecutionTaskStatus::Ready, + ) + .await?; + refresh_run_after_wait_settlement_in_conn(&mut conn, run_uid, task_id, resolved_at).await?; + if !matches!( + run.status, + ExecutionRunStatus::PauseRequested + | ExecutionRunStatus::Pausing + | ExecutionRunStatus::Paused + ) { + enqueue_run_activation_in_conn( + conn.as_mut(), + run.tenant_id, + run_uid, + run.controller_generation, + resolved_at, + json!({ + "source": "task_action_review_resolution", + "task_id": task_id, + "review_uid": review_uid, + }), + ) + .await?; + } + conn.commit().await.map_err(storage_error)?; + Ok(TaskAttemptReviewResolutionOutcome::Applied { + task, + checkpoint: Box::new(resolved_checkpoint), + }) + } + + /// Persists an exact provider job and fences its task before sandbox teardown begins. + pub async fn begin_task_attempt_external_release( + &self, + fence: TaskAttemptFence, + expected_task_generation: u64, + external_job_uid: Uuid, + claimed_at: DateTime, + ) -> Result { + if expected_task_generation == 0 { + return Err(Error::InvalidRepositoryInput { + message: "external release must name the exact active task attempt".to_string(), + }); + } + let mut conn = ExecutionScope::ControlPlane.begin(&self.pool).await?; + let Some(persisted_job) = + load_external_job_for_update_in_conn(conn.as_mut(), external_job_uid).await? + else { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptReleaseClaimOutcome::NotFound); + }; + if persisted_job.tenant_id != fence.tenant_id + || persisted_job.run_uid != fence.run_uid + || persisted_job.owner + != (ExecutionExternalJobOwner::Task { + task_id: fence.task_id.as_uuid(), + attempt_generation: fence.attempt_generation, + }) + || persisted_job.state == ExecutionExternalJobState::Unbound + { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptReleaseClaimOutcome::Stale); + } + let Some(run_row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) + .bind(fence.run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptReleaseClaimOutcome::NotFound); + }; + let run = run_from_row(&run_row)?; + let Some(task_row) = sqlx::query(LOAD_TASK_FOR_UPDATE_SQL) + .bind(fence.run_uid) + .bind(fence.task_id.as_uuid()) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptReleaseClaimOutcome::NotFound); + }; + let task = task_from_row(&task_row)?; + if !task_attempt_fence_matches(&run, &task, &fence) + || task.generation != expected_task_generation + || !task_attempt_resources_match(&mut conn, &fence).await? + { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptReleaseClaimOutcome::Stale); + } + if task.status == ExecutionTaskStatus::Running + && task.attempt_state == ExecutionAttemptState::Cancelling + && task.external_job_uid == Some(external_job_uid) + { + conn.commit().await.map_err(storage_error)?; + return Ok(TaskAttemptReleaseClaimOutcome::Replayed(Box::new( + TaskAttemptRecord { run, task }, + ))); + } + if task.status != ExecutionTaskStatus::Running + || task.attempt_state != ExecutionAttemptState::Running + || task.external_job_uid.is_some() + || run.status.is_terminal() + { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptReleaseClaimOutcome::InvalidState); + } + let row = sqlx::query( + "UPDATE moa.execution_task SET attempt_state='cancelling', external_job_uid=$5, \ + last_progress_at=GREATEST(last_progress_at,$6), \ + generation_history=generation_history || \ + jsonb_build_array(jsonb_build_object( \ + 'kind','external_job_release_claimed','dispatch_uid',$4::TEXT, \ + 'attempt_generation',$3,'external_job_uid',$5::TEXT,'recorded_at',$6)), \ + updated_at=NOW() \ + WHERE run_uid=$1 AND task_id=$2 AND attempt_generation=$3 \ + AND active_dispatch_uid=$4 AND attempt_state='running' \ + AND external_job_uid IS NULL RETURNING *", + ) + .bind(fence.run_uid) + .bind(fence.task_id.as_uuid()) + .bind(to_i64(fence.attempt_generation, "attempt generation")?) + .bind(fence.dispatch_uid) + .bind(external_job_uid) + .bind(claimed_at) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(row) = row else { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptReleaseClaimOutcome::Stale); + }; + let task = task_from_row(&row)?; + conn.commit().await.map_err(storage_error)?; + Ok(TaskAttemptReleaseClaimOutcome::Applied(Box::new( + TaskAttemptRecord { run, task }, + ))) + } + + /// Claims one exact active attempt for checkpoint-and-release before provider I/O. + pub async fn begin_task_attempt_release( + &self, + fence: TaskAttemptFence, + expected_task_generation: u64, + reason: &str, + claimed_at: DateTime, + ) -> Result { + if expected_task_generation == 0 || reason.trim().is_empty() { + return Err(Error::InvalidRepositoryInput { + message: "task-attempt release requires a generation and reason".to_string(), + }); + } + let mut conn = ExecutionScope::ControlPlane.begin(&self.pool).await?; + let Some(run_row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) + .bind(fence.run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + conn.commit().await.map_err(storage_error)?; + return Ok(TaskAttemptReleaseClaimOutcome::NotFound); + }; + let run = run_from_row(&run_row)?; + let Some(task_row) = sqlx::query(LOAD_TASK_FOR_UPDATE_SQL) + .bind(fence.run_uid) + .bind(fence.task_id.as_uuid()) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + conn.commit().await.map_err(storage_error)?; + return Ok(TaskAttemptReleaseClaimOutcome::NotFound); + }; + let task = task_from_row(&task_row)?; + if !task_attempt_fence_matches(&run, &task, &fence) + || task.generation != expected_task_generation + || !task_attempt_resources_match(&mut conn, &fence).await? + { + conn.commit().await.map_err(storage_error)?; + return Ok(TaskAttemptReleaseClaimOutcome::Stale); + } + if task.status == ExecutionTaskStatus::Running + && task.attempt_state == ExecutionAttemptState::Cancelling + { + conn.commit().await.map_err(storage_error)?; + return Ok(TaskAttemptReleaseClaimOutcome::Replayed(Box::new( + TaskAttemptRecord { run, task }, + ))); + } + if task.status != ExecutionTaskStatus::Running + || task.attempt_state != ExecutionAttemptState::Running + { + conn.commit().await.map_err(storage_error)?; + return Ok(TaskAttemptReleaseClaimOutcome::InvalidState); + } + let row = sqlx::query( + "UPDATE moa.execution_task SET attempt_state='cancelling', \ + last_progress_at=GREATEST(last_progress_at,$5), \ + generation_history=generation_history || jsonb_build_array(jsonb_build_object( \ + 'kind','bounded_attempt_release_claimed','dispatch_uid',$4::TEXT, \ + 'attempt_generation',$3,'reason',$6,'recorded_at',$5)), updated_at=NOW() \ + WHERE run_uid=$1 AND task_id=$2 AND attempt_generation=$3 \ + AND active_dispatch_uid=$4 AND attempt_state='running' RETURNING *", + ) + .bind(fence.run_uid) + .bind(fence.task_id.as_uuid()) + .bind(to_i64(fence.attempt_generation, "attempt generation")?) + .bind(fence.dispatch_uid) + .bind(claimed_at) + .bind(reason) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(row) = row else { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptReleaseClaimOutcome::Stale); + }; + let task = task_from_row(&row)?; + conn.commit().await.map_err(storage_error)?; + Ok(TaskAttemptReleaseClaimOutcome::Applied(Box::new( + TaskAttemptRecord { run, task }, + ))) + } + + /// Persists one reviewed continuation and releases active capacity atomically. + pub async fn park_task_attempt_on_review( + &self, + checkpoint: NewTaskAttemptCheckpoint, + review_uid: Uuid, + ) -> Result { + if review_uid.is_nil() || checkpoint_review_uid(&checkpoint.payload) != Some(review_uid) { + return Ok(TaskAttemptReviewParkOutcome::InvalidState); + } + let fence = checkpoint.fence; + let mut conn = ExecutionScope::ControlPlane.begin(&self.pool).await?; + let capacity = release_task_capacity_in_tx( + &mut conn, + fence.capacity_reservation_uid, + fence.run_uid, + fence.task_id, + fence.attempt_generation, + ) + .await?; + if capacity == CapacityReleaseOutcome::NotFound { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptReviewParkOutcome::NotFound); + } + if capacity == CapacityReleaseOutcome::Stale { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptReviewParkOutcome::Stale); + } + if capacity == CapacityReleaseOutcome::AlreadyReleased { + let task_row = sqlx::query(LOAD_TASK_FOR_UPDATE_SQL) + .bind(fence.run_uid) + .bind(fence.task_id.as_uuid()) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let current_checkpoint = sqlx::query( + "SELECT * FROM moa.execution_task_checkpoint WHERE tenant_id=$1 AND run_uid=$2 \ + AND task_id=$3 AND superseded_at IS NULL FOR UPDATE", + ) + .bind(fence.tenant_id.0) + .bind(fence.run_uid) + .bind(fence.task_id.as_uuid()) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(task_row) = task_row else { + conn.commit().await.map_err(storage_error)?; + return Ok(TaskAttemptReviewParkOutcome::NotFound); + }; + let Some(current_checkpoint) = current_checkpoint else { + conn.commit().await.map_err(storage_error)?; + return Ok(TaskAttemptReviewParkOutcome::NotFound); + }; + let task = task_from_row(&task_row)?; + let current_checkpoint = task_checkpoint_from_row(¤t_checkpoint)?; + let replay = task.status == ExecutionTaskStatus::WaitingReview + && task.attempt_generation == fence.attempt_generation + && task.active_dispatch_uid.is_none() + && current_checkpoint.dispatch_uid == fence.dispatch_uid + && checkpoint_review_uid(¤t_checkpoint.payload) == Some(review_uid); + conn.commit().await.map_err(storage_error)?; + return Ok(if replay { + TaskAttemptReviewParkOutcome::Replayed { + task, + checkpoint: Box::new(current_checkpoint), + } + } else { + TaskAttemptReviewParkOutcome::Stale + }); + } + if let Some(receipt) = checkpoint.workspace_release_receipt.as_ref() + && !persisted_task_release_receipt_matches( + &mut conn, + &fence, + checkpoint.task_generation, + receipt, + ) + .await? + { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptReviewParkOutcome::Stale); + } + + let persisted = + match persist_task_attempt_checkpoint_in_conn(&mut conn, &checkpoint).await? { + TaskAttemptCheckpointWriteOutcome::Applied(checkpoint) + | TaskAttemptCheckpointWriteOutcome::Replayed(checkpoint) => checkpoint, + TaskAttemptCheckpointWriteOutcome::NotFound => { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptReviewParkOutcome::NotFound); + } + TaskAttemptCheckpointWriteOutcome::Stale => { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptReviewParkOutcome::Stale); + } + TaskAttemptCheckpointWriteOutcome::InvalidState => { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptReviewParkOutcome::InvalidState); + } + }; + if supersede_trigger_in_conn( + conn.as_mut(), + fence.watchdog_trigger_uid, + ExecutionTriggerKind::TaskWatchdog, + Some(fence.controller_generation), + Some(fence.attempt_generation), + None, + None, + ) + .await? + == ExecutionTriggerSupersedeOutcome::StaleOrMissing + { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptReviewParkOutcome::Stale); + } + let row = sqlx::query( + "UPDATE moa.execution_task SET status='waiting_review', attempt_state='waiting', \ + waiting_since=$5, active_dispatch_uid=NULL, attempt_deadline_at=NULL, \ + last_progress_at=GREATEST(last_progress_at,$5), updated_at=NOW() \ + WHERE run_uid=$1 AND task_id=$2 AND attempt_generation=$3 \ + AND active_dispatch_uid=$4 AND status='running' RETURNING *", + ) + .bind(fence.run_uid) + .bind(fence.task_id.as_uuid()) + .bind(to_i64(fence.attempt_generation, "attempt generation")?) + .bind(fence.dispatch_uid) + .bind(checkpoint.created_at) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(row) = row else { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptReviewParkOutcome::Stale); + }; + let task = task_from_row(&row)?; + transition_node_counters_in_tx( + &mut conn, + fence.run_uid, + &task.node_id, + &task.item_key, + ExecutionTaskStatus::Running, + ExecutionTaskStatus::WaitingReview, + ) + .await?; + let review_reason = checkpoint_review_waiting_reason(&persisted, &task)?; + append_run_wait_reason_in_tx( + &mut conn, + fence.run_uid, + &review_reason, + checkpoint.created_at, + ) + .await?; + enqueue_run_activation_in_conn( + conn.as_mut(), + fence.tenant_id, + fence.run_uid, + fence.controller_generation, + checkpoint.created_at, + json!({ + "source": "task_attempt_review_park", + "task_id": fence.task_id, + "dispatch_uid": fence.dispatch_uid, + "attempt_generation": fence.attempt_generation, + }), + ) + .await?; + conn.commit().await.map_err(storage_error)?; + Ok(TaskAttemptReviewParkOutcome::Applied { + task, + checkpoint: persisted, + }) + } + + /// Checkpoints one bounded agent slice and atomically returns the task to ready storage. + pub async fn yield_task_attempt_continuation( + &self, + checkpoint: NewTaskAttemptCheckpoint, + ) -> Result { + let fence = checkpoint.fence; + let mut conn = ExecutionScope::ControlPlane.begin(&self.pool).await?; + let capacity = release_task_capacity_in_tx( + &mut conn, + fence.capacity_reservation_uid, + fence.run_uid, + fence.task_id, + fence.attempt_generation, + ) + .await?; + if capacity == CapacityReleaseOutcome::NotFound { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptContinuationYieldOutcome::NotFound); + } + if capacity == CapacityReleaseOutcome::Stale { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptContinuationYieldOutcome::Stale); + } + if capacity == CapacityReleaseOutcome::AlreadyReleased { + let task_row = sqlx::query(LOAD_TASK_FOR_UPDATE_SQL) + .bind(fence.run_uid) + .bind(fence.task_id.as_uuid()) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let checkpoint_row = sqlx::query( + "SELECT * FROM moa.execution_task_checkpoint WHERE tenant_id=$1 AND run_uid=$2 \ + AND task_id=$3 AND superseded_at IS NULL FOR UPDATE", + ) + .bind(fence.tenant_id.0) + .bind(fence.run_uid) + .bind(fence.task_id.as_uuid()) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let (Some(task_row), Some(checkpoint_row)) = (task_row, checkpoint_row) else { + conn.commit().await.map_err(storage_error)?; + return Ok(TaskAttemptContinuationYieldOutcome::NotFound); + }; + let task = task_from_row(&task_row)?; + let persisted = task_checkpoint_from_row(&checkpoint_row)?; + let replay = task.status == ExecutionTaskStatus::Ready + && task.attempt_state == ExecutionAttemptState::Idle + && task.active_dispatch_uid.is_none() + && persisted.dispatch_uid == fence.dispatch_uid; + conn.commit().await.map_err(storage_error)?; + return Ok(if replay { + TaskAttemptContinuationYieldOutcome::Replayed { + task, + checkpoint: Box::new(persisted), + } + } else { + TaskAttemptContinuationYieldOutcome::Stale + }); + } + if let Some(receipt) = checkpoint.workspace_release_receipt.as_ref() + && !persisted_task_release_receipt_matches( + &mut conn, + &fence, + checkpoint.task_generation, + receipt, + ) + .await? + { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptContinuationYieldOutcome::Stale); + } + let persisted = + match persist_task_attempt_checkpoint_in_conn(&mut conn, &checkpoint).await? { + TaskAttemptCheckpointWriteOutcome::Applied(checkpoint) + | TaskAttemptCheckpointWriteOutcome::Replayed(checkpoint) => checkpoint, + TaskAttemptCheckpointWriteOutcome::NotFound => { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptContinuationYieldOutcome::NotFound); + } + TaskAttemptCheckpointWriteOutcome::Stale => { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptContinuationYieldOutcome::Stale); + } + TaskAttemptCheckpointWriteOutcome::InvalidState => { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptContinuationYieldOutcome::InvalidState); + } + }; + if supersede_trigger_in_conn( + conn.as_mut(), + fence.watchdog_trigger_uid, + ExecutionTriggerKind::TaskWatchdog, + Some(fence.controller_generation), + Some(fence.attempt_generation), + None, + None, + ) + .await? + == ExecutionTriggerSupersedeOutcome::StaleOrMissing + { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptContinuationYieldOutcome::Stale); + } + let next_attempt_generation = fence.attempt_generation.checked_add(1).ok_or_else(|| { + Error::InvalidRepositoryData { + message: "task attempt generation overflow".to_string(), + } + })?; + let row = sqlx::query( + "UPDATE moa.execution_task SET status='ready', attempt_state='idle', ready_at=$5, \ + waiting_since=NULL, active_dispatch_uid=NULL, attempt_deadline_at=NULL, \ + attempt_generation=$6, last_progress_at=GREATEST(last_progress_at,$5), \ + updated_at=NOW() \ + WHERE run_uid=$1 AND task_id=$2 AND attempt_generation=$3 \ + AND active_dispatch_uid=$4 AND status='running' AND attempt_state='cancelling' \ + RETURNING *", + ) + .bind(fence.run_uid) + .bind(fence.task_id.as_uuid()) + .bind(to_i64(fence.attempt_generation, "attempt generation")?) + .bind(fence.dispatch_uid) + .bind(checkpoint.created_at) + .bind(to_i64(next_attempt_generation, "next attempt generation")?) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(row) = row else { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptContinuationYieldOutcome::Stale); + }; + let task = task_from_row(&row)?; + transition_node_counters_in_tx( + &mut conn, + fence.run_uid, + &task.node_id, + &task.item_key, + ExecutionTaskStatus::Running, + ExecutionTaskStatus::Ready, + ) + .await?; + enqueue_run_activation_in_conn( + conn.as_mut(), + fence.tenant_id, + fence.run_uid, + fence.controller_generation, + checkpoint.created_at, + json!({ + "source": "task_attempt_continuation", + "task_id": fence.task_id, + "dispatch_uid": fence.dispatch_uid, + "attempt_generation": fence.attempt_generation, + }), + ) + .await?; + conn.commit().await.map_err(storage_error)?; + Ok(TaskAttemptContinuationYieldOutcome::Applied { + task, + checkpoint: persisted, + }) + } + + /// Starts one exact admitted task attempt in a control-plane transaction. + pub async fn start_task_attempt( + &self, + fence: TaskAttemptFence, + ) -> Result { + let mut conn = ExecutionScope::ControlPlane.begin(&self.pool).await?; + let Some(run_row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) + .bind(fence.run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + conn.commit().await.map_err(storage_error)?; + return Ok(TaskAttemptStartOutcome::NotFound); + }; + let run = run_from_row(&run_row)?; + let Some(task_row) = sqlx::query(LOAD_TASK_FOR_UPDATE_SQL) + .bind(fence.run_uid) + .bind(fence.task_id.as_uuid()) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + conn.commit().await.map_err(storage_error)?; + return Ok(TaskAttemptStartOutcome::NotFound); + }; + let task = task_from_row(&task_row)?; + if !task_attempt_fence_matches(&run, &task, &fence) + || !task_attempt_resources_match(&mut conn, &fence).await? + { + conn.commit().await.map_err(storage_error)?; + return Ok(TaskAttemptStartOutcome::Stale); + } + if task.status == ExecutionTaskStatus::Running + && task.attempt_state == ExecutionAttemptState::Running + { + conn.commit().await.map_err(storage_error)?; + return Ok(TaskAttemptStartOutcome::AlreadyStarted(Box::new( + TaskAttemptRecord { run, task }, + ))); + } + if task.status != ExecutionTaskStatus::Dispatching + || task.attempt_state != ExecutionAttemptState::Dispatching + || !matches!( + run.status, + ExecutionRunStatus::Queued | ExecutionRunStatus::Running + ) + || run.pending_terminal.is_some() + || storage_only_task_kind(&task.kind) + { + conn.commit().await.map_err(storage_error)?; + return Ok(TaskAttemptStartOutcome::InvalidState); + } + let row = sqlx::query( + "UPDATE moa.execution_task \ + SET status = 'running', attempt_state = 'running', \ + attempt_started_at = COALESCE(attempt_started_at, NOW()), \ + started_at = COALESCE(started_at, NOW()), last_progress_at = NOW(), \ + updated_at = NOW() \ + WHERE run_uid = $1 AND task_id = $2 AND status = 'dispatching' \ + AND attempt_generation = $3 AND active_dispatch_uid = $4 \ + RETURNING *", + ) + .bind(fence.run_uid) + .bind(fence.task_id.as_uuid()) + .bind(to_i64(fence.attempt_generation, "attempt generation")?) + .bind(fence.dispatch_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + sqlx::query( + "UPDATE moa.execution_run \ + SET status = CASE WHEN status = 'queued' THEN 'running' ELSE status END, \ + started_at = COALESCE(started_at, NOW()), last_progress_at = NOW(), \ + updated_at = NOW() WHERE run_uid = $1", + ) + .bind(fence.run_uid) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let task = task_from_row(&row)?; + conn.commit().await.map_err(storage_error)?; + Ok(TaskAttemptStartOutcome::Started(Box::new( + TaskAttemptRecord { run, task }, + ))) + } + + /// Releases and settles an exact dispatch whose receiver never committed its start fence. + pub async fn settle_unstarted_task_attempt( + &self, + fence: TaskAttemptFence, + disposition: UnstartedTaskAttemptDisposition, + settled_at: DateTime, + ) -> Result { + if let UnstartedTaskAttemptDisposition::Cancelled { reason } = &disposition + && (reason.trim().is_empty() || reason.chars().count() > 1_024) + { + return Err(Error::InvalidRepositoryInput { + message: "unstarted task cancellation reason must contain 1..=1024 characters" + .to_string(), + }); + } + if matches!( + &disposition, + UnstartedTaskAttemptDisposition::Paused { + controller_generation: 0 + } + ) { + return Err(Error::InvalidRepositoryInput { + message: "unstarted task pause controller generation must be positive".to_string(), + }); + } + if disposition == UnstartedTaskAttemptDisposition::DispatchDeliveryLost { + let mut conn = ExecutionScope::ControlPlane.begin(&self.pool).await?; + let outcome = + settle_unstarted_task_attempt_in_conn(&mut conn, fence, settled_at).await?; + if matches!( + outcome, + TaskAttemptSettlementOutcome::Applied { .. } + | TaskAttemptSettlementOutcome::Replayed { .. } + ) { + conn.commit().await.map_err(storage_error)?; + } else { + conn.rollback().await.map_err(storage_error)?; + } + return Ok(outcome); + } + let mut conn = ExecutionScope::ControlPlane.begin(&self.pool).await?; + let capacity = release_task_capacity_in_tx( + &mut conn, + fence.capacity_reservation_uid, + fence.run_uid, + fence.task_id, + fence.attempt_generation, + ) + .await?; + if capacity == CapacityReleaseOutcome::NotFound { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptSettlementOutcome::NotFound); + } + if capacity == CapacityReleaseOutcome::Stale { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptSettlementOutcome::Stale); + } + let Some(run_row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) + .bind(fence.run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptSettlementOutcome::NotFound); + }; + let run = run_from_row(&run_row)?; + let Some(task_row) = sqlx::query(LOAD_TASK_FOR_UPDATE_SQL) + .bind(fence.run_uid) + .bind(fence.task_id.as_uuid()) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptSettlementOutcome::NotFound); + }; + let task = task_from_row(&task_row)?; + if capacity == CapacityReleaseOutcome::AlreadyReleased { + let replay = unstarted_task_attempt_run_fence_matches(&run, &fence, &disposition) + && unstarted_task_attempt_settlement_replayed(&task, &fence, &disposition); + conn.commit().await.map_err(storage_error)?; + return Ok(if replay { + TaskAttemptSettlementOutcome::Replayed { run, task } + } else { + TaskAttemptSettlementOutcome::Stale + }); + } + if !unstarted_task_attempt_fence_matches(&run, &task, &fence, &disposition) + || run.status.is_terminal() + { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptSettlementOutcome::Stale); + } + if matches!(&disposition, UnstartedTaskAttemptDisposition::Paused { .. }) + && !matches!( + run.status, + ExecutionRunStatus::PauseRequested + | ExecutionRunStatus::Pausing + | ExecutionRunStatus::Paused + ) + { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptSettlementOutcome::InvalidState); + } + let expected_attempt_state = match &disposition { + UnstartedTaskAttemptDisposition::Cancelled { .. } + | UnstartedTaskAttemptDisposition::Paused { .. } => ExecutionAttemptState::Cancelling, + UnstartedTaskAttemptDisposition::DispatchDeliveryLost => { + ExecutionAttemptState::Dispatching + } + }; + if task.status != ExecutionTaskStatus::Dispatching + || task.attempt_state != expected_attempt_state + { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptSettlementOutcome::InvalidState); + } + if supersede_trigger_in_conn( + conn.as_mut(), + fence.watchdog_trigger_uid, + ExecutionTriggerKind::TaskWatchdog, + Some(fence.controller_generation), + Some(fence.attempt_generation), + None, + None, + ) + .await? + == ExecutionTriggerSupersedeOutcome::StaleOrMissing + { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptSettlementOutcome::Stale); + } + let history = json!({ + "kind": "unstarted_attempt_settlement", + "dispatch_uid": fence.dispatch_uid, + "attempt_generation": fence.attempt_generation, + "disposition": unstarted_disposition_label(&disposition), + "reason": match &disposition { + UnstartedTaskAttemptDisposition::Cancelled { reason } => Some(reason.as_str()), + UnstartedTaskAttemptDisposition::Paused { .. } + | UnstartedTaskAttemptDisposition::DispatchDeliveryLost => None, + }, + "controller_generation": match &disposition { + UnstartedTaskAttemptDisposition::Paused { controller_generation } => { + Some(*controller_generation) + } + UnstartedTaskAttemptDisposition::Cancelled { .. } + | UnstartedTaskAttemptDisposition::DispatchDeliveryLost => None, + }, + "recorded_at": settled_at, + }); + let task = match &disposition { + UnstartedTaskAttemptDisposition::Cancelled { reason } => { + let outcome = cancelled_task_outcome(reason.clone(), task.actual.clone()); + let accepted = match record_task_outcome_in_conn( + &mut conn, + fence.run_uid, + fence.task_id, + task.generation, + outcome, + ) + .await? + { + TaskOutcomeWrite::Applied { task, .. } => task, + TaskOutcomeWrite::Replayed { run, task, .. } => { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptSettlementOutcome::Replayed { run, task }); + } + TaskOutcomeWrite::NotFound => { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptSettlementOutcome::NotFound); + } + TaskOutcomeWrite::Rejected { .. } => { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptSettlementOutcome::InvalidState); + } + }; + let row = sqlx::query( + "UPDATE moa.execution_task SET active_dispatch_uid=NULL, \ + attempt_deadline_at=NULL, generation_history=generation_history || \ + jsonb_build_array($5::JSONB), \ + last_progress_at=GREATEST(last_progress_at,$6), updated_at=NOW() \ + WHERE run_uid=$1 AND task_id=$2 AND attempt_generation=$3 \ + AND active_dispatch_uid=$4 AND status='cancelled' RETURNING *", + ) + .bind(fence.run_uid) + .bind(fence.task_id.as_uuid()) + .bind(to_i64(fence.attempt_generation, "attempt generation")?) + .bind(fence.dispatch_uid) + .bind(&history) + .bind(settled_at) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(row) = row else { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptSettlementOutcome::Stale); + }; + let task = task_from_row(&row)?; + debug_assert_eq!(task.task_id, accepted.task_id); + task + } + UnstartedTaskAttemptDisposition::Paused { .. } + | UnstartedTaskAttemptDisposition::DispatchDeliveryLost => { + let next_attempt_generation = + fence.attempt_generation.checked_add(1).ok_or_else(|| { + Error::InvalidRepositoryData { + message: "task attempt generation overflow".to_string(), + } + })?; + let expected_attempt_state = + if matches!(&disposition, UnstartedTaskAttemptDisposition::Paused { .. }) { + "cancelling" + } else { + "dispatching" + }; + let row = sqlx::query( + "UPDATE moa.execution_task SET status='ready', attempt_state='idle', \ + attempt_generation=$5, active_dispatch_uid=NULL, \ + attempt_deadline_at=NULL, ready_at=$6, waiting_since=NULL, \ + generation_history=generation_history || jsonb_build_array($7::JSONB), \ + last_progress_at=GREATEST(last_progress_at,$6), updated_at=NOW() \ + WHERE run_uid=$1 AND task_id=$2 AND attempt_generation=$3 \ + AND active_dispatch_uid=$4 AND status='dispatching' \ + AND attempt_state=$8 RETURNING *", + ) + .bind(fence.run_uid) + .bind(fence.task_id.as_uuid()) + .bind(to_i64(fence.attempt_generation, "attempt generation")?) + .bind(fence.dispatch_uid) + .bind(to_i64(next_attempt_generation, "next attempt generation")?) + .bind(settled_at) + .bind(&history) + .bind(expected_attempt_state) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(row) = row else { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptSettlementOutcome::Stale); + }; + task_from_row(&row)? + } + }; + transition_node_counters_in_tx( + &mut conn, + fence.run_uid, + &task.node_id, + &task.item_key, + ExecutionTaskStatus::Dispatching, + task.status, + ) + .await?; + if !matches!(&disposition, UnstartedTaskAttemptDisposition::Paused { .. }) { + enqueue_run_activation_in_conn( + conn.as_mut(), + fence.tenant_id, + fence.run_uid, + fence.controller_generation, + settled_at, + json!({ + "source": "unstarted_task_attempt_settlement", + "task_id": fence.task_id, + "dispatch_uid": fence.dispatch_uid, + "attempt_generation": fence.attempt_generation, + "disposition": unstarted_disposition_label(&disposition), + }), + ) + .await?; + } + let run_row = sqlx::query(LOAD_RUN_SQL) + .bind(fence.run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let run = run_from_row(&run_row)?; + conn.commit().await.map_err(storage_error)?; + Ok(TaskAttemptSettlementOutcome::Applied { run, task }) + } + + /// Records monotonic progress for one exact active task attempt. + pub async fn record_task_attempt_progress( + &self, + fence: TaskAttemptFence, + observed_at: DateTime, + ) -> Result { + let mut conn = ExecutionScope::ControlPlane.begin(&self.pool).await?; + let Some(run_row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) + .bind(fence.run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + conn.commit().await.map_err(storage_error)?; + return Ok(TaskAttemptProgressOutcome::NotFound); + }; + let run = run_from_row(&run_row)?; + let Some(task_row) = sqlx::query(LOAD_TASK_FOR_UPDATE_SQL) + .bind(fence.run_uid) + .bind(fence.task_id.as_uuid()) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + conn.commit().await.map_err(storage_error)?; + return Ok(TaskAttemptProgressOutcome::NotFound); + }; + let task = task_from_row(&task_row)?; + if !task_attempt_fence_matches(&run, &task, &fence) { + conn.commit().await.map_err(storage_error)?; + return Ok(TaskAttemptProgressOutcome::Stale); + } + if task.status != ExecutionTaskStatus::Running + || task.attempt_state != ExecutionAttemptState::Running + { + conn.commit().await.map_err(storage_error)?; + return Ok(TaskAttemptProgressOutcome::InvalidState); + } + if observed_at <= task.last_progress_at { + conn.commit().await.map_err(storage_error)?; + return Ok(TaskAttemptProgressOutcome::Replayed); + } + sqlx::query( + "UPDATE moa.execution_task SET last_progress_at = $5, updated_at = NOW() \ + WHERE run_uid = $1 AND task_id = $2 AND attempt_generation = $3 \ + AND active_dispatch_uid = $4 AND attempt_state = 'running'", + ) + .bind(fence.run_uid) + .bind(fence.task_id.as_uuid()) + .bind(to_i64(fence.attempt_generation, "attempt generation")?) + .bind(fence.dispatch_uid) + .bind(observed_at) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + sqlx::query( + "UPDATE moa.execution_run SET last_progress_at = GREATEST(last_progress_at, $2), \ + updated_at = NOW() WHERE run_uid = $1", + ) + .bind(fence.run_uid) + .bind(observed_at) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + conn.commit().await.map_err(storage_error)?; + Ok(TaskAttemptProgressOutcome::Applied) + } + + /// Persists one bounded task outcome, releases active capacity, and wakes its controller. + /// + /// `retry_at` must be present exactly for a retryable failure. The logical and + /// attempt generations are advanced before that delayed controller activation + /// can admit the next slice. + pub async fn settle_task_attempt( + &self, + config: &ExecutionConfig, + fence: TaskAttemptFence, + outcome: ExecutionTaskOutcome, + retry_at: Option>, + settled_at: DateTime, + ) -> Result { + self.settle_task_attempt_inner(SettleTaskAttemptRequest { + config, + fence, + outcome, + retry_at, + settled_at, + expected_attempt_state: ExecutionAttemptState::Running, + workspace_release_receipt: None, + continuation_checkpoint: None, + }) + .await + } + + /// Finalizes a cancelling attempt only after exact sandbox release proof is available. + pub async fn settle_released_task_attempt( + &self, + config: &ExecutionConfig, + fence: TaskAttemptFence, + outcome: ExecutionTaskOutcome, + retry_at: Option>, + settled_at: DateTime, + workspace_release_receipt: Option, + ) -> Result { + if workspace_release_receipt.as_ref().is_some_and(|receipt| { + receipt.tenant_id != fence.tenant_id + || receipt.run_id.0 != fence.run_uid + || !matches!( + receipt.owner, + ExecutionHandReleaseOwner::Task { task_id, .. } + if task_id.0 == fence.task_id.as_uuid() + ) + || receipt.attempt_generation != fence.attempt_generation + }) { + return Ok(TaskAttemptSettlementOutcome::Stale); + } + self.settle_task_attempt_inner(SettleTaskAttemptRequest { + config, + fence, + outcome, + retry_at, + settled_at, + expected_attempt_state: ExecutionAttemptState::Cancelling, + workspace_release_receipt, + continuation_checkpoint: None, + }) + .await + } + + /// Releases the exact normal-outcome task capacity after durable sandbox-release proof. + pub async fn release_released_task_attempt_capacity( + &self, + fence: TaskAttemptFence, + logical_generation: u64, + workspace_release_receipt: ExecutionHandReleaseReceipt, + ) -> Result { + if workspace_release_receipt.tenant_id != fence.tenant_id + || workspace_release_receipt.run_id.0 != fence.run_uid + || !matches!( + workspace_release_receipt.owner, + ExecutionHandReleaseOwner::Task { task_id, logical_generation: receipt_generation } + if task_id.0 == fence.task_id.as_uuid() + && receipt_generation == logical_generation + ) + || workspace_release_receipt.attempt_generation != fence.attempt_generation + { + return Ok(ReleasedTaskAttemptCapacityOutcome::Stale); + } + let mut conn = ExecutionScope::ControlPlane.begin(&self.pool).await?; + // The exact admitted task and watchdog receipts prove these four buckets already exist. + // Release must preserve their persisted limits rather than reconcile current config. + prelock_existing_capacity_dimensions_in_tx( + conn.as_mut(), + fence.tenant_id, + &[ + ExecutionCapacityDimension::ActiveTasks, + ExecutionCapacityDimension::ScheduledTriggers, + ], + ) + .await?; + let capacity = release_task_capacity_in_tx( + &mut conn, + fence.capacity_reservation_uid, + fence.run_uid, + fence.task_id, + fence.attempt_generation, + ) + .await?; + if capacity == CapacityReleaseOutcome::NotFound { + conn.rollback().await.map_err(storage_error)?; + return Ok(ReleasedTaskAttemptCapacityOutcome::NotFound); + } + if capacity == CapacityReleaseOutcome::Stale { + conn.rollback().await.map_err(storage_error)?; + return Ok(ReleasedTaskAttemptCapacityOutcome::Stale); + } + let Some(run_row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) + .bind(fence.run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + conn.rollback().await.map_err(storage_error)?; + return Ok(ReleasedTaskAttemptCapacityOutcome::NotFound); + }; + let run = run_from_row(&run_row)?; + let Some(task_row) = sqlx::query(LOAD_TASK_FOR_UPDATE_SQL) + .bind(fence.run_uid) + .bind(fence.task_id.as_uuid()) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + conn.rollback().await.map_err(storage_error)?; + return Ok(ReleasedTaskAttemptCapacityOutcome::NotFound); + }; + let task = task_from_row(&task_row)?; + if !task_attempt_fence_matches(&run, &task, &fence) + || task.generation != logical_generation + || !persisted_task_release_receipt_matches( + &mut conn, + &fence, + logical_generation, + &workspace_release_receipt, + ) + .await? + { + conn.rollback().await.map_err(storage_error)?; + return Ok(ReleasedTaskAttemptCapacityOutcome::Stale); + } + if task.status != ExecutionTaskStatus::Running + || task.attempt_state != ExecutionAttemptState::Cancelling + || run.status.is_terminal() + { + conn.rollback().await.map_err(storage_error)?; + return Ok(ReleasedTaskAttemptCapacityOutcome::InvalidState); + } + let superseded = supersede_trigger_in_conn( + conn.as_mut(), + fence.watchdog_trigger_uid, + ExecutionTriggerKind::TaskWatchdog, + Some(fence.controller_generation), + Some(fence.attempt_generation), + None, + None, + ) + .await?; + if superseded == ExecutionTriggerSupersedeOutcome::StaleOrMissing { + conn.rollback().await.map_err(storage_error)?; + return Ok(ReleasedTaskAttemptCapacityOutcome::Stale); + } + conn.commit().await.map_err(storage_error)?; + Ok(if capacity == CapacityReleaseOutcome::Released { + ReleasedTaskAttemptCapacityOutcome::Applied + } else { + ReleasedTaskAttemptCapacityOutcome::Replayed + }) + } + + /// Settles an input wait while preserving the exact resumable agent continuation. + pub async fn settle_released_task_attempt_with_checkpoint( + &self, + config: &ExecutionConfig, + fence: TaskAttemptFence, + outcome: ExecutionTaskOutcome, + settled_at: DateTime, + workspace_release_receipt: Option, + checkpoint: NewTaskAttemptCheckpoint, + ) -> Result { + if !matches!(outcome.result, ExecutionTaskResult::NeedsInput { .. }) + || checkpoint.fence != fence + || checkpoint.workspace_release_receipt != workspace_release_receipt + { + return Ok(TaskAttemptSettlementOutcome::InvalidState); + } + self.settle_task_attempt_inner(SettleTaskAttemptRequest { + config, + fence, + outcome, + retry_at: None, + settled_at, + expected_attempt_state: ExecutionAttemptState::Cancelling, + workspace_release_receipt, + continuation_checkpoint: Some(checkpoint), + }) + .await + } + + /// Finalizes a pause-owned release by returning the logical task to Ready without dispatch. + pub async fn finalize_paused_task_attempt_release( + &self, + current_controller_generation: u64, + fence: TaskAttemptFence, + settled_at: DateTime, + workspace_release_receipt: Option, + ) -> Result { + if workspace_release_receipt.as_ref().is_some_and(|receipt| { + receipt.tenant_id != fence.tenant_id + || receipt.run_id.0 != fence.run_uid + || !matches!( + receipt.owner, + ExecutionHandReleaseOwner::Task { task_id, .. } + if task_id.0 == fence.task_id.as_uuid() + ) + || receipt.attempt_generation != fence.attempt_generation + }) { + return Ok(TaskAttemptSettlementOutcome::Stale); + } + let mut conn = ExecutionScope::ControlPlane.begin(&self.pool).await?; + let capacity = release_task_capacity_in_tx( + &mut conn, + fence.capacity_reservation_uid, + fence.run_uid, + fence.task_id, + fence.attempt_generation, + ) + .await?; + if capacity == CapacityReleaseOutcome::NotFound { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptSettlementOutcome::NotFound); + } + if capacity == CapacityReleaseOutcome::Stale { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptSettlementOutcome::Stale); + } + let Some(run_row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) + .bind(fence.run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptSettlementOutcome::NotFound); + }; + let run = run_from_row(&run_row)?; + let Some(task_row) = sqlx::query(LOAD_TASK_FOR_UPDATE_SQL) + .bind(fence.run_uid) + .bind(fence.task_id.as_uuid()) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptSettlementOutcome::NotFound); + }; + let task = task_from_row(&task_row)?; + if capacity == CapacityReleaseOutcome::AlreadyReleased { + let replay = task.status == ExecutionTaskStatus::Ready + && task.active_dispatch_uid.is_none() + && task.attempt_generation == fence.attempt_generation.saturating_add(1) + && run.controller_generation == current_controller_generation + && matches!( + run.status, + ExecutionRunStatus::PauseRequested + | ExecutionRunStatus::Pausing + | ExecutionRunStatus::Paused + ) + && paused_task_attempt_release_history_matches( + &task.generation_history, + &fence, + current_controller_generation, + ); + conn.commit().await.map_err(storage_error)?; + return Ok(if replay { + TaskAttemptSettlementOutcome::Replayed { run, task } + } else { + TaskAttemptSettlementOutcome::Stale + }); + } + if !task_attempt_resource_fence_matches(&run, &task, &fence) + || run.controller_generation != current_controller_generation + || task.status != ExecutionTaskStatus::Running + || task.attempt_state != ExecutionAttemptState::Cancelling + || !matches!( + run.status, + ExecutionRunStatus::PauseRequested + | ExecutionRunStatus::Pausing + | ExecutionRunStatus::Paused + ) + { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptSettlementOutcome::InvalidState); + } + if let Some(receipt) = workspace_release_receipt.as_ref() + && !persisted_task_release_receipt_matches(&mut conn, &fence, task.generation, receipt) + .await? + { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptSettlementOutcome::Stale); + } + if supersede_trigger_in_conn( + conn.as_mut(), + fence.watchdog_trigger_uid, + ExecutionTriggerKind::TaskWatchdog, + Some(fence.controller_generation), + Some(fence.attempt_generation), + None, + None, + ) + .await? + == ExecutionTriggerSupersedeOutcome::StaleOrMissing + { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptSettlementOutcome::Stale); + } + let row = sqlx::query( + "UPDATE moa.execution_task SET status='ready', attempt_state='idle', \ + attempt_generation=attempt_generation+1, active_dispatch_uid=NULL, \ + attempt_deadline_at=NULL, ready_at=$5, waiting_since=NULL, \ + last_progress_at=GREATEST(last_progress_at,$5), \ + generation_history=generation_history || jsonb_build_array(jsonb_build_object( \ + 'kind','pause_release_finalized','dispatch_uid',$4::TEXT, \ + 'attempt_generation',$3,'attempt_controller_generation',$7, \ + 'controller_generation',$8,'workspace_release_receipt_id',$6::TEXT, \ + 'recorded_at',$5)), updated_at=NOW() \ + WHERE run_uid=$1 AND task_id=$2 AND attempt_generation=$3 \ + AND active_dispatch_uid=$4 AND status='running' \ + AND attempt_state='cancelling' RETURNING *", + ) + .bind(fence.run_uid) + .bind(fence.task_id.as_uuid()) + .bind(to_i64(fence.attempt_generation, "attempt generation")?) + .bind(fence.dispatch_uid) + .bind(settled_at) + .bind( + workspace_release_receipt + .as_ref() + .map(|receipt| receipt.receipt_id), + ) + .bind(to_i64( + fence.controller_generation, + "attempt controller generation", + )?) + .bind(to_i64( + current_controller_generation, + "current controller generation", + )?) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let task = task_from_row(&row)?; + transition_node_counters_in_tx( + &mut conn, + fence.run_uid, + &task.node_id, + &task.item_key, + ExecutionTaskStatus::Running, + ExecutionTaskStatus::Ready, + ) + .await?; + let run_row = sqlx::query(LOAD_RUN_SQL) + .bind(fence.run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let run = run_from_row(&run_row)?; + conn.commit().await.map_err(storage_error)?; + Ok(TaskAttemptSettlementOutcome::Applied { run, task }) + } + + async fn settle_task_attempt_inner( + &self, + request: SettleTaskAttemptRequest<'_>, + ) -> Result { + let SettleTaskAttemptRequest { + config, + fence, + outcome, + retry_at, + settled_at, + expected_attempt_state, + workspace_release_receipt, + continuation_checkpoint, + } = request; + let is_retry = matches!( + outcome.result, + ExecutionTaskResult::Failed { + class: moa_artifacts::execution_plan::ExecutionFailureClass::Retryable, + .. + } + ); + if is_retry != retry_at.is_some() || retry_at.is_some_and(|at| at < settled_at) { + return Err(Error::InvalidRepositoryInput { + message: "task attempt retry time must be present exactly for a future retry" + .to_string(), + }); + } + let mut conn = ExecutionScope::ControlPlane.begin(&self.pool).await?; + let capacity_was_pre_released = workspace_release_receipt.is_some() + && sqlx::query_scalar::<_, bool>( + "SELECT EXISTS (SELECT 1 FROM moa.execution_capacity_reservation \ + WHERE reservation_uid=$1 AND tenant_id=$2 AND run_uid=$3 AND task_id=$4 \ + AND controller_generation=$5 AND attempt_generation=$6 \ + AND resource_dimension='active_tasks' AND state='released')", + ) + .bind(fence.capacity_reservation_uid) + .bind(fence.tenant_id.0) + .bind(fence.run_uid) + .bind(fence.task_id.as_uuid()) + .bind(to_i64( + fence.controller_generation, + "controller generation", + )?) + .bind(to_i64(fence.attempt_generation, "attempt generation")?) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let capacity = if capacity_was_pre_released { + CapacityReleaseOutcome::AlreadyReleased + } else { + release_task_capacity_in_tx( + &mut conn, + fence.capacity_reservation_uid, + fence.run_uid, + fence.task_id, + fence.attempt_generation, + ) + .await? + }; + if matches!(capacity, CapacityReleaseOutcome::NotFound) { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptSettlementOutcome::NotFound); + } + if matches!(capacity, CapacityReleaseOutcome::Stale) { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptSettlementOutcome::Stale); + } + let Some(run_row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) + .bind(fence.run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptSettlementOutcome::NotFound); + }; + let run = run_from_row(&run_row)?; + let Some(task_row) = sqlx::query(LOAD_TASK_FOR_UPDATE_SQL) + .bind(fence.run_uid) + .bind(fence.task_id.as_uuid()) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptSettlementOutcome::NotFound); + }; + let task = task_from_row(&task_row)?; + if capacity == CapacityReleaseOutcome::AlreadyReleased { + let replay = task_attempt_settlement_replayed(&task, &fence, &outcome); + if replay { + conn.commit().await.map_err(storage_error)?; + return Ok(TaskAttemptSettlementOutcome::Replayed { run, task }); + } + if !capacity_was_pre_released { + conn.commit().await.map_err(storage_error)?; + return Ok(TaskAttemptSettlementOutcome::Stale); + } + } + if !task_attempt_fence_matches(&run, &task, &fence) { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptSettlementOutcome::Stale); + } + if let Some(receipt) = workspace_release_receipt.as_ref() + && !persisted_task_release_receipt_matches(&mut conn, &fence, task.generation, receipt) + .await? + { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptSettlementOutcome::Stale); + } + if task.status != ExecutionTaskStatus::Running + || task.attempt_state != expected_attempt_state + || run.status.is_terminal() + || (is_retry && task.attempt >= task.retry.max_attempts) + { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptSettlementOutcome::InvalidState); + } + if capacity_was_pre_released { + let watchdog_was_superseded = sqlx::query_scalar::<_, bool>( + "SELECT EXISTS (SELECT 1 FROM moa.execution_trigger \ + WHERE trigger_uid=$1 AND tenant_id=$2 AND run_uid=$3 AND task_id=$4 \ + AND trigger_kind='task_watchdog' AND controller_generation=$5 \ + AND attempt_generation=$6 AND state='superseded')", + ) + .bind(fence.watchdog_trigger_uid) + .bind(fence.tenant_id.0) + .bind(fence.run_uid) + .bind(fence.task_id.as_uuid()) + .bind(to_i64( + fence.controller_generation, + "controller generation", + )?) + .bind(to_i64(fence.attempt_generation, "attempt generation")?) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if !watchdog_was_superseded { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptSettlementOutcome::Stale); + } + } else { + let superseded = supersede_trigger_in_conn( + conn.as_mut(), + fence.watchdog_trigger_uid, + ExecutionTriggerKind::TaskWatchdog, + Some(fence.controller_generation), + Some(fence.attempt_generation), + None, + None, + ) + .await?; + if superseded == ExecutionTriggerSupersedeOutcome::StaleOrMissing { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptSettlementOutcome::Stale); + } + } + if let Some(checkpoint) = &continuation_checkpoint { + match persist_task_attempt_checkpoint_in_conn(&mut conn, checkpoint).await? { + TaskAttemptCheckpointWriteOutcome::Applied(_) + | TaskAttemptCheckpointWriteOutcome::Replayed(_) => {} + TaskAttemptCheckpointWriteOutcome::NotFound => { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptSettlementOutcome::NotFound); + } + TaskAttemptCheckpointWriteOutcome::Stale => { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptSettlementOutcome::Stale); + } + TaskAttemptCheckpointWriteOutcome::InvalidState => { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptSettlementOutcome::InvalidState); + } + } + } + let logical_generation = task.generation; + let write = record_task_outcome_in_conn( + &mut conn, + fence.run_uid, + fence.task_id, + logical_generation, + outcome, + ) + .await?; + let accepted_task = match write { + TaskOutcomeWrite::Applied { task, .. } => task, + TaskOutcomeWrite::NotFound => { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptSettlementOutcome::NotFound); + } + TaskOutcomeWrite::Rejected { .. } => { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptSettlementOutcome::InvalidState); + } + TaskOutcomeWrite::Replayed { run, task, .. } => { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptSettlementOutcome::Replayed { run, task }); + } + }; + let (status, attempt_state, next_generation, next_attempt_generation, ready_at) = + if let Some(retry_at) = retry_at { + ( + ExecutionTaskStatus::Ready, + ExecutionAttemptState::Idle, + logical_generation.checked_add(1).ok_or_else(|| { + Error::InvalidRepositoryInput { + message: "task logical generation overflow".to_string(), + } + })?, + fence.attempt_generation.checked_add(1).ok_or_else(|| { + Error::InvalidRepositoryInput { + message: "task attempt generation overflow".to_string(), + } + })?, + Some(retry_at), + ) + } else { + let accepted_outcome = accepted_task.current_outcome.as_ref().ok_or_else(|| { + Error::InvalidRepositoryData { + message: "accepted task outcome is missing from its projection".to_string(), + } + })?; + let status = task_status_from_outcome(accepted_outcome, false); + let attempt_state = if status == ExecutionTaskStatus::UnknownOutcome { + ExecutionAttemptState::UnknownOutcome + } else if status.is_terminal() { + ExecutionAttemptState::Terminal + } else { + ExecutionAttemptState::Waiting + }; + ( + status, + attempt_state, + logical_generation, + fence.attempt_generation, + None, + ) + }; + let row = sqlx::query( + "UPDATE moa.execution_task \ + SET status = $3, attempt_state = $4, generation = $5, \ + attempt_generation = $6, attempt = CASE WHEN $7::BOOLEAN THEN attempt + 1 ELSE attempt END, \ + active_dispatch_uid = NULL, attempt_deadline_at = NULL, \ + waiting_since = CASE WHEN $4 = 'waiting' THEN $8 ELSE NULL END, \ + ready_at = $9, last_progress_at = GREATEST(last_progress_at, $8), \ + generation_history = generation_history || jsonb_build_array($11::JSONB), \ + updated_at = NOW() \ + WHERE run_uid = $1 AND task_id = $2 AND active_dispatch_uid = $10 \ + RETURNING *", + ) + .bind(fence.run_uid) + .bind(fence.task_id.as_uuid()) + .bind(status.as_str()) + .bind(attempt_state.as_str()) + .bind(to_i64(next_generation, "next task generation")?) + .bind(to_i64( + next_attempt_generation, + "next attempt generation", + )?) + .bind(is_retry) + .bind(settled_at) + .bind(ready_at) + .bind(fence.dispatch_uid) + .bind(json!({ + "kind": "bounded_attempt_settlement", + "dispatch_uid": fence.dispatch_uid, + "attempt_generation": fence.attempt_generation, + "retry_scheduled": is_retry, + "workspace_release_receipt_id": workspace_release_receipt + .as_ref() + .map(|receipt| receipt.receipt_id), + "recorded_at": settled_at, + })) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let task = task_from_row(&row)?; + let input_wait_due_at = if task.status == ExecutionTaskStatus::WaitingInput { + let run_deadline_at = + run.approved_budget + .deadline_at + .ok_or_else(|| Error::InvalidRepositoryInput { + message: "input waits require an absolute run deadline".to_string(), + })?; + Some(crate::interpreter::resolve_temporal_target( + &run.active_plan.definition.input_wait_policy.expiry, + settled_at, + run_deadline_at, + )?) + } else { + None + }; + if task.status == ExecutionTaskStatus::WaitingInput { + let input_audience = task + .current_outcome + .as_ref() + .and_then(|outcome| match &outcome.result { + ExecutionTaskResult::NeedsInput { audience, .. } => Some(audience), + _ => None, + }) + .ok_or_else(|| Error::InvalidRepositoryData { + message: "waiting-input task is missing its typed input audience".to_string(), + })?; + transition_node_counters_with_input_audience_in_tx( + &mut conn, + fence.run_uid, + &task.node_id, + &task.item_key, + ExecutionTaskStatus::Running, + task.status, + input_audience, + ) + .await?; + let ExecutionTaskResult::NeedsInput { question, audience } = task + .current_outcome + .as_ref() + .ok_or_else(|| Error::InvalidRepositoryData { + message: "waiting-input task is missing its typed outcome".to_string(), + })? + .result + .clone() + else { + return Err(Error::InvalidRepositoryData { + message: "waiting-input task is missing its typed input request".to_string(), + }); + }; + append_run_wait_reason_in_tx( + &mut conn, + fence.run_uid, + &WaitingReason::Input { + task_id: task.task_id, + audience, + question, + wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { + expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::At { + at: input_wait_due_at.ok_or_else(|| Error::InvalidRepositoryData { + message: "waiting-input task is missing its resolved expiry" + .to_string(), + })?, + }, + on_expiry: run + .active_plan + .definition + .input_wait_policy + .on_expiry + .clone(), + }, + }, + settled_at, + ) + .await?; + } else { + transition_node_counters_in_tx( + &mut conn, + fence.run_uid, + &task.node_id, + &task.item_key, + ExecutionTaskStatus::Running, + task.status, + ) + .await?; + } + if task.status == ExecutionTaskStatus::WaitingInput { + let due_at = input_wait_due_at.ok_or_else(|| Error::InvalidRepositoryData { + message: "waiting-input task is missing its resolved expiry".to_string(), + })?; + let trigger_uid = Uuid::new_v5( + &TASK_INPUT_WAIT_TRIGGER_NAMESPACE, + format!( + "{}:{}:{}:{}", + fence.run_uid, task.task_id, task.generation, settled_at + ) + .as_bytes(), + ); + create_trigger_with_dispatch_in_conn( + conn.as_mut(), + config, + &NewExecutionTrigger { + trigger_uid, + tenant_id: fence.tenant_id, + run_uid: Some(fence.run_uid), + task_id: Some(fence.task_id.as_uuid()), + compensation_id: None, + schedule_uid: None, + schedule_incarnation: None, + kind: ExecutionTriggerKind::WaitExpiry, + controller_generation: Some(fence.controller_generation), + attempt_generation: Some(task.generation), + compensation_generation: None, + compensation_attempt_generation: None, + occurrence_sequence: None, + due_at, + payload: json!({ + "task_generation": task.generation, + "waiting_since": settled_at, + "source": "active_task_input_wait", + }), + }, + ) + .await?; + } + let activation_at = retry_at.unwrap_or(settled_at); + enqueue_run_activation_in_conn( + conn.as_mut(), + fence.tenant_id, + fence.run_uid, + fence.controller_generation, + activation_at, + json!({ + "source": "task_attempt_settlement", + "task_id": fence.task_id, + "dispatch_uid": fence.dispatch_uid, + "attempt_generation": fence.attempt_generation, + }), + ) + .await?; + let run_row = sqlx::query(LOAD_RUN_SQL) + .bind(fence.run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let run = run_from_row(&run_row)?; + conn.commit().await.map_err(storage_error)?; + Ok(TaskAttemptSettlementOutcome::Applied { run, task }) + } + + /// Requeues a running task after provider recovery proved its reserved start did not happen. + /// + /// The caller must release the exact unbound external-job intent in the same transaction. + /// The provisional checkpoint deliberately remains current and is consumed by the successor + /// attempt so neither the model turn nor the direct capability call is reconstructed. + pub(super) async fn requeue_task_external_start_not_started_in_conn( + conn: &mut ScopedConn<'_>, + intent: &NewExecutionExternalJobIntent, + recovered_at: DateTime, + ) -> Result { + let ExecutionExternalJobOwner::Task { + task_id, + attempt_generation, + } = intent.owner + else { + return Ok(TaskExternalStartRetryOutcome::InvalidState); + }; + let task_id = ExecutionTaskId::from_uuid(task_id); + let Some(fence) = load_running_external_start_fence( + conn, + intent.tenant_id, + intent.run_uid, + task_id, + attempt_generation, + ) + .await? + else { + return Ok(TaskExternalStartRetryOutcome::NotFound); + }; + let capacity = release_task_capacity_in_tx( + conn, + fence.capacity_reservation_uid, + fence.run_uid, + fence.task_id, + fence.attempt_generation, + ) + .await?; + if capacity == CapacityReleaseOutcome::NotFound { + return Ok(TaskExternalStartRetryOutcome::NotFound); + } + if capacity == CapacityReleaseOutcome::Stale { + return Ok(TaskExternalStartRetryOutcome::Stale); + } + let Some((run, task, checkpoint)) = load_locked_external_start_owner(conn, fence).await? + else { + return Ok(TaskExternalStartRetryOutcome::NotFound); + }; + if capacity == CapacityReleaseOutcome::AlreadyReleased { + let external_job_uid = intent.external_job_uid.to_string(); + let replay = task.status == ExecutionTaskStatus::Ready + && task.attempt_state == ExecutionAttemptState::Idle + && task.attempt_generation == fence.attempt_generation.saturating_add(1) + && task.active_dispatch_uid.is_none() + && task.generation_history.iter().any(|entry| { + entry.get("kind").and_then(Value::as_str) == Some("external_start_not_started") + && entry.get("external_job_uid").and_then(Value::as_str) + == Some(external_job_uid.as_str()) + }); + return Ok(if replay { + TaskExternalStartRetryOutcome::Replayed { + task, + checkpoint: Box::new(checkpoint), + } + } else { + TaskExternalStartRetryOutcome::Stale + }); + } + if !matches!( + run.status, + ExecutionRunStatus::Queued | ExecutionRunStatus::Running + ) || task.status != ExecutionTaskStatus::Running + || task.attempt_state != ExecutionAttemptState::Running + { + return Ok(TaskExternalStartRetryOutcome::InvalidState); + } + if supersede_trigger_in_conn( + conn.as_mut(), + fence.watchdog_trigger_uid, + ExecutionTriggerKind::TaskWatchdog, + Some(fence.controller_generation), + Some(fence.attempt_generation), + None, + None, + ) + .await? + == ExecutionTriggerSupersedeOutcome::StaleOrMissing + { + return Ok(TaskExternalStartRetryOutcome::Stale); + } + let next_attempt_generation = fence.attempt_generation.checked_add(1).ok_or_else(|| { + Error::InvalidRepositoryData { + message: "task attempt generation overflow during external-start recovery" + .to_string(), + } + })?; + let row = sqlx::query( + "UPDATE moa.execution_task SET status='ready',attempt_state='idle', \ + attempt_generation=$5,active_dispatch_uid=NULL,attempt_deadline_at=NULL, \ + waiting_since=NULL,ready_at=$6, \ + last_progress_at=GREATEST(last_progress_at,$6), \ + generation_history=generation_history || jsonb_build_array(jsonb_build_object( \ + 'kind','external_start_not_started','external_job_uid',$7::TEXT, \ + 'dispatch_uid',$4::TEXT,'attempt_generation',$3,'recorded_at',$6)), \ + updated_at=NOW() \ + WHERE run_uid=$1 AND task_id=$2 AND attempt_generation=$3 \ + AND active_dispatch_uid=$4 AND status='running' AND attempt_state='running' \ + RETURNING *", + ) + .bind(fence.run_uid) + .bind(fence.task_id.as_uuid()) + .bind(to_i64(fence.attempt_generation, "attempt generation")?) + .bind(fence.dispatch_uid) + .bind(to_i64(next_attempt_generation, "next attempt generation")?) + .bind(recovered_at) + .bind(intent.external_job_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(row) = row else { + return Ok(TaskExternalStartRetryOutcome::Stale); + }; + let task = task_from_row(&row)?; + transition_node_counters_in_tx( + conn, + fence.run_uid, + &task.node_id, + &task.item_key, + ExecutionTaskStatus::Running, + ExecutionTaskStatus::Ready, + ) + .await?; + enqueue_run_activation_in_conn( + conn.as_mut(), + fence.tenant_id, + fence.run_uid, + fence.controller_generation, + recovered_at, + json!({ + "source": "external_start_not_started", + "task_id": fence.task_id, + "external_job_uid": intent.external_job_uid, + "attempt_generation": fence.attempt_generation, + }), + ) + .await?; + Ok(TaskExternalStartRetryOutcome::Applied { + task, + checkpoint: Box::new(checkpoint), + }) + } + + /// Adopts one recovered provider job and parks its exact running task atomically. + /// + /// The caller must bind the job and prelock `active_tasks` before `external_jobs` in the same + /// scoped transaction. Async-capable task execution never owns a sandbox hand, so this path + /// releases capacity and the watchdog without accepting a caller-supplied hand receipt. + pub(super) async fn adopt_recovered_task_external_job_in_conn( + conn: &mut ScopedConn<'_>, + job: &ExecutionExternalJobRecord, + adopted_at: DateTime, + ) -> Result { + let ExecutionExternalJobOwner::Task { + task_id, + attempt_generation, + } = job.owner + else { + return Ok(TaskAttemptExternalOutcome::InvalidState); + }; + if job.state == ExecutionExternalJobState::Unbound { + return Ok(TaskAttemptExternalOutcome::InvalidState); + } + let task_id = ExecutionTaskId::from_uuid(task_id); + let Some(fence) = load_running_external_start_fence( + conn, + job.tenant_id, + job.run_uid, + task_id, + attempt_generation, + ) + .await? + else { + return Ok(TaskAttemptExternalOutcome::NotFound); + }; + let capacity = release_task_capacity_in_tx( + conn, + fence.capacity_reservation_uid, + fence.run_uid, + fence.task_id, + fence.attempt_generation, + ) + .await?; + if capacity == CapacityReleaseOutcome::NotFound { + return Ok(TaskAttemptExternalOutcome::NotFound); + } + if capacity == CapacityReleaseOutcome::Stale { + return Ok(TaskAttemptExternalOutcome::Stale); + } + let Some((run, task, checkpoint)) = load_locked_external_start_owner(conn, fence).await? + else { + return Ok(TaskAttemptExternalOutcome::NotFound); + }; + if capacity == CapacityReleaseOutcome::AlreadyReleased { + let replay = task.attempt_generation == fence.attempt_generation + && task.active_dispatch_uid.is_none() + && (task.status == ExecutionTaskStatus::WaitingExternal + || task.status.is_terminal()) + && task.external_job_uid == Some(job.external_job_uid); + return Ok(if replay { + TaskAttemptExternalOutcome::Replayed { + run, + task, + external_job: job.clone(), + } + } else { + TaskAttemptExternalOutcome::Stale + }); + } + if !matches!( + run.status, + ExecutionRunStatus::Queued | ExecutionRunStatus::Running + ) || task.status != ExecutionTaskStatus::Running + || task.attempt_state != ExecutionAttemptState::Running + { + return Ok(TaskAttemptExternalOutcome::InvalidState); + } + let mut payload = checkpoint.payload.clone(); + bind_recovered_external_job_in_checkpoint( + &mut payload, + checkpoint.kind, + job.external_job_uid, + )?; + if payload != checkpoint.payload { + insert_resolved_task_checkpoint_in_conn( + conn, + &checkpoint, + run.controller_generation, + checkpoint.task_generation, + checkpoint.attempt_generation, + payload, + adopted_at, + ) + .await?; + } + if supersede_trigger_in_conn( + conn.as_mut(), + fence.watchdog_trigger_uid, + ExecutionTriggerKind::TaskWatchdog, + Some(fence.controller_generation), + Some(fence.attempt_generation), + None, + None, + ) + .await? + == ExecutionTriggerSupersedeOutcome::StaleOrMissing + { + return Ok(TaskAttemptExternalOutcome::Stale); + } + let row = sqlx::query( + "UPDATE moa.execution_task SET status='waiting_external',attempt_state='waiting', \ + waiting_since=$5,external_job_uid=$6,active_dispatch_uid=NULL, \ + attempt_deadline_at=NULL, \ + last_progress_at=GREATEST(last_progress_at,$5),updated_at=NOW() \ + WHERE run_uid=$1 AND task_id=$2 AND attempt_generation=$3 \ + AND active_dispatch_uid=$4 AND status='running' AND attempt_state='running' \ + RETURNING *", + ) + .bind(fence.run_uid) + .bind(fence.task_id.as_uuid()) + .bind(to_i64(fence.attempt_generation, "attempt generation")?) + .bind(fence.dispatch_uid) + .bind(adopted_at) + .bind(job.external_job_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(row) = row else { + return Ok(TaskAttemptExternalOutcome::Stale); + }; + let task = task_from_row(&row)?; + transition_node_counters_in_tx( + conn, + fence.run_uid, + &task.node_id, + &task.item_key, + ExecutionTaskStatus::Running, + ExecutionTaskStatus::WaitingExternal, + ) + .await?; + append_run_wait_reason_in_tx( + conn, + fence.run_uid, + &WaitingReason::External { + task_id: task.task_id, + }, + adopted_at, + ) + .await?; + enqueue_run_activation_in_conn( + conn.as_mut(), + fence.tenant_id, + fence.run_uid, + fence.controller_generation, + adopted_at, + json!({ + "source": "external_start_recovered", + "task_id": fence.task_id, + "external_job_uid": job.external_job_uid, + }), + ) + .await?; + let run_row = sqlx::query(LOAD_RUN_SQL) + .bind(fence.run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + Ok(TaskAttemptExternalOutcome::Applied { + run: run_from_row(&run_row)?, + task, + external_job: job.clone(), + }) + } + + /// Commits one provider job before parking the task and releasing active resources. + pub async fn yield_task_attempt_to_external_job( + &self, + fence: TaskAttemptFence, + external_job_uid: Uuid, + continuation_checkpoint: Option, + workspace_release_receipt: Option, + yielded_at: DateTime, + ) -> Result { + if workspace_release_receipt.as_ref().is_some_and(|receipt| { + receipt.tenant_id != fence.tenant_id + || receipt.run_id.0 != fence.run_uid + || !matches!( + receipt.owner, + ExecutionHandReleaseOwner::Task { task_id, .. } + if task_id.0 == fence.task_id.as_uuid() + ) + || receipt.attempt_generation != fence.attempt_generation + }) { + return Ok(TaskAttemptExternalOutcome::Stale); + } + if continuation_checkpoint.as_ref().is_some_and(|checkpoint| { + checkpoint.fence != fence + || checkpoint_external_job_uid(&checkpoint.payload) != Some(external_job_uid) + || checkpoint.workspace_release_receipt != workspace_release_receipt + }) { + return Ok(TaskAttemptExternalOutcome::Stale); + } + let mut conn = ExecutionScope::ControlPlane.begin(&self.pool).await?; + let capacity = release_task_capacity_in_tx( + &mut conn, + fence.capacity_reservation_uid, + fence.run_uid, + fence.task_id, + fence.attempt_generation, + ) + .await?; + if matches!(capacity, CapacityReleaseOutcome::NotFound) { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptExternalOutcome::NotFound); + } + if matches!(capacity, CapacityReleaseOutcome::Stale) { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptExternalOutcome::Stale); + } + let Some(persisted_job) = + load_external_job_for_update_in_conn(conn.as_mut(), external_job_uid).await? + else { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptExternalOutcome::NotFound); + }; + if persisted_job.tenant_id != fence.tenant_id + || persisted_job.run_uid != fence.run_uid + || persisted_job.owner + != (ExecutionExternalJobOwner::Task { + task_id: fence.task_id.as_uuid(), + attempt_generation: fence.attempt_generation, + }) + || persisted_job.state == ExecutionExternalJobState::Unbound + { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptExternalOutcome::Stale); + } + let Some(run_row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) + .bind(fence.run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptExternalOutcome::NotFound); + }; + let run = run_from_row(&run_row)?; + let Some(task_row) = sqlx::query(LOAD_TASK_FOR_UPDATE_SQL) + .bind(fence.run_uid) + .bind(fence.task_id.as_uuid()) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptExternalOutcome::NotFound); + }; + let task = task_from_row(&task_row)?; + if capacity == CapacityReleaseOutcome::AlreadyReleased { + let replay = task.attempt_generation == fence.attempt_generation + && task.active_dispatch_uid.is_none() + && (task.status == ExecutionTaskStatus::WaitingExternal + || task.status.is_terminal()) + && task.external_job_uid == Some(external_job_uid); + conn.commit().await.map_err(storage_error)?; + return Ok(if replay { + TaskAttemptExternalOutcome::Replayed { + run, + task, + external_job: persisted_job, + } + } else { + TaskAttemptExternalOutcome::Stale + }); + } + if !task_attempt_fence_matches(&run, &task, &fence) { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptExternalOutcome::Stale); + } + if let Some(receipt) = workspace_release_receipt.as_ref() + && !persisted_task_release_receipt_matches(&mut conn, &fence, task.generation, receipt) + .await? + { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptExternalOutcome::Stale); + } + if let Some(checkpoint) = &continuation_checkpoint { + match persist_task_attempt_checkpoint_in_conn(&mut conn, checkpoint).await? { + TaskAttemptCheckpointWriteOutcome::Applied(_) + | TaskAttemptCheckpointWriteOutcome::Replayed(_) => {} + TaskAttemptCheckpointWriteOutcome::NotFound => { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptExternalOutcome::NotFound); + } + TaskAttemptCheckpointWriteOutcome::Stale => { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptExternalOutcome::Stale); + } + TaskAttemptCheckpointWriteOutcome::InvalidState => { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptExternalOutcome::InvalidState); + } + } + } + if task.status != ExecutionTaskStatus::Running + || task.attempt_state != ExecutionAttemptState::Cancelling + || run.status.is_terminal() + { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptExternalOutcome::InvalidState); + } + let superseded = supersede_trigger_in_conn( + conn.as_mut(), + fence.watchdog_trigger_uid, + ExecutionTriggerKind::TaskWatchdog, + Some(fence.controller_generation), + Some(fence.attempt_generation), + None, + None, + ) + .await?; + if superseded == ExecutionTriggerSupersedeOutcome::StaleOrMissing { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptExternalOutcome::Stale); + } + let row = sqlx::query( + "UPDATE moa.execution_task \ + SET status = 'waiting_external', attempt_state = 'waiting', \ + waiting_since = $5, external_job_uid = $6, active_dispatch_uid = NULL, \ + attempt_deadline_at = NULL, \ + last_progress_at = GREATEST(last_progress_at, $5), updated_at = NOW() \ + WHERE run_uid = $1 AND task_id = $2 AND attempt_generation = $3 \ + AND active_dispatch_uid = $4 RETURNING *", + ) + .bind(fence.run_uid) + .bind(fence.task_id.as_uuid()) + .bind(to_i64(fence.attempt_generation, "attempt generation")?) + .bind(fence.dispatch_uid) + .bind(yielded_at) + .bind(external_job_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let task = task_from_row(&row)?; + transition_node_counters_in_tx( + &mut conn, + fence.run_uid, + &task.node_id, + &task.item_key, + ExecutionTaskStatus::Running, + ExecutionTaskStatus::WaitingExternal, + ) + .await?; + append_run_wait_reason_in_tx( + &mut conn, + fence.run_uid, + &WaitingReason::External { + task_id: task.task_id, + }, + yielded_at, + ) + .await?; + let task = if persisted_job.state.is_terminal() { + match settle_external_job_terminal_in_conn(&mut conn, &persisted_job, yielded_at) + .await? + { + ExternalJobTaskSettlementOutcome::Applied(task) + | ExternalJobTaskSettlementOutcome::Replayed(task) => task, + ExternalJobTaskSettlementOutcome::DeferredRelease(_) => { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptExternalOutcome::InvalidState); + } + ExternalJobTaskSettlementOutcome::Stale => { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptExternalOutcome::Stale); + } + ExternalJobTaskSettlementOutcome::NotFound => { + conn.rollback().await.map_err(storage_error)?; + return Ok(TaskAttemptExternalOutcome::NotFound); + } + } + } else { + task + }; + enqueue_run_activation_in_conn( + conn.as_mut(), + fence.tenant_id, + fence.run_uid, + fence.controller_generation, + yielded_at, + json!({ + "source": if persisted_job.state.is_terminal() { + "external_job_terminal_after_release" + } else { + "external_job_started" + }, + "task_id": fence.task_id, + "external_job_uid": external_job_uid, + }), + ) + .await?; + let run_row = sqlx::query(LOAD_RUN_SQL) + .bind(fence.run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let run = run_from_row(&run_row)?; + conn.commit().await.map_err(storage_error)?; + Ok(TaskAttemptExternalOutcome::Applied { + run, + task, + external_job: persisted_job, + }) + } +} + +/// Settles one exact terminal external job into its waiting logical task. +pub async fn settle_external_job_terminal_in_conn( + conn: &mut ScopedConn<'_>, + job: &ExecutionExternalJobRecord, + settled_at: DateTime, +) -> Result { + if !job.state.is_terminal() { + return Err(Error::InvalidRepositoryInput { + message: "external-job task settlement requires a terminal job".to_string(), + }); + } + let Some(run_row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) + .bind(job.run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + return Ok(ExternalJobTaskSettlementOutcome::NotFound); + }; + let run = run_from_row(&run_row)?; + let ExecutionExternalJobOwner::Task { + task_id: external_task_id, + attempt_generation: external_attempt_generation, + } = job.owner + else { + return Ok(ExternalJobTaskSettlementOutcome::Stale); + }; + let task_id = ExecutionTaskId::from_uuid(external_task_id); + let Some(task_row) = sqlx::query(LOAD_TASK_FOR_UPDATE_SQL) + .bind(job.run_uid) + .bind(external_task_id) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + return Ok(ExternalJobTaskSettlementOutcome::NotFound); + }; + let task = task_from_row(&task_row)?; + if task.status.is_terminal() { + return Ok(if task.external_job_uid == Some(job.external_job_uid) { + ExternalJobTaskSettlementOutcome::Replayed(task) + } else { + ExternalJobTaskSettlementOutcome::Stale + }); + } + let exact_active_owner = run.tenant_id == job.tenant_id + && task.tenant_id == job.tenant_id + && task.attempt_generation == external_attempt_generation + && task.status == ExecutionTaskStatus::Running; + let release_pending = task.attempt_state == ExecutionAttemptState::Cancelling + && task.external_job_uid == Some(job.external_job_uid); + let release_not_started = task.attempt_state == ExecutionAttemptState::Running + && task.external_job_uid.is_none() + && task.active_dispatch_uid.is_some() + && task.attempt_deadline_at.is_some(); + if exact_active_owner && (release_pending || release_not_started) { + return Ok(ExternalJobTaskSettlementOutcome::DeferredRelease(task)); + } + if run.tenant_id != job.tenant_id + || task.tenant_id != job.tenant_id + || task.attempt_generation != external_attempt_generation + || task.external_job_uid != Some(job.external_job_uid) + || task.status != ExecutionTaskStatus::WaitingExternal + || task.attempt_state != ExecutionAttemptState::Waiting + { + return Ok(ExternalJobTaskSettlementOutcome::Stale); + } + let terminal_resolution = match job.state { + super::external_job::ExecutionExternalJobState::Completed => { + moa_core::types::tools::AsyncToolJobTerminalOutcome::Completed { + output: job.output.clone().unwrap_or(Value::Null), + } + } + super::external_job::ExecutionExternalJobState::Failed => { + moa_core::types::tools::AsyncToolJobTerminalOutcome::Failed { + error: job + .error + .clone() + .unwrap_or_else(|| json!({"message": "asynchronous provider job failed"})), + } + } + super::external_job::ExecutionExternalJobState::Cancelled => { + moa_core::types::tools::AsyncToolJobTerminalOutcome::Cancelled + } + super::external_job::ExecutionExternalJobState::UnknownOutcome => { + moa_core::types::tools::AsyncToolJobTerminalOutcome::UnknownOutcome { + error: job.error.clone().unwrap_or_else( + || json!({"message": "asynchronous provider outcome is unknown"}), + ), + } + } + super::external_job::ExecutionExternalJobState::Unbound + | super::external_job::ExecutionExternalJobState::Starting + | super::external_job::ExecutionExternalJobState::Running + | super::external_job::ExecutionExternalJobState::WaitingReconcile + | super::external_job::ExecutionExternalJobState::CancelRequested => { + return Err(Error::InvalidRepositoryInput { + message: "external-job task settlement observed a nonterminal state".to_string(), + }); + } + }; + let current_checkpoint = sqlx::query( + "SELECT * FROM moa.execution_task_checkpoint WHERE tenant_id=$1 AND run_uid=$2 \ + AND task_id=$3 AND superseded_at IS NULL FOR UPDATE", + ) + .bind(job.tenant_id.0) + .bind(job.run_uid) + .bind(external_task_id) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if let Some(current_checkpoint) = current_checkpoint { + let current_checkpoint = task_checkpoint_from_row(¤t_checkpoint)?; + if current_checkpoint.kind == TaskAttemptCheckpointKind::AgentContinuation + && checkpoint_external_job_uid(¤t_checkpoint.payload) + == Some(job.external_job_uid) + { + let mut payload = current_checkpoint.payload.clone(); + payload["external_job_resolution"] = serde_json::to_value(&terminal_resolution)?; + insert_resolved_task_checkpoint_in_conn( + conn, + ¤t_checkpoint, + run.controller_generation, + current_checkpoint.task_generation, + current_checkpoint.attempt_generation, + payload, + settled_at, + ) + .await?; + let next_attempt_generation = + task.attempt_generation.checked_add(1).ok_or_else(|| { + Error::InvalidRepositoryData { + message: "task attempt generation overflow".to_string(), + } + })?; + let row = sqlx::query( + "UPDATE moa.execution_task SET status='ready', attempt_state='idle', \ + waiting_since=NULL, ready_at=$3, external_job_uid=NULL, \ + attempt_generation=$4, last_progress_at=GREATEST(last_progress_at,$3), \ + updated_at=NOW() \ + WHERE run_uid=$1 AND task_id=$2 AND status='waiting_external' \ + RETURNING *", + ) + .bind(job.run_uid) + .bind(external_task_id) + .bind(settled_at) + .bind(to_i64(next_attempt_generation, "next attempt generation")?) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let resumed = task_from_row(&row)?; + transition_node_counters_in_tx( + conn, + job.run_uid, + &resumed.node_id, + &resumed.item_key, + ExecutionTaskStatus::WaitingExternal, + ExecutionTaskStatus::Ready, + ) + .await?; + refresh_run_after_wait_settlement_in_conn(conn, job.run_uid, task.task_id, settled_at) + .await?; + return Ok(ExternalJobTaskSettlementOutcome::Applied(resumed)); + } + } + let outcome = match job.state { + super::external_job::ExecutionExternalJobState::Completed => completed_task_outcome( + job.output.clone().unwrap_or(Value::Null), + task.actual.clone(), + ), + super::external_job::ExecutionExternalJobState::Failed => failed_task_outcome( + moa_artifacts::execution_plan::ExecutionFailureClass::Terminal, + job.error + .as_ref() + .map(Value::to_string) + .unwrap_or_else(|| "asynchronous provider job failed".to_string()), + task.actual.clone(), + ), + super::external_job::ExecutionExternalJobState::Cancelled => cancelled_task_outcome( + "asynchronous provider job was cancelled".to_string(), + task.actual.clone(), + ), + super::external_job::ExecutionExternalJobState::UnknownOutcome => ExecutionTaskOutcome { + schema_version: 1, + usage: task.actual.clone(), + result: ExecutionTaskResult::UnknownOutcome { + message: job.error.as_ref().map(Value::to_string).unwrap_or_else(|| { + "asynchronous provider job has an unknown outcome".to_string() + }), + }, + }, + super::external_job::ExecutionExternalJobState::Unbound + | super::external_job::ExecutionExternalJobState::Starting + | super::external_job::ExecutionExternalJobState::Running + | super::external_job::ExecutionExternalJobState::WaitingReconcile + | super::external_job::ExecutionExternalJobState::CancelRequested => { + return Err(Error::InvalidRepositoryInput { + message: "external-job task settlement observed a nonterminal state".to_string(), + }); + } + }; + let write = record_waiting_external_task_outcome_in_conn( + conn, + job.run_uid, + task_id, + task.generation, + outcome, + ) + .await?; + let settled = match write { + TaskOutcomeWrite::Applied { task, .. } | TaskOutcomeWrite::Replayed { task, .. } => task, + TaskOutcomeWrite::NotFound => return Ok(ExternalJobTaskSettlementOutcome::NotFound), + TaskOutcomeWrite::Rejected { .. } => { + return Ok(ExternalJobTaskSettlementOutcome::Stale); + } + }; + transition_node_counters_in_tx( + conn, + job.run_uid, + &settled.node_id, + &settled.item_key, + ExecutionTaskStatus::WaitingExternal, + settled.status, + ) + .await?; + refresh_run_after_wait_settlement_in_conn(conn, job.run_uid, task.task_id, settled_at).await?; + Ok(ExternalJobTaskSettlementOutcome::Applied(settled)) +} + +async fn persist_task_attempt_checkpoint_in_conn( + conn: &mut ScopedConn<'_>, + request: &NewTaskAttemptCheckpoint, +) -> Result { + persist_task_attempt_checkpoint_for_state_in_conn( + conn, + request, + ExecutionAttemptState::Cancelling, + ) + .await +} + +async fn persist_task_attempt_checkpoint_for_state_in_conn( + conn: &mut ScopedConn<'_>, + request: &NewTaskAttemptCheckpoint, + expected_attempt_state: ExecutionAttemptState, +) -> Result { + if request.task_generation == 0 || request.schema_version == 0 || !request.payload.is_object() { + return Ok(TaskAttemptCheckpointWriteOutcome::InvalidState); + } + let payload_bytes = canonical_json_bytes(&request.payload).map_err(Error::from)?; + if payload_bytes.len() > 1024 * 1024 { + return Ok(TaskAttemptCheckpointWriteOutcome::InvalidState); + } + let payload_hash = blake3::hash(&payload_bytes).to_hex().to_string(); + let workspace_release_receipt = request + .workspace_release_receipt + .as_ref() + .map(serde_json::to_value) + .transpose()?; + if workspace_release_receipt.as_ref().is_some_and(|receipt| { + serde_json::to_vec(receipt) + .map(|bytes| bytes.len() > 256 * 1024) + .unwrap_or(true) + }) { + return Ok(TaskAttemptCheckpointWriteOutcome::InvalidState); + } + if request + .workspace_release_receipt + .as_ref() + .is_some_and(|receipt| { + receipt.tenant_id != request.fence.tenant_id + || receipt.run_id.0 != request.fence.run_uid + || !matches!( + receipt.owner, + ExecutionHandReleaseOwner::Task { task_id, logical_generation } + if task_id.0 == request.fence.task_id.as_uuid() + && logical_generation == request.task_generation + ) + || receipt.attempt_generation != request.fence.attempt_generation + }) + { + return Ok(TaskAttemptCheckpointWriteOutcome::Stale); + } + let Some(run_row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) + .bind(request.fence.run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + return Ok(TaskAttemptCheckpointWriteOutcome::NotFound); + }; + let run = run_from_row(&run_row)?; + let Some(task_row) = sqlx::query(LOAD_TASK_FOR_UPDATE_SQL) + .bind(request.fence.run_uid) + .bind(request.fence.task_id.as_uuid()) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + return Ok(TaskAttemptCheckpointWriteOutcome::NotFound); + }; + let task = task_from_row(&task_row)?; + if !task_attempt_fence_matches(&run, &task, &request.fence) + || task.generation != request.task_generation + { + return Ok(TaskAttemptCheckpointWriteOutcome::Stale); + } + if task.status != ExecutionTaskStatus::Running || task.attempt_state != expected_attempt_state { + return Ok(TaskAttemptCheckpointWriteOutcome::InvalidState); + } + + let current = sqlx::query( + "SELECT * FROM moa.execution_task_checkpoint \ + WHERE tenant_id = $1 AND run_uid = $2 AND task_id = $3 \ + AND superseded_at IS NULL FOR UPDATE", + ) + .bind(request.fence.tenant_id.0) + .bind(request.fence.run_uid) + .bind(request.fence.task_id.as_uuid()) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if let Some(current) = current { + let current = task_checkpoint_from_row(¤t)?; + let replay = current.controller_generation == request.fence.controller_generation + && current.task_generation == request.task_generation + && current.attempt_generation == request.fence.attempt_generation + && current.dispatch_uid == request.fence.dispatch_uid + && current.kind == request.kind + && current.schema_version == request.schema_version + && current.payload_hash == payload_hash + && current.workspace_release_receipt == request.workspace_release_receipt; + if replay { + return Ok(TaskAttemptCheckpointWriteOutcome::Replayed(Box::new( + current, + ))); + } + sqlx::query( + "UPDATE moa.execution_task_checkpoint SET superseded_at = $4 \ + WHERE tenant_id = $1 AND run_uid = $2 AND task_id = $3 \ + AND superseded_at IS NULL", + ) + .bind(request.fence.tenant_id.0) + .bind(request.fence.run_uid) + .bind(request.fence.task_id.as_uuid()) + .bind(request.created_at) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + } + let checkpoint_sequence = sqlx::query_scalar::<_, i64>( + "SELECT COALESCE(MAX(checkpoint_sequence), 0) + 1 \ + FROM moa.execution_task_checkpoint WHERE tenant_id = $1 AND run_uid = $2 AND task_id = $3", + ) + .bind(request.fence.tenant_id.0) + .bind(request.fence.run_uid) + .bind(request.fence.task_id.as_uuid()) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let checkpoint_sequence = to_u64(checkpoint_sequence, "task checkpoint sequence")?; + let checkpoint_uid = Uuid::new_v5( + &request.fence.task_id.as_uuid(), + format!("task-checkpoint-v1:{checkpoint_sequence}:{payload_hash}").as_bytes(), + ); + let row = sqlx::query( + "INSERT INTO moa.execution_task_checkpoint (\ + checkpoint_uid, tenant_id, run_uid, task_id, checkpoint_sequence, \ + controller_generation, task_generation, attempt_generation, dispatch_uid, \ + checkpoint_kind, schema_version, payload, payload_hash, \ + workspace_release_receipt, created_at\ + ) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) RETURNING *", + ) + .bind(checkpoint_uid) + .bind(request.fence.tenant_id.0) + .bind(request.fence.run_uid) + .bind(request.fence.task_id.as_uuid()) + .bind(to_i64(checkpoint_sequence, "task checkpoint sequence")?) + .bind(to_i64( + request.fence.controller_generation, + "controller generation", + )?) + .bind(to_i64(request.task_generation, "task generation")?) + .bind(to_i64( + request.fence.attempt_generation, + "attempt generation", + )?) + .bind(request.fence.dispatch_uid) + .bind(request.kind.as_str()) + .bind(i64::from(request.schema_version)) + .bind(&request.payload) + .bind(payload_hash) + .bind(workspace_release_receipt) + .bind(request.created_at) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + Ok(TaskAttemptCheckpointWriteOutcome::Applied(Box::new( + task_checkpoint_from_row(&row)?, + ))) +} + +fn external_start_checkpoint_payload_is_provisional( + kind: TaskAttemptCheckpointKind, + payload: &Value, +) -> bool { + let Some(state) = payload.get("state").and_then(Value::as_object) else { + return false; + }; + match kind { + TaskAttemptCheckpointKind::AgentContinuation => { + state.get("kind").and_then(Value::as_str) == Some("agent") + && state + .get("pending_external") + .and_then(Value::as_object) + .is_some_and(|pending| { + pending.get("external_job_uid").is_some_and(Value::is_null) + && pending.get("invocation").is_some_and(Value::is_object) + }) + } + TaskAttemptCheckpointKind::CapabilityExternalStart => { + state.get("kind").and_then(Value::as_str) == Some("capability_external_start") + && state.get("tool_id").and_then(Value::as_str).is_some() + } + TaskAttemptCheckpointKind::CapabilityReview => false, + } +} + +async fn insert_resolved_task_checkpoint_in_conn( + conn: &mut ScopedConn<'_>, + current: &TaskAttemptCheckpointRecord, + controller_generation: u64, + task_generation: u64, + attempt_generation: u64, + payload: Value, + resolved_at: DateTime, +) -> Result { + let canonical = canonical_json_bytes(&payload).map_err(Error::from)?; + if canonical.len() > 1024 * 1024 { + return Err(Error::InvalidRepositoryInput { + message: "resolved task continuation exceeds one MiB".to_string(), + }); + } + let payload_hash = blake3::hash(&canonical).to_hex().to_string(); + let updated = sqlx::query( + "UPDATE moa.execution_task_checkpoint SET superseded_at=$4 \ + WHERE tenant_id=$1 AND run_uid=$2 AND task_id=$3 AND checkpoint_uid=$5 \ + AND superseded_at IS NULL", + ) + .bind(current.tenant_id.0) + .bind(current.run_uid) + .bind(current.task_id.as_uuid()) + .bind(resolved_at) + .bind(current.checkpoint_uid) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if updated.rows_affected() != 1 { + return Err(Error::InvalidRepositoryData { + message: "current task review checkpoint was concurrently superseded".to_string(), + }); + } + let checkpoint_sequence = current.checkpoint_sequence.checked_add(1).ok_or_else(|| { + Error::InvalidRepositoryInput { + message: "task checkpoint sequence overflow".to_string(), + } + })?; + let checkpoint_uid = Uuid::new_v5( + ¤t.task_id.as_uuid(), + format!("task-checkpoint-v1:{checkpoint_sequence}:{payload_hash}").as_bytes(), + ); + let workspace_release_receipt = current + .workspace_release_receipt + .as_ref() + .map(serde_json::to_value) + .transpose()?; + let row = sqlx::query( + "INSERT INTO moa.execution_task_checkpoint (checkpoint_uid,tenant_id,run_uid,task_id, \ + checkpoint_sequence,controller_generation,task_generation,attempt_generation, \ + dispatch_uid,checkpoint_kind,schema_version,payload,payload_hash, \ + workspace_release_receipt,created_at) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) RETURNING *", + ) + .bind(checkpoint_uid) + .bind(current.tenant_id.0) + .bind(current.run_uid) + .bind(current.task_id.as_uuid()) + .bind(to_i64(checkpoint_sequence, "task checkpoint sequence")?) + .bind(to_i64(controller_generation, "controller generation")?) + .bind(to_i64(task_generation, "task generation")?) + .bind(to_i64(attempt_generation, "attempt generation")?) + .bind(current.dispatch_uid) + .bind(current.kind.as_str()) + .bind(i64::from(current.schema_version)) + .bind(payload) + .bind(payload_hash) + .bind(workspace_release_receipt) + .bind(resolved_at) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + task_checkpoint_from_row(&row) +} + +fn append_agent_resume_input(payload: &mut Value, input: &Value) -> Result<()> { + let state = payload + .get_mut("state") + .and_then(Value::as_object_mut) + .ok_or_else(|| Error::InvalidRepositoryData { + message: "agent continuation checkpoint is missing its state object".to_string(), + })?; + if state.get("kind").and_then(Value::as_str) != Some("agent") { + return Err(Error::InvalidRepositoryData { + message: "waiting-input checkpoint is not an agent continuation".to_string(), + }); + } + let messages = state + .get_mut("messages") + .and_then(Value::as_array_mut) + .ok_or_else(|| Error::InvalidRepositoryData { + message: "agent continuation checkpoint is missing its message history".to_string(), + })?; + let content = match input { + Value::String(content) => content.clone(), + other => String::from_utf8(canonical_json_bytes(other).map_err(Error::from)?).map_err( + |error| Error::InvalidRepositoryData { + message: format!("canonical input is not UTF-8: {error}"), + }, + )?, + }; + messages.push(serde_json::to_value(ContextMessage::user(content))?); + Ok(()) +} + +pub(super) fn task_checkpoint_from_row(row: &PgRow) -> Result { + let payload: Value = row.try_get("payload").map_err(row_error)?; + let payload_hash: String = row.try_get("payload_hash").map_err(row_error)?; + let canonical = canonical_json_bytes(&payload).map_err(Error::from)?; + if blake3::hash(&canonical).to_hex().as_str() != payload_hash { + return Err(Error::InvalidRepositoryData { + message: "task-attempt checkpoint payload hash mismatch".to_string(), + }); + } + let workspace_release_receipt = row + .try_get::, _>("workspace_release_receipt") + .map_err(row_error)? + .map(serde_json::from_value) + .transpose()?; + Ok(TaskAttemptCheckpointRecord { + checkpoint_uid: row.try_get("checkpoint_uid").map_err(row_error)?, + checkpoint_sequence: required_u64(row, "checkpoint_sequence")?, + tenant_id: TenantId(row.try_get("tenant_id").map_err(row_error)?), + run_uid: row.try_get("run_uid").map_err(row_error)?, + task_id: ExecutionTaskId::from_uuid(row.try_get("task_id").map_err(row_error)?), + controller_generation: required_u64(row, "controller_generation")?, + task_generation: required_u64(row, "task_generation")?, + attempt_generation: required_u64(row, "attempt_generation")?, + dispatch_uid: row.try_get("dispatch_uid").map_err(row_error)?, + kind: TaskAttemptCheckpointKind::parse( + &row.try_get::("checkpoint_kind") + .map_err(row_error)?, + )?, + schema_version: u32::try_from(row.try_get::("schema_version").map_err(row_error)?) + .map_err(|_| Error::InvalidRepositoryData { + message: "task checkpoint schema version is outside u32".to_string(), + })?, + payload, + payload_hash, + workspace_release_receipt, + created_at: row.try_get("created_at").map_err(row_error)?, + }) +} + +pub(super) fn checkpoint_review_uid(payload: &Value) -> Option { + payload + .get("state")? + .get("pending_review")? + .get("review_uid")? + .as_str() + .and_then(|value| Uuid::parse_str(value).ok()) +} + +fn checkpoint_review_waiting_reason( + checkpoint: &TaskAttemptCheckpointRecord, + task: &ExecutionTaskRecord, +) -> Result { + let pending = checkpoint + .payload + .get("state") + .and_then(|state| state.get("pending_review")) + .ok_or_else(|| Error::InvalidRepositoryData { + message: "review checkpoint is missing its pending review".to_string(), + })?; + let expires_at = pending + .get("expires_at") + .cloned() + .map(serde_json::from_value) + .transpose()? + .ok_or_else(|| Error::InvalidRepositoryData { + message: "review checkpoint is missing its expiry".to_string(), + })?; + let invocation_name = pending + .get("invocation") + .and_then(|invocation| invocation.get("name")) + .and_then(Value::as_str) + .ok_or_else(|| Error::InvalidRepositoryData { + message: "review checkpoint is missing its invocation name".to_string(), + })?; + Ok(WaitingReason::Review { + task_id: task.task_id, + prompt: format!("Review governed capability `{invocation_name}`"), + wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { + expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::At { at: expires_at }, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, + }, + }) +} + +fn checkpoint_review_resolution(payload: &Value) -> Option<&Value> { + payload.get("review_resolution") +} + +fn checkpoint_external_job_uid(payload: &Value) -> Option { + payload + .get("state")? + .get("pending_external")? + .get("external_job_uid")? + .as_str() + .and_then(|value| Uuid::parse_str(value).ok()) +} + +fn task_attempt_fence_matches( + run: &ExecutionRunRecord, + task: &ExecutionTaskRecord, + fence: &TaskAttemptFence, +) -> bool { + run.controller_generation == fence.controller_generation + && task_attempt_resource_fence_matches(run, task, fence) +} + +fn task_attempt_resource_fence_matches( + run: &ExecutionRunRecord, + task: &ExecutionTaskRecord, + fence: &TaskAttemptFence, +) -> bool { + run.run_uid == fence.run_uid + && run.tenant_id == fence.tenant_id + && task.run_uid == fence.run_uid + && task.tenant_id == fence.tenant_id + && task.task_id == fence.task_id + && task.attempt_generation == fence.attempt_generation + && task.active_dispatch_uid == Some(fence.dispatch_uid) + && task.attempt_deadline_at == Some(fence.attempt_deadline_at) +} + +fn unstarted_task_attempt_fence_matches( + run: &ExecutionRunRecord, + task: &ExecutionTaskRecord, + fence: &TaskAttemptFence, + disposition: &UnstartedTaskAttemptDisposition, +) -> bool { + unstarted_task_attempt_run_fence_matches(run, fence, disposition) + && task_attempt_resource_fence_matches(run, task, fence) +} + +fn unstarted_task_attempt_run_fence_matches( + run: &ExecutionRunRecord, + fence: &TaskAttemptFence, + disposition: &UnstartedTaskAttemptDisposition, +) -> bool { + let controller_generation = match disposition { + UnstartedTaskAttemptDisposition::Paused { + controller_generation, + } => *controller_generation, + UnstartedTaskAttemptDisposition::Cancelled { .. } + | UnstartedTaskAttemptDisposition::DispatchDeliveryLost => fence.controller_generation, + }; + run.run_uid == fence.run_uid + && run.tenant_id == fence.tenant_id + && run.controller_generation == controller_generation +} + +fn task_attempt_settlement_replayed( + task: &ExecutionTaskRecord, + fence: &TaskAttemptFence, + outcome: &ExecutionTaskOutcome, +) -> bool { + let dispatch_uid = fence.dispatch_uid.to_string(); + task.active_dispatch_uid.is_none() + && task.current_outcome.as_ref() == Some(outcome) + && task.generation_history.iter().any(|entry| { + entry.get("kind").and_then(Value::as_str) == Some("bounded_attempt_settlement") + && entry.get("dispatch_uid").and_then(Value::as_str) == Some(dispatch_uid.as_str()) + && entry.get("attempt_generation").and_then(Value::as_u64) + == Some(fence.attempt_generation) + }) +} + +/// Atomically repairs one admitted dispatch whose receiver never committed its start fence. +/// +/// The caller owns the transaction and must commit only `Applied` or `Replayed`; every other +/// outcome may follow a tentative capacity release and therefore requires rollback. +pub(super) async fn settle_unstarted_task_attempt_in_conn( + conn: &mut ScopedConn<'_>, + fence: TaskAttemptFence, + settled_at: DateTime, +) -> Result { + let disposition = UnstartedTaskAttemptDisposition::DispatchDeliveryLost; + let capacity = release_task_capacity_in_tx( + conn, + fence.capacity_reservation_uid, + fence.run_uid, + fence.task_id, + fence.attempt_generation, + ) + .await?; + if capacity == CapacityReleaseOutcome::NotFound { + return Ok(TaskAttemptSettlementOutcome::NotFound); + } + if capacity == CapacityReleaseOutcome::Stale { + return Ok(TaskAttemptSettlementOutcome::Stale); + } + let Some(run_row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) + .bind(fence.run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + return Ok(TaskAttemptSettlementOutcome::NotFound); + }; + let run = run_from_row(&run_row)?; + let Some(task_row) = sqlx::query(LOAD_TASK_FOR_UPDATE_SQL) + .bind(fence.run_uid) + .bind(fence.task_id.as_uuid()) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + return Ok(TaskAttemptSettlementOutcome::NotFound); + }; + let task = task_from_row(&task_row)?; + if capacity == CapacityReleaseOutcome::AlreadyReleased { + return Ok( + if unstarted_task_attempt_settlement_replayed(&task, &fence, &disposition) { + TaskAttemptSettlementOutcome::Replayed { run, task } + } else { + TaskAttemptSettlementOutcome::Stale + }, + ); + } + if !task_attempt_fence_matches(&run, &task, &fence) || run.status.is_terminal() { + return Ok(TaskAttemptSettlementOutcome::Stale); + } + if task.status != ExecutionTaskStatus::Dispatching + || task.attempt_state != ExecutionAttemptState::Dispatching + { + return Ok(TaskAttemptSettlementOutcome::InvalidState); + } + if supersede_trigger_in_conn( + conn.as_mut(), + fence.watchdog_trigger_uid, + ExecutionTriggerKind::TaskWatchdog, + Some(fence.controller_generation), + Some(fence.attempt_generation), + None, + None, + ) + .await? + == ExecutionTriggerSupersedeOutcome::StaleOrMissing + { + return Ok(TaskAttemptSettlementOutcome::Stale); + } + let next_attempt_generation = + fence + .attempt_generation + .checked_add(1) + .ok_or_else(|| Error::InvalidRepositoryData { + message: "task attempt generation overflow".to_string(), + })?; + let history = json!({ + "kind": "unstarted_attempt_settlement", + "dispatch_uid": fence.dispatch_uid, + "attempt_generation": fence.attempt_generation, + "disposition": unstarted_disposition_label(&disposition), + "reason": Value::Null, + "recorded_at": settled_at, + }); + let row = sqlx::query( + "UPDATE moa.execution_task SET status='ready', attempt_state='idle', \ + attempt_generation=$5, active_dispatch_uid=NULL, attempt_deadline_at=NULL, \ + ready_at=$6, waiting_since=NULL, generation_history=generation_history || \ + jsonb_build_array($7::JSONB), \ + last_progress_at=GREATEST(last_progress_at,$6), updated_at=NOW() \ + WHERE run_uid=$1 AND task_id=$2 AND attempt_generation=$3 \ + AND active_dispatch_uid=$4 AND status='dispatching' \ + AND attempt_state='dispatching' RETURNING *", + ) + .bind(fence.run_uid) + .bind(fence.task_id.as_uuid()) + .bind(to_i64(fence.attempt_generation, "attempt generation")?) + .bind(fence.dispatch_uid) + .bind(to_i64(next_attempt_generation, "next attempt generation")?) + .bind(settled_at) + .bind(history) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(row) = row else { + return Ok(TaskAttemptSettlementOutcome::Stale); + }; + let task = task_from_row(&row)?; + transition_node_counters_in_tx( + conn, + fence.run_uid, + &task.node_id, + &task.item_key, + ExecutionTaskStatus::Dispatching, + ExecutionTaskStatus::Ready, + ) + .await?; + enqueue_run_activation_in_conn( + conn.as_mut(), + fence.tenant_id, + fence.run_uid, + fence.controller_generation, + settled_at, + json!({ + "source": "unstarted_task_attempt_settlement", + "task_id": fence.task_id, + "dispatch_uid": fence.dispatch_uid, + "attempt_generation": fence.attempt_generation, + "disposition": unstarted_disposition_label(&disposition), + }), + ) + .await?; + let run_row = sqlx::query(LOAD_RUN_SQL) + .bind(fence.run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + Ok(TaskAttemptSettlementOutcome::Applied { + run: run_from_row(&run_row)?, + task, + }) +} + +fn unstarted_disposition_label(disposition: &UnstartedTaskAttemptDisposition) -> &'static str { + match disposition { + UnstartedTaskAttemptDisposition::Cancelled { .. } => "cancelled", + UnstartedTaskAttemptDisposition::Paused { .. } => "paused", + UnstartedTaskAttemptDisposition::DispatchDeliveryLost => "dispatch_delivery_lost", + } +} + +fn unstarted_task_attempt_settlement_replayed( + task: &ExecutionTaskRecord, + fence: &TaskAttemptFence, + disposition: &UnstartedTaskAttemptDisposition, +) -> bool { + unstarted_task_attempt_history_matches(&task.generation_history, fence, disposition) +} + +fn unstarted_task_attempt_history_matches( + generation_history: &[Value], + fence: &TaskAttemptFence, + disposition: &UnstartedTaskAttemptDisposition, +) -> bool { + let dispatch_uid = fence.dispatch_uid.to_string(); + let expected_reason = match disposition { + UnstartedTaskAttemptDisposition::Cancelled { reason } => Some(reason.as_str()), + UnstartedTaskAttemptDisposition::Paused { .. } + | UnstartedTaskAttemptDisposition::DispatchDeliveryLost => None, + }; + generation_history.iter().any(|entry| { + entry.get("kind").and_then(Value::as_str) == Some("unstarted_attempt_settlement") + && entry.get("dispatch_uid").and_then(Value::as_str) == Some(dispatch_uid.as_str()) + && entry.get("attempt_generation").and_then(Value::as_u64) + == Some(fence.attempt_generation) + && entry.get("disposition").and_then(Value::as_str) + == Some(unstarted_disposition_label(disposition)) + && entry.get("reason").and_then(Value::as_str) == expected_reason + && match disposition { + UnstartedTaskAttemptDisposition::Paused { + controller_generation, + } => { + entry.get("controller_generation").and_then(Value::as_u64) + == Some(*controller_generation) + } + UnstartedTaskAttemptDisposition::Cancelled { .. } + | UnstartedTaskAttemptDisposition::DispatchDeliveryLost => true, + } + }) +} + +fn paused_task_attempt_release_history_matches( + generation_history: &[Value], + fence: &TaskAttemptFence, + controller_generation: u64, +) -> bool { + let dispatch_uid = fence.dispatch_uid.to_string(); + generation_history.iter().any(|entry| { + entry.get("kind").and_then(Value::as_str) == Some("pause_release_finalized") + && entry.get("dispatch_uid").and_then(Value::as_str) == Some(dispatch_uid.as_str()) + && entry.get("attempt_generation").and_then(Value::as_u64) + == Some(fence.attempt_generation) + && entry + .get("attempt_controller_generation") + .and_then(Value::as_u64) + == Some(fence.controller_generation) + && entry.get("controller_generation").and_then(Value::as_u64) + == Some(controller_generation) + }) +} + +fn storage_only_task_kind(kind: &LogicalTaskKind) -> bool { + matches!( + kind, + LogicalTaskKind::Review { .. } + | LogicalTaskKind::WaitSignal { .. } + | LogicalTaskKind::WaitUntil { .. } + ) +} + +async fn load_running_external_start_fence( + conn: &mut ScopedConn<'_>, + tenant_id: TenantId, + run_uid: Uuid, + task_id: ExecutionTaskId, + attempt_generation: u64, +) -> Result> { + let payload = sqlx::query_scalar::<_, Value>( + "SELECT dispatch.payload \ + FROM moa.execution_task_checkpoint AS checkpoint \ + JOIN moa.execution_dispatch_outbox AS dispatch \ + ON dispatch.dispatch_uid=checkpoint.dispatch_uid \ + AND dispatch.tenant_id=checkpoint.tenant_id \ + AND dispatch.run_uid=checkpoint.run_uid \ + AND dispatch.task_id=checkpoint.task_id \ + WHERE checkpoint.tenant_id=$1 AND checkpoint.run_uid=$2 \ + AND checkpoint.task_id=$3 AND checkpoint.attempt_generation=$4 \ + AND checkpoint.superseded_at IS NULL \ + AND dispatch.dispatch_kind='task_attempt'", + ) + .bind(tenant_id.0) + .bind(run_uid) + .bind(task_id.as_uuid()) + .bind(to_i64(attempt_generation, "attempt generation")?) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(payload) = payload else { + return Ok(None); + }; + let request: ExecutionTaskAttemptRequest = + serde_json::from_value(payload).map_err(|error| Error::InvalidRepositoryData { + message: format!("task external-start checkpoint has invalid dispatch: {error}"), + })?; + if request.tenant_id != tenant_id + || request.run_uid != run_uid + || request.task_id != task_id + || request.attempt_generation != attempt_generation + { + return Err(Error::InvalidRepositoryData { + message: "task external-start checkpoint dispatch changed immutable owner coordinates" + .to_string(), + }); + } + Ok(Some(TaskAttemptFence { + tenant_id, + run_uid, + task_id, + controller_generation: request.controller_generation, + attempt_generation, + dispatch_uid: request.dispatch_uid, + capacity_reservation_uid: request.capacity_reservation_uid, + watchdog_trigger_uid: request.watchdog_trigger_uid, + attempt_deadline_at: request.attempt_deadline_at, + })) +} + +async fn load_locked_external_start_owner( + conn: &mut ScopedConn<'_>, + fence: TaskAttemptFence, +) -> Result< + Option<( + ExecutionRunRecord, + ExecutionTaskRecord, + TaskAttemptCheckpointRecord, + )>, +> { + let Some(run_row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) + .bind(fence.run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + return Ok(None); + }; + let run = run_from_row(&run_row)?; + let Some(task_row) = sqlx::query(LOAD_TASK_FOR_UPDATE_SQL) + .bind(fence.run_uid) + .bind(fence.task_id.as_uuid()) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + return Ok(None); + }; + let task = task_from_row(&task_row)?; + let checkpoint = sqlx::query( + "SELECT * FROM moa.execution_task_checkpoint \ + WHERE tenant_id=$1 AND run_uid=$2 AND task_id=$3 \ + AND attempt_generation=$4 AND dispatch_uid=$5 AND superseded_at IS NULL \ + FOR UPDATE", + ) + .bind(fence.tenant_id.0) + .bind(fence.run_uid) + .bind(fence.task_id.as_uuid()) + .bind(to_i64(fence.attempt_generation, "attempt generation")?) + .bind(fence.dispatch_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(checkpoint) = checkpoint else { + return Ok(None); + }; + let checkpoint = task_checkpoint_from_row(&checkpoint)?; + if run.tenant_id != fence.tenant_id + || run.controller_generation != fence.controller_generation + || task.tenant_id != fence.tenant_id + || checkpoint.controller_generation != fence.controller_generation + || checkpoint.task_generation != task.generation + || !external_start_checkpoint_payload_is_provisional(checkpoint.kind, &checkpoint.payload) + { + return Ok(None); + } + Ok(Some((run, task, checkpoint))) +} + +fn bind_recovered_external_job_in_checkpoint( + payload: &mut Value, + kind: TaskAttemptCheckpointKind, + external_job_uid: Uuid, +) -> Result<()> { + let external_job_uid_text = external_job_uid.to_string(); + match kind { + TaskAttemptCheckpointKind::AgentContinuation => { + let pending = payload + .get_mut("state") + .and_then(|state| state.get_mut("pending_external")) + .and_then(Value::as_object_mut) + .ok_or_else(|| Error::InvalidRepositoryData { + message: "agent external-start checkpoint lost its pending invocation" + .to_string(), + })?; + match pending.get("external_job_uid") { + Some(value) if value.is_null() => { + pending.insert("external_job_uid".to_string(), json!(external_job_uid)); + } + Some(value) if value.as_str() == Some(external_job_uid_text.as_str()) => {} + _ => { + return Err(Error::InvalidRepositoryData { + message: "agent external-start checkpoint is bound to another job" + .to_string(), + }); + } + } + Ok(()) + } + TaskAttemptCheckpointKind::CapabilityExternalStart => Ok(()), + TaskAttemptCheckpointKind::CapabilityReview => Err(Error::InvalidRepositoryData { + message: "review checkpoint cannot adopt a direct recovered external start".to_string(), + }), + } +} + +async fn task_attempt_resources_match( + conn: &mut ScopedConn<'_>, + fence: &TaskAttemptFence, +) -> Result { + sqlx::query_scalar::<_, bool>( + "SELECT EXISTS ( \ + SELECT 1 FROM moa.execution_capacity_reservation \ + WHERE reservation_uid = $1 AND tenant_id = $2 AND run_uid = $3 AND task_id = $4 \ + AND controller_generation = $5 AND attempt_generation = $6 \ + AND resource_dimension = 'active_tasks' AND state = 'reserved' \ + ) AND EXISTS ( \ + SELECT 1 FROM moa.execution_trigger \ + WHERE trigger_uid = $7 AND tenant_id = $2 AND run_uid = $3 AND task_id = $4 \ + AND trigger_kind = 'task_watchdog' AND controller_generation = $5 \ + AND attempt_generation = $6 AND state IN ('pending', 'dispatching') \ + )", + ) + .bind(fence.capacity_reservation_uid) + .bind(fence.tenant_id.0) + .bind(fence.run_uid) + .bind(fence.task_id.as_uuid()) + .bind(to_i64( + fence.controller_generation, + "controller generation", + )?) + .bind(to_i64(fence.attempt_generation, "attempt generation")?) + .bind(fence.watchdog_trigger_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error) +} + +async fn persisted_task_release_receipt_matches( + conn: &mut ScopedConn<'_>, + fence: &TaskAttemptFence, + logical_generation: u64, + receipt: &ExecutionHandReleaseReceipt, +) -> Result { + let ExecutionHandReleaseOwner::Task { + task_id, + logical_generation: receipt_logical_generation, + } = &receipt.owner + else { + return Ok(false); + }; + if receipt.tenant_id != fence.tenant_id + || receipt.run_id.0 != fence.run_uid + || task_id.0 != fence.task_id.as_uuid() + || *receipt_logical_generation != logical_generation + || receipt.attempt_generation != fence.attempt_generation + { + return Ok(false); + } + let verified_absence = task_release_receipt_is_verified_absence(receipt); + if verified_absence { + return sqlx::query_scalar::<_, bool>( + "SELECT EXISTS (SELECT 1 FROM moa.sandbox_execution_hand_release_receipts \ + WHERE receipt_id=$1 AND tenant_id=$2 AND run_uid=$3 AND owner_kind='task' \ + AND task_id=$4 AND compensation_id IS NULL AND logical_generation=$5 \ + AND attempt_generation=$6 AND receipt_state='released' \ + AND destroy_outcome='verified_absent' AND released_at IS NOT NULL \ + AND workspace_id IS NULL AND writer_epoch IS NULL \ + AND instance_generation IS NULL \ + AND hand_provisioning_operation_id IS NULL \ + AND hand_lease_generation IS NULL AND checkpoint_id IS NULL \ + AND checkpoint_generation IS NULL \ + AND checkpoint_manifest_digest IS NULL \ + AND checkpoint_logical_bytes IS NULL)", + ) + .bind(receipt.receipt_id) + .bind(fence.tenant_id.0) + .bind(fence.run_uid) + .bind(fence.task_id.as_uuid()) + .bind(to_i64( + logical_generation, + "release receipt logical generation", + )?) + .bind(to_i64( + fence.attempt_generation, + "release receipt attempt generation", + )?) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error); + } + let ( + Some(workspace_id), + Some(writer_epoch), + Some(instance_generation), + Some(hand_provisioning_operation_id), + Some(hand_lease_generation), + Some(checkpoint_id), + Some(checkpoint_generation), + Some(checkpoint_manifest_digest), + Some(checkpoint_logical_bytes), + ) = ( + receipt.workspace_id, + receipt.writer_epoch, + receipt.instance_generation, + receipt.hand_provisioning_operation_id, + receipt.hand_lease_generation, + receipt.checkpoint_id, + receipt.checkpoint_generation, + receipt.checkpoint_manifest_digest.as_deref(), + receipt.checkpoint_logical_bytes, + ) + else { + return Ok(false); + }; + sqlx::query_scalar::<_, bool>( + "SELECT EXISTS (SELECT 1 FROM moa.sandbox_execution_hand_release_receipts \ + WHERE receipt_id=$1 AND tenant_id=$2 AND run_uid=$3 AND owner_kind='task' \ + AND task_id=$4 AND compensation_id IS NULL AND logical_generation=$5 \ + AND attempt_generation=$6 AND receipt_state='released' \ + AND destroy_outcome='verified_absent' AND released_at IS NOT NULL \ + AND workspace_id=$7 AND writer_epoch=$8 AND instance_generation=$9 \ + AND hand_provisioning_operation_id=$10 AND hand_lease_generation=$11 \ + AND checkpoint_id=$12 AND checkpoint_generation=$13 \ + AND checkpoint_manifest_digest=$14 AND checkpoint_logical_bytes=$15)", + ) + .bind(receipt.receipt_id) + .bind(fence.tenant_id.0) + .bind(fence.run_uid) + .bind(fence.task_id.as_uuid()) + .bind(to_i64( + logical_generation, + "release receipt logical generation", + )?) + .bind(to_i64( + fence.attempt_generation, + "release receipt attempt generation", + )?) + .bind(workspace_id.0) + .bind(to_i64(writer_epoch, "release receipt writer epoch")?) + .bind(to_i64( + instance_generation, + "release receipt instance generation", + )?) + .bind(hand_provisioning_operation_id.0) + .bind(to_i64( + hand_lease_generation, + "release receipt hand lease generation", + )?) + .bind(checkpoint_id.0) + .bind(to_i64( + checkpoint_generation, + "release receipt checkpoint generation", + )?) + .bind(checkpoint_manifest_digest) + .bind(to_i64( + checkpoint_logical_bytes, + "release receipt checkpoint logical bytes", + )?) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error) +} + +fn task_release_receipt_is_verified_absence(receipt: &ExecutionHandReleaseReceipt) -> bool { + receipt.workspace_id.is_none() + && receipt.writer_epoch.is_none() + && receipt.instance_generation.is_none() + && receipt.hand_provisioning_operation_id.is_none() + && receipt.hand_lease_generation.is_none() + && receipt.checkpoint_id.is_none() + && receipt.checkpoint_generation.is_none() + && receipt.checkpoint_manifest_digest.is_none() + && receipt.checkpoint_logical_bytes.is_none() +} impl ExecutionRepository { /// Atomically reserves all five resource dimensions for one pending task. @@ -35,7 +4312,7 @@ impl ExecutionRepository { return Ok(ReservationOutcome::NotFound); }; let task = task_from_row(&task_row)?; - if task.generation != generation { + if task.generation != generation || task.attempt_generation != generation { conn.commit().await.map_err(storage_error)?; return Ok(ReservationOutcome::Rejected( ReservationRejection::GenerationMismatch, @@ -157,7 +4434,7 @@ impl ExecutionRepository { return Ok(TransitionOutcome::NotFound); }; let task = task_from_row(&task_row)?; - if task.generation != generation { + if task.generation != generation || task.attempt_generation != generation { conn.commit().await.map_err(storage_error)?; return Ok(TransitionOutcome::Rejected( TransitionRejection::GenerationMismatch, @@ -216,27 +4493,37 @@ impl ExecutionRepository { generation: u64, kind: ResumeKind, ) -> Result { - self.resume_task_inner(scope, run_uid, task_id, generation, kind, None) - .await + self.resume_task_inner(ResumeTaskRequest { + scope, + config: None, + run_uid, + task_id, + generation, + kind, + resume_input: None, + }) + .await } /// Resumes one waiting-input task and atomically appends the exact supplied payload. pub async fn resume_task_with_input( &self, scope: ExecutionScope, + config: &ExecutionConfig, run_uid: Uuid, task_id: ExecutionTaskId, generation: u64, input: Value, ) -> Result { - self.resume_task_inner( + self.resume_task_inner(ResumeTaskRequest { scope, + config: Some(config), run_uid, task_id, generation, - ResumeKind::Input, - Some(input), - ) + kind: ResumeKind::Input, + resume_input: Some(input), + }) .await } @@ -248,20 +4535,77 @@ impl ExecutionRepository { task_id: ExecutionTaskId, generation: u64, ) -> Result { - self.resume_task_inner(scope, run_uid, task_id, generation, ResumeKind::Retry, None) - .await + self.resume_task_inner(ResumeTaskRequest { + scope, + config: None, + run_uid, + task_id, + generation, + kind: ResumeKind::Retry, + resume_input: None, + }) + .await } - async fn resume_task_inner( - &self, - scope: ExecutionScope, - run_uid: Uuid, - task_id: ExecutionTaskId, - generation: u64, - kind: ResumeKind, - resume_input: Option, - ) -> Result { + async fn resume_task_inner(&self, request: ResumeTaskRequest<'_>) -> Result { + let ResumeTaskRequest { + scope, + config, + run_uid, + task_id, + generation, + kind, + resume_input, + } = request; let mut conn = scope.begin(&self.pool).await?; + let locked_wait_trigger_uid = if kind == ResumeKind::Input { + let config = config.ok_or_else(|| Error::InvalidRepositoryInput { + message: "input resume requires validated execution capacity configuration" + .to_string(), + })?; + let tenant_id = sqlx::query_scalar::<_, Uuid>( + "SELECT tenant_id FROM moa.execution_task WHERE run_uid=$1 AND task_id=$2", + ) + .bind(run_uid) + .bind(task_id.as_uuid()) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(tenant_id) = tenant_id else { + conn.commit().await.map_err(storage_error)?; + return Ok(TransitionOutcome::NotFound); + }; + prelock_capacity_dimensions_in_tx( + conn.as_mut(), + config, + TenantId(tenant_id), + &[ + ExecutionCapacityDimension::ActiveRuns, + ExecutionCapacityDimension::ParkedRuns, + ExecutionCapacityDimension::ScheduledTriggers, + ], + ) + .await?; + let trigger_uids = sqlx::query_scalar::<_, Uuid>( + "SELECT trigger_uid FROM moa.execution_trigger \ + WHERE run_uid=$1 AND task_id=$2 AND trigger_kind='wait_expiry' \ + AND state IN ('pending','dispatching') \ + ORDER BY trigger_uid LIMIT 2 FOR UPDATE", + ) + .bind(run_uid) + .bind(task_id.as_uuid()) + .fetch_all(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if trigger_uids.len() > 1 { + return Err(Error::InvalidRepositoryData { + message: "waiting-input task owns multiple active expiry triggers".to_string(), + }); + } + trigger_uids.into_iter().next() + } else { + None + }; let Some(run_row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) .bind(run_uid) .fetch_optional(conn.as_mut()) @@ -291,7 +4635,9 @@ impl ExecutionRepository { conn.commit().await.map_err(storage_error)?; return Ok(TransitionOutcome::AlreadyApplied(task)); } - if task.generation != generation { + if task.generation != generation + || (kind == ResumeKind::Retry && task.attempt_generation != generation) + { conn.commit().await.map_err(storage_error)?; return Ok(TransitionOutcome::Rejected( TransitionRejection::GenerationMismatch, @@ -309,7 +4655,11 @@ impl ExecutionRepository { ExecutionTaskStatus::WaitingInput, matches!( run.status, - ExecutionRunStatus::WaitingInput | ExecutionRunStatus::Running + ExecutionRunStatus::WaitingInput + | ExecutionRunStatus::Running + | ExecutionRunStatus::PauseRequested + | ExecutionRunStatus::Pausing + | ExecutionRunStatus::Paused ), task.attempt, ), @@ -362,6 +4712,102 @@ impl ExecutionRepository { TransitionRejection::CounterOverflow, )); }; + let Some(next_attempt_generation) = task.attempt_generation.checked_add(1) else { + conn.commit().await.map_err(storage_error)?; + return Ok(TransitionOutcome::Rejected( + TransitionRejection::CounterOverflow, + )); + }; + let resumed_at = Utc::now(); + let input_audience = if kind == ResumeKind::Input { + Some( + task.current_outcome + .as_ref() + .and_then(|outcome| match &outcome.result { + ExecutionTaskResult::NeedsInput { audience, .. } => Some(audience.clone()), + _ => None, + }) + .ok_or_else(|| Error::InvalidRepositoryData { + message: "waiting-input task is missing its typed input audience" + .to_string(), + })?, + ) + } else { + None + }; + if kind == ResumeKind::Input { + let Some(trigger_uid) = locked_wait_trigger_uid else { + return Err(Error::InvalidRepositoryData { + message: "waiting-input task is missing its active expiry trigger".to_string(), + }); + }; + match supersede_trigger_in_conn( + conn.as_mut(), + trigger_uid, + ExecutionTriggerKind::WaitExpiry, + Some(run.controller_generation), + Some(task.generation), + None, + None, + ) + .await? + { + ExecutionTriggerSupersedeOutcome::Superseded + | ExecutionTriggerSupersedeOutcome::AlreadySuperseded + | ExecutionTriggerSupersedeOutcome::AlreadyInactive => {} + ExecutionTriggerSupersedeOutcome::StaleOrMissing => { + return Err(Error::InvalidRepositoryData { + message: "waiting-input expiry trigger lost its generation fence" + .to_string(), + }); + } + } + + let current_checkpoint = sqlx::query( + "SELECT * FROM moa.execution_task_checkpoint WHERE tenant_id=$1 AND run_uid=$2 \ + AND task_id=$3 AND superseded_at IS NULL FOR UPDATE", + ) + .bind(run.tenant_id.0) + .bind(run_uid) + .bind(task_id.as_uuid()) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if let Some(current_checkpoint) = current_checkpoint { + let current_checkpoint = task_checkpoint_from_row(¤t_checkpoint)?; + if current_checkpoint.kind != TaskAttemptCheckpointKind::AgentContinuation + || current_checkpoint.task_generation != generation + { + return Err(Error::InvalidRepositoryData { + message: "waiting-input agent checkpoint is generation-stale".to_string(), + }); + } + let mut payload = current_checkpoint.payload.clone(); + append_agent_resume_input( + &mut payload, + resume_input + .as_ref() + .ok_or_else(|| Error::InvalidRepositoryInput { + message: "input resume is missing its payload".to_string(), + })?, + )?; + insert_resolved_task_checkpoint_in_conn( + &mut conn, + ¤t_checkpoint, + run.controller_generation, + next_generation, + next_attempt_generation, + payload, + resumed_at, + ) + .await?; + } else if matches!(task.kind, LogicalTaskKind::Agent { .. }) { + return Err(Error::InvalidRepositoryData { + message: "waiting-input agent task is missing its durable continuation" + .to_string(), + }); + } + } let history = json!({ "kind": history_kind, "requested_generation": generation, @@ -385,35 +4831,71 @@ impl ExecutionRepository { .bind(to_i64(next_generation, "next task generation")?) .bind(history) .bind(resume_input) + .bind(to_i64(next_attempt_generation, "next attempt generation")?) + .bind(resumed_at) .fetch_one(conn.as_mut()) .await .map_err(sqlx_error)?; - let task = task_from_row(&row)?; - if let Some((class, reason)) = admission_rejection { - let task = terminalize_redispatch_rejection( + let resumed_task = task_from_row(&row)?; + let task = if let Some((class, reason)) = admission_rejection { + terminalize_redispatch_rejection( &mut conn, &run, - &task, + &resumed_task, history_kind, class, reason, ) + .await? + } else { + resumed_task + }; + if let Some(input_audience) = input_audience.as_ref() { + transition_node_counters_with_input_audience_in_tx( + &mut conn, + run_uid, + &task.node_id, + &task.item_key, + ExecutionTaskStatus::WaitingInput, + task.status, + input_audience, + ) + .await?; + refresh_run_after_wait_settlement_in_conn(&mut conn, run_uid, task_id, resumed_at) + .await?; + } else { + transition_node_counters_in_tx( + &mut conn, + run_uid, + &task.node_id, + &task.item_key, + expected_status, + task.status, + ) + .await?; + } + if !matches!( + run.status, + ExecutionRunStatus::PauseRequested + | ExecutionRunStatus::Pausing + | ExecutionRunStatus::Paused + ) { + enqueue_run_activation_in_conn( + conn.as_mut(), + run.tenant_id, + run_uid, + run.controller_generation, + resumed_at, + json!({ + "source": history_kind, + "task_id": task_id, + "requested_generation": generation, + "next_generation": task.generation, + "next_attempt_generation": task.attempt_generation, + }), + ) .await?; - conn.commit().await.map_err(storage_error)?; - return Ok(TransitionOutcome::Applied(task)); } - sqlx::query( - "UPDATE moa.execution_run \ - SET status = CASE WHEN status IN ('waiting_input', 'waiting_replan') \ - THEN 'running' ELSE status END, \ - waiting_reasons = '[]'::JSONB, wake_epoch = wake_epoch + 1, \ - updated_at = NOW() \ - WHERE run_uid = $1", - ) - .bind(run_uid) - .execute(conn.as_mut()) - .await - .map_err(sqlx_error)?; conn.commit().await.map_err(storage_error)?; Ok(TransitionOutcome::Applied(task)) } @@ -462,179 +4944,446 @@ impl ExecutionRepository { Ok(ExecutionTaskPage { tasks, next_cursor }) } - /// Transitions a run into one scheduler wait state under a source-status fence. - pub async fn transition_run_wait( + /// Records a generation-fenced zero- or nonzero-usage outcome for a parked external wait. + pub async fn complete_external_wait( &self, scope: ExecutionScope, + config: &ExecutionConfig, run_uid: Uuid, - expected_status: ExecutionRunStatus, - waiting_status: ExecutionRunStatus, - ) -> Result { - self.transition_run_wait_with_reasons( - scope, - run_uid, - expected_status, - waiting_status, - Vec::new(), + task_id: ExecutionTaskId, + generation: u64, + outcome: ExecutionTaskOutcome, + ) -> Result { + let mut conn = scope.begin(&self.pool).await?; + let tenant_id = sqlx::query_scalar::<_, Uuid>( + "SELECT tenant_id FROM moa.execution_task WHERE run_uid=$1 AND task_id=$2", ) + .bind(run_uid) + .bind(task_id.as_uuid()) + .fetch_optional(conn.as_mut()) .await - } - - /// Transitions a run and persists the exact scheduler wait reasons atomically. - pub async fn transition_run_wait_with_reasons( - &self, - scope: ExecutionScope, - run_uid: Uuid, - expected_status: ExecutionRunStatus, - waiting_status: ExecutionRunStatus, - waiting_reasons: Vec, - ) -> Result { - if !matches!( - waiting_status, - ExecutionRunStatus::WaitingInput - | ExecutionRunStatus::WaitingReview - | ExecutionRunStatus::WaitingReplan - | ExecutionRunStatus::Running - ) { - return Err(Error::InvalidRepositoryInput { - message: "run wait target must be running or one waiting status".to_string(), + .map_err(sqlx_error)?; + let Some(tenant_id) = tenant_id else { + conn.commit().await.map_err(storage_error)?; + return Ok(TaskOutcomeWrite::NotFound); + }; + prelock_capacity_dimensions_in_tx( + conn.as_mut(), + config, + TenantId(tenant_id), + &[ + ExecutionCapacityDimension::ActiveRuns, + ExecutionCapacityDimension::ParkedRuns, + ExecutionCapacityDimension::ScheduledTriggers, + ], + ) + .await?; + let trigger_uids = sqlx::query_scalar::<_, Uuid>( + "SELECT trigger_uid FROM moa.execution_trigger \ + WHERE run_uid=$1 AND task_id=$2 AND trigger_kind='wait_expiry' \ + AND state IN ('pending','dispatching') \ + ORDER BY trigger_uid LIMIT 2 FOR UPDATE", + ) + .bind(run_uid) + .bind(task_id.as_uuid()) + .fetch_all(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if trigger_uids.len() > 1 { + return Err(Error::InvalidRepositoryData { + message: "storage wait owns multiple active expiry triggers".to_string(), }); } - let waiting_value = serde_json::to_value(&waiting_reasons)?; - let mut conn = scope.begin(&self.pool).await?; - let Some(row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) + let locked_wait_trigger_uid = trigger_uids.into_iter().next(); + let Some(run_row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) .bind(run_uid) .fetch_optional(conn.as_mut()) .await .map_err(sqlx_error)? else { conn.commit().await.map_err(storage_error)?; - return Ok(TransitionOutcome::NotFound); + return Ok(TaskOutcomeWrite::NotFound); }; - let current = run_from_row(&row)?; - if current.status == waiting_status && current.waiting_reasons == waiting_reasons { + let run = run_from_row(&run_row)?; + let Some(task_row) = sqlx::query(LOAD_TASK_FOR_UPDATE_SQL) + .bind(run_uid) + .bind(task_id.as_uuid()) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { conn.commit().await.map_err(storage_error)?; - return Ok(TransitionOutcome::RunAlreadyApplied(current)); - } - if current.status != expected_status - || current.status.is_terminal() - || current.pending_terminal.is_some() + return Ok(TaskOutcomeWrite::NotFound); + }; + let task = task_from_row(&task_row)?; + if task_outcome_is_exact_replay(&task, generation, &outcome) + || task.generation != generation + || !matches!( + task.status, + ExecutionTaskStatus::WaitingReview | ExecutionTaskStatus::WaitingSignal + ) { + let write = + record_task_outcome_in_conn(&mut conn, run_uid, task_id, generation, outcome) + .await?; conn.commit().await.map_err(storage_error)?; - return Ok(TransitionOutcome::Rejected( - TransitionRejection::InvalidRunStatus, - )); + return Ok(write); } - if current.status == ExecutionRunStatus::Queued - && waiting_status != ExecutionRunStatus::Running + let Some(trigger_uid) = locked_wait_trigger_uid else { + return Err(Error::InvalidRepositoryData { + message: "storage wait is missing its active expiry trigger".to_string(), + }); + }; + match supersede_trigger_in_conn( + conn.as_mut(), + trigger_uid, + ExecutionTriggerKind::WaitExpiry, + Some(run.controller_generation), + Some(task.generation), + None, + None, + ) + .await? { - sqlx::query( - "UPDATE moa.execution_run SET status = 'running', \ - started_at = COALESCE(started_at, NOW()), updated_at = NOW() \ - WHERE run_uid = $1 AND status = 'queued'", - ) - .bind(run_uid) - .execute(conn.as_mut()) - .await - .map_err(sqlx_error)?; + ExecutionTriggerSupersedeOutcome::Superseded + | ExecutionTriggerSupersedeOutcome::AlreadySuperseded + | ExecutionTriggerSupersedeOutcome::AlreadyInactive => {} + ExecutionTriggerSupersedeOutcome::StaleOrMissing => { + return Err(Error::InvalidRepositoryData { + message: "storage-wait expiry trigger lost its generation fence".to_string(), + }); + } } - let row = sqlx::query( - "UPDATE moa.execution_run \ - SET status = $2, waiting_reasons = $3, wake_epoch = wake_epoch + 1, \ - started_at = CASE WHEN $2 = 'running' THEN COALESCE(started_at, NOW()) \ - ELSE started_at END, \ - updated_at = NOW() \ - WHERE run_uid = $1 \ - RETURNING *", + let settled_at = Utc::now(); + let transitioned = sqlx::query( + "UPDATE moa.execution_task SET status='running', attempt_state='running', \ + generation_history=generation_history || jsonb_build_array(jsonb_build_object( \ + 'kind','external_wait_resolution','generation',$3,'recorded_at',$4)), \ + last_progress_at=GREATEST(last_progress_at,$4), updated_at=$4 \ + WHERE run_uid=$1 AND task_id=$2 AND generation=$3 \ + AND status IN ('waiting_review','waiting_signal') AND attempt_state='waiting'", ) .bind(run_uid) - .bind(waiting_status.as_str()) - .bind(waiting_value) - .fetch_one(conn.as_mut()) + .bind(task_id.as_uuid()) + .bind(to_i64(generation, "task generation")?) + .bind(settled_at) + .execute(conn.as_mut()) .await .map_err(sqlx_error)?; - let run = run_from_row(&row)?; + if transitioned.rows_affected() != 1 { + return Err(Error::InvalidRepositoryData { + message: "storage wait lost its locked transition fence".to_string(), + }); + } + let write = + record_task_outcome_in_conn(&mut conn, run_uid, task_id, generation, outcome).await?; + let settled_task = match &write { + TaskOutcomeWrite::Applied { task, .. } | TaskOutcomeWrite::Replayed { task, .. } => { + task + } + TaskOutcomeWrite::NotFound | TaskOutcomeWrite::Rejected { .. } => { + return Err(Error::InvalidRepositoryData { + message: "locked storage-wait outcome was not accepted".to_string(), + }); + } + }; + transition_node_counters_in_tx( + &mut conn, + run_uid, + &settled_task.node_id, + &settled_task.item_key, + task.status, + settled_task.status, + ) + .await?; + refresh_run_after_wait_settlement_in_conn(&mut conn, run_uid, task_id, settled_at).await?; + if !matches!( + run.status, + ExecutionRunStatus::PauseRequested + | ExecutionRunStatus::Pausing + | ExecutionRunStatus::Paused + ) { + enqueue_run_activation_in_conn( + conn.as_mut(), + run.tenant_id, + run_uid, + run.controller_generation, + settled_at, + json!({ + "source": "external_wait_resolution", + "task_id": task_id, + "task_generation": generation, + }), + ) + .await?; + } conn.commit().await.map_err(storage_error)?; - Ok(TransitionOutcome::RunApplied(run)) + Ok(write) } +} - /// Records a generation-fenced zero- or nonzero-usage outcome for a parked external wait. - pub async fn complete_external_wait( - &self, - scope: ExecutionScope, - run_uid: Uuid, - task_id: ExecutionTaskId, - generation: u64, - outcome: ExecutionTaskOutcome, - ) -> Result { - self.record_task_outcome(scope, run_uid, task_id, generation, outcome) - .await +#[cfg(test)] +mod tests { + use super::{ + TaskAttemptCheckpointKind, TaskAttemptFence, UnstartedTaskAttemptDisposition, + append_agent_resume_input, external_start_checkpoint_payload_is_provisional, + paused_task_attempt_release_history_matches, task_release_receipt_is_verified_absence, + unstarted_task_attempt_history_matches, + }; + use chrono::Utc; + use moa_core::types::context::ContextMessage; + use moa_core::types::{ + identifiers::{ExecutionRunScopeId, ExecutionTaskScopeId, TenantId}, + sandbox_workspace::{ExecutionHandReleaseOwner, ExecutionHandReleaseReceipt}, + }; + use serde_json::json; + use uuid::Uuid; + + use crate::state::ExecutionTaskId; + + #[test] + fn agent_input_resume_appends_exact_user_message_to_durable_checkpoint() { + // Pins: a public input acknowledgement is not complete unless the next bounded Agent + // attempt can observe the exact supplied reply from its canonical checkpoint. + let mut checkpoint = json!({ + "schema_version": 1, + "state": { + "kind": "agent", + "messages": [serde_json::to_value(ContextMessage::assistant("question")) + .expect("fixture message serializes")], + "next_turn": 1, + "usage": {}, + "security_circuit": {}, + "disabled_capabilities": {}, + "pending_review": null, + "pending_external": null + }, + "review_resolution": null, + "external_job_resolution": null, + "workspace_release_receipt_id": null + }); + + append_agent_resume_input(&mut checkpoint, &json!({"answer": "approved"})) + .expect("agent input appends"); + + let messages = checkpoint["state"]["messages"] + .as_array() + .expect("messages remain an array"); + assert_eq!(messages.len(), 2); + assert_eq!( + messages[1], + serde_json::to_value(ContextMessage::user(r#"{"answer":"approved"}"#)) + .expect("expected message serializes") + ); + assert_eq!(checkpoint["state"]["next_turn"], 1); } - /// Idempotently audits one action-review resolution under its task generation fence. - pub async fn record_action_review_resolution( - &self, - scope: ExecutionScope, - run_uid: Uuid, - task_id: ExecutionTaskId, - generation: u64, - review_uid: Uuid, - resolution: &ExecutionActionReviewResolution, - ) -> Result { - let mut conn = scope.begin(&self.pool).await?; - let Some(row) = sqlx::query(LOAD_TASK_FOR_UPDATE_SQL) - .bind(run_uid) - .bind(task_id.as_uuid()) - .fetch_optional(conn.as_mut()) - .await - .map_err(sqlx_error)? - else { - conn.commit().await.map_err(storage_error)?; - return Ok(ActionReviewResolutionWrite::NotFound); - }; - let task = task_from_row(&row)?; - let review_uid_text = review_uid.to_string(); - if let Some(existing) = task.outcome_audit.iter().find(|entry| { - entry.get("kind").and_then(Value::as_str) == Some("execution_action_review_resolution") - && entry.get("review_uid").and_then(Value::as_str) == Some(review_uid_text.as_str()) - && entry.get("generation").and_then(Value::as_u64) == Some(generation) - }) { - let existing_resolution: ExecutionActionReviewResolution = - serde_json::from_value(existing.get("resolution").cloned().ok_or_else(|| { - Error::InvalidRepositoryData { - message: "persisted task review audit is missing its resolution" - .to_string(), - } - })?)?; - if existing_resolution != *resolution { - return Err(Error::InvalidRepositoryData { - message: "task review UID was replayed with a different resolution".to_string(), - }); + #[test] + fn external_start_checkpoint_requires_exact_typed_provisional_shape() { + // Pins: provider start cannot precede a durable continuation that identifies either the + // exact pending Agent invocation or the stable direct capability tool-call identity. + let agent = json!({ + "state": { + "kind": "agent", + "pending_external": { + "external_job_uid": null, + "invocation": {"id": "call-1"} + } } - conn.commit().await.map_err(storage_error)?; - return Ok(ActionReviewResolutionWrite::Replayed); - } - let accepted = task.generation == generation && task.status == ExecutionTaskStatus::Running; - let audit = json!({ - "kind": "execution_action_review_resolution", - "review_uid": review_uid, - "generation": generation, - "accepted": accepted, - "resolution": resolution, - "recorded_at": Utc::now(), }); - sqlx::query(APPEND_TASK_OUTCOME_AUDIT_SQL) - .bind(run_uid) - .bind(task_id.as_uuid()) - .bind(audit) - .fetch_one(conn.as_mut()) - .await - .map_err(sqlx_error)?; - conn.commit().await.map_err(storage_error)?; - Ok(if accepted { - ActionReviewResolutionWrite::Applied - } else { - ActionReviewResolutionWrite::AuditedStale - }) + assert!(external_start_checkpoint_payload_is_provisional( + TaskAttemptCheckpointKind::AgentContinuation, + &agent, + )); + let bound_agent = json!({ + "state": { + "kind": "agent", + "pending_external": { + "external_job_uid": Uuid::new_v4(), + "invocation": {"id": "call-1"} + } + } + }); + assert!(!external_start_checkpoint_payload_is_provisional( + TaskAttemptCheckpointKind::AgentContinuation, + &bound_agent, + )); + assert!(external_start_checkpoint_payload_is_provisional( + TaskAttemptCheckpointKind::CapabilityExternalStart, + &json!({"state":{"kind":"capability_external_start","tool_id":Uuid::new_v4()}}), + )); + assert!(!external_start_checkpoint_payload_is_provisional( + TaskAttemptCheckpointKind::CapabilityReview, + &json!({"state":{"kind":"capability_external_start","tool_id":Uuid::new_v4()}}), + )); + } + + #[test] + fn unstarted_attempt_replay_requires_exact_dispatch_generation_and_disposition() { + // Pins: a dead-letter or cancel replay must never consume another admitted attempt's + // capacity or treat a different terminal disposition as already settled. + let fence = TaskAttemptFence { + tenant_id: TenantId(Uuid::new_v4()), + run_uid: Uuid::new_v4(), + task_id: ExecutionTaskId::from_uuid(Uuid::new_v4()), + controller_generation: 4, + attempt_generation: 7, + dispatch_uid: Uuid::new_v4(), + capacity_reservation_uid: Uuid::new_v4(), + watchdog_trigger_uid: Uuid::new_v4(), + attempt_deadline_at: Utc::now(), + }; + let history = vec![json!({ + "kind": "unstarted_attempt_settlement", + "dispatch_uid": fence.dispatch_uid, + "attempt_generation": fence.attempt_generation, + "disposition": "dispatch_delivery_lost", + "reason": null, + })]; + + assert!(unstarted_task_attempt_history_matches( + &history, + &fence, + &UnstartedTaskAttemptDisposition::DispatchDeliveryLost, + )); + assert!(!unstarted_task_attempt_history_matches( + &history, + &fence, + &UnstartedTaskAttemptDisposition::Cancelled { + reason: "pause requested".to_string(), + }, + )); + assert!(!unstarted_task_attempt_history_matches( + &history, + &fence, + &UnstartedTaskAttemptDisposition::Paused { + controller_generation: 5, + }, + )); + let mut stale_fence = fence; + stale_fence.attempt_generation += 1; + assert!(!unstarted_task_attempt_history_matches( + &history, + &stale_fence, + &UnstartedTaskAttemptDisposition::DispatchDeliveryLost, + )); + } + + #[test] + fn pause_release_replay_requires_both_controller_generations() { + // Pins: pause increments the run generation, while the released capacity and watchdog + // remain owned by the prior admission generation; neither coordinate is interchangeable. + let fence = TaskAttemptFence { + tenant_id: TenantId(Uuid::new_v4()), + run_uid: Uuid::new_v4(), + task_id: ExecutionTaskId::from_uuid(Uuid::new_v4()), + controller_generation: 4, + attempt_generation: 7, + dispatch_uid: Uuid::new_v4(), + capacity_reservation_uid: Uuid::new_v4(), + watchdog_trigger_uid: Uuid::new_v4(), + attempt_deadline_at: Utc::now(), + }; + let history = vec![json!({ + "kind": "pause_release_finalized", + "dispatch_uid": fence.dispatch_uid, + "attempt_generation": fence.attempt_generation, + "attempt_controller_generation": fence.controller_generation, + "controller_generation": 5, + })]; + + assert!(paused_task_attempt_release_history_matches( + &history, &fence, 5, + )); + assert!(!paused_task_attempt_release_history_matches( + &history, &fence, 6, + )); + let mut stale_fence = fence; + stale_fence.controller_generation += 1; + assert!(!paused_task_attempt_release_history_matches( + &history, + &stale_fence, + 5, + )); + } + + #[test] + fn unstarted_pause_replay_is_exact_and_distinct_from_terminal_cancel() { + // Pins: a pause received before start requeues the task and replay detection must not + // reinterpret that durable disposition as a terminal cancellation. + let fence = TaskAttemptFence { + tenant_id: TenantId(Uuid::new_v4()), + run_uid: Uuid::new_v4(), + task_id: ExecutionTaskId::from_uuid(Uuid::new_v4()), + controller_generation: 8, + attempt_generation: 3, + dispatch_uid: Uuid::new_v4(), + capacity_reservation_uid: Uuid::new_v4(), + watchdog_trigger_uid: Uuid::new_v4(), + attempt_deadline_at: Utc::now(), + }; + let history = vec![json!({ + "kind": "unstarted_attempt_settlement", + "dispatch_uid": fence.dispatch_uid, + "attempt_generation": fence.attempt_generation, + "disposition": "paused", + "reason": null, + "controller_generation": 9, + })]; + + assert!(unstarted_task_attempt_history_matches( + &history, + &fence, + &UnstartedTaskAttemptDisposition::Paused { + controller_generation: 9, + }, + )); + assert!(!unstarted_task_attempt_history_matches( + &history, + &fence, + &UnstartedTaskAttemptDisposition::Paused { + controller_generation: 10, + }, + )); + assert!(!unstarted_task_attempt_history_matches( + &history, + &fence, + &UnstartedTaskAttemptDisposition::Cancelled { + reason: "pause requested".to_string(), + }, + )); + } + + #[test] + fn verified_absence_receipt_shape_rejects_partial_workspace_identity() { + // Pins: a no-hand task may settle only with the canonical all-NULL durable absence proof; + // dropping part of a real checkpoint identity must never be treated as absence. + let task_id = ExecutionTaskScopeId(Uuid::new_v4()); + let now = Utc::now(); + let mut receipt = ExecutionHandReleaseReceipt { + receipt_id: Uuid::new_v4(), + tenant_id: TenantId(Uuid::new_v4()), + run_id: ExecutionRunScopeId(Uuid::new_v4()), + owner: ExecutionHandReleaseOwner::Task { + task_id, + logical_generation: 3, + }, + attempt_generation: 5, + workspace_id: None, + writer_epoch: None, + instance_generation: None, + hand_provisioning_operation_id: None, + hand_lease_generation: None, + checkpoint_id: None, + checkpoint_generation: None, + checkpoint_manifest_digest: None, + checkpoint_logical_bytes: None, + requested_at: now, + released_at: now, + }; + + assert!(task_release_receipt_is_verified_absence(&receipt)); + receipt.writer_epoch = Some(1); + assert!(!task_release_receipt_is_verified_absence(&receipt)); } } diff --git a/crates/moa-execution/src/repository/terminal.rs b/crates/moa-execution/src/repository/terminal.rs index 2f5341436..7e84d58bb 100644 --- a/crates/moa-execution/src/repository/terminal.rs +++ b/crates/moa-execution/src/repository/terminal.rs @@ -1,9 +1,335 @@ //! Successful run finalization and shared state-projection helpers. use super::*; -use super::{projection::*, rows::*, sql::*}; + +/// Exact amendment identity that caused a compensation-safe replan-stop fence. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ReplanStopReceipt { + /// Waiting-replan task whose current outcome triggered amendment evaluation. + pub task_id: ExecutionTaskId, + /// Exact generation of the waiting-replan task. + pub task_generation: u64, + /// Plan revision against which the amendment was evaluated. + pub base_plan_revision: u64, + /// Domain-separated hash of the exact amendment request. + pub amendment_hash: ExecutionHash, +} + +/// Durable phase reached by one bounded pending-terminal advancement. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum PendingTerminalAdvanceStage { + /// A bounded page settled or claimed work and another controller activation was queued. + EnqueuedPage, + /// Exact active attempts are relinquishing hands and still own capacity. + Draining, + /// Forward work drained and bounded reverse-order compensation was activated. + CompensationQueued, + /// The held terminal intent became the run's final state. + Finalized, + /// Compensation evidence requires governed repair instead of a clean terminal result. + ManualRepairRequired, +} + +/// Durable receipts committed by one bounded pending-terminal advancement. +#[derive(Clone, Debug, PartialEq)] +pub struct PendingTerminalAdvanceCommit { + /// Current run after the exact wake checkpoint was committed. + pub run: ExecutionRunRecord, + /// Lifecycle phase reached by this bounded page. + pub stage: PendingTerminalAdvanceStage, + /// Number of storage-only forward tasks settled by this page. + pub settled_task_count: u64, + /// Number of durable triggers superseded within the same bounded page. + pub drained_trigger_count: u32, + /// Exact task or compensation cancellation deliveries enqueued by this page. + pub cancellation_dispatches: Vec, + /// At most one reverse-order compensation slice admitted by this activation. + pub compensation_admission: Option>, + /// Optional controller continuation created in the same transaction. + pub continuation: Option>, + /// Whether durable work still prevents installing the held terminal result. + pub work_remaining: bool, +} + +/// Generation-and-wake-fenced result of advancing a bounded terminal drain page. +#[derive(Clone, Debug, PartialEq)] +pub enum PendingTerminalAdvanceOutcome { + /// This invocation committed one bounded monotonic page. + Applied(Box), + /// The exact wake was already acknowledged and no delivery was duplicated. + Replayed(Box), + /// No visible run exists under the supplied scope. + NotFound, + /// Generation, wake, terminal intent, or lifecycle state did not match. + Conflict, +} + +/// Result of terminal run finalization. +#[derive(Clone, Debug, PartialEq)] +pub enum FinalizationOutcome { + /// Terminal state and completion evidence were persisted. + Finalized(ExecutionRunRecord), + /// The same terminal projection was already persisted. + Replayed(ExecutionRunRecord), + /// No visible run exists. + NotFound, + /// Revision, status, or completion evaluation did not match. + Conflict, +} + +/// Durable receipt for one bounded page of active run-trigger settlement. +#[derive(Clone, Debug, PartialEq)] +pub struct RunTriggerDrainCommit { + /// Run after the exact wake was acknowledged and its continuation was queued. + pub run: ExecutionRunRecord, + /// Exact number of triggers superseded by this bounded page. + pub drained_trigger_count: u32, + /// Controller continuation committed in the same transaction. + pub continuation: Box, +} + +/// Exact controller fence and page bound for one terminal trigger-drain activation. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct RunTriggerDrainRequest { + /// Run whose remaining durable triggers must be settled. + pub run_uid: Uuid, + /// Exact controller generation that owns this drain activation. + pub controller_generation: u64, + /// Exact unprocessed wake epoch claimed by the controller. + pub wake_epoch: u64, + /// Maximum number of triggers settled by this bounded page. + pub page_limit: u32, + /// Database-operation timestamp shared by the page transaction. + pub now: DateTime, +} + +/// Generation-and-wake-fenced result of draining active run triggers before finalization. +#[derive(Clone, Debug, PartialEq)] +pub enum RunTriggerDrainOutcome { + /// One bounded page committed and another activation owns the remaining work. + PageDrained(Box), + /// No active trigger or trigger delivery remains; the same activation may finalize. + ReadyToFinalize { + /// Current locked run projection. + run: Box, + /// Exact number of triggers settled by this final page. + drained_trigger_count: u32, + }, + /// The exact wake was already acknowledged; its durable continuation must not be duplicated. + Replayed(Box), + /// No visible run exists under the supplied scope. + NotFound, + /// The supplied controller generation is stale. + StaleGeneration { + /// Current persisted generation. + current_generation: u64, + }, + /// The supplied wake is not the current unprocessed wake. + StaleWake { + /// Current persisted wake epoch. + current_wake_epoch: u64, + /// Greatest wake epoch already acknowledged by the controller. + processed_wake_epoch: u64, + }, + /// The run lifecycle cannot accept trigger drain work. + InvalidState, +} + +/// Optimistically fenced request to atomically persist one terminal run projection. +#[derive(Clone, Debug, PartialEq)] +pub struct RunFinalizationRequest { + /// Run to finalize. + pub run_uid: Uuid, + /// Active plan revision used for completion evaluation. + pub expected_revision: u64, + /// Wake epoch of the structured projection used for completion evaluation. + pub expected_wake_epoch: u64, + /// Exact terminal projection selected by the scheduler. + pub terminal_projection: TerminalProjection, + /// Deterministic completion evaluation over the observed projection. + pub completion_evaluation: CompletionEvaluation, + /// Exact typed cause and requirement-count replay identity. + pub terminal_evidence: ExecutionTerminalEvidence, + /// Exact normalized terminal reason selected from typed evidence. + pub terminal_reason: ExecutionTerminalReason, +} +use super::{ + capacity::{ + ExecutionCapacityDimension, prelock_existing_capacity_dimensions_in_tx, + release_owned_run_capacity_in_tx, + }, + projection::*, + rows::*, + run::{complete_controller_wake_in_conn, enqueue_run_activation_in_conn}, + sql::*, + trigger::{ExecutionTriggerKind, ExecutionTriggerSupersedeOutcome, supersede_trigger_in_conn}, +}; + +const MAX_TRIGGER_DRAIN_PAGE_SIZE: u32 = 1_000; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(super) struct RunTriggerDrainPage { + pub(super) drained_trigger_count: u32, + pub(super) work_remaining: bool, + pub(super) next_wake_at: Option>, +} impl ExecutionRepository { + /// Settles one bounded page of run triggers before terminal finalization. + pub async fn drain_run_triggers_page( + &self, + scope: ExecutionScope, + config: &moa_config::ExecutionConfig, + request: RunTriggerDrainRequest, + ) -> Result { + let RunTriggerDrainRequest { + run_uid, + controller_generation, + wake_epoch, + page_limit, + now, + } = request; + validate_trigger_drain_page_limit(page_limit)?; + let mut conn = scope.begin(&self.pool).await?; + let tenant_id = sqlx::query_scalar::<_, Uuid>( + "SELECT tenant_id FROM moa.execution_run WHERE run_uid=$1", + ) + .bind(run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(tenant_id) = tenant_id else { + conn.commit().await.map_err(storage_error)?; + return Ok(RunTriggerDrainOutcome::NotFound); + }; + super::capacity::prelock_capacity_dimensions_in_tx( + conn.as_mut(), + config, + TenantId(tenant_id), + &[ + ExecutionCapacityDimension::ActiveRuns, + ExecutionCapacityDimension::ParkedRuns, + ExecutionCapacityDimension::ScheduledTriggers, + ], + ) + .await?; + let Some(row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) + .bind(run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + conn.commit().await.map_err(storage_error)?; + return Ok(RunTriggerDrainOutcome::NotFound); + }; + let run = run_from_row(&row)?; + if run.controller_generation != controller_generation { + conn.commit().await.map_err(storage_error)?; + return Ok(RunTriggerDrainOutcome::StaleGeneration { + current_generation: run.controller_generation, + }); + } + if wake_epoch <= run.processed_wake_epoch { + conn.commit().await.map_err(storage_error)?; + return Ok(RunTriggerDrainOutcome::Replayed(Box::new(run))); + } + if run.wake_epoch != wake_epoch { + conn.commit().await.map_err(storage_error)?; + return Ok(RunTriggerDrainOutcome::StaleWake { + current_wake_epoch: run.wake_epoch, + processed_wake_epoch: run.processed_wake_epoch, + }); + } + if run.status.is_terminal() || run.activation_state != ExecutionActivationState::Advancing { + conn.commit().await.map_err(storage_error)?; + return Ok(RunTriggerDrainOutcome::InvalidState); + } + + let page = drain_run_triggers_page_in_conn(&mut conn, &run, page_limit).await?; + if !page.work_remaining { + conn.commit().await.map_err(storage_error)?; + return Ok(RunTriggerDrainOutcome::ReadyToFinalize { + run: Box::new(run), + drained_trigger_count: page.drained_trigger_count, + }); + } + + let checkpoint = ExecutionRunActivationCheckpoint { + status: run.status, + activation_state: ExecutionActivationState::Idle, + next_wake_at: page.next_wake_at, + waiting_since: run.waiting_since, + ready_task_count: run.ready_task_count, + active_task_count: run.active_task_count, + }; + let checkpointed = match complete_controller_wake_in_conn( + &mut conn, + run_uid, + controller_generation, + wake_epoch, + checkpoint, + ) + .await? + { + RunControllerCompletionOutcome::Applied { run, .. } => run, + RunControllerCompletionOutcome::Replayed(run) => { + conn.commit().await.map_err(storage_error)?; + return Ok(RunTriggerDrainOutcome::Replayed(run)); + } + RunControllerCompletionOutcome::NotFound => { + conn.rollback().await.map_err(storage_error)?; + return Err(Error::InvalidRepositoryData { + message: "row-locked run disappeared during trigger-drain checkpoint" + .to_string(), + }); + } + RunControllerCompletionOutcome::StaleGeneration { current_generation } => { + conn.commit().await.map_err(storage_error)?; + return Ok(RunTriggerDrainOutcome::StaleGeneration { current_generation }); + } + RunControllerCompletionOutcome::StaleWake { + current_wake_epoch, + processed_wake_epoch, + } => { + conn.commit().await.map_err(storage_error)?; + return Ok(RunTriggerDrainOutcome::StaleWake { + current_wake_epoch, + processed_wake_epoch, + }); + } + RunControllerCompletionOutcome::InvalidState + | RunControllerCompletionOutcome::CapacitySaturated { .. } => { + conn.rollback().await.map_err(storage_error)?; + return Err(Error::InvalidRepositoryData { + message: "trigger-drain checkpoint rejected a locked advancing run".to_string(), + }); + } + }; + let continuation = enqueue_run_activation_in_conn( + conn.as_mut(), + checkpointed.tenant_id, + checkpointed.run_uid, + checkpointed.controller_generation, + now, + json!({"reason": "terminal_trigger_drain"}), + ) + .await?; + let row = sqlx::query(LOAD_RUN_SQL) + .bind(run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let checkpointed = run_from_row(&row)?; + conn.commit().await.map_err(storage_error)?; + Ok(RunTriggerDrainOutcome::PageDrained(Box::new( + RunTriggerDrainCommit { + run: checkpointed, + drained_trigger_count: page.drained_trigger_count, + continuation: Box::new(continuation), + }, + ))) + } + /// Atomically finalizes one successfully completed revision with deterministic evidence. pub async fn finalize_run( &self, @@ -47,6 +373,49 @@ impl ExecutionRepository { let gaps = serde_json::to_value(&completion_evaluation.gaps)?; let terminal_cause = serde_json::to_value(&terminal_evidence.cause)?; let mut conn = scope.begin(&self.pool).await?; + let tenant_id = sqlx::query_scalar::<_, Uuid>( + "SELECT tenant_id FROM moa.execution_run WHERE run_uid=$1", + ) + .bind(run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(tenant_id) = tenant_id else { + conn.commit().await.map_err(storage_error)?; + return Ok(FinalizationOutcome::NotFound); + }; + let capacity_labels = sqlx::query_scalar::<_, String>( + "SELECT DISTINCT resource_dimension \ + FROM moa.execution_capacity_reservation \ + WHERE run_uid = $1 AND state IN ('reserved', 'reconciling') \ + AND resource_dimension IN ('active_runs', 'parked_runs', 'scheduled_triggers')", + ) + .bind(run_uid) + .fetch_all(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let mut capacity_dimensions = Vec::with_capacity(capacity_labels.len()); + for label in capacity_labels { + capacity_dimensions.push(match label.as_str() { + "active_runs" => ExecutionCapacityDimension::ActiveRuns, + "parked_runs" => ExecutionCapacityDimension::ParkedRuns, + "scheduled_triggers" => ExecutionCapacityDimension::ScheduledTriggers, + _ => { + conn.rollback().await.map_err(storage_error)?; + return Err(Error::InvalidRepositoryData { + message: format!( + "terminal capacity prelock found unexpected dimension `{label}`" + ), + }); + } + }); + } + prelock_existing_capacity_dimensions_in_tx( + conn.as_mut(), + TenantId(tenant_id), + &capacity_dimensions, + ) + .await?; let Some(row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) .bind(run_uid) .fetch_optional(conn.as_mut()) @@ -76,14 +445,31 @@ impl ExecutionRepository { conn.commit().await.map_err(storage_error)?; return Ok(FinalizationOutcome::Conflict); } - let nonterminal_tasks = sqlx::query(LOAD_NONTERMINAL_TASKS_FOR_UPDATE_SQL) - .bind(run_uid) - .fetch_all(conn.as_mut()) - .await - .map_err(sqlx_error)?; + let terminal_boundary = sqlx::query( + "SELECT \ + EXISTS (SELECT 1 FROM moa.execution_task \ + WHERE run_uid = $1 AND status NOT IN \ + ('completed', 'skipped', 'failed', 'cancelled', 'unknown_outcome')) \ + AS has_nonterminal_tasks, \ + EXISTS (SELECT 1 FROM moa.execution_node_state \ + WHERE run_uid = $1 AND node_status NOT IN \ + ('completed', 'skipped', 'failed', 'cancelled')) \ + AS has_unfinished_nodes", + ) + .bind(run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let has_nonterminal_tasks: bool = terminal_boundary + .try_get("has_nonterminal_tasks") + .map_err(row_error)?; + let has_unfinished_nodes: bool = terminal_boundary + .try_get("has_unfinished_nodes") + .map_err(row_error)?; if current.pending_terminal.is_some() || current.manual_repair_required - || !nonterminal_tasks.is_empty() + || has_nonterminal_tasks + || has_unfinished_nodes { conn.commit().await.map_err(storage_error)?; return Ok(FinalizationOutcome::Conflict); @@ -96,14 +482,52 @@ impl ExecutionRepository { conn.commit().await.map_err(storage_error)?; return Ok(FinalizationOutcome::Conflict); } + let has_pending_trigger_work: bool = sqlx::query_scalar( + "SELECT \ + EXISTS (SELECT 1 FROM moa.execution_trigger \ + WHERE run_uid = $1 AND state IN ('pending', 'dispatching')) \ + OR EXISTS (SELECT 1 FROM moa.execution_dispatch_outbox \ + WHERE run_uid = $1 AND trigger_uid IS NOT NULL \ + AND state IN ('pending', 'dispatching')) \ + OR EXISTS (SELECT 1 FROM moa.execution_capacity_reservation \ + WHERE run_uid = $1 AND resource_dimension = 'scheduled_triggers' \ + AND state IN ('reserved', 'reconciling'))", + ) + .bind(current.run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if has_pending_trigger_work { + conn.commit().await.map_err(storage_error)?; + return Ok(FinalizationOutcome::Conflict); + } + release_owned_run_capacity_in_tx( + conn.as_mut(), + current.tenant_id, + current.run_uid, + current.controller_generation, + ) + .await?; let row = sqlx::query( "UPDATE moa.execution_run \ SET status = $3, output = $4, completion_check_results = $5, \ terminal_gaps = $6, terminal_cause = $7, \ terminal_satisfied_requirement_count = $8, \ terminal_requirement_count = $9, terminal_reason = $10, \ - waiting_reasons = '[]'::JSONB, \ - wake_epoch = wake_epoch + 1, completed_at = NOW(), updated_at = NOW() \ + reserved_cost_microusd = 0, reserved_tokens = 0, \ + reserved_tasks = 0, reserved_tool_calls = 0, \ + reserved_retrieved_bytes = 0, \ + next_wake_at = NULL, waiting_since = NULL, \ + waiting_reasons = '[]'::JSONB, waiting_task_count = 0, \ + waiting_input_task_count = 0, waiting_review_task_count = 0, \ + waiting_signal_task_count = 0, waiting_timer_task_count = 0, \ + waiting_external_task_count = 0, waiting_replan_task_count = 0, \ + waiting_input_user_task_count = 0, \ + waiting_input_tenant_admin_task_count = 0, \ + waiting_input_external_task_count = 0, \ + waiting_reasons_truncated = FALSE, \ + processed_wake_epoch = $11, wake_epoch = wake_epoch + 1, \ + activation_state = 'terminal', completed_at = NOW(), updated_at = NOW() \ WHERE run_uid = $1 AND plan_revision = $2 \ RETURNING *", ) @@ -123,6 +547,7 @@ impl ExecutionRepository { "terminal requirement count", )?) .bind(terminal_reason.as_str()) + .bind(to_i64(expected_wake_epoch, "expected wake epoch")?) .fetch_one(conn.as_mut()) .await .map_err(sqlx_error)?; @@ -132,6 +557,88 @@ impl ExecutionRepository { } } +/// Supersedes one stable bounded page of active run triggers in the caller transaction. +/// +/// This is the single trigger/outbox/capacity settlement path shared by successful and held +/// terminal flows. Callers own the run row lock and must checkpoint or finalize before commit. +pub(super) async fn drain_run_triggers_page_in_conn( + conn: &mut ScopedConn<'_>, + run: &ExecutionRunRecord, + page_limit: u32, +) -> Result { + validate_trigger_drain_page_limit(page_limit)?; + let trigger_rows = sqlx::query( + "SELECT trigger_uid, trigger_kind, controller_generation, attempt_generation, \ + compensation_generation, compensation_attempt_generation \ + FROM moa.execution_trigger \ + WHERE run_uid = $1 AND state IN ('pending', 'dispatching') \ + ORDER BY trigger_kind, trigger_uid LIMIT $2 FOR UPDATE", + ) + .bind(run.run_uid) + .bind(i64::from(page_limit)) + .fetch_all(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let drained_trigger_count = + u32::try_from(trigger_rows.len()).map_err(|_| Error::InvalidRepositoryData { + message: "bounded trigger-drain page count exceeds u32".to_string(), + })?; + for trigger in trigger_rows { + let trigger_uid = trigger + .try_get::("trigger_uid") + .map_err(row_error)?; + let trigger_kind = trigger + .try_get::("trigger_kind") + .map_err(row_error)? + .parse::()?; + match supersede_trigger_in_conn( + conn.as_mut(), + trigger_uid, + trigger_kind, + optional_u64(&trigger, "controller_generation")?, + optional_u64(&trigger, "attempt_generation")?, + optional_u64(&trigger, "compensation_generation")?, + optional_u64(&trigger, "compensation_attempt_generation")?, + ) + .await? + { + ExecutionTriggerSupersedeOutcome::Superseded + | ExecutionTriggerSupersedeOutcome::AlreadySuperseded + | ExecutionTriggerSupersedeOutcome::AlreadyInactive => {} + ExecutionTriggerSupersedeOutcome::StaleOrMissing => { + return Err(Error::InvalidRepositoryData { + message: "row-locked terminal trigger disappeared during bounded drain" + .to_string(), + }); + } + } + } + let next_wake_at: Option> = sqlx::query_scalar( + "SELECT MIN(due_at) FROM moa.execution_trigger \ + WHERE run_uid = $1 AND state IN ('pending', 'dispatching')", + ) + .bind(run.run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + Ok(RunTriggerDrainPage { + drained_trigger_count, + work_remaining: next_wake_at.is_some(), + next_wake_at, + }) +} + +fn validate_trigger_drain_page_limit(page_limit: u32) -> Result<()> { + if page_limit == 0 || page_limit > MAX_TRIGGER_DRAIN_PAGE_SIZE { + return Err(Error::InvalidRepositoryInput { + message: format!( + "trigger-drain page limit must be between 1 and {MAX_TRIGGER_DRAIN_PAGE_SIZE}" + ), + }); + } + Ok(()) +} + #[cfg(test)] mod tests { #[test] diff --git a/crates/moa-execution/src/repository/transition.rs b/crates/moa-execution/src/repository/transition.rs index 890b99305..2b855f0b4 100644 --- a/crates/moa-execution/src/repository/transition.rs +++ b/crates/moa-execution/src/repository/transition.rs @@ -1,6 +1,1201 @@ //! Shared task transition evidence. +use super::outbox::{ExecutionDispatchKind, NewExecutionDispatch, enqueue_dispatch_in_conn}; +use super::outcome::record_task_outcome_in_conn; +use super::ready::transition_node_counters_in_tx; +use super::rows::{required_u64, run_from_row}; +use super::run::enqueue_run_activation_in_conn; +use super::sql::{LOAD_RUN_FOR_UPDATE_SQL, LOAD_TASK_FOR_UPDATE_SQL}; +use super::trigger::{ + ExecutionTriggerKind, ExecutionTriggerNoOp, ExecutionWaitTriggerDeliveryOutcome, + deliver_wait_trigger_in_conn, +}; use super::*; +use crate::state::{WaitSettlement, completed_task_outcome, failed_task_outcome}; +use crate::wire::{ + ExecutionAttemptCancelReason, ExecutionCompensationAttemptCancelRequest, + ExecutionTaskAttemptCancelRequest, +}; +use moa_artifacts::execution_plan::{ExecutionFailureClass, ExecutionWaitExpiryAction}; +use moa_config::ExecutionConfig; + +use super::capacity::{ + CapacityReleaseOutcome, CapacityReserveOutcome, ExecutionCapacityDimension, + parked_run_capacity_request, prelock_capacity_dimensions_in_tx, + release_parked_run_capacity_in_tx, reserve_capacity_in_tx, transfer_active_run_to_parked_in_tx, + transfer_parked_run_to_active_in_tx, +}; + +const PAUSE_CANCEL_NAMESPACE: Uuid = Uuid::from_u128(0x7c90_7811_f496_5ace_a244_b645_8cc1_0a73); + +impl ExecutionRepository { + /// Fences one current run from new reservations and drains its active bounded attempts. + pub async fn pause_run( + &self, + scope: ExecutionScope, + config: &ExecutionConfig, + run_uid: Uuid, + expected_controller_generation: u64, + ) -> Result { + let mut conn = scope.begin(&self.pool).await?; + let tenant_id = sqlx::query_scalar::<_, Uuid>( + "SELECT tenant_id FROM moa.execution_run WHERE run_uid=$1", + ) + .bind(run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(tenant_id) = tenant_id else { + conn.commit().await.map_err(storage_error)?; + return Ok(TransitionOutcome::NotFound); + }; + prelock_capacity_dimensions_in_tx( + conn.as_mut(), + config, + TenantId(tenant_id), + &[ + ExecutionCapacityDimension::ActiveRuns, + ExecutionCapacityDimension::ParkedRuns, + ], + ) + .await?; + let Some(row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) + .bind(run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + conn.commit().await.map_err(storage_error)?; + return Ok(TransitionOutcome::NotFound); + }; + let run = run_from_row(&row)?; + if run.controller_generation != expected_controller_generation { + let replay_generation = expected_controller_generation.checked_add(1); + let replayed = replay_generation == Some(run.controller_generation) + && matches!( + run.status, + ExecutionRunStatus::PauseRequested + | ExecutionRunStatus::Pausing + | ExecutionRunStatus::Paused + ); + conn.commit().await.map_err(storage_error)?; + return Ok(if replayed { + TransitionOutcome::RunAlreadyApplied(run) + } else { + TransitionOutcome::Rejected(TransitionRejection::GenerationMismatch) + }); + } + if matches!( + run.status, + ExecutionRunStatus::PauseRequested + | ExecutionRunStatus::Pausing + | ExecutionRunStatus::Paused + ) { + conn.commit().await.map_err(storage_error)?; + return Ok(TransitionOutcome::RunAlreadyApplied(run)); + } + if run.status.is_terminal() || run.status == ExecutionRunStatus::AwaitingConfirmation { + conn.commit().await.map_err(storage_error)?; + return Ok(TransitionOutcome::Rejected( + TransitionRejection::InvalidRunStatus, + )); + } + + let next_generation = run.controller_generation.checked_add(1).ok_or_else(|| { + Error::InvalidRepositoryData { + message: "execution controller generation overflow".to_string(), + } + })?; + let row = sqlx::query( + "UPDATE moa.execution_run SET status='pause_requested', \ + controller_generation=$2, activation_state='paused', \ + pause_requested_at=COALESCE(pause_requested_at, NOW()), \ + last_progress_at=NOW(), updated_at=NOW() WHERE run_uid=$1 RETURNING *", + ) + .bind(run_uid) + .bind(to_i64(next_generation, "controller generation")?) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let pause_fence = run_from_row(&row)?; + enqueue_pause_cancellations( + conn.as_mut(), + &pause_fence, + run.controller_generation, + config.max_fleet_active_tasks, + ) + .await?; + let target_status = if run.active_task_count == 0 { + "paused" + } else { + "pausing" + }; + let row = sqlx::query( + "UPDATE moa.execution_run SET status=$2, \ + paused_at=CASE WHEN $2='paused' THEN NOW() ELSE NULL END, \ + updated_at=NOW() WHERE run_uid=$1 RETURNING *", + ) + .bind(run_uid) + .bind(target_status) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let updated = run_from_row(&row)?; + let released_parked = match release_parked_run_capacity_in_tx( + conn.as_mut(), + run.tenant_id, + run.run_uid, + run.controller_generation, + ) + .await? + { + CapacityReleaseOutcome::Released => true, + CapacityReleaseOutcome::AlreadyReleased | CapacityReleaseOutcome::NotFound => false, + CapacityReleaseOutcome::Stale => { + return Err(Error::InvalidRepositoryData { + message: "pause encountered a stale parked-run capacity receipt".to_string(), + }); + } + }; + let parked = if released_parked { + reserve_capacity_in_tx( + conn.as_mut(), + config, + parked_run_capacity_request(&updated, updated.wake_epoch), + ) + .await? + } else { + transfer_active_run_to_parked_in_tx(conn.as_mut(), config, &updated, updated.wake_epoch) + .await? + }; + if parked == CapacityReserveOutcome::Saturated { + conn.rollback().await.map_err(storage_error)?; + return Err(Error::CapacitySaturated { + dimension: ExecutionCapacityDimension::ParkedRuns.as_str(), + }); + } + conn.commit().await.map_err(storage_error)?; + Ok(TransitionOutcome::RunApplied(updated)) + } + + /// Resumes one fully drained paused run and enqueues exactly one new controller activation. + pub async fn resume_run( + &self, + scope: ExecutionScope, + config: &ExecutionConfig, + run_uid: Uuid, + expected_controller_generation: u64, + ) -> Result { + let mut conn = scope.begin(&self.pool).await?; + let tenant_id = sqlx::query_scalar::<_, Uuid>( + "SELECT tenant_id FROM moa.execution_run WHERE run_uid=$1", + ) + .bind(run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(tenant_id) = tenant_id else { + conn.commit().await.map_err(storage_error)?; + return Ok(TransitionOutcome::NotFound); + }; + prelock_capacity_dimensions_in_tx( + conn.as_mut(), + config, + TenantId(tenant_id), + &[ + ExecutionCapacityDimension::ActiveRuns, + ExecutionCapacityDimension::ParkedRuns, + ], + ) + .await?; + let Some(row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) + .bind(run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + conn.commit().await.map_err(storage_error)?; + return Ok(TransitionOutcome::NotFound); + }; + let run = run_from_row(&row)?; + if run.controller_generation != expected_controller_generation { + let replay_generation = expected_controller_generation.checked_add(1); + let replayed = replay_generation == Some(run.controller_generation) + && run.pause_requested_at.is_some() + && matches!( + run.status, + ExecutionRunStatus::Queued + | ExecutionRunStatus::Running + | ExecutionRunStatus::Compensating + ); + conn.commit().await.map_err(storage_error)?; + return Ok(if replayed { + TransitionOutcome::RunAlreadyApplied(run) + } else { + TransitionOutcome::Rejected(TransitionRejection::GenerationMismatch) + }); + } + if run.status != ExecutionRunStatus::Paused + || run.active_task_count != 0 + || run.activation_state != ExecutionActivationState::Paused + { + conn.commit().await.map_err(storage_error)?; + return Ok(TransitionOutcome::Rejected( + TransitionRejection::InvalidRunStatus, + )); + } + let owns_active_capacity: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM moa.execution_capacity_reservation \ + WHERE run_uid=$1 AND state IN ('reserved','reconciling') \ + AND resource_dimension='active_tasks')", + ) + .bind(run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if owns_active_capacity { + conn.commit().await.map_err(storage_error)?; + return Ok(TransitionOutcome::Rejected( + TransitionRejection::InvalidRunStatus, + )); + } + if transfer_parked_run_to_active_in_tx( + conn.as_mut(), + run.tenant_id, + run.run_uid, + run.controller_generation, + ) + .await? + == CapacityReserveOutcome::Saturated + { + conn.rollback().await.map_err(storage_error)?; + return Err(Error::CapacitySaturated { + dimension: ExecutionCapacityDimension::ActiveRuns.as_str(), + }); + } + let next_generation = run.controller_generation.checked_add(1).ok_or_else(|| { + Error::InvalidRepositoryData { + message: "execution controller generation overflow".to_string(), + } + })?; + sqlx::query( + "UPDATE moa.execution_run SET \ + status=CASE WHEN pending_terminal_status IS NULL THEN 'queued' \ + ELSE 'compensating' END, controller_generation=$2, \ + activation_state='idle', paused_at=NULL, last_progress_at=NOW(), updated_at=NOW() \ + WHERE run_uid=$1", + ) + .bind(run_uid) + .bind(to_i64(next_generation, "controller generation")?) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + enqueue_run_activation_in_conn( + conn.as_mut(), + run.tenant_id, + run_uid, + next_generation, + Utc::now(), + json!({"reason":"run_resumed"}), + ) + .await?; + let row = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) + .bind(run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let updated = run_from_row(&row)?; + conn.commit().await.map_err(storage_error)?; + Ok(TransitionOutcome::RunApplied(updated)) + } + + /// Atomically settles one due storage-only wait under its run, task, and wait-entry fences. + pub async fn settle_wait( + &self, + scope: ExecutionScope, + run_uid: Uuid, + expected_task_generation: u64, + expected_waiting_since: DateTime, + settlement: WaitSettlement, + settled_at: DateTime, + ) -> Result { + let task_id = match &settlement { + WaitSettlement::TimerElapsed { task_id, .. } + | WaitSettlement::WaitExpired { task_id, .. } => *task_id, + }; + let mut conn = scope.begin(&self.pool).await?; + let Some(run_row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) + .bind(run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + conn.commit().await.map_err(storage_error)?; + return Ok(TransitionOutcome::NotFound); + }; + let run = run_from_row(&run_row)?; + let Some(task_row) = sqlx::query(LOAD_TASK_FOR_UPDATE_SQL) + .bind(run_uid) + .bind(task_id.as_uuid()) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + conn.commit().await.map_err(storage_error)?; + return Ok(TransitionOutcome::NotFound); + }; + let task = super::rows::task_from_row(&task_row)?; + let outcome = settle_wait_locked_in_conn( + &mut conn, + &run, + &task, + expected_task_generation, + expected_waiting_since, + settlement, + settled_at, + ) + .await?; + conn.commit().await.map_err(storage_error)?; + Ok(outcome) + } + + /// Delivers and settles one exact due task wait, activating only a non-paused run. + pub async fn fire_wait_trigger( + &self, + scope: ExecutionScope, + config: &ExecutionConfig, + trigger_uid: Uuid, + ) -> Result<(TransitionOutcome, Option)> { + let mut conn = scope.begin(&self.pool).await?; + let tenant_id = sqlx::query_scalar::<_, Uuid>( + "SELECT tenant_id FROM moa.execution_trigger WHERE trigger_uid=$1", + ) + .bind(trigger_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(tenant_id) = tenant_id else { + conn.commit().await.map_err(storage_error)?; + return Ok((TransitionOutcome::NotFound, None)); + }; + prelock_capacity_dimensions_in_tx( + conn.as_mut(), + config, + TenantId(tenant_id), + &[ + ExecutionCapacityDimension::ActiveRuns, + ExecutionCapacityDimension::ParkedRuns, + ExecutionCapacityDimension::ScheduledTriggers, + ], + ) + .await?; + let delivery = deliver_wait_trigger_in_conn(conn.as_mut(), trigger_uid).await?; + let (trigger, observed_at) = match delivery { + ExecutionWaitTriggerDeliveryOutcome::Delivered { + trigger, + observed_at, + } => (trigger, observed_at), + ExecutionWaitTriggerDeliveryOutcome::NoOp(ExecutionTriggerNoOp::NotFound) => { + conn.commit().await.map_err(storage_error)?; + return Ok((TransitionOutcome::NotFound, None)); + } + ExecutionWaitTriggerDeliveryOutcome::NoOp(ExecutionTriggerNoOp::Duplicate) => { + let replay_row = sqlx::query( + "SELECT task.* FROM moa.execution_trigger AS trigger \ + JOIN moa.execution_task AS task \ + ON task.run_uid=trigger.run_uid AND task.task_id=trigger.task_id \ + AND task.tenant_id=trigger.tenant_id \ + WHERE trigger.trigger_uid=$1 FOR UPDATE OF task", + ) + .bind(trigger_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let outcome = match replay_row { + Some(row) => { + let task = super::rows::task_from_row(&row)?; + if task.status.is_terminal() + && task.generation_history.iter().any(|entry| { + entry.get("kind").and_then(Value::as_str) + == Some("storage_wait_settlement") + }) + { + TransitionOutcome::AlreadyApplied(task) + } else { + TransitionOutcome::Rejected(TransitionRejection::InvalidTaskStatus) + } + } + None => TransitionOutcome::NotFound, + }; + conn.commit().await.map_err(storage_error)?; + return Ok((outcome, None)); + } + ExecutionWaitTriggerDeliveryOutcome::NoOp( + ExecutionTriggerNoOp::Inactive + | ExecutionTriggerNoOp::StaleGeneration + | ExecutionTriggerNoOp::NotDue, + ) => { + conn.commit().await.map_err(storage_error)?; + return Ok(( + TransitionOutcome::Rejected(TransitionRejection::InvalidTaskStatus), + None, + )); + } + }; + let run_uid = trigger + .run_uid + .ok_or_else(|| Error::InvalidRepositoryData { + message: "wait trigger is missing run identity".to_string(), + })?; + let task_id = ExecutionTaskId::from_uuid(trigger.task_id.ok_or_else(|| { + Error::InvalidRepositoryData { + message: "wait trigger is missing task identity".to_string(), + } + })?); + let run_row = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) + .bind(run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let run = run_from_row(&run_row)?; + let task_row = sqlx::query(LOAD_TASK_FOR_UPDATE_SQL) + .bind(run_uid) + .bind(task_id.as_uuid()) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let task = super::rows::task_from_row(&task_row)?; + let waiting_since = task + .waiting_since + .ok_or_else(|| Error::InvalidRepositoryData { + message: "current wait trigger task has no wait-entry timestamp".to_string(), + })?; + let settlement = settlement_for_delivered_trigger(&run, &task, trigger.kind)?; + let outcome = settle_wait_locked_in_conn( + &mut conn, + &run, + &task, + task.generation, + waiting_since, + settlement, + observed_at, + ) + .await?; + if matches!(outcome, TransitionOutcome::Rejected(_)) { + conn.rollback().await.map_err(storage_error)?; + return Ok((outcome, None)); + } + let current_run_row = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) + .bind(run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let current_run = run_from_row(¤t_run_row)?; + let activation = if matches!(outcome, TransitionOutcome::Applied(_)) + && !matches!( + current_run.status, + ExecutionRunStatus::PauseRequested + | ExecutionRunStatus::Pausing + | ExecutionRunStatus::Paused + ) { + Some( + enqueue_run_activation_in_conn( + conn.as_mut(), + current_run.tenant_id, + run_uid, + current_run.controller_generation, + observed_at, + json!({ "trigger_uid": trigger_uid, "reason": "storage_wait_settled" }), + ) + .await?, + ) + } else { + None + }; + conn.commit().await.map_err(storage_error)?; + Ok((outcome, activation)) + } +} + +async fn settle_wait_locked_in_conn( + conn: &mut ScopedConn<'_>, + run: &ExecutionRunRecord, + task: &ExecutionTaskRecord, + expected_task_generation: u64, + expected_waiting_since: DateTime, + settlement: WaitSettlement, + settled_at: DateTime, +) -> Result { + if wait_settlement_is_exact_replay( + task, + expected_task_generation, + expected_waiting_since, + &settlement, + ) { + return Ok(TransitionOutcome::AlreadyApplied(task.clone())); + } + if task.generation != expected_task_generation { + return Ok(TransitionOutcome::Rejected( + TransitionRejection::GenerationMismatch, + )); + } + let outcome = wait_settlement_outcome(run, task, &settlement)?; + if run.status.is_terminal() { + return Ok(TransitionOutcome::Rejected( + TransitionRejection::InvalidRunStatus, + )); + } + if task.waiting_since != Some(expected_waiting_since) + || task.attempt_state != ExecutionAttemptState::Waiting + { + return Ok(TransitionOutcome::Rejected( + TransitionRejection::InvalidTaskStatus, + )); + } + let Some(run_deadline_at) = run.approved_budget.deadline_at else { + return Err(Error::InvalidRepositoryData { + message: "storage-only wait run has no absolute deadline".to_string(), + }); + }; + if settled_at >= run_deadline_at { + return Ok(TransitionOutcome::Rejected( + TransitionRejection::DeadlineElapsed, + )); + } + let due_at = wait_settlement_due_at( + run, + task, + &settlement, + expected_waiting_since, + run_deadline_at, + )?; + if settled_at < due_at { + return Ok(TransitionOutcome::Rejected( + TransitionRejection::InvalidTaskStatus, + )); + } + + let waiting_status = task.status; + let input_audience = if waiting_status == ExecutionTaskStatus::WaitingInput { + Some( + task.current_outcome + .as_ref() + .and_then(|outcome| match &outcome.result { + ExecutionTaskResult::NeedsInput { audience, .. } => Some(audience.clone()), + _ => None, + }) + .ok_or_else(|| Error::InvalidRepositoryData { + message: "waiting-input task is missing its typed input audience".to_string(), + })?, + ) + } else { + None + }; + let transitioned = sqlx::query( + "UPDATE moa.execution_task SET status='running', attempt_state='running', \ + generation_history = generation_history || jsonb_build_array($5::JSONB), \ + last_progress_at=GREATEST(last_progress_at, $6), updated_at=NOW() \ + WHERE run_uid=$1 AND task_id=$2 AND generation=$3 \ + AND waiting_since=$4 AND attempt_state='waiting'", + ) + .bind(run.run_uid) + .bind(task.task_id.as_uuid()) + .bind(to_i64(expected_task_generation, "wait task generation")?) + .bind(expected_waiting_since) + .bind(json!({ + "kind": "storage_wait_settlement", + "controller_generation_at_settlement": run.controller_generation, + "task_generation": expected_task_generation, + "waiting_since": expected_waiting_since, + "settled_at": settled_at, + "settlement": settlement, + })) + .bind(settled_at) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if transitioned.rows_affected() != 1 { + return Err(Error::InvalidRepositoryData { + message: "current storage wait lost its exact settlement fence".to_string(), + }); + } + let write = record_task_outcome_in_conn( + conn, + run.run_uid, + task.task_id, + expected_task_generation, + outcome, + ) + .await?; + let settled_task = match write { + TaskOutcomeWrite::Applied { task, .. } | TaskOutcomeWrite::Replayed { task, .. } => task, + TaskOutcomeWrite::NotFound => { + return Err(Error::InvalidRepositoryData { + message: "storage wait task disappeared during settlement".to_string(), + }); + } + TaskOutcomeWrite::Rejected { reason, .. } => { + return Err(Error::InvalidRepositoryData { + message: format!("canonical storage wait outcome was rejected: {reason:?}"), + }); + } + }; + if let Some(input_audience) = input_audience.as_ref() { + super::ready::transition_node_counters_with_input_audience_in_tx( + conn, + run.run_uid, + &settled_task.node_id, + &settled_task.item_key, + waiting_status, + settled_task.status, + input_audience, + ) + .await?; + } else { + transition_node_counters_in_tx( + conn, + run.run_uid, + &settled_task.node_id, + &settled_task.item_key, + waiting_status, + settled_task.status, + ) + .await?; + } + refresh_run_after_wait_settlement_in_conn(conn, run.run_uid, task.task_id, settled_at).await?; + Ok(TransitionOutcome::Applied(settled_task)) +} + +pub(super) async fn refresh_run_after_wait_settlement_in_conn( + conn: &mut ScopedConn<'_>, + run_uid: Uuid, + task_id: ExecutionTaskId, + settled_at: DateTime, +) -> Result<()> { + let updated = sqlx::query( + r#" + WITH remaining AS ( + SELECT COALESCE(jsonb_agg(reason ORDER BY ordinal), '[]'::JSONB) AS reasons + FROM moa.execution_run AS run, + jsonb_array_elements(run.waiting_reasons) + WITH ORDINALITY AS item(reason, ordinal) + WHERE run.run_uid = $1 + AND COALESCE(reason ->> 'task_id', '') <> $2 + ), next_trigger AS ( + SELECT ( + SELECT due_at FROM moa.execution_trigger + WHERE run_uid = $1 AND state IN ('pending', 'dispatching') + ORDER BY due_at, trigger_uid LIMIT 1 + ) AS due_at + ), remaining_wait AS ( + SELECT ( + SELECT waiting_since FROM moa.execution_task + WHERE run_uid = $1 + AND status IN ( + 'waiting_input', 'waiting_review', 'waiting_signal', + 'waiting_timer', 'waiting_external', 'waiting_replan' + ) + AND waiting_since IS NOT NULL + ORDER BY waiting_since, task_id LIMIT 1 + ) AS waiting_since + ) + UPDATE moa.execution_run AS run + SET waiting_reasons = remaining.reasons, + status = CASE + WHEN run.status IN ('pause_requested', 'pausing', 'paused') THEN run.status + WHEN run.waiting_input_task_count > 0 THEN 'waiting_input' + WHEN run.waiting_review_task_count > 0 THEN 'waiting_review' + WHEN run.waiting_signal_task_count > 0 THEN 'waiting_signal' + WHEN run.waiting_timer_task_count > 0 THEN 'waiting_timer' + WHEN run.waiting_external_task_count > 0 THEN 'waiting_external' + WHEN run.waiting_replan_task_count > 0 THEN 'waiting_replan' + ELSE 'running' + END, + next_wake_at = next_trigger.due_at, + waiting_since = remaining_wait.waiting_since, + waiting_reasons_truncated = jsonb_array_length( + jsonb_path_query_array( + remaining.reasons, + '$[*] ? (exists(@.task_id))' + ) + ) < run.waiting_task_count, + last_progress_at = GREATEST(run.last_progress_at, $3), + updated_at = NOW() + FROM remaining, next_trigger, remaining_wait + WHERE run.run_uid = $1 + "#, + ) + .bind(run_uid) + .bind(task_id.as_uuid().to_string()) + .bind(settled_at) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if updated.rows_affected() != 1 { + return Err(Error::InvalidRepositoryData { + message: "storage wait settlement lost its run projection".to_string(), + }); + } + Ok(()) +} + +fn settlement_for_delivered_trigger( + run: &ExecutionRunRecord, + task: &ExecutionTaskRecord, + kind: ExecutionTriggerKind, +) -> Result { + match kind { + ExecutionTriggerKind::TaskTimer => match &task.kind { + LogicalTaskKind::WaitUntil { result, .. } => Ok(WaitSettlement::TimerElapsed { + task_id: task.task_id, + output: result.clone(), + }), + _ => Err(Error::InvalidRepositoryData { + message: "task_timer trigger targets a non-timer task".to_string(), + }), + }, + ExecutionTriggerKind::WaitExpiry => { + let action = match (&task.status, &task.kind) { + (ExecutionTaskStatus::WaitingInput, _) => run + .active_plan + .definition + .input_wait_policy + .on_expiry + .clone(), + ( + ExecutionTaskStatus::WaitingReview, + LogicalTaskKind::Review { wait_policy, .. }, + ) + | ( + ExecutionTaskStatus::WaitingSignal, + LogicalTaskKind::WaitSignal { wait_policy, .. }, + ) => wait_policy.on_expiry.clone(), + _ => { + return Err(Error::InvalidRepositoryData { + message: "wait_expiry trigger targets a non-expiring wait".to_string(), + }); + } + }; + Ok(WaitSettlement::WaitExpired { + task_id: task.task_id, + action, + }) + } + ExecutionTriggerKind::RunDeadline + | ExecutionTriggerKind::TaskWatchdog + | ExecutionTriggerKind::ExternalReconcile + | ExecutionTriggerKind::ExternalStartRecovery + | ExecutionTriggerKind::ScheduleOccurrence + | ExecutionTriggerKind::CompensationWatchdog => Err(Error::InvalidRepositoryInput { + message: "only task wait triggers have storage wait settlements".to_string(), + }), + } +} + +fn wait_settlement_is_exact_replay( + task: &ExecutionTaskRecord, + task_generation: u64, + waiting_since: DateTime, + settlement: &WaitSettlement, +) -> bool { + task.status.is_terminal() + && task.current_outcome.is_some() + && task.generation_history.iter().any(|entry| { + entry.get("kind").and_then(Value::as_str) == Some("storage_wait_settlement") + && entry.get("task_generation").and_then(Value::as_u64) == Some(task_generation) + && entry + .get("waiting_since") + .cloned() + .and_then(|value| serde_json::from_value::>(value).ok()) + == Some(waiting_since) + && entry + .get("settlement") + .cloned() + .and_then(|value| serde_json::from_value::(value).ok()) + .as_ref() + == Some(settlement) + }) + && task.outcome_audit.iter().any(|entry| { + entry.get("received_generation").and_then(Value::as_u64) == Some(task_generation) + && entry.get("accepted").and_then(Value::as_bool) == Some(true) + }) +} + +fn wait_settlement_outcome( + run: &ExecutionRunRecord, + task: &ExecutionTaskRecord, + settlement: &WaitSettlement, +) -> Result { + let usage = task.actual.clone(); + match settlement { + WaitSettlement::TimerElapsed { output, .. } => match &task.kind { + LogicalTaskKind::WaitUntil { result, .. } + if task.status == ExecutionTaskStatus::WaitingTimer && result == output => + { + Ok(completed_task_outcome(output.clone(), usage)) + } + _ => Err(Error::InvalidRepositoryInput { + message: "timer settlement does not match the persisted waiting task".to_string(), + }), + }, + WaitSettlement::WaitExpired { action, .. } => { + let persisted_action = match (&task.status, &task.kind) { + (ExecutionTaskStatus::WaitingInput, _) => { + &run.active_plan.definition.input_wait_policy.on_expiry + } + ( + ExecutionTaskStatus::WaitingReview, + LogicalTaskKind::Review { wait_policy, .. }, + ) + | ( + ExecutionTaskStatus::WaitingSignal, + LogicalTaskKind::WaitSignal { wait_policy, .. }, + ) => &wait_policy.on_expiry, + _ => { + return Err(Error::InvalidRepositoryInput { + message: "wait expiry does not match the persisted waiting task" + .to_string(), + }); + } + }; + if persisted_action != action { + return Err(Error::InvalidRepositoryInput { + message: "wait expiry action differs from the immutable compiled plan" + .to_string(), + }); + } + match action { + ExecutionWaitExpiryAction::ContinueWith { output } => { + Ok(completed_task_outcome(output.clone(), usage)) + } + ExecutionWaitExpiryAction::FailTask => Ok(failed_task_outcome( + ExecutionFailureClass::Terminal, + "storage wait expired with fail_task policy".to_string(), + usage, + )), + ExecutionWaitExpiryAction::FailRun => Ok(failed_task_outcome( + ExecutionFailureClass::Terminal, + "storage wait expired with fail_run policy".to_string(), + usage, + )), + } + } + } +} + +fn wait_settlement_due_at( + run: &ExecutionRunRecord, + task: &ExecutionTaskRecord, + settlement: &WaitSettlement, + waiting_since: DateTime, + run_deadline_at: DateTime, +) -> Result> { + let target = match settlement { + WaitSettlement::TimerElapsed { .. } => match &task.kind { + LogicalTaskKind::WaitUntil { wake, .. } => wake, + _ => { + return Err(Error::InvalidRepositoryInput { + message: "timer settlement target is missing".to_string(), + }); + } + }, + WaitSettlement::WaitExpired { .. } => match (&task.status, &task.kind) { + (ExecutionTaskStatus::WaitingInput, _) => { + &run.active_plan.definition.input_wait_policy.expiry + } + (ExecutionTaskStatus::WaitingReview, LogicalTaskKind::Review { wait_policy, .. }) + | ( + ExecutionTaskStatus::WaitingSignal, + LogicalTaskKind::WaitSignal { wait_policy, .. }, + ) => &wait_policy.expiry, + _ => { + return Err(Error::InvalidRepositoryInput { + message: "wait expiry target is missing".to_string(), + }); + } + }, + }; + crate::interpreter::resolve_temporal_target(target, waiting_since, run_deadline_at).map_err( + |_| Error::InvalidRepositoryInput { + message: "wait expiry target is invalid or not before the run deadline".to_string(), + }, + ) +} + +async fn enqueue_pause_cancellations( + conn: &mut PgConnection, + run: &ExecutionRunRecord, + attempt_controller_generation: u64, + max_active_attempts: u32, +) -> Result<()> { + let mut matched_cancellations = 0_u64; + let task_rows = sqlx::query( + "SELECT task.task_id, task.generation, task.attempt_generation, \ + task.active_dispatch_uid, reservation.reservation_uid, trigger.trigger_uid \ + FROM moa.execution_task AS task \ + JOIN moa.execution_capacity_reservation AS reservation \ + ON reservation.run_uid=task.run_uid AND reservation.task_id=task.task_id \ + AND reservation.attempt_generation=task.attempt_generation \ + AND reservation.controller_generation=$2 \ + AND reservation.state IN ('reserved','reconciling') \ + JOIN moa.execution_trigger AS trigger \ + ON trigger.run_uid=task.run_uid AND trigger.task_id=task.task_id \ + AND trigger.attempt_generation=task.attempt_generation \ + AND trigger.controller_generation=$2 \ + AND trigger.trigger_kind='task_watchdog' \ + AND trigger.state IN ('pending','dispatching') \ + WHERE task.run_uid=$1 AND task.active_dispatch_uid IS NOT NULL \ + AND task.attempt_state IN ('dispatching','running') \ + ORDER BY task.task_id LIMIT $3", + ) + .bind(run.run_uid) + .bind(to_i64( + attempt_controller_generation, + "attempt controller generation", + )?) + .bind(i64::from(max_active_attempts) + 1) + .fetch_all(&mut *conn) + .await + .map_err(sqlx_error)?; + if task_rows.len() > max_active_attempts as usize { + return Err(Error::InvalidRepositoryData { + message: "pause found more task attempts than fleet ActiveTasks capacity".to_string(), + }); + } + let remaining_limit = max_active_attempts + .checked_sub( + u32::try_from(task_rows.len()).map_err(|_| Error::InvalidRepositoryData { + message: "pause task attempt count does not fit in u32".to_string(), + })?, + ) + .ok_or_else(|| Error::InvalidRepositoryData { + message: "pause task attempt count exceeds fleet ActiveTasks capacity".to_string(), + })?; + let compensation_rows = sqlx::query( + "SELECT compensation.compensation_id, compensation.generation, \ + compensation.attempt_generation, compensation.active_dispatch_uid, \ + reservation.reservation_uid, trigger.trigger_uid \ + FROM moa.execution_compensation AS compensation \ + JOIN moa.execution_capacity_reservation AS reservation \ + ON reservation.run_uid=compensation.run_uid \ + AND reservation.compensation_id=compensation.compensation_id \ + AND reservation.compensation_generation=compensation.generation \ + AND reservation.compensation_attempt_generation=compensation.attempt_generation \ + AND reservation.controller_generation=$2 \ + AND reservation.state IN ('reserved','reconciling') \ + JOIN moa.execution_trigger AS trigger \ + ON trigger.run_uid=compensation.run_uid \ + AND trigger.compensation_id=compensation.compensation_id \ + AND trigger.compensation_generation=compensation.generation \ + AND trigger.compensation_attempt_generation=compensation.attempt_generation \ + AND trigger.controller_generation=$2 \ + AND trigger.trigger_kind='compensation_watchdog' \ + AND trigger.state IN ('pending','dispatching') \ + WHERE compensation.run_uid=$1 AND compensation.active_dispatch_uid IS NOT NULL \ + AND compensation.attempt_state IN ('dispatching','running') \ + ORDER BY compensation.compensation_id LIMIT $3", + ) + .bind(run.run_uid) + .bind(to_i64( + attempt_controller_generation, + "attempt controller generation", + )?) + .bind(i64::from(remaining_limit) + 1) + .fetch_all(&mut *conn) + .await + .map_err(sqlx_error)?; + if compensation_rows.len() > remaining_limit as usize { + return Err(Error::InvalidRepositoryData { + message: "pause found more combined attempts than fleet ActiveTasks capacity" + .to_string(), + }); + } + let bounded_attempt_count = task_rows + .len() + .checked_add(compensation_rows.len()) + .ok_or_else(|| Error::InvalidRepositoryData { + message: "pause active attempt count overflow".to_string(), + })?; + if u64::try_from(bounded_attempt_count).map_err(|_| Error::InvalidRepositoryData { + message: "pause active attempt count does not fit in u64".to_string(), + })? != run.active_task_count + { + return Err(Error::InvalidRepositoryData { + message: format!( + "pause cancellation matched {bounded_attempt_count} active attempts but run counter records {}", + run.active_task_count + ), + }); + } + for row in task_rows { + let task_id: Uuid = row.try_get("task_id").map_err(row_error)?; + let task_generation = required_u64(&row, "generation")?; + let attempt_generation = required_u64(&row, "attempt_generation")?; + let active_dispatch_uid: Uuid = row.try_get("active_dispatch_uid").map_err(row_error)?; + let capacity_reservation_uid: Uuid = row.try_get("reservation_uid").map_err(row_error)?; + let watchdog_trigger_uid: Uuid = row.try_get("trigger_uid").map_err(row_error)?; + let cancellation_dispatch_uid = Uuid::new_v5( + &PAUSE_CANCEL_NAMESPACE, + format!( + "{}:{}:pause_requested", + active_dispatch_uid, run.controller_generation + ) + .as_bytes(), + ); + let payload = serde_json::to_value(ExecutionTaskAttemptCancelRequest { + cancellation_dispatch_uid, + tenant_id: run.tenant_id, + run_uid: run.run_uid, + task_id: ExecutionTaskId::from_uuid(task_id), + controller_generation: run.controller_generation, + attempt_controller_generation, + task_generation, + attempt_generation, + active_dispatch_uid, + capacity_reservation_uid, + watchdog_trigger_uid, + reason: ExecutionAttemptCancelReason::PauseRequested, + })?; + enqueue_dispatch_in_conn( + conn, + &NewExecutionDispatch { + dispatch_uid: cancellation_dispatch_uid, + tenant_id: run.tenant_id, + run_uid: Some(run.run_uid), + task_id: Some(task_id), + compensation_id: None, + trigger_uid: None, + external_job_uid: None, + kind: ExecutionDispatchKind::TaskAttemptCancel, + controller_generation: Some(run.controller_generation), + wake_epoch: None, + attempt_generation: Some(attempt_generation), + compensation_generation: None, + compensation_attempt_generation: None, + not_before_at: Utc::now(), + payload, + }, + ) + .await?; + let cancelling = sqlx::query( + "UPDATE moa.execution_task SET attempt_state='cancelling', \ + last_progress_at=NOW(), updated_at=NOW() \ + WHERE run_uid=$1 AND task_id=$2 AND generation=$3 \ + AND attempt_generation=$4 AND active_dispatch_uid=$5 \ + AND attempt_state IN ('dispatching','running')", + ) + .bind(run.run_uid) + .bind(task_id) + .bind(to_i64(task_generation, "task cancellation generation")?) + .bind(to_i64( + attempt_generation, + "task cancellation attempt generation", + )?) + .bind(active_dispatch_uid) + .execute(&mut *conn) + .await + .map_err(sqlx_error)?; + if cancelling.rows_affected() != 1 { + return Err(Error::InvalidRepositoryData { + message: format!("task `{task_id}` lost its pause cancellation fence"), + }); + } + matched_cancellations = + matched_cancellations + .checked_add(1) + .ok_or_else(|| Error::InvalidRepositoryData { + message: "pause cancellation count overflow".to_string(), + })?; + } + + for row in compensation_rows { + let compensation_id: Uuid = row.try_get("compensation_id").map_err(row_error)?; + let compensation_generation = required_u64(&row, "generation")?; + let compensation_attempt_generation = required_u64(&row, "attempt_generation")?; + let active_dispatch_uid: Uuid = row.try_get("active_dispatch_uid").map_err(row_error)?; + let capacity_reservation_uid: Uuid = row.try_get("reservation_uid").map_err(row_error)?; + let watchdog_trigger_uid: Uuid = row.try_get("trigger_uid").map_err(row_error)?; + let cancellation_dispatch_uid = Uuid::new_v5( + &PAUSE_CANCEL_NAMESPACE, + format!( + "{}:{}:pause_requested", + active_dispatch_uid, run.controller_generation + ) + .as_bytes(), + ); + let payload = serde_json::to_value(ExecutionCompensationAttemptCancelRequest { + cancellation_dispatch_uid, + tenant_id: run.tenant_id, + run_uid: run.run_uid, + compensation_id: CompensationId::from_uuid(compensation_id), + controller_generation: run.controller_generation, + attempt_controller_generation, + compensation_generation, + compensation_attempt_generation, + active_dispatch_uid, + capacity_reservation_uid, + watchdog_trigger_uid, + intent: crate::wire::ExecutionCompensationReleaseIntent::Pause, + })?; + enqueue_dispatch_in_conn( + conn, + &NewExecutionDispatch { + dispatch_uid: cancellation_dispatch_uid, + tenant_id: run.tenant_id, + run_uid: Some(run.run_uid), + task_id: None, + compensation_id: Some(compensation_id), + trigger_uid: None, + external_job_uid: None, + kind: ExecutionDispatchKind::CompensationAttemptCancel, + controller_generation: Some(run.controller_generation), + wake_epoch: None, + attempt_generation: None, + compensation_generation: Some(compensation_generation), + compensation_attempt_generation: Some(compensation_attempt_generation), + not_before_at: Utc::now(), + payload, + }, + ) + .await?; + let cancelling = sqlx::query( + "UPDATE moa.execution_compensation SET attempt_state='cancelling', \ + last_progress_at=NOW(), updated_at=NOW() \ + WHERE run_uid=$1 AND compensation_id=$2 AND generation=$3 \ + AND attempt_generation=$4 AND active_dispatch_uid=$5 \ + AND attempt_state IN ('dispatching','running')", + ) + .bind(run.run_uid) + .bind(compensation_id) + .bind(to_i64( + compensation_generation, + "compensation cancellation generation", + )?) + .bind(to_i64( + compensation_attempt_generation, + "compensation cancellation attempt generation", + )?) + .bind(active_dispatch_uid) + .execute(&mut *conn) + .await + .map_err(sqlx_error)?; + if cancelling.rows_affected() != 1 { + return Err(Error::InvalidRepositoryData { + message: format!( + "compensation `{compensation_id}` lost its pause cancellation fence" + ), + }); + } + matched_cancellations = + matched_cancellations + .checked_add(1) + .ok_or_else(|| Error::InvalidRepositoryData { + message: "pause cancellation count overflow".to_string(), + })?; + } + debug_assert_eq!(matched_cancellations, run.active_task_count); + Ok(()) +} pub(super) fn task_outcome_is_exact_replay( task: &ExecutionTaskRecord, diff --git a/crates/moa-execution/src/repository/trigger.rs b/crates/moa-execution/src/repository/trigger.rs new file mode 100644 index 000000000..236dca574 --- /dev/null +++ b/crates/moa-execution/src/repository/trigger.rs @@ -0,0 +1,2450 @@ +//! Generation-fenced temporal trigger persistence and delivery activation. + +use crate::wire::{ + ExecutionExternalJobStartRecoveryOwner, ExecutionExternalJobStartRecoveryRequest, +}; +use std::str::FromStr; + +use chrono::{DateTime, Utc}; +use moa_config::ExecutionConfig; +use moa_core::types::identifiers::TenantId; +use serde::Deserialize; +use serde_json::{Value, json}; +use sqlx::{PgConnection, Row}; +use uuid::Uuid; + +use super::{ + Error, ExecutionRepository, ExecutionScope, Result, + capacity::{ + CapacityReleaseOutcome, CapacityReserveOutcome, ExecutionCapacityDimension, + ExecutionCapacityOwner, ExecutionCapacityRequest, execution_capacity_reservation_uid, + prelock_existing_capacity_dimensions_in_tx, release_capacity_in_tx, reserve_capacity_in_tx, + }, + outbox::{ + ExecutionDeliveryState, ExecutionDispatchKind, ExecutionDispatchRecord, + NewExecutionDispatch, enqueue_dispatch_in_conn, + requeue_current_accepted_dispatches_in_conn, requeue_current_run_activations_in_conn, + requeue_delivered_dispatch_in_conn, + }, + run::enqueue_run_activation_in_conn, + sqlx_error, storage_error, to_i64, to_optional_i64, +}; + +const MAX_RECONCILE_BATCH_SIZE: u32 = 1_000; +const RESTATE_STATE_LOSS_REDRIVE_GRACE_SECONDS: i64 = 30; +const TRIGGER_DISPATCH_NAMESPACE: Uuid = Uuid::from_u128(0xa431_37f6_2bd7_5bdd_8a24_3da7_0fa1_c017); + +/// Kind of durable temporal condition represented by a trigger row. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ExecutionTriggerKind { + /// Absolute run-budget deadline. + RunDeadline, + /// Plan-authored task timer. + TaskTimer, + /// Expiry for a human or external-signal wait. + WaitExpiry, + /// Liveness deadline for one active task attempt. + TaskWatchdog, + /// Sparse reconciliation wake for one asynchronous provider job. + ExternalReconcile, + /// Recovers an unbound provider start after its reservation deadline. + ExternalStartRecovery, + /// One immutable tenant-schedule occurrence. + ScheduleOccurrence, + /// Liveness deadline for one active compensation attempt. + CompensationWatchdog, +} + +impl ExecutionTriggerKind { + /// Returns the canonical database label. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::RunDeadline => "run_deadline", + Self::TaskTimer => "task_timer", + Self::WaitExpiry => "wait_expiry", + Self::TaskWatchdog => "task_watchdog", + Self::ExternalReconcile => "external_reconcile", + Self::ExternalStartRecovery => "external_start_recovery", + Self::ScheduleOccurrence => "schedule_occurrence", + Self::CompensationWatchdog => "compensation_watchdog", + } + } +} + +impl FromStr for ExecutionTriggerKind { + type Err = Error; + + fn from_str(value: &str) -> Result { + match value { + "run_deadline" => Ok(Self::RunDeadline), + "task_timer" => Ok(Self::TaskTimer), + "wait_expiry" => Ok(Self::WaitExpiry), + "task_watchdog" => Ok(Self::TaskWatchdog), + "external_reconcile" => Ok(Self::ExternalReconcile), + "external_start_recovery" => Ok(Self::ExternalStartRecovery), + "schedule_occurrence" => Ok(Self::ScheduleOccurrence), + "compensation_watchdog" => Ok(Self::CompensationWatchdog), + _ => Err(Error::InvalidRepositoryData { + message: format!("unknown execution trigger kind `{value}`"), + }), + } + } +} + +/// Immutable temporal trigger inserted with its delayed-delivery outbox row. +#[derive(Clone, Debug, PartialEq)] +pub struct NewExecutionTrigger { + /// Stable trigger identity. + pub trigger_uid: Uuid, + /// Tenant that owns every target row. + pub tenant_id: TenantId, + /// Owning execution run. + pub run_uid: Option, + /// Owning logical task. + pub task_id: Option, + /// Owning compensation registration. + pub compensation_id: Option, + /// Owning tenant schedule. + pub schedule_uid: Option, + /// Exact tenant-schedule incarnation for immutable occurrence fencing. + pub schedule_incarnation: Option, + /// Temporal condition kind. + pub kind: ExecutionTriggerKind, + /// Current run-controller generation. + pub controller_generation: Option, + /// Current task-attempt generation. + pub attempt_generation: Option, + /// Current compensation registration generation. + pub compensation_generation: Option, + /// Current compensation-attempt generation. + pub compensation_attempt_generation: Option, + /// Immutable schedule occurrence sequence. + pub occurrence_sequence: Option, + /// Exact absolute delivery time. + pub due_at: DateTime, + /// Bounded structured trigger payload. + pub payload: Value, +} + +/// One persisted temporal trigger. +#[derive(Clone, Debug, PartialEq)] +pub struct ExecutionTriggerRecord { + /// Stable trigger identity. + pub trigger_uid: Uuid, + /// Tenant that owns the row. + pub tenant_id: TenantId, + /// Owning execution run. + pub run_uid: Option, + /// Owning logical task. + pub task_id: Option, + /// Owning compensation registration. + pub compensation_id: Option, + /// Owning tenant schedule. + pub schedule_uid: Option, + /// Tenant-schedule incarnation fence. + pub schedule_incarnation: Option, + /// Temporal condition kind. + pub kind: ExecutionTriggerKind, + /// Delivery lifecycle state. + pub state: ExecutionDeliveryState, + /// Run-controller generation fence. + pub controller_generation: Option, + /// Task-attempt generation fence. + pub attempt_generation: Option, + /// Compensation registration generation fence. + pub compensation_generation: Option, + /// Compensation-attempt generation fence. + pub compensation_attempt_generation: Option, + /// Tenant-schedule occurrence sequence. + pub occurrence_sequence: Option, + /// Exact absolute delivery time. + pub due_at: DateTime, + /// Structured trigger payload. + pub payload: Value, + /// Successful delivery time. + pub delivered_at: Option>, + /// Creation time. + pub created_at: DateTime, + /// Last mutation time. + pub updated_at: DateTime, +} + +/// Durable trigger plus the delayed dispatch that targets its immutable ID. +#[derive(Clone, Debug, PartialEq)] +pub struct ExecutionTriggerWrite { + /// Persisted trigger row. + pub trigger: ExecutionTriggerRecord, + /// Persisted delayed trigger-delivery dispatch. + pub dispatch: ExecutionDispatchRecord, +} + +/// Why a trigger delivery completed without advancing canonical state. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ExecutionTriggerNoOp { + /// No visible trigger has this immutable ID. + NotFound, + /// Delivery was already acknowledged. + Duplicate, + /// Cancellation or supersession fenced delivery. + Inactive, + /// The canonical database deadline has not arrived yet. + NotDue, + /// The referenced run, task, schedule, or generation is no longer current. + StaleGeneration, +} + +/// Result of idempotently superseding one exact trigger generation. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ExecutionTriggerSupersedeOutcome { + /// An active trigger was superseded by this call. + Superseded, + /// The exact trigger generation was already superseded. + AlreadySuperseded, + /// The exact trigger generation had already reached another inactive state. + AlreadyInactive, + /// No trigger matched the immutable identity and generation fence. + StaleOrMissing, +} + +/// Result of atomically firing one immutable trigger. +#[derive(Clone, Debug, PartialEq)] +pub enum ExecutionTriggerFireOutcome { + /// The trigger was current and delivered; run-scoped triggers enqueue one activation. + Delivered { + /// Run activation created by the same transaction, if the trigger owns a run. + activation: Option>, + }, + /// Delivery is an idempotent success with no canonical advancement. + NoOp(ExecutionTriggerNoOp), +} + +/// Read-only disposition for one exact sparse external-job reconciliation trigger. +#[derive(Clone, Debug, PartialEq)] +pub enum ExecutionExternalReconcileTriggerOutcome { + /// The trigger is due and still owns the exact provider-job generation. + Ready(crate::wire::ExecutionExternalJobReconcileRequest), + /// Canonical state makes delivery an idempotent success without a provider call. + NoOp(ExecutionTriggerNoOp), +} + +/// Preparation result for one crash-safe external provider start recovery. +#[derive(Clone, Debug, PartialEq)] +pub enum ExecutionExternalStartRecoveryTriggerOutcome { + /// The exact due unbound intent is ready for its typed receiver. + Ready(ExecutionExternalJobStartRecoveryRequest), + /// Delivery is early, stale, inactive, duplicated, or missing. + NoOp(ExecutionTriggerNoOp), +} + +/// Result of preserving an ambiguous provider start behind bounded retry. +#[derive(Clone, Debug, PartialEq)] +pub enum ExecutionExternalStartRecoveryRearmOutcome { + /// The same immutable trigger and delivery were rearmed for sparse retry. + Rearmed(Box), + /// The trigger or unbound intent no longer matched the exact recovery request. + StaleOrMissing, +} + +/// Read-only disposition for one exact absolute run-deadline trigger. +#[derive(Clone, Debug, PartialEq)] +pub enum ExecutionRunDeadlineTriggerOutcome { + /// The deadline is due for the run's current controller generation and wake. + Ready { + /// Owning run. + run_uid: Uuid, + /// Current locked controller generation, independent of the trigger's arm generation. + controller_generation: u64, + /// Current locked wake epoch consumed by deadline fencing. + wake_epoch: u64, + /// Canonical database observation time proving the absolute deadline is due. + observed_at: DateTime, + }, + /// Canonical state makes delivery an idempotent no-op. + NoOp(ExecutionTriggerNoOp), +} + +/// Read-only disposition for one exact bounded-attempt watchdog trigger. +#[derive(Clone, Debug, PartialEq)] +pub enum ExecutionWatchdogTriggerOutcome { + /// Due watchdog for a forward task attempt. + Task(crate::wire::ExecutionTaskAttemptWatchdogRequest), + /// Due watchdog for a compensation attempt. + Compensation(crate::wire::ExecutionCompensationAttemptWatchdogRequest), + /// Canonical state makes delivery an idempotent no-op. + NoOp(ExecutionTriggerNoOp), +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct ExternalReconcileTriggerPayload { + external_job_uid: Uuid, + job_generation: u64, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct ExternalStartRecoveryTriggerPayload { + external_job_uid: Uuid, + job_generation: u64, + declared_provider: String, + idempotency_key: String, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct RunDeadlineTriggerPayload { + run_uid: Uuid, + deadline_at: DateTime, +} + +/// Atomic storage-wait trigger delivery without controller activation. +#[derive(Clone, Debug, PartialEq)] +pub enum ExecutionWaitTriggerDeliveryOutcome { + /// The exact due wait generation was marked delivered at the database clock. + Delivered { + /// Immutable delivered trigger. + trigger: Box, + /// PostgreSQL observation time used for due validation and settlement. + observed_at: DateTime, + }, + /// Delivery was safely ignored without changing live task state. + NoOp(ExecutionTriggerNoOp), +} + +impl ExecutionRepository { + /// Loads one visible immutable trigger for trusted delivery routing. + pub async fn load_trigger( + &self, + scope: ExecutionScope, + trigger_uid: Uuid, + ) -> Result> { + let mut conn = scope.begin(&self.pool).await?; + let row = sqlx::query("SELECT * FROM moa.execution_trigger WHERE trigger_uid = $1") + .bind(trigger_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let trigger = row.as_ref().map(trigger_from_row).transpose()?; + conn.commit().await.map_err(storage_error)?; + Ok(trigger) + } + + /// Validates one due external reconciliation trigger without settling it before provider work. + pub async fn prepare_external_reconcile_trigger( + &self, + scope: ExecutionScope, + trigger_uid: Uuid, + ) -> Result { + let mut conn = scope.begin(&self.pool).await?; + prelock_trigger_scheduled_capacity_in_conn(conn.as_mut(), trigger_uid).await?; + let row = + sqlx::query("SELECT * FROM moa.execution_trigger WHERE trigger_uid = $1 FOR UPDATE") + .bind(trigger_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(row) = row else { + conn.commit().await.map_err(storage_error)?; + return Ok(ExecutionExternalReconcileTriggerOutcome::NoOp( + ExecutionTriggerNoOp::NotFound, + )); + }; + let trigger = trigger_from_row(&row)?; + if trigger.kind != ExecutionTriggerKind::ExternalReconcile { + return Err(Error::InvalidRepositoryInput { + message: "external reconcile preparation requires an external_reconcile trigger" + .to_string(), + }); + } + let inactive = match trigger.state { + ExecutionDeliveryState::Pending | ExecutionDeliveryState::Dispatching => None, + ExecutionDeliveryState::Delivered => Some(ExecutionTriggerNoOp::Duplicate), + ExecutionDeliveryState::Superseded + | ExecutionDeliveryState::Cancelled + | ExecutionDeliveryState::DeadLetter => Some(ExecutionTriggerNoOp::Inactive), + }; + if let Some(reason) = inactive { + conn.commit().await.map_err(storage_error)?; + return Ok(ExecutionExternalReconcileTriggerOutcome::NoOp(reason)); + } + let observed_at: DateTime = sqlx::query_scalar("SELECT NOW()") + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if trigger.due_at > observed_at { + conn.commit().await.map_err(storage_error)?; + return Ok(ExecutionExternalReconcileTriggerOutcome::NoOp( + ExecutionTriggerNoOp::NotDue, + )); + } + if !trigger_is_current(conn.as_mut(), &trigger).await? { + supersede_trigger_in_conn( + conn.as_mut(), + trigger_uid, + ExecutionTriggerKind::ExternalReconcile, + trigger.controller_generation, + trigger.attempt_generation, + trigger.compensation_generation, + trigger.compensation_attempt_generation, + ) + .await?; + conn.commit().await.map_err(storage_error)?; + return Ok(ExecutionExternalReconcileTriggerOutcome::NoOp( + ExecutionTriggerNoOp::StaleGeneration, + )); + } + let payload: ExternalReconcileTriggerPayload = + serde_json::from_value(trigger.payload.clone()).map_err(|error| { + Error::InvalidRepositoryData { + message: format!("invalid external reconcile trigger payload: {error}"), + } + })?; + if payload.external_job_uid.is_nil() || payload.job_generation == 0 { + return Err(Error::InvalidRepositoryData { + message: "external reconcile trigger payload has an invalid identity".to_string(), + }); + } + let run_uid = trigger + .run_uid + .ok_or_else(|| Error::InvalidRepositoryData { + message: "external reconcile trigger is missing run identity".to_string(), + })?; + let job = sqlx::query_as::<_, (String, String, String)>( + "SELECT provider, provider_job_id, idempotency_key \ + FROM moa.execution_external_job \ + WHERE external_job_uid=$1 AND tenant_id=$2 AND run_uid=$3 \ + AND job_generation=$4 \ + AND ( \ + (task_id=$5 AND attempt_generation=$6 AND compensation_id IS NULL) \ + OR \ + (compensation_id=$7 AND compensation_generation=$8 \ + AND compensation_attempt_generation=$9 AND task_id IS NULL) \ + )", + ) + .bind(payload.external_job_uid) + .bind(trigger.tenant_id.0) + .bind(run_uid) + .bind(to_i64(payload.job_generation, "job generation")?) + .bind(trigger.task_id) + .bind(to_optional_i64( + trigger.attempt_generation, + "attempt generation", + )?) + .bind(trigger.compensation_id) + .bind(to_optional_i64( + trigger.compensation_generation, + "compensation generation", + )?) + .bind(to_optional_i64( + trigger.compensation_attempt_generation, + "compensation attempt generation", + )?) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some((provider, provider_job_id, idempotency_key)) = job else { + supersede_trigger_in_conn( + conn.as_mut(), + trigger_uid, + ExecutionTriggerKind::ExternalReconcile, + trigger.controller_generation, + trigger.attempt_generation, + trigger.compensation_generation, + trigger.compensation_attempt_generation, + ) + .await?; + conn.commit().await.map_err(storage_error)?; + return Ok(ExecutionExternalReconcileTriggerOutcome::NoOp( + ExecutionTriggerNoOp::StaleGeneration, + )); + }; + conn.commit().await.map_err(storage_error)?; + Ok(ExecutionExternalReconcileTriggerOutcome::Ready( + crate::wire::ExecutionExternalJobReconcileRequest { + tenant_id: trigger.tenant_id, + external_job_uid: payload.external_job_uid, + trigger_uid, + job_generation: payload.job_generation, + provider, + provider_job_id, + idempotency_key, + }, + )) + } + + /// Validates one due unbound provider-start recovery trigger. + pub async fn prepare_external_start_recovery_trigger( + &self, + scope: ExecutionScope, + trigger_uid: Uuid, + ) -> Result { + let mut conn = scope.begin(&self.pool).await?; + prelock_trigger_scheduled_capacity_in_conn(conn.as_mut(), trigger_uid).await?; + let row = sqlx::query( + "SELECT trigger.*, now() AS observed_at FROM moa.execution_trigger AS trigger \ + WHERE trigger_uid=$1 FOR UPDATE", + ) + .bind(trigger_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(row) = row else { + conn.commit().await.map_err(storage_error)?; + return Ok(ExecutionExternalStartRecoveryTriggerOutcome::NoOp( + ExecutionTriggerNoOp::NotFound, + )); + }; + let trigger = trigger_from_row(&row)?; + if trigger.kind != ExecutionTriggerKind::ExternalStartRecovery { + return Err(Error::InvalidRepositoryInput { + message: "start recovery preparation requires external_start_recovery".to_string(), + }); + } + let inactive = match trigger.state { + ExecutionDeliveryState::Pending | ExecutionDeliveryState::Dispatching => None, + ExecutionDeliveryState::Delivered => Some(ExecutionTriggerNoOp::Duplicate), + ExecutionDeliveryState::Superseded + | ExecutionDeliveryState::Cancelled + | ExecutionDeliveryState::DeadLetter => Some(ExecutionTriggerNoOp::Inactive), + }; + if let Some(reason) = inactive { + conn.commit().await.map_err(storage_error)?; + return Ok(ExecutionExternalStartRecoveryTriggerOutcome::NoOp(reason)); + } + let observed_at = row + .try_get::, _>("observed_at") + .map_err(super::row_error)?; + if trigger.due_at > observed_at { + conn.commit().await.map_err(storage_error)?; + return Ok(ExecutionExternalStartRecoveryTriggerOutcome::NoOp( + ExecutionTriggerNoOp::NotDue, + )); + } + if !trigger_is_current(conn.as_mut(), &trigger).await? { + supersede_trigger_in_conn( + conn.as_mut(), + trigger_uid, + trigger.kind, + trigger.controller_generation, + trigger.attempt_generation, + trigger.compensation_generation, + trigger.compensation_attempt_generation, + ) + .await?; + conn.commit().await.map_err(storage_error)?; + return Ok(ExecutionExternalStartRecoveryTriggerOutcome::NoOp( + ExecutionTriggerNoOp::StaleGeneration, + )); + } + let payload: ExternalStartRecoveryTriggerPayload = + serde_json::from_value(trigger.payload.clone()).map_err(|error| { + Error::InvalidRepositoryData { + message: format!("invalid external start recovery payload: {error}"), + } + })?; + let run_uid = trigger + .run_uid + .ok_or_else(|| Error::InvalidRepositoryData { + message: "external start recovery trigger is missing run identity".to_string(), + })?; + let owner = match ( + trigger.task_id, + trigger.attempt_generation, + trigger.compensation_id, + trigger.compensation_generation, + trigger.compensation_attempt_generation, + ) { + (Some(task_id), Some(attempt_generation), None, None, None) => { + ExecutionExternalJobStartRecoveryOwner::Task { + task_id, + attempt_generation, + } + } + (None, None, Some(compensation_id), Some(generation), Some(attempt_generation)) => { + ExecutionExternalJobStartRecoveryOwner::Compensation { + compensation_id, + compensation_generation: generation, + compensation_attempt_generation: attempt_generation, + } + } + _ => { + return Err(Error::InvalidRepositoryData { + message: "external start recovery trigger has invalid owner shape".to_string(), + }); + } + }; + conn.commit().await.map_err(storage_error)?; + Ok(ExecutionExternalStartRecoveryTriggerOutcome::Ready( + ExecutionExternalJobStartRecoveryRequest { + tenant_id: trigger.tenant_id, + run_uid, + owner, + external_job_uid: payload.external_job_uid, + job_generation: payload.job_generation, + provider: payload.declared_provider, + idempotency_key: payload.idempotency_key, + trigger_uid, + }, + )) + } + + /// Validates an absolute run deadline against DB time and reloads current run fences. + pub async fn prepare_run_deadline_trigger( + &self, + scope: ExecutionScope, + trigger_uid: Uuid, + ) -> Result { + let mut conn = scope.begin(&self.pool).await?; + prelock_trigger_scheduled_capacity_in_conn(conn.as_mut(), trigger_uid).await?; + let row = sqlx::query( + "SELECT trigger.*, now() AS observed_at, trigger.due_at <= now() AS is_due \ + FROM moa.execution_trigger AS trigger WHERE trigger_uid=$1 FOR UPDATE", + ) + .bind(trigger_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(row) = row else { + conn.commit().await.map_err(storage_error)?; + return Ok(ExecutionRunDeadlineTriggerOutcome::NoOp( + ExecutionTriggerNoOp::NotFound, + )); + }; + let trigger = trigger_from_row(&row)?; + if trigger.kind != ExecutionTriggerKind::RunDeadline { + return Err(Error::InvalidRepositoryInput { + message: "run deadline preparation requires a run_deadline trigger".to_string(), + }); + } + let inactive = match trigger.state { + ExecutionDeliveryState::Pending | ExecutionDeliveryState::Dispatching => None, + ExecutionDeliveryState::Delivered => Some(ExecutionTriggerNoOp::Duplicate), + ExecutionDeliveryState::Superseded + | ExecutionDeliveryState::Cancelled + | ExecutionDeliveryState::DeadLetter => Some(ExecutionTriggerNoOp::Inactive), + }; + if let Some(reason) = inactive { + conn.commit().await.map_err(storage_error)?; + return Ok(ExecutionRunDeadlineTriggerOutcome::NoOp(reason)); + } + if !row.try_get::("is_due").map_err(super::row_error)? { + conn.commit().await.map_err(storage_error)?; + return Ok(ExecutionRunDeadlineTriggerOutcome::NoOp( + ExecutionTriggerNoOp::NotDue, + )); + } + let run_uid = trigger + .run_uid + .ok_or_else(|| Error::InvalidRepositoryData { + message: "run deadline trigger is missing run identity".to_string(), + })?; + let run = sqlx::query_as::<_, (i64, i64, String, Option>)>( + "SELECT controller_generation, wake_epoch, status, budget_deadline_at \ + FROM moa.execution_run \ + WHERE tenant_id=$1 AND run_uid=$2 FOR UPDATE", + ) + .bind(trigger.tenant_id.0) + .bind(run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some((controller_generation, wake_epoch, status, approved_deadline_at)) = run else { + conn.commit().await.map_err(storage_error)?; + return Ok(ExecutionRunDeadlineTriggerOutcome::NoOp( + ExecutionTriggerNoOp::NotFound, + )); + }; + if matches!( + status.as_str(), + "completed" | "partial" | "blocked" | "unsupported" | "failed" | "cancelled" + ) { + supersede_trigger_in_conn( + conn.as_mut(), + trigger_uid, + ExecutionTriggerKind::RunDeadline, + trigger.controller_generation, + None, + None, + None, + ) + .await?; + conn.commit().await.map_err(storage_error)?; + return Ok(ExecutionRunDeadlineTriggerOutcome::NoOp( + ExecutionTriggerNoOp::StaleGeneration, + )); + } + if approved_deadline_at != Some(trigger.due_at) || !run_deadline_payload_matches(&trigger)? + { + supersede_trigger_in_conn( + conn.as_mut(), + trigger_uid, + ExecutionTriggerKind::RunDeadline, + trigger.controller_generation, + None, + None, + None, + ) + .await?; + conn.commit().await.map_err(storage_error)?; + return Ok(ExecutionRunDeadlineTriggerOutcome::NoOp( + ExecutionTriggerNoOp::StaleGeneration, + )); + } + let observed_at = row + .try_get::, _>("observed_at") + .map_err(super::row_error)?; + conn.commit().await.map_err(storage_error)?; + Ok(ExecutionRunDeadlineTriggerOutcome::Ready { + run_uid, + controller_generation: super::to_u64(controller_generation, "controller generation")?, + wake_epoch: super::to_u64(wake_epoch, "wake epoch")?, + observed_at, + }) + } + + /// Validates one due watchdog and resolves its exact active dispatch and capacity receipt. + pub async fn prepare_watchdog_trigger( + &self, + scope: ExecutionScope, + trigger_uid: Uuid, + ) -> Result { + let mut conn = scope.begin(&self.pool).await?; + prelock_trigger_scheduled_capacity_in_conn(conn.as_mut(), trigger_uid).await?; + let row = sqlx::query( + "SELECT trigger.*, trigger.due_at <= now() AS is_due \ + FROM moa.execution_trigger AS trigger WHERE trigger_uid=$1 FOR UPDATE", + ) + .bind(trigger_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(row) = row else { + conn.commit().await.map_err(storage_error)?; + return Ok(ExecutionWatchdogTriggerOutcome::NoOp( + ExecutionTriggerNoOp::NotFound, + )); + }; + let trigger = trigger_from_row(&row)?; + if !matches!( + trigger.kind, + ExecutionTriggerKind::TaskWatchdog | ExecutionTriggerKind::CompensationWatchdog + ) { + return Err(Error::InvalidRepositoryInput { + message: "watchdog preparation requires a task or compensation watchdog" + .to_string(), + }); + } + let inactive = match trigger.state { + ExecutionDeliveryState::Pending | ExecutionDeliveryState::Dispatching => None, + ExecutionDeliveryState::Delivered => Some(ExecutionTriggerNoOp::Duplicate), + ExecutionDeliveryState::Superseded + | ExecutionDeliveryState::Cancelled + | ExecutionDeliveryState::DeadLetter => Some(ExecutionTriggerNoOp::Inactive), + }; + if let Some(reason) = inactive { + conn.commit().await.map_err(storage_error)?; + return Ok(ExecutionWatchdogTriggerOutcome::NoOp(reason)); + } + if !row.try_get::("is_due").map_err(super::row_error)? { + conn.commit().await.map_err(storage_error)?; + return Ok(ExecutionWatchdogTriggerOutcome::NoOp( + ExecutionTriggerNoOp::NotDue, + )); + } + if !trigger_is_current(conn.as_mut(), &trigger).await? { + supersede_trigger_in_conn( + conn.as_mut(), + trigger_uid, + trigger.kind, + trigger.controller_generation, + trigger.attempt_generation, + trigger.compensation_generation, + trigger.compensation_attempt_generation, + ) + .await?; + conn.commit().await.map_err(storage_error)?; + return Ok(ExecutionWatchdogTriggerOutcome::NoOp( + ExecutionTriggerNoOp::StaleGeneration, + )); + } + let run_uid = trigger + .run_uid + .ok_or_else(|| Error::InvalidRepositoryData { + message: "watchdog trigger is missing run identity".to_string(), + })?; + let controller_generation = + trigger + .controller_generation + .ok_or_else(|| Error::InvalidRepositoryData { + message: "watchdog trigger is missing controller generation".to_string(), + })?; + let outcome = match trigger.kind { + ExecutionTriggerKind::TaskWatchdog => { + let task_id = trigger + .task_id + .ok_or_else(|| Error::InvalidRepositoryData { + message: "task watchdog is missing task identity".to_string(), + })?; + let attempt_generation = + trigger + .attempt_generation + .ok_or_else(|| Error::InvalidRepositoryData { + message: "task watchdog is missing attempt generation".to_string(), + })?; + let owner = sqlx::query_as::<_, (Uuid, Uuid)>( + "SELECT task.active_dispatch_uid, reservation.reservation_uid \ + FROM moa.execution_task AS task \ + JOIN moa.execution_capacity_reservation AS reservation \ + ON reservation.tenant_id=task.tenant_id \ + AND reservation.run_uid=task.run_uid \ + AND reservation.task_id=task.task_id \ + AND reservation.attempt_generation=task.attempt_generation \ + AND reservation.controller_generation=$5 \ + AND reservation.resource_dimension='active_tasks' \ + AND reservation.state IN ('reserved','reconciling') \ + WHERE task.tenant_id=$1 AND task.run_uid=$2 AND task.task_id=$3 \ + AND task.attempt_generation=$4 AND task.active_dispatch_uid IS NOT NULL", + ) + .bind(trigger.tenant_id.0) + .bind(run_uid) + .bind(task_id) + .bind(to_i64(attempt_generation, "attempt generation")?) + .bind(to_i64(controller_generation, "controller generation")?) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + owner.map(|(dispatch_uid, capacity_reservation_uid)| { + ExecutionWatchdogTriggerOutcome::Task( + crate::wire::ExecutionTaskAttemptWatchdogRequest { + dispatch_uid, + capacity_reservation_uid, + watchdog_trigger_uid: trigger_uid, + run_uid, + task_id: crate::state::ExecutionTaskId::from_uuid(task_id), + controller_generation, + attempt_generation, + tenant_id: trigger.tenant_id, + }, + ) + }) + } + ExecutionTriggerKind::CompensationWatchdog => { + let compensation_id = + trigger + .compensation_id + .ok_or_else(|| Error::InvalidRepositoryData { + message: "compensation watchdog is missing compensation identity" + .to_string(), + })?; + let compensation_generation = trigger.compensation_generation.ok_or_else(|| { + Error::InvalidRepositoryData { + message: "compensation watchdog is missing logical generation".to_string(), + } + })?; + let attempt_generation = + trigger.compensation_attempt_generation.ok_or_else(|| { + Error::InvalidRepositoryData { + message: "compensation watchdog is missing attempt generation" + .to_string(), + } + })?; + let owner = sqlx::query_as::<_, (Uuid, Uuid)>( + "SELECT compensation.active_dispatch_uid, reservation.reservation_uid \ + FROM moa.execution_compensation AS compensation \ + JOIN moa.execution_capacity_reservation AS reservation \ + ON reservation.tenant_id=compensation.tenant_id \ + AND reservation.run_uid=compensation.run_uid \ + AND reservation.compensation_id=compensation.compensation_id \ + AND reservation.compensation_generation=compensation.generation \ + AND reservation.compensation_attempt_generation=compensation.attempt_generation \ + AND reservation.controller_generation=$6 \ + AND reservation.resource_dimension='active_tasks' \ + AND reservation.state IN ('reserved','reconciling') \ + WHERE compensation.tenant_id=$1 AND compensation.run_uid=$2 \ + AND compensation.compensation_id=$3 AND compensation.generation=$4 \ + AND compensation.attempt_generation=$5 \ + AND compensation.active_dispatch_uid IS NOT NULL", + ) + .bind(trigger.tenant_id.0) + .bind(run_uid) + .bind(compensation_id) + .bind(to_i64( + compensation_generation, + "compensation generation", + )?) + .bind(to_i64( + attempt_generation, + "compensation attempt generation", + )?) + .bind(to_i64(controller_generation, "controller generation")?) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + owner.map(|(dispatch_uid, capacity_reservation_uid)| { + ExecutionWatchdogTriggerOutcome::Compensation( + crate::wire::ExecutionCompensationAttemptWatchdogRequest { + dispatch_uid, + capacity_reservation_uid, + watchdog_trigger_uid: trigger_uid, + run_uid, + compensation_id: crate::state::CompensationId::from_uuid( + compensation_id, + ), + compensation_generation, + compensation_attempt_generation: attempt_generation, + controller_generation, + tenant_id: trigger.tenant_id, + }, + ) + }) + } + _ => None, + }; + let Some(outcome) = outcome else { + supersede_trigger_in_conn( + conn.as_mut(), + trigger_uid, + trigger.kind, + trigger.controller_generation, + trigger.attempt_generation, + trigger.compensation_generation, + trigger.compensation_attempt_generation, + ) + .await?; + conn.commit().await.map_err(storage_error)?; + return Ok(ExecutionWatchdogTriggerOutcome::NoOp( + ExecutionTriggerNoOp::StaleGeneration, + )); + }; + conn.commit().await.map_err(storage_error)?; + Ok(outcome) + } + + /// Supersedes one external reconciliation trigger after its provider result is durable. + pub async fn settle_external_reconcile_trigger( + &self, + scope: ExecutionScope, + trigger_uid: Uuid, + ) -> Result { + let mut conn = scope.begin(&self.pool).await?; + prelock_trigger_scheduled_capacity_in_conn(conn.as_mut(), trigger_uid).await?; + let row = + sqlx::query("SELECT * FROM moa.execution_trigger WHERE trigger_uid = $1 FOR UPDATE") + .bind(trigger_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(row) = row else { + conn.commit().await.map_err(storage_error)?; + return Ok(ExecutionTriggerSupersedeOutcome::StaleOrMissing); + }; + let trigger = trigger_from_row(&row)?; + if trigger.kind != ExecutionTriggerKind::ExternalReconcile { + return Err(Error::InvalidRepositoryInput { + message: "external reconcile settlement requires an external_reconcile trigger" + .to_string(), + }); + } + let outcome = supersede_trigger_in_conn( + conn.as_mut(), + trigger_uid, + ExecutionTriggerKind::ExternalReconcile, + trigger.controller_generation, + trigger.attempt_generation, + trigger.compensation_generation, + trigger.compensation_attempt_generation, + ) + .await?; + conn.commit().await.map_err(storage_error)?; + Ok(outcome) + } + + /// Supersedes one provider-start recovery trigger after recovery is durable. + pub async fn settle_external_start_recovery_trigger( + &self, + scope: ExecutionScope, + trigger_uid: Uuid, + ) -> Result { + let mut conn = scope.begin(&self.pool).await?; + prelock_trigger_scheduled_capacity_in_conn(conn.as_mut(), trigger_uid).await?; + let row = + sqlx::query("SELECT * FROM moa.execution_trigger WHERE trigger_uid=$1 FOR UPDATE") + .bind(trigger_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(row) = row else { + conn.commit().await.map_err(storage_error)?; + return Ok(ExecutionTriggerSupersedeOutcome::StaleOrMissing); + }; + let trigger = trigger_from_row(&row)?; + if trigger.kind != ExecutionTriggerKind::ExternalStartRecovery { + return Err(Error::InvalidRepositoryInput { + message: "start recovery settlement requires external_start_recovery".to_string(), + }); + } + let outcome = supersede_trigger_in_conn( + conn.as_mut(), + trigger_uid, + trigger.kind, + trigger.controller_generation, + trigger.attempt_generation, + trigger.compensation_generation, + trigger.compensation_attempt_generation, + ) + .await?; + conn.commit().await.map_err(storage_error)?; + Ok(outcome) + } + + /// Rearms the same exact start-recovery trigger after an ambiguous provider lookup. + pub async fn rearm_external_start_recovery( + &self, + scope: ExecutionScope, + request: &ExecutionExternalJobStartRecoveryRequest, + retry_at: DateTime, + error: &str, + ) -> Result { + let mut conn = scope.begin(&self.pool).await?; + prelock_trigger_scheduled_capacity_in_conn(conn.as_mut(), request.trigger_uid).await?; + let row = sqlx::query( + "SELECT trigger.*, $2::TIMESTAMPTZ > now() AS retry_is_future \ + FROM moa.execution_trigger AS trigger WHERE trigger_uid=$1 FOR UPDATE", + ) + .bind(request.trigger_uid) + .bind(retry_at) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(row) = row else { + conn.commit().await.map_err(storage_error)?; + return Ok(ExecutionExternalStartRecoveryRearmOutcome::StaleOrMissing); + }; + let trigger = trigger_from_row(&row)?; + if trigger.kind != ExecutionTriggerKind::ExternalStartRecovery + || !row + .try_get::("retry_is_future") + .map_err(super::row_error)? + || !start_recovery_request_matches_trigger(request, &trigger) + || !trigger_is_current(conn.as_mut(), &trigger).await? + { + conn.commit().await.map_err(storage_error)?; + return Ok(ExecutionExternalStartRecoveryRearmOutcome::StaleOrMissing); + } + let last_error = error.chars().take(4_096).collect::(); + // Restate journals both ExecutionTrigger/fire and provider recovery by dispatch UID. A + // rearm therefore needs a new persisted delivery identity or the next due claim would + // replay the completed Unknown/NotDue invocation without another provider lookup. + let next_dispatch_uid = + rearmed_trigger_delivery_dispatch_uid(request.trigger_uid, retry_at); + sqlx::query( + "UPDATE moa.execution_trigger SET state='pending', due_at=$2, \ + claim_owner=NULL,claimed_at=NULL,claim_expires_at=NULL,delivered_at=NULL, \ + last_error=$3,updated_at=now() WHERE trigger_uid=$1", + ) + .bind(request.trigger_uid) + .bind(retry_at) + .bind(&last_error) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let row = sqlx::query( + "UPDATE moa.execution_dispatch_outbox SET dispatch_uid=$4,state='pending', \ + not_before_at=$2,delivery_attempts=0, \ + claim_owner=NULL,claimed_at=NULL,claim_expires_at=NULL,delivered_at=NULL, \ + last_error=$3,updated_at=now() \ + WHERE trigger_uid=$1 AND dispatch_kind='trigger_delivery' \ + AND state IN ('pending','dispatching','delivered') RETURNING *", + ) + .bind(request.trigger_uid) + .bind(retry_at) + .bind(last_error) + .bind(next_dispatch_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + .ok_or_else(|| Error::InvalidRepositoryData { + message: "start recovery trigger is missing its exact delivery outbox".to_string(), + })?; + let dispatch = super::outbox::dispatch_from_row_for_repository(&row)?; + conn.commit().await.map_err(storage_error)?; + Ok(ExecutionExternalStartRecoveryRearmOutcome::Rearmed( + Box::new(dispatch), + )) + } + + /// Supersedes one run deadline trigger after its bounded terminal fence is durable. + pub async fn settle_run_deadline_trigger( + &self, + scope: ExecutionScope, + trigger_uid: Uuid, + ) -> Result { + let mut conn = scope.begin(&self.pool).await?; + prelock_trigger_scheduled_capacity_in_conn(conn.as_mut(), trigger_uid).await?; + let row = + sqlx::query("SELECT * FROM moa.execution_trigger WHERE trigger_uid=$1 FOR UPDATE") + .bind(trigger_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(row) = row else { + conn.commit().await.map_err(storage_error)?; + return Ok(ExecutionTriggerSupersedeOutcome::StaleOrMissing); + }; + let trigger = trigger_from_row(&row)?; + if trigger.kind != ExecutionTriggerKind::RunDeadline { + return Err(Error::InvalidRepositoryInput { + message: "run deadline settlement requires a run_deadline trigger".to_string(), + }); + } + let outcome = supersede_trigger_in_conn( + conn.as_mut(), + trigger_uid, + ExecutionTriggerKind::RunDeadline, + trigger.controller_generation, + None, + None, + None, + ) + .await?; + conn.commit().await.map_err(storage_error)?; + Ok(outcome) + } + + /// Supersedes one watchdog after its keyed attempt receiver durably completes. + pub async fn settle_watchdog_trigger( + &self, + scope: ExecutionScope, + trigger_uid: Uuid, + ) -> Result { + let mut conn = scope.begin(&self.pool).await?; + prelock_trigger_scheduled_capacity_in_conn(conn.as_mut(), trigger_uid).await?; + let row = + sqlx::query("SELECT * FROM moa.execution_trigger WHERE trigger_uid=$1 FOR UPDATE") + .bind(trigger_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(row) = row else { + conn.commit().await.map_err(storage_error)?; + return Ok(ExecutionTriggerSupersedeOutcome::StaleOrMissing); + }; + let trigger = trigger_from_row(&row)?; + if !matches!( + trigger.kind, + ExecutionTriggerKind::TaskWatchdog | ExecutionTriggerKind::CompensationWatchdog + ) { + return Err(Error::InvalidRepositoryInput { + message: "watchdog settlement requires a task or compensation watchdog".to_string(), + }); + } + let outcome = supersede_trigger_in_conn( + conn.as_mut(), + trigger_uid, + trigger.kind, + trigger.controller_generation, + trigger.attempt_generation, + trigger.compensation_generation, + trigger.compensation_attempt_generation, + ) + .await?; + conn.commit().await.map_err(storage_error)?; + Ok(outcome) + } + + /// Creates a trigger and its delayed delivery dispatch atomically. + pub async fn create_trigger( + &self, + scope: ExecutionScope, + config: &ExecutionConfig, + request: NewExecutionTrigger, + ) -> Result { + let mut conn = scope.begin(&self.pool).await?; + let write = create_trigger_with_dispatch_in_conn(conn.as_mut(), config, &request).await?; + conn.commit().await.map_err(storage_error)?; + Ok(write) + } + + /// Fires one trigger under current run/task/schedule generation fences. + pub async fn fire_trigger( + &self, + scope: ExecutionScope, + trigger_uid: Uuid, + ) -> Result { + let mut conn = scope.begin(&self.pool).await?; + let owner = sqlx::query_as::<_, (Uuid, Option)>( + "SELECT tenant_id,run_uid FROM moa.execution_trigger WHERE trigger_uid=$1", + ) + .bind(trigger_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some((tenant_id, run_uid)) = owner else { + conn.commit().await.map_err(storage_error)?; + return Ok(ExecutionTriggerFireOutcome::NoOp( + ExecutionTriggerNoOp::NotFound, + )); + }; + let dimensions = if run_uid.is_some() { + vec![ + ExecutionCapacityDimension::ActiveRuns, + ExecutionCapacityDimension::ParkedRuns, + ExecutionCapacityDimension::ScheduledTriggers, + ] + } else { + vec![ExecutionCapacityDimension::ScheduledTriggers] + }; + prelock_existing_capacity_dimensions_in_tx(conn.as_mut(), TenantId(tenant_id), &dimensions) + .await?; + let outcome = fire_trigger_in_conn(conn.as_mut(), trigger_uid).await?; + conn.commit().await.map_err(storage_error)?; + Ok(outcome) + } + + /// Repairs due trigger deliveries and queued run activations after dispatch-state loss. + pub async fn reconcile_due_trigger_dispatches( + &self, + scope: ExecutionScope, + batch_size: u32, + ) -> Result> { + if !(3..=MAX_RECONCILE_BATCH_SIZE).contains(&batch_size) { + return Err(Error::InvalidRepositoryInput { + message: format!( + "execution trigger reconciliation batch must be 3..={MAX_RECONCILE_BATCH_SIZE}" + ), + }); + } + let mut conn = scope.begin(&self.pool).await?; + let (trigger_budget, accepted_budget, run_budget) = reconcile_lane_budgets(batch_size); + let candidates = sqlx::query_as::<_, (Uuid, Uuid)>( + r#" + SELECT trigger.tenant_id, trigger.trigger_uid + FROM moa.execution_trigger AS trigger + WHERE trigger.state = 'pending' + AND trigger.due_at <= now() - make_interval(secs => $2) + AND ( + NOT EXISTS ( + SELECT 1 + FROM moa.execution_dispatch_outbox AS dispatch + WHERE dispatch.tenant_id = trigger.tenant_id + AND dispatch.trigger_uid = trigger.trigger_uid + AND dispatch.dispatch_kind = 'trigger_delivery' + ) + OR EXISTS ( + SELECT 1 + FROM moa.execution_dispatch_outbox AS dispatch + WHERE dispatch.tenant_id = trigger.tenant_id + AND dispatch.trigger_uid = trigger.trigger_uid + AND dispatch.dispatch_kind = 'trigger_delivery' + AND dispatch.state = 'delivered' + AND dispatch.delivered_at + <= now() - make_interval(secs => $2) + ) + ) + AND ( + ( + trigger.run_uid IS NOT NULL + AND EXISTS ( + SELECT 1 FROM moa.execution_run AS run + WHERE run.tenant_id = trigger.tenant_id + AND run.run_uid = trigger.run_uid + AND run.status NOT IN ( + 'completed', 'partial', 'blocked', 'unsupported', + 'failed', 'cancelled' + ) + ) + ) OR ( + trigger.schedule_uid IS NOT NULL + AND EXISTS ( + SELECT 1 FROM moa.execution_schedule AS schedule + WHERE schedule.tenant_id = trigger.tenant_id + AND schedule.schedule_uid = trigger.schedule_uid + AND schedule.status = 'active' + ) + ) + ) + ORDER BY trigger.due_at, trigger.tenant_id, trigger.trigger_uid + LIMIT $1 + "#, + ) + .bind(i64::from(trigger_budget)) + .bind(RESTATE_STATE_LOSS_REDRIVE_GRACE_SECONDS) + .fetch_all(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let candidate_trigger_uids = candidates + .iter() + .map(|(_, trigger_uid)| *trigger_uid) + .collect::>(); + let mut candidate_tenant_ids = candidates + .into_iter() + .map(|(tenant_id, _)| tenant_id) + .collect::>(); + candidate_tenant_ids.sort_unstable(); + candidate_tenant_ids.dedup(); + for tenant_id in candidate_tenant_ids { + prelock_existing_capacity_dimensions_in_tx( + conn.as_mut(), + TenantId(tenant_id), + &[ExecutionCapacityDimension::ScheduledTriggers], + ) + .await?; + } + let rows = if candidate_trigger_uids.is_empty() { + Vec::new() + } else { + sqlx::query( + r#" + SELECT trigger.* + FROM moa.execution_trigger AS trigger + WHERE trigger.trigger_uid = ANY($1) + AND trigger.state = 'pending' + AND trigger.due_at <= now() - make_interval(secs => $2) + AND ( + NOT EXISTS ( + SELECT 1 + FROM moa.execution_dispatch_outbox AS dispatch + WHERE dispatch.tenant_id = trigger.tenant_id + AND dispatch.trigger_uid = trigger.trigger_uid + AND dispatch.dispatch_kind = 'trigger_delivery' + ) + OR EXISTS ( + SELECT 1 + FROM moa.execution_dispatch_outbox AS dispatch + WHERE dispatch.tenant_id = trigger.tenant_id + AND dispatch.trigger_uid = trigger.trigger_uid + AND dispatch.dispatch_kind = 'trigger_delivery' + AND dispatch.state = 'delivered' + AND dispatch.delivered_at + <= now() - make_interval(secs => $2) + ) + ) + ORDER BY trigger.due_at, trigger.tenant_id, trigger.trigger_uid + FOR UPDATE OF trigger SKIP LOCKED + "#, + ) + .bind(&candidate_trigger_uids) + .bind(RESTATE_STATE_LOSS_REDRIVE_GRACE_SECONDS) + .fetch_all(conn.as_mut()) + .await + .map_err(sqlx_error)? + }; + let mut dispatches = Vec::with_capacity(batch_size as usize); + for row in rows { + let mut trigger = trigger_from_row(&row)?; + if !trigger_is_current(conn.as_mut(), &trigger).await? { + sqlx::query( + "UPDATE moa.execution_trigger SET state = 'superseded', updated_at = now() \ + WHERE trigger_uid = $1 AND state = 'pending'", + ) + .bind(trigger.trigger_uid) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + release_trigger_capacity_in_conn(conn.as_mut(), &trigger).await?; + continue; + } + let persisted_dispatch_uid: Option = sqlx::query_scalar( + "SELECT dispatch_uid FROM moa.execution_dispatch_outbox \ + WHERE tenant_id=$1 AND trigger_uid=$2 \ + AND dispatch_kind='trigger_delivery'", + ) + .bind(trigger.tenant_id.0) + .bind(trigger.trigger_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let dispatch_uid = match persisted_dispatch_uid { + Some(dispatch_uid) => dispatch_uid, + None => { + trigger.updated_at = sqlx::query_scalar( + "UPDATE moa.execution_trigger \ + SET updated_at=GREATEST(now(), updated_at + INTERVAL '1 microsecond') \ + WHERE trigger_uid=$1 RETURNING updated_at", + ) + .bind(trigger.trigger_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + repaired_trigger_delivery_dispatch_uid(&trigger) + } + }; + let request = trigger_delivery_dispatch_with_uid(&trigger, dispatch_uid); + dispatches.push( + match requeue_delivered_dispatch_in_conn(conn.as_mut(), &request).await? { + Some(dispatch) => dispatch, + None => enqueue_dispatch_in_conn(conn.as_mut(), &request).await?, + }, + ); + } + if run_budget > 0 { + let run_dispatches = requeue_current_run_activations_in_conn( + conn.as_mut(), + run_budget, + RESTATE_STATE_LOSS_REDRIVE_GRACE_SECONDS, + ) + .await?; + dispatches.extend(run_dispatches); + } + if accepted_budget > 0 { + dispatches.extend( + requeue_current_accepted_dispatches_in_conn( + conn.as_mut(), + accepted_budget, + RESTATE_STATE_LOSS_REDRIVE_GRACE_SECONDS, + ) + .await?, + ); + } + conn.commit().await.map_err(storage_error)?; + Ok(dispatches) + } +} + +async fn prelock_trigger_scheduled_capacity_in_conn( + conn: &mut PgConnection, + trigger_uid: Uuid, +) -> Result<()> { + let tenant_id: Option = + sqlx::query_scalar("SELECT tenant_id FROM moa.execution_trigger WHERE trigger_uid=$1") + .bind(trigger_uid) + .fetch_optional(&mut *conn) + .await + .map_err(sqlx_error)?; + if let Some(tenant_id) = tenant_id { + prelock_existing_capacity_dimensions_in_tx( + conn, + TenantId(tenant_id), + &[ExecutionCapacityDimension::ScheduledTriggers], + ) + .await?; + } + Ok(()) +} + +fn reconcile_lane_budgets(batch_size: u32) -> (u32, u32, u32) { + let lane_base = batch_size / 3; + let lane_remainder = batch_size % 3; + ( + lane_base + u32::from(lane_remainder > 0), + lane_base + u32::from(lane_remainder > 1), + lane_base, + ) +} + +/// Inserts a trigger and matching delayed outbox row in the caller transaction. +pub async fn create_trigger_with_dispatch_in_conn( + conn: &mut PgConnection, + config: &ExecutionConfig, + request: &NewExecutionTrigger, +) -> Result { + let trigger = create_trigger_in_conn(conn, request).await?; + if matches!( + trigger.state, + ExecutionDeliveryState::Pending | ExecutionDeliveryState::Dispatching + ) && reserve_capacity_in_tx(conn, config, trigger_capacity_request(&trigger)).await? + == CapacityReserveOutcome::Saturated + { + return Err(Error::CapacitySaturated { + dimension: ExecutionCapacityDimension::ScheduledTriggers.as_str(), + }); + } + let dispatch = enqueue_dispatch_in_conn(conn, &trigger_delivery_dispatch(&trigger)).await?; + Ok(ExecutionTriggerWrite { trigger, dispatch }) +} + +/// Fires a trigger and enqueues its run activation in the caller transaction. +pub async fn fire_trigger_in_conn( + conn: &mut PgConnection, + trigger_uid: Uuid, +) -> Result { + let row = sqlx::query( + "SELECT trigger.*, trigger.due_at <= now() AS is_due \ + FROM moa.execution_trigger AS trigger WHERE trigger_uid = $1 FOR UPDATE", + ) + .bind(trigger_uid) + .fetch_optional(&mut *conn) + .await + .map_err(sqlx_error)?; + let Some(row) = row else { + return Ok(ExecutionTriggerFireOutcome::NoOp( + ExecutionTriggerNoOp::NotFound, + )); + }; + let trigger = trigger_from_row(&row)?; + if matches!( + trigger.kind, + ExecutionTriggerKind::TaskTimer + | ExecutionTriggerKind::WaitExpiry + | ExecutionTriggerKind::TaskWatchdog + | ExecutionTriggerKind::CompensationWatchdog + | ExecutionTriggerKind::ExternalReconcile + | ExecutionTriggerKind::ExternalStartRecovery + ) { + return Err(Error::InvalidRepositoryInput { + message: "wait, watchdog, and external-job triggers require typed settlement" + .to_string(), + }); + } + match trigger.state { + ExecutionDeliveryState::Delivered => { + settle_trigger_dispatch(conn, trigger_uid, ExecutionDeliveryState::Delivered).await?; + release_trigger_capacity_in_conn(conn, &trigger).await?; + return Ok(ExecutionTriggerFireOutcome::NoOp( + ExecutionTriggerNoOp::Duplicate, + )); + } + ExecutionDeliveryState::Superseded + | ExecutionDeliveryState::Cancelled + | ExecutionDeliveryState::DeadLetter => { + settle_trigger_dispatch(conn, trigger_uid, ExecutionDeliveryState::Cancelled).await?; + release_trigger_capacity_in_conn(conn, &trigger).await?; + return Ok(ExecutionTriggerFireOutcome::NoOp( + ExecutionTriggerNoOp::Inactive, + )); + } + ExecutionDeliveryState::Pending | ExecutionDeliveryState::Dispatching => {} + } + let is_due = row.try_get::("is_due").map_err(super::row_error)?; + if !is_due { + return Ok(ExecutionTriggerFireOutcome::NoOp( + ExecutionTriggerNoOp::NotDue, + )); + } + if !trigger_is_current(conn, &trigger).await? { + sqlx::query( + r#" + UPDATE moa.execution_trigger + SET state = 'superseded', claim_owner = NULL, claimed_at = NULL, + claim_expires_at = NULL, updated_at = now() + WHERE trigger_uid = $1 AND state IN ('pending', 'dispatching') + "#, + ) + .bind(trigger_uid) + .execute(&mut *conn) + .await + .map_err(sqlx_error)?; + settle_trigger_dispatch(conn, trigger_uid, ExecutionDeliveryState::Cancelled).await?; + release_trigger_capacity_in_conn(conn, &trigger).await?; + return Ok(ExecutionTriggerFireOutcome::NoOp( + ExecutionTriggerNoOp::StaleGeneration, + )); + } + + sqlx::query( + r#" + UPDATE moa.execution_trigger + SET state = 'delivered', delivered_at = now(), claim_owner = NULL, + claimed_at = NULL, claim_expires_at = NULL, last_error = NULL, + updated_at = now() + WHERE trigger_uid = $1 AND state IN ('pending', 'dispatching') + "#, + ) + .bind(trigger_uid) + .execute(&mut *conn) + .await + .map_err(sqlx_error)?; + settle_trigger_dispatch(conn, trigger_uid, ExecutionDeliveryState::Delivered).await?; + release_trigger_capacity_in_conn(conn, &trigger).await?; + + let activation = if let (Some(run_uid), Some(controller_generation)) = + (trigger.run_uid, trigger.controller_generation) + { + let activation_allowed = sqlx::query_scalar::<_, bool>( + "SELECT status NOT IN ('pause_requested','pausing','paused') \ + FROM moa.execution_run WHERE tenant_id = $1 AND run_uid = $2 \ + AND controller_generation = $3", + ) + .bind(trigger.tenant_id.0) + .bind(run_uid) + .bind(to_i64(controller_generation, "controller generation")?) + .fetch_optional(&mut *conn) + .await + .map_err(sqlx_error)? + .unwrap_or(false); + if activation_allowed { + Some(Box::new( + enqueue_run_activation_in_conn( + conn, + trigger.tenant_id, + run_uid, + controller_generation, + Utc::now(), + json!({ "trigger_uid": trigger.trigger_uid }), + ) + .await?, + )) + } else { + None + } + } else { + None + }; + Ok(ExecutionTriggerFireOutcome::Delivered { activation }) +} + +/// Delivers a due task wait trigger without committing or enqueueing controller work. +pub(super) async fn deliver_wait_trigger_in_conn( + conn: &mut PgConnection, + trigger_uid: Uuid, +) -> Result { + let row = sqlx::query( + "SELECT trigger.*, now() AS observed_at, trigger.due_at <= now() AS is_due \ + FROM moa.execution_trigger AS trigger WHERE trigger_uid = $1 FOR UPDATE", + ) + .bind(trigger_uid) + .fetch_optional(&mut *conn) + .await + .map_err(sqlx_error)?; + let Some(row) = row else { + return Ok(ExecutionWaitTriggerDeliveryOutcome::NoOp( + ExecutionTriggerNoOp::NotFound, + )); + }; + let trigger = trigger_from_row(&row)?; + if !matches!( + trigger.kind, + ExecutionTriggerKind::TaskTimer | ExecutionTriggerKind::WaitExpiry + ) { + return Err(Error::InvalidRepositoryInput { + message: "wait-trigger delivery accepts only task_timer or wait_expiry".to_string(), + }); + } + match trigger.state { + ExecutionDeliveryState::Delivered => { + settle_trigger_dispatch(conn, trigger_uid, ExecutionDeliveryState::Delivered).await?; + release_trigger_capacity_in_conn(conn, &trigger).await?; + return Ok(ExecutionWaitTriggerDeliveryOutcome::NoOp( + ExecutionTriggerNoOp::Duplicate, + )); + } + ExecutionDeliveryState::Superseded + | ExecutionDeliveryState::Cancelled + | ExecutionDeliveryState::DeadLetter => { + settle_trigger_dispatch(conn, trigger_uid, ExecutionDeliveryState::Cancelled).await?; + release_trigger_capacity_in_conn(conn, &trigger).await?; + return Ok(ExecutionWaitTriggerDeliveryOutcome::NoOp( + ExecutionTriggerNoOp::Inactive, + )); + } + ExecutionDeliveryState::Pending | ExecutionDeliveryState::Dispatching => {} + } + if !row.try_get::("is_due").map_err(super::row_error)? { + return Ok(ExecutionWaitTriggerDeliveryOutcome::NoOp( + ExecutionTriggerNoOp::NotDue, + )); + } + if !trigger_is_current(conn, &trigger).await? { + sqlx::query( + "UPDATE moa.execution_trigger SET state='superseded', claim_owner=NULL, \ + claimed_at=NULL, claim_expires_at=NULL, updated_at=now() \ + WHERE trigger_uid=$1 AND state IN ('pending','dispatching')", + ) + .bind(trigger_uid) + .execute(&mut *conn) + .await + .map_err(sqlx_error)?; + settle_trigger_dispatch(conn, trigger_uid, ExecutionDeliveryState::Cancelled).await?; + release_trigger_capacity_in_conn(conn, &trigger).await?; + return Ok(ExecutionWaitTriggerDeliveryOutcome::NoOp( + ExecutionTriggerNoOp::StaleGeneration, + )); + } + let observed_at = row + .try_get::, _>("observed_at") + .map_err(super::row_error)?; + sqlx::query( + "UPDATE moa.execution_trigger SET state='delivered', delivered_at=$2, \ + claim_owner=NULL, claimed_at=NULL, claim_expires_at=NULL, last_error=NULL, \ + updated_at=now() WHERE trigger_uid=$1 AND state IN ('pending','dispatching')", + ) + .bind(trigger_uid) + .bind(observed_at) + .execute(&mut *conn) + .await + .map_err(sqlx_error)?; + settle_trigger_dispatch(conn, trigger_uid, ExecutionDeliveryState::Delivered).await?; + release_trigger_capacity_in_conn(conn, &trigger).await?; + Ok(ExecutionWaitTriggerDeliveryOutcome::Delivered { + trigger: Box::new(trigger), + observed_at, + }) +} + +/// Supersedes one exact trigger generation without committing the caller transaction. +#[allow(clippy::too_many_arguments)] +pub async fn supersede_trigger_in_conn( + conn: &mut PgConnection, + trigger_uid: Uuid, + expected_kind: ExecutionTriggerKind, + expected_controller_generation: Option, + expected_attempt_generation: Option, + expected_compensation_generation: Option, + expected_compensation_attempt_generation: Option, +) -> Result { + let row = sqlx::query( + r#" + SELECT * + FROM moa.execution_trigger + WHERE trigger_uid = $1 + AND trigger_kind = $2 + AND controller_generation IS NOT DISTINCT FROM $3 + AND attempt_generation IS NOT DISTINCT FROM $4 + AND compensation_generation IS NOT DISTINCT FROM $5 + AND compensation_attempt_generation IS NOT DISTINCT FROM $6 + FOR UPDATE + "#, + ) + .bind(trigger_uid) + .bind(expected_kind.as_str()) + .bind(to_optional_i64( + expected_controller_generation, + "controller generation", + )?) + .bind(to_optional_i64( + expected_attempt_generation, + "attempt generation", + )?) + .bind(to_optional_i64( + expected_compensation_generation, + "compensation generation", + )?) + .bind(to_optional_i64( + expected_compensation_attempt_generation, + "compensation attempt generation", + )?) + .fetch_optional(&mut *conn) + .await + .map_err(sqlx_error)?; + let Some(row) = row else { + return Ok(ExecutionTriggerSupersedeOutcome::StaleOrMissing); + }; + let trigger = trigger_from_row(&row)?; + let state = trigger.state; + match state { + ExecutionDeliveryState::Pending | ExecutionDeliveryState::Dispatching => { + sqlx::query( + r#" + UPDATE moa.execution_trigger + SET state = 'superseded', claim_owner = NULL, claimed_at = NULL, + claim_expires_at = NULL, updated_at = now() + WHERE trigger_uid = $1 + "#, + ) + .bind(trigger_uid) + .execute(&mut *conn) + .await + .map_err(sqlx_error)?; + settle_trigger_dispatch(conn, trigger_uid, ExecutionDeliveryState::Cancelled).await?; + release_trigger_capacity_in_conn(conn, &trigger).await?; + Ok(ExecutionTriggerSupersedeOutcome::Superseded) + } + ExecutionDeliveryState::Superseded => { + settle_trigger_dispatch(conn, trigger_uid, ExecutionDeliveryState::Cancelled).await?; + release_trigger_capacity_in_conn(conn, &trigger).await?; + Ok(ExecutionTriggerSupersedeOutcome::AlreadySuperseded) + } + ExecutionDeliveryState::Delivered => { + settle_trigger_dispatch(conn, trigger_uid, ExecutionDeliveryState::Delivered).await?; + release_trigger_capacity_in_conn(conn, &trigger).await?; + Ok(ExecutionTriggerSupersedeOutcome::AlreadyInactive) + } + ExecutionDeliveryState::Cancelled | ExecutionDeliveryState::DeadLetter => { + settle_trigger_dispatch(conn, trigger_uid, ExecutionDeliveryState::Cancelled).await?; + release_trigger_capacity_in_conn(conn, &trigger).await?; + Ok(ExecutionTriggerSupersedeOutcome::AlreadyInactive) + } + } +} + +/// Releases the exact capacity receipt for one trigger owner fence. +pub(super) async fn release_trigger_capacity_in_conn( + conn: &mut PgConnection, + trigger: &ExecutionTriggerRecord, +) -> Result<()> { + match release_capacity_in_tx(conn, trigger_capacity_request(trigger)).await? { + CapacityReleaseOutcome::Released | CapacityReleaseOutcome::AlreadyReleased => Ok(()), + CapacityReleaseOutcome::NotFound | CapacityReleaseOutcome::Stale => { + Err(Error::InvalidRepositoryData { + message: "trigger capacity release lost its exact owner fence".to_string(), + }) + } + } +} + +fn trigger_capacity_request(trigger: &ExecutionTriggerRecord) -> ExecutionCapacityRequest { + ExecutionCapacityRequest { + reservation_uid: execution_capacity_reservation_uid( + ExecutionCapacityDimension::ScheduledTriggers, + trigger.trigger_uid, + None, + ), + tenant_id: trigger.tenant_id, + run_uid: trigger.run_uid, + controller_generation: trigger.controller_generation, + dimension: ExecutionCapacityDimension::ScheduledTriggers, + owner: ExecutionCapacityOwner::Trigger { + trigger_uid: trigger.trigger_uid, + }, + expires_at: None, + } +} + +async fn settle_trigger_dispatch( + conn: &mut PgConnection, + trigger_uid: Uuid, + state: ExecutionDeliveryState, +) -> Result<()> { + let delivered_at = (state == ExecutionDeliveryState::Delivered).then(Utc::now); + sqlx::query( + r#" + UPDATE moa.execution_dispatch_outbox + SET state = $2, claim_owner = NULL, claimed_at = NULL, claim_expires_at = NULL, + delivered_at = $3, last_error = NULL, updated_at = now() + WHERE trigger_uid = $1 AND dispatch_kind = 'trigger_delivery' + AND state IN ('pending', 'dispatching') + "#, + ) + .bind(trigger_uid) + .bind(state.as_str()) + .bind(delivered_at) + .execute(&mut *conn) + .await + .map_err(sqlx_error)?; + Ok(()) +} + +async fn create_trigger_in_conn( + conn: &mut PgConnection, + request: &NewExecutionTrigger, +) -> Result { + validate_trigger(request)?; + let inserted = sqlx::query( + r#" + INSERT INTO moa.execution_trigger ( + trigger_uid, tenant_id, run_uid, task_id, compensation_id, schedule_uid, + schedule_incarnation, + trigger_kind, controller_generation, attempt_generation, + compensation_generation, compensation_attempt_generation, + occurrence_sequence, due_at, payload + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15) + ON CONFLICT (trigger_uid) DO NOTHING + RETURNING * + "#, + ) + .bind(request.trigger_uid) + .bind(request.tenant_id.0) + .bind(request.run_uid) + .bind(request.task_id) + .bind(request.compensation_id) + .bind(request.schedule_uid) + .bind(to_optional_i64( + request.schedule_incarnation, + "schedule incarnation", + )?) + .bind(request.kind.as_str()) + .bind(to_optional_i64( + request.controller_generation, + "controller generation", + )?) + .bind(to_optional_i64( + request.attempt_generation, + "attempt generation", + )?) + .bind(to_optional_i64( + request.compensation_generation, + "compensation generation", + )?) + .bind(to_optional_i64( + request.compensation_attempt_generation, + "compensation attempt generation", + )?) + .bind(to_optional_i64( + request.occurrence_sequence, + "occurrence sequence", + )?) + .bind(request.due_at) + .bind(&request.payload) + .fetch_optional(&mut *conn) + .await + .map_err(sqlx_error)?; + let row = match inserted { + Some(row) => row, + None => sqlx::query("SELECT * FROM moa.execution_trigger WHERE trigger_uid = $1") + .bind(request.trigger_uid) + .fetch_optional(&mut *conn) + .await + .map_err(sqlx_error)? + .ok_or_else(|| Error::Storage { + message: "trigger insert conflicted without a visible replay row".to_string(), + })?, + }; + let record = trigger_from_row(&row)?; + if !trigger_matches_request(&record, request) { + return Err(Error::InvalidRepositoryInput { + message: "trigger UID is already bound to different immutable semantics".to_string(), + }); + } + Ok(record) +} + +async fn trigger_is_current( + conn: &mut PgConnection, + trigger: &ExecutionTriggerRecord, +) -> Result { + let current = match trigger.kind { + ExecutionTriggerKind::ScheduleOccurrence => sqlx::query_scalar::<_, bool>( + r#" + SELECT EXISTS ( + SELECT 1 FROM moa.execution_schedule + WHERE schedule_uid = $1 AND tenant_id = $2 AND status = 'active' + AND schedule_incarnation = $3 + AND last_occurrence_sequence + 1 = $4 + ) + "#, + ) + .bind(trigger.schedule_uid) + .bind(trigger.tenant_id.0) + .bind(to_optional_i64( + trigger.schedule_incarnation, + "schedule incarnation", + )?) + .bind(to_optional_i64( + trigger.occurrence_sequence, + "occurrence sequence", + )?) + .fetch_one(&mut *conn) + .await + .map_err(sqlx_error)?, + ExecutionTriggerKind::RunDeadline => run_deadline_is_current(conn, trigger).await?, + ExecutionTriggerKind::TaskTimer + | ExecutionTriggerKind::WaitExpiry + | ExecutionTriggerKind::TaskWatchdog => { + let expected_statuses: &[&str] = match trigger.kind { + ExecutionTriggerKind::TaskTimer => &["waiting_timer"], + ExecutionTriggerKind::WaitExpiry => { + &["waiting_input", "waiting_review", "waiting_signal"] + } + ExecutionTriggerKind::TaskWatchdog => &["dispatching", "running"], + ExecutionTriggerKind::RunDeadline + | ExecutionTriggerKind::ExternalReconcile + | ExecutionTriggerKind::ExternalStartRecovery + | ExecutionTriggerKind::ScheduleOccurrence + | ExecutionTriggerKind::CompensationWatchdog => { + unreachable!("non-task trigger handled above") + } + }; + let task_is_current = sqlx::query_scalar::<_, bool>( + r#" + SELECT EXISTS ( + SELECT 1 + FROM moa.execution_task AS task + JOIN moa.execution_run AS run + ON run.run_uid = task.run_uid AND run.tenant_id = task.tenant_id + WHERE task.task_id = $1 AND task.run_uid = $2 AND task.tenant_id = $3 + AND ( + ($7 AND task.generation = $4) + OR (NOT $7 AND task.attempt_generation = $4) + ) + AND task.status = ANY($5) + AND ($7 OR run.controller_generation = $6) + AND run.status NOT IN ( + 'completed', 'partial', 'blocked', 'unsupported', 'failed', 'cancelled' + ) + ) + "#, + ) + .bind(trigger.task_id) + .bind(trigger.run_uid) + .bind(trigger.tenant_id.0) + .bind(to_optional_i64( + trigger.attempt_generation, + "attempt generation", + )?) + .bind(expected_statuses) + .bind(to_optional_i64( + trigger.controller_generation, + "controller generation", + )?) + .bind(matches!( + trigger.kind, + ExecutionTriggerKind::TaskTimer | ExecutionTriggerKind::WaitExpiry + )) + .fetch_one(&mut *conn) + .await + .map_err(sqlx_error)?; + if trigger.kind == ExecutionTriggerKind::TaskWatchdog && task_is_current { + sqlx::query_scalar::<_, bool>( + r#" + SELECT EXISTS ( + SELECT 1 FROM moa.execution_task + WHERE task_id = $1 AND tenant_id = $2 + AND attempt_state IN ('dispatching','running') + ) + "#, + ) + .bind(trigger.task_id) + .bind(trigger.tenant_id.0) + .fetch_one(&mut *conn) + .await + .map_err(sqlx_error)? + } else { + task_is_current + } + } + ExecutionTriggerKind::ExternalReconcile => { + external_job_trigger_is_current(conn, trigger, false).await? + } + ExecutionTriggerKind::ExternalStartRecovery => { + external_job_trigger_is_current(conn, trigger, true).await? + } + ExecutionTriggerKind::CompensationWatchdog => sqlx::query_scalar::<_, bool>( + r#" + SELECT EXISTS ( + SELECT 1 + FROM moa.execution_compensation AS compensation + JOIN moa.execution_run AS run + ON run.run_uid = compensation.run_uid + AND run.tenant_id = compensation.tenant_id + WHERE compensation.compensation_id = $1 + AND compensation.run_uid = $2 + AND compensation.tenant_id = $3 + AND compensation.generation = $4 + AND compensation.attempt_generation = $5 + AND compensation.status = 'running' + AND compensation.attempt_state = ANY($6) + AND run.controller_generation = $7 + AND run.status = 'compensating' + ) + "#, + ) + .bind(trigger.compensation_id) + .bind(trigger.run_uid) + .bind(trigger.tenant_id.0) + .bind(to_optional_i64( + trigger.compensation_generation, + "compensation generation", + )?) + .bind(to_optional_i64( + trigger.compensation_attempt_generation, + "compensation attempt generation", + )?) + .bind(["dispatching", "running"]) + .bind(to_optional_i64( + trigger.controller_generation, + "controller generation", + )?) + .fetch_one(&mut *conn) + .await + .map_err(sqlx_error)?, + }; + Ok(current) +} + +async fn external_job_trigger_is_current( + conn: &mut PgConnection, + trigger: &ExecutionTriggerRecord, + require_unbound: bool, +) -> Result { + let external_job_uid = trigger + .payload + .get("external_job_uid") + .and_then(Value::as_str) + .and_then(|value| Uuid::parse_str(value).ok()); + let job_generation = trigger + .payload + .get("job_generation") + .and_then(Value::as_u64) + .map(|value| to_i64(value, "external job generation")) + .transpose()?; + if external_job_uid.is_none() || job_generation.is_none() { + return Ok(false); + } + let declared_provider = trigger + .payload + .get("declared_provider") + .and_then(Value::as_str); + let idempotency_key = trigger + .payload + .get("idempotency_key") + .and_then(Value::as_str); + if require_unbound && (declared_provider.is_none() || idempotency_key.is_none()) { + return Ok(false); + } + sqlx::query_scalar::<_, bool>( + r#" + SELECT EXISTS ( + SELECT 1 FROM moa.execution_external_job AS job + WHERE job.external_job_uid=$1 AND job.tenant_id=$2 AND job.run_uid=$3 + AND job.job_generation=$4 + AND job.task_id IS NOT DISTINCT FROM $5 + AND job.attempt_generation IS NOT DISTINCT FROM $6 + AND job.compensation_id IS NOT DISTINCT FROM $7 + AND job.compensation_generation IS NOT DISTINCT FROM $8 + AND job.compensation_attempt_generation IS NOT DISTINCT FROM $9 + AND ( + ($10 AND job.state='unbound' AND job.declared_provider=$11 + AND job.idempotency_key=$12) + OR (NOT $10 AND job.state IN ( + 'starting','running','waiting_reconcile','cancel_requested' + )) + ) + ) + "#, + ) + .bind(external_job_uid) + .bind(trigger.tenant_id.0) + .bind(trigger.run_uid) + .bind(job_generation) + .bind(trigger.task_id) + .bind(to_optional_i64( + trigger.attempt_generation, + "attempt generation", + )?) + .bind(trigger.compensation_id) + .bind(to_optional_i64( + trigger.compensation_generation, + "compensation generation", + )?) + .bind(to_optional_i64( + trigger.compensation_attempt_generation, + "compensation attempt generation", + )?) + .bind(require_unbound) + .bind(declared_provider) + .bind(idempotency_key) + .fetch_one(&mut *conn) + .await + .map_err(sqlx_error) +} + +fn start_recovery_request_matches_trigger( + request: &ExecutionExternalJobStartRecoveryRequest, + trigger: &ExecutionTriggerRecord, +) -> bool { + let owner_matches = match (&request.owner, trigger.task_id, trigger.compensation_id) { + ( + ExecutionExternalJobStartRecoveryOwner::Task { + task_id, + attempt_generation, + }, + Some(trigger_task_id), + None, + ) => { + *task_id == trigger_task_id + && Some(*attempt_generation) == trigger.attempt_generation + && trigger.compensation_generation.is_none() + && trigger.compensation_attempt_generation.is_none() + } + ( + ExecutionExternalJobStartRecoveryOwner::Compensation { + compensation_id, + compensation_generation, + compensation_attempt_generation, + }, + None, + Some(trigger_compensation_id), + ) => { + *compensation_id == trigger_compensation_id + && Some(*compensation_generation) == trigger.compensation_generation + && Some(*compensation_attempt_generation) == trigger.compensation_attempt_generation + && trigger.attempt_generation.is_none() + } + _ => false, + }; + owner_matches + && request.trigger_uid == trigger.trigger_uid + && request.tenant_id == trigger.tenant_id + && Some(request.run_uid) == trigger.run_uid + && trigger.payload + == json!({ + "external_job_uid": request.external_job_uid, + "job_generation": request.job_generation, + "declared_provider": request.provider, + "idempotency_key": request.idempotency_key, + }) +} + +async fn run_deadline_is_current( + conn: &mut PgConnection, + trigger: &ExecutionTriggerRecord, +) -> Result { + if !run_deadline_payload_matches(trigger)? { + return Ok(false); + } + sqlx::query_scalar::<_, bool>( + r#" + SELECT EXISTS ( + SELECT 1 FROM moa.execution_run + WHERE run_uid = $1 AND tenant_id = $2 AND budget_deadline_at = $3 + AND status NOT IN ( + 'completed', 'partial', 'blocked', 'unsupported', 'failed', 'cancelled' + ) + ) + "#, + ) + .bind(trigger.run_uid) + .bind(trigger.tenant_id.0) + .bind(trigger.due_at) + .fetch_one(&mut *conn) + .await + .map_err(sqlx_error) +} + +fn run_deadline_payload_matches(trigger: &ExecutionTriggerRecord) -> Result { + let payload: RunDeadlineTriggerPayload = serde_json::from_value(trigger.payload.clone()) + .map_err(|error| Error::InvalidRepositoryData { + message: format!("invalid run deadline trigger payload: {error}"), + })?; + Ok(Some(payload.run_uid) == trigger.run_uid && payload.deadline_at == trigger.due_at) +} + +fn trigger_delivery_dispatch(trigger: &ExecutionTriggerRecord) -> NewExecutionDispatch { + trigger_delivery_dispatch_with_uid( + trigger, + Uuid::new_v5(&TRIGGER_DISPATCH_NAMESPACE, trigger.trigger_uid.as_bytes()), + ) +} + +fn trigger_delivery_dispatch_with_uid( + trigger: &ExecutionTriggerRecord, + dispatch_uid: Uuid, +) -> NewExecutionDispatch { + NewExecutionDispatch { + dispatch_uid, + tenant_id: trigger.tenant_id, + run_uid: None, + task_id: None, + compensation_id: None, + trigger_uid: Some(trigger.trigger_uid), + external_job_uid: None, + kind: ExecutionDispatchKind::TriggerDelivery, + controller_generation: None, + wake_epoch: None, + attempt_generation: None, + compensation_generation: None, + compensation_attempt_generation: None, + not_before_at: trigger.due_at, + payload: json!({ + "trigger_uid": trigger.trigger_uid, + "trigger_kind": trigger.kind.as_str(), + }), + } +} + +fn repaired_trigger_delivery_dispatch_uid(trigger: &ExecutionTriggerRecord) -> Uuid { + Uuid::new_v5( + &TRIGGER_DISPATCH_NAMESPACE, + format!( + "{}:repair:{}:{}", + trigger.trigger_uid, + trigger.due_at.timestamp_micros(), + trigger.updated_at.timestamp_micros() + ) + .as_bytes(), + ) +} + +fn rearmed_trigger_delivery_dispatch_uid(trigger_uid: Uuid, retry_at: DateTime) -> Uuid { + Uuid::new_v5( + &TRIGGER_DISPATCH_NAMESPACE, + format!("{trigger_uid}:rearm:{}", retry_at.timestamp_micros()).as_bytes(), + ) +} + +fn validate_trigger(request: &NewExecutionTrigger) -> Result<()> { + let generation_is_zero = [ + request.schedule_incarnation, + request.controller_generation, + request.attempt_generation, + request.compensation_generation, + request.compensation_attempt_generation, + request.occurrence_sequence, + ] + .into_iter() + .flatten() + .any(|generation| generation == 0); + if request.trigger_uid.is_nil() || !request.payload.is_object() || generation_is_zero { + return Err(Error::InvalidRepositoryInput { + message: + "execution trigger requires a non-nil UID, positive generations, and object payload" + .to_string(), + }); + } + let valid = match request.kind { + ExecutionTriggerKind::ScheduleOccurrence => { + request.schedule_uid.is_some() + && request.schedule_incarnation.is_some() + && request.occurrence_sequence.is_some() + && request.run_uid.is_none() + && request.task_id.is_none() + && request.compensation_id.is_none() + && request.controller_generation.is_none() + && request.attempt_generation.is_none() + && request.compensation_generation.is_none() + && request.compensation_attempt_generation.is_none() + } + ExecutionTriggerKind::RunDeadline => { + request.run_uid.is_some() + && request.task_id.is_none() + && request.compensation_id.is_none() + && request.schedule_uid.is_none() + && request.schedule_incarnation.is_none() + && request.controller_generation.is_some() + && request.attempt_generation.is_none() + && request.compensation_generation.is_none() + && request.compensation_attempt_generation.is_none() + && request.occurrence_sequence.is_none() + } + ExecutionTriggerKind::TaskTimer + | ExecutionTriggerKind::WaitExpiry + | ExecutionTriggerKind::TaskWatchdog => { + request.run_uid.is_some() + && request.task_id.is_some() + && request.compensation_id.is_none() + && request.schedule_uid.is_none() + && request.schedule_incarnation.is_none() + && request.controller_generation.is_some() + && request.attempt_generation.is_some() + && request.compensation_generation.is_none() + && request.compensation_attempt_generation.is_none() + && request.occurrence_sequence.is_none() + } + ExecutionTriggerKind::ExternalReconcile | ExecutionTriggerKind::ExternalStartRecovery => { + let task_owner = request.task_id.is_some() + && request.attempt_generation.is_some() + && request.compensation_id.is_none() + && request.compensation_generation.is_none() + && request.compensation_attempt_generation.is_none(); + let compensation_owner = request.task_id.is_none() + && request.attempt_generation.is_none() + && request.compensation_id.is_some() + && request.compensation_generation.is_some() + && request.compensation_attempt_generation.is_some(); + request.run_uid.is_some() + && request.schedule_uid.is_none() + && request.schedule_incarnation.is_none() + && request.controller_generation.is_some() + && request.occurrence_sequence.is_none() + && (task_owner || compensation_owner) + } + ExecutionTriggerKind::CompensationWatchdog => { + request.run_uid.is_some() + && request.task_id.is_none() + && request.compensation_id.is_some() + && request.schedule_uid.is_none() + && request.schedule_incarnation.is_none() + && request.controller_generation.is_some() + && request.attempt_generation.is_none() + && request.compensation_generation.is_some() + && request.compensation_attempt_generation.is_some() + && request.occurrence_sequence.is_none() + } + }; + if !valid { + return Err(Error::InvalidRepositoryInput { + message: format!( + "execution trigger target shape does not match {}", + request.kind.as_str() + ), + }); + } + Ok(()) +} + +fn trigger_matches_request(record: &ExecutionTriggerRecord, request: &NewExecutionTrigger) -> bool { + record.trigger_uid == request.trigger_uid + && record.tenant_id == request.tenant_id + && record.run_uid == request.run_uid + && record.task_id == request.task_id + && record.compensation_id == request.compensation_id + && record.schedule_uid == request.schedule_uid + && record.schedule_incarnation == request.schedule_incarnation + && record.kind == request.kind + && record.controller_generation == request.controller_generation + && record.attempt_generation == request.attempt_generation + && record.compensation_generation == request.compensation_generation + && record.compensation_attempt_generation == request.compensation_attempt_generation + && record.occurrence_sequence == request.occurrence_sequence + && record.due_at == request.due_at + && record.payload == request.payload +} + +fn trigger_from_row(row: &sqlx::postgres::PgRow) -> Result { + let controller_generation = row + .try_get::, _>("controller_generation") + .map_err(super::row_error)?; + let attempt_generation = row + .try_get::, _>("attempt_generation") + .map_err(super::row_error)?; + let compensation_generation = row + .try_get::, _>("compensation_generation") + .map_err(super::row_error)?; + let compensation_attempt_generation = row + .try_get::, _>("compensation_attempt_generation") + .map_err(super::row_error)?; + let occurrence_sequence = row + .try_get::, _>("occurrence_sequence") + .map_err(super::row_error)?; + let schedule_incarnation = row + .try_get::, _>("schedule_incarnation") + .map_err(super::row_error)?; + Ok(ExecutionTriggerRecord { + trigger_uid: row.try_get("trigger_uid").map_err(super::row_error)?, + tenant_id: TenantId(row.try_get("tenant_id").map_err(super::row_error)?), + run_uid: row.try_get("run_uid").map_err(super::row_error)?, + task_id: row.try_get("task_id").map_err(super::row_error)?, + compensation_id: row.try_get("compensation_id").map_err(super::row_error)?, + schedule_uid: row.try_get("schedule_uid").map_err(super::row_error)?, + schedule_incarnation: schedule_incarnation + .map(|value| super::to_u64(value, "schedule incarnation")) + .transpose()?, + kind: row + .try_get::("trigger_kind") + .map_err(super::row_error)? + .parse()?, + state: row + .try_get::("state") + .map_err(super::row_error)? + .parse()?, + controller_generation: controller_generation + .map(|value| super::to_u64(value, "controller generation")) + .transpose()?, + attempt_generation: attempt_generation + .map(|value| super::to_u64(value, "attempt generation")) + .transpose()?, + compensation_generation: compensation_generation + .map(|value| super::to_u64(value, "compensation generation")) + .transpose()?, + compensation_attempt_generation: compensation_attempt_generation + .map(|value| super::to_u64(value, "compensation attempt generation")) + .transpose()?, + occurrence_sequence: occurrence_sequence + .map(|value| super::to_u64(value, "occurrence sequence")) + .transpose()?, + due_at: row.try_get("due_at").map_err(super::row_error)?, + payload: row.try_get("payload").map_err(super::row_error)?, + delivered_at: row.try_get("delivered_at").map_err(super::row_error)?, + created_at: row.try_get("created_at").map_err(super::row_error)?, + updated_at: row.try_get("updated_at").map_err(super::row_error)?, + }) +} + +#[cfg(test)] +mod tests { + use super::reconcile_lane_budgets; + + #[test] + fn every_reconciliation_lane_has_budget_under_sustained_backlog() { + // Pins: a due-trigger backlog cannot consume the entire bounded pass and starve + // accepted-before-start dispatches or queued run activations. + for batch_size in [3, 4, 5, 32, 1_000] { + let (triggers, accepted, runs) = reconcile_lane_budgets(batch_size); + assert!(triggers > 0, "batch {batch_size}"); + assert!(accepted > 0, "batch {batch_size}"); + assert!(runs > 0, "batch {batch_size}"); + assert_eq!(triggers + accepted + runs, batch_size); + } + } +} diff --git a/crates/moa-execution/src/state.rs b/crates/moa-execution/src/state.rs index 7ecf93dfc..a532eda05 100644 --- a/crates/moa-execution/src/state.rs +++ b/crates/moa-execution/src/state.rs @@ -1,11 +1,16 @@ //! Public pure execution projection, task, waiting, and terminal state types. -use std::{collections::BTreeMap, fmt, str::FromStr}; +use std::{ + collections::{BTreeMap, BTreeSet}, + fmt, + str::FromStr, +}; use moa_artifacts::{ execution_plan::{ CapabilityReference, ExecutionCitation, ExecutionCompensation, ExecutionFailureClass, - ExecutionTaskOutcome, ExecutionTaskResult, ExecutionUsage, InputAudience, RetryPolicy, + ExecutionTaskOutcome, ExecutionTaskResult, ExecutionTemporalTarget, ExecutionUsage, + ExecutionWaitPolicy, InputAudience, RetryPolicy, }, reference::ArtifactRef, }; @@ -123,8 +128,20 @@ pub enum ExecutionRunStatus { WaitingInput, /// The run is waiting for a tenant review decision. WaitingReview, + /// The run is waiting for a named external signal. + WaitingSignal, + /// The run is waiting for an exact durable timer. + WaitingTimer, + /// The run is waiting for an asynchronous external job. + WaitingExternal, /// The run is waiting for a compiler-validated amendment. WaitingReplan, + /// An authorized caller requested a safe pause. + PauseRequested, + /// Active work is reaching safe checkpoint boundaries before pausing. + Pausing, + /// The run is durably parked without active compute. + Paused, /// Forward work is fenced while committed effects are undone in reverse order. Compensating, /// Every required completion check passed. @@ -151,7 +168,13 @@ impl ExecutionRunStatus { Self::Running => "running", Self::WaitingInput => "waiting_input", Self::WaitingReview => "waiting_review", + Self::WaitingSignal => "waiting_signal", + Self::WaitingTimer => "waiting_timer", + Self::WaitingExternal => "waiting_external", Self::WaitingReplan => "waiting_replan", + Self::PauseRequested => "pause_requested", + Self::Pausing => "pausing", + Self::Paused => "paused", Self::Compensating => "compensating", Self::Completed => "completed", Self::Partial => "partial", @@ -187,7 +210,13 @@ impl FromStr for ExecutionRunStatus { "running" => Ok(Self::Running), "waiting_input" => Ok(Self::WaitingInput), "waiting_review" => Ok(Self::WaitingReview), + "waiting_signal" => Ok(Self::WaitingSignal), + "waiting_timer" => Ok(Self::WaitingTimer), + "waiting_external" => Ok(Self::WaitingExternal), "waiting_replan" => Ok(Self::WaitingReplan), + "pause_requested" => Ok(Self::PauseRequested), + "pausing" => Ok(Self::Pausing), + "paused" => Ok(Self::Paused), "compensating" => Ok(Self::Compensating), "completed" => Ok(Self::Completed), "partial" => Ok(Self::Partial), @@ -649,12 +678,24 @@ pub struct ExecutionTerminalEvidence { pub enum ExecutionTaskStatus { /// Task is materialized and ready for reservation. Pending, + /// Task is admitted to the bounded durable ready queue. + Ready, /// Worst-case budget is reserved. Reserved, + /// One generation-fenced attempt is awaiting durable delivery. + Dispatching, /// Current generation is executing or has a retry scheduled. Running, /// Task is waiting for audience input. WaitingInput, + /// Task is waiting for a tenant review decision. + WaitingReview, + /// Task is waiting for a named external signal. + WaitingSignal, + /// Task is waiting for an exact durable timer. + WaitingTimer, + /// Task is waiting for an asynchronous external job. + WaitingExternal, /// Task is waiting for a compiler-validated amendment. WaitingReplan, /// Task completed successfully. @@ -663,6 +704,8 @@ pub enum ExecutionTaskStatus { Skipped, /// Task ended in terminal failure. Failed, + /// A non-idempotent attempt may have committed and requires reconciliation. + UnknownOutcome, /// Task was cancelled. Cancelled, } @@ -673,13 +716,20 @@ impl ExecutionTaskStatus { pub const fn as_str(self) -> &'static str { match self { Self::Pending => "pending", + Self::Ready => "ready", Self::Reserved => "reserved", + Self::Dispatching => "dispatching", Self::Running => "running", Self::WaitingInput => "waiting_input", + Self::WaitingReview => "waiting_review", + Self::WaitingSignal => "waiting_signal", + Self::WaitingTimer => "waiting_timer", + Self::WaitingExternal => "waiting_external", Self::WaitingReplan => "waiting_replan", Self::Completed => "completed", Self::Skipped => "skipped", Self::Failed => "failed", + Self::UnknownOutcome => "unknown_outcome", Self::Cancelled => "cancelled", } } @@ -689,7 +739,7 @@ impl ExecutionTaskStatus { pub const fn is_terminal(self) -> bool { matches!( self, - Self::Completed | Self::Skipped | Self::Failed | Self::Cancelled + Self::Completed | Self::Skipped | Self::Failed | Self::UnknownOutcome | Self::Cancelled ) } } @@ -700,13 +750,20 @@ impl FromStr for ExecutionTaskStatus { fn from_str(value: &str) -> Result { match value { "pending" => Ok(Self::Pending), + "ready" => Ok(Self::Ready), "reserved" => Ok(Self::Reserved), + "dispatching" => Ok(Self::Dispatching), "running" => Ok(Self::Running), "waiting_input" => Ok(Self::WaitingInput), + "waiting_review" => Ok(Self::WaitingReview), + "waiting_signal" => Ok(Self::WaitingSignal), + "waiting_timer" => Ok(Self::WaitingTimer), + "waiting_external" => Ok(Self::WaitingExternal), "waiting_replan" => Ok(Self::WaitingReplan), "completed" => Ok(Self::Completed), "skipped" => Ok(Self::Skipped), "failed" => Ok(Self::Failed), + "unknown_outcome" => Ok(Self::UnknownOutcome), "cancelled" => Ok(Self::Cancelled), _ => Err(Error::InvalidRepositoryData { message: format!("unknown execution task status `{value}`"), @@ -749,6 +806,20 @@ pub struct ExecutionProjection { pub tasks: Vec, } +/// Compact bounded node/task evidence accepted by restricted plan amendments. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ExecutionAmendmentProjection { + /// Active immutable plan revision. + pub plan_revision: u64, + /// Exact aggregate status for every compiler-bounded plan node. + pub node_statuses: BTreeMap, + /// Nodes with any persisted task materialization or non-pending aggregate state. + pub started_node_ids: BTreeSet, + /// Bounded current replan origins; repository correctness requires exactly one. + pub replan_tasks: Vec, +} + /// Pure description of one logical task ready for durable materialization or dispatch. #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] #[serde(deny_unknown_fields)] @@ -801,11 +872,22 @@ pub enum LogicalTaskKind { Review { /// Review prompt. prompt: String, + /// Absolute expiry and deterministic expiry action. + wait_policy: ExecutionWaitPolicy, }, /// Pause for one named signal. WaitSignal { /// Stable signal name. signal_name: String, + /// Absolute expiry and deterministic expiry action. + wait_policy: ExecutionWaitPolicy, + }, + /// Park until an exact or wait-entry-relative timestamp. + WaitUntil { + /// Exact or wait-entry-relative wake target. + wake: ExecutionTemporalTarget, + /// Structured output installed when the timer fires. + result: Value, }, /// Validate and persist terminal output. Output { @@ -832,6 +914,7 @@ impl LogicalTaskKind { Self::Agent { .. } => "agent", Self::Review { .. } => "review", Self::WaitSignal { .. } => "wait_signal", + Self::WaitUntil { .. } => "wait_until", Self::Output { .. } => "output", Self::CompletionVerifier { .. } => "completion_verifier", } @@ -844,6 +927,8 @@ impl LogicalTaskKind { pub enum ScheduleDecision { /// Newly ready logical tasks. Ready(Vec), + /// One storage-only wait reached its deterministic settlement time. + SettleWait(WaitSettlement), /// Durable work is waiting on execution or an external condition. Waiting(Vec), /// The run has a terminal projection. @@ -855,6 +940,26 @@ pub enum ScheduleDecision { }, } +/// One deterministic storage-only wait transition selected by the scheduler. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum WaitSettlement { + /// A `WaitUntil` node reached its resolved wake time. + TimerElapsed { + /// Stable waiting task ID. + task_id: ExecutionTaskId, + /// Structured node output declared by the immutable plan. + output: Value, + }, + /// An input, review, or signal wait reached its resolved expiry time. + WaitExpired { + /// Stable waiting task ID. + task_id: ExecutionTaskId, + /// Deterministic expiry action declared by the immutable plan. + action: moa_artifacts::execution_plan::ExecutionWaitExpiryAction, + }, +} + /// Reason the pure scheduler cannot currently advance. #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] @@ -869,6 +974,8 @@ pub enum WaitingReason { audience: InputAudience, /// Exact task question. question: String, + /// Absolute expiry and deterministic expiry action. + wait_policy: ExecutionWaitPolicy, }, /// One task needs a tenant review decision. Review { @@ -876,6 +983,8 @@ pub enum WaitingReason { task_id: ExecutionTaskId, /// Exact review prompt. prompt: String, + /// Absolute expiry and deterministic expiry action. + wait_policy: ExecutionWaitPolicy, }, /// One task awaits a named signal. Signal { @@ -883,6 +992,20 @@ pub enum WaitingReason { task_id: ExecutionTaskId, /// Stable signal name. signal_name: String, + /// Absolute expiry and deterministic expiry action. + wait_policy: ExecutionWaitPolicy, + }, + /// One task is parked until an exact or wait-entry-relative timestamp. + Timer { + /// Stable waiting task ID. + task_id: ExecutionTaskId, + /// Exact or wait-entry-relative wake target. + wake: ExecutionTemporalTarget, + }, + /// One task is awaiting completion of an asynchronous external job. + External { + /// Stable waiting task ID. + task_id: ExecutionTaskId, }, /// Pending nodes still depend on unfinished predecessors. Dependencies { @@ -1030,7 +1153,7 @@ pub fn task_status_from_outcome( ExecutionTaskResult::NeedsInput { .. } => ExecutionTaskStatus::WaitingInput, ExecutionTaskResult::NeedsReplan { .. } => ExecutionTaskStatus::WaitingReplan, ExecutionTaskResult::Cancelled { .. } => ExecutionTaskStatus::Cancelled, - ExecutionTaskResult::UnknownOutcome { .. } => ExecutionTaskStatus::Failed, + ExecutionTaskResult::UnknownOutcome { .. } => ExecutionTaskStatus::UnknownOutcome, ExecutionTaskResult::Failed { class: ExecutionFailureClass::Retryable, .. @@ -1059,7 +1182,13 @@ pub fn run_status_after_task_outcome( current: ExecutionRunStatus, outcome: &ExecutionTaskOutcome, ) -> ExecutionRunStatus { - if current == ExecutionRunStatus::Compensating { + if matches!( + current, + ExecutionRunStatus::Compensating + | ExecutionRunStatus::PauseRequested + | ExecutionRunStatus::Pausing + | ExecutionRunStatus::Paused + ) { return current; } match &outcome.result { @@ -1069,7 +1198,14 @@ pub fn run_status_after_task_outcome( | ExecutionTaskResult::Cancelled { .. } | ExecutionTaskResult::UnknownOutcome { .. } | ExecutionTaskResult::Failed { .. } - if current == ExecutionRunStatus::WaitingReview => + if matches!( + current, + ExecutionRunStatus::WaitingInput + | ExecutionRunStatus::WaitingReview + | ExecutionRunStatus::WaitingSignal + | ExecutionRunStatus::WaitingTimer + | ExecutionRunStatus::WaitingExternal + ) => { ExecutionRunStatus::Running } diff --git a/crates/moa-execution/src/wire.rs b/crates/moa-execution/src/wire.rs index bf4c0227f..cf2b74832 100644 --- a/crates/moa-execution/src/wire.rs +++ b/crates/moa-execution/src/wire.rs @@ -10,14 +10,15 @@ use moa_artifacts::{ }; use moa_core::events::Event; use moa_core::events::{ - ExecutionFailureDisposition, ExecutionProgress, ExecutionTaskResultsRef, + ExecutionBlockerAudience, ExecutionFailureDisposition, ExecutionProgress, + ExecutionProgressPhase, ExecutionRemainingBudget, ExecutionTaskResultsRef, ExecutionTerminalSummary, }; -use moa_core::traits::Identity; use moa_core::types::{ contact::ContactId, execution_planning::{ExecutionSourceProvenance, PinnedExecutionTemplateRef}, identifiers::{SessionId, TenantId, UserId}, + tools::{AsyncToolJob, AsyncToolJobCallbackOutcome, AsyncToolJobCancelOutcome}, }; use serde::{Deserialize, Serialize, de::DeserializeOwned}; use serde_json::Value; @@ -93,6 +94,8 @@ pub struct ExecutionPlanningContextRequest { pub session_id: SessionId, /// Exact persisted user-message sequence that supplies the objective. pub originating_user_sequence_num: u64, + /// Absolute authorized deadline frozen into the Durable planning context. + pub deadline_at: DateTime, /// Optional exact template selection hint; this grants no authority. pub requested_template: Option, } @@ -658,19 +661,109 @@ pub struct ExecutionSynthesisEvidence { } /// Builds compact aggregate progress from one canonical execution-run row. -#[must_use] pub fn execution_progress_from_run( run: &crate::repository::ExecutionRunRecord, -) -> ExecutionProgress { - ExecutionProgress { +) -> Result { + let phase = execution_progress_phase(run); + let remaining = BudgetLedger { + limit: run.approved_budget.clone(), + reserved: run.reserved, + consumed: run.consumed, + overrun: run.budget_overrun, + } + .remaining_limit()?; + Ok(ExecutionProgress { run_uid: run.run_uid, originating_user_sequence_num: run.originating_user_sequence_num, plan_revision: run.plan_revision, status: run.status.as_str().to_string(), + phase, + waiting_since: run.waiting_since, + next_wake_at: run.next_wake_at, + last_progress_at: run.last_progress_at, + // A run can own multiple concurrent external jobs. Only a task-qualified + // transition can name one exact job; the aggregate run row cannot. + external_job_uid: None, + ready_tasks: run.ready_task_count, + active_tasks: run.active_task_count, + parked_tasks: run.waiting_task_count, + blocker_audience: execution_blocker_audience(run), + remaining_budget: ExecutionRemainingBudget { + cost_microusd: remaining.max_cost_microusd, + tokens: remaining.max_tokens, + tasks: remaining.max_tasks, + tool_calls: remaining.max_tool_calls, + retrieved_bytes: remaining.max_retrieved_bytes, + deadline_at: remaining.deadline_at, + }, total: run.progress_total_tasks, completed: run.progress_completed_tasks, failed: run.progress_failed_tasks, cancelled: run.progress_cancelled_tasks, + }) +} + +fn execution_blocker_audience( + run: &crate::repository::ExecutionRunRecord, +) -> Option { + execution_blocker_audience_from_flags( + run.waiting_input_user_task_count > 0, + run.waiting_input_tenant_admin_task_count > 0 || run.waiting_review_task_count > 0, + run.waiting_input_external_task_count > 0 + || run.waiting_signal_task_count > 0 + || run.waiting_external_task_count > 0, + run.waiting_timer_task_count > 0 || run.waiting_replan_task_count > 0, + ) +} + +fn execution_blocker_audience_from_flags( + user: bool, + tenant_reviewer: bool, + external: bool, + system: bool, +) -> Option { + if user { + Some(ExecutionBlockerAudience::User) + } else if tenant_reviewer { + Some(ExecutionBlockerAudience::TenantReviewer) + } else if external { + Some(ExecutionBlockerAudience::External) + } else if system { + Some(ExecutionBlockerAudience::System) + } else { + None + } +} + +fn execution_progress_phase(run: &crate::repository::ExecutionRunRecord) -> ExecutionProgressPhase { + execution_progress_phase_from_flags( + run.status, + run.waiting_input_task_count > 0, + run.waiting_review_task_count > 0, + run.waiting_signal_task_count > 0, + run.waiting_timer_task_count > 0, + run.waiting_external_task_count > 0, + ) +} + +fn execution_progress_phase_from_flags( + status: ExecutionRunStatus, + waiting_input: bool, + waiting_review: bool, + waiting_signal: bool, + waiting_timer: bool, + waiting_external: bool, +) -> ExecutionProgressPhase { + match status { + ExecutionRunStatus::PauseRequested => ExecutionProgressPhase::PauseRequested, + ExecutionRunStatus::Pausing => ExecutionProgressPhase::Pausing, + ExecutionRunStatus::Paused => ExecutionProgressPhase::Paused, + _ if waiting_input => ExecutionProgressPhase::WaitingInput, + _ if waiting_review => ExecutionProgressPhase::WaitingReview, + _ if waiting_signal => ExecutionProgressPhase::WaitingSignal, + _ if waiting_timer => ExecutionProgressPhase::WaitingTimer, + _ if waiting_external => ExecutionProgressPhase::WaitingExternal, + _ => ExecutionProgressPhase::Running, } } @@ -920,6 +1013,13 @@ pub enum ExecutionActionReviewResolution { /// Serialized governed tool output. tool_output: Value, }, + /// The approved capability committed asynchronous provider work. + ExternalJob { + /// MOA-owned job identity reserved before provider dispatch. + external_job_uid: Uuid, + /// Immutable provider job identity and recovery contract. + job: AsyncToolJob, + }, /// The approved tool failed during dispatch. Failed { /// Typed task failure classification. @@ -949,18 +1049,6 @@ pub enum ExecutionActionReviewResolution { }, } -/// Idempotent acknowledgement returned to the action-review outbox dispatcher. -#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] -#[serde(rename_all = "snake_case")] -pub enum ExecutionActionReviewAcknowledgement { - /// The resolution was applied to the current task generation. - Applied, - /// This review UID was already applied to the same task generation. - Replayed, - /// The resolution was durably audited but its generation is stale or terminal. - AuditedStale, -} - /// Typed terminal action-policy review delivery to a compensation generation. #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] #[serde(deny_unknown_fields)] @@ -977,106 +1065,397 @@ pub struct ExecutionCompensationReviewResolutionRequest { pub resolution: ExecutionActionReviewResolution, } -/// Idempotent acknowledgement returned to a compensation-review dispatcher. -#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] -#[serde(rename_all = "snake_case")] -pub enum ExecutionCompensationReviewAcknowledgement { - /// The resolution was applied to the current compensation generation. - Applied, - /// This review UID was already applied to the same compensation generation. - Replayed, - /// The resolution was durably audited but its generation is stale or settled. - AuditedStale, +/// Immutable request that executes one bounded task-attempt slice. +/// +/// The Restate workflow key is [`Self::dispatch_uid`]. Re-delivery of this +/// request can replay the same slice, but a different dispatch UID must never +/// continue or replace it. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ExecutionTaskAttemptRequest { + /// Immutable durable-dispatch identity and workflow key. + pub dispatch_uid: Uuid, + /// Exact active-capacity receipt released when this slice yields or settles. + pub capacity_reservation_uid: Uuid, + /// Exact watchdog trigger owned only while this slice is active. + pub watchdog_trigger_uid: Uuid, + /// Durable delayed-delivery dispatch for the exact watchdog trigger. + pub watchdog_dispatch_uid: Uuid, + /// Owning run. + pub run_uid: Uuid, + /// Stable logical task. + pub task_id: ExecutionTaskId, + /// Run-controller generation that admitted the slice. + pub controller_generation: u64, + /// Exact bounded attempt generation. + pub attempt_generation: u64, + /// Absolute deadline committed by admission. + pub attempt_deadline_at: DateTime, + /// Owning tenant. + pub tenant_id: TenantId, } -/// Internal request that starts the keyed run workflow. +/// Delivery of the exact watchdog owned by one active task attempt. #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] #[serde(deny_unknown_fields)] -pub struct ExecutionRunWorkflowRequest { - /// Durable run identifier and workflow key. +pub struct ExecutionTaskAttemptWatchdogRequest { + /// Immutable durable-dispatch identity and workflow key. + pub dispatch_uid: Uuid, + /// Exact active-capacity receipt released by watchdog settlement. + pub capacity_reservation_uid: Uuid, + /// Trigger whose delivery caused this check. + pub watchdog_trigger_uid: Uuid, + /// Owning run. pub run_uid: Uuid, + /// Stable logical task. + pub task_id: ExecutionTaskId, + /// Run-controller generation that admitted the slice. + pub controller_generation: u64, + /// Exact bounded attempt generation. + pub attempt_generation: u64, /// Owning tenant. pub tenant_id: TenantId, - /// Optional owning contact. - pub contact_id: Option, - /// Parent session. - pub session_id: SessionId, - /// Exact authenticated identity admitted when the run was launched. - pub identity: Identity, } -/// Internal request that notifies a keyed run of a persisted scheduling change. +/// Immutable request that executes one bounded compensation-attempt slice. +/// +/// The Restate workflow key is [`Self::dispatch_uid`]. Logical compensation +/// generation and attempt generation are distinct fences. #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] #[serde(deny_unknown_fields)] -pub struct ExecutionRunWakeRequest { - /// Durable run identifier and workflow key. +pub struct ExecutionCompensationAttemptRequest { + /// Immutable durable-dispatch identity and workflow key. + pub dispatch_uid: Uuid, + /// Exact shared active-capacity receipt released when this slice returns. + pub capacity_reservation_uid: Uuid, + /// Exact watchdog trigger owned only while this slice is active. + pub watchdog_trigger_uid: Uuid, + /// Durable delayed-delivery dispatch for the exact watchdog trigger. + pub watchdog_dispatch_uid: Uuid, + /// Owning execution run. pub run_uid: Uuid, - /// Exact persisted monotonic wake epoch. - pub wake_epoch: u64, - /// Mutation that caused the wake. - pub reason: ExecutionRunWakeReason, + /// Stable compensation registration. + pub compensation_id: CompensationId, + /// Logical compensation generation selected in strict reverse order. + pub compensation_generation: u64, + /// Exact bounded compensation-attempt generation. + pub compensation_attempt_generation: u64, + /// Run-controller generation that admitted the slice. + pub controller_generation: u64, + /// Absolute deadline committed by admission. + pub attempt_deadline_at: DateTime, + /// Owning tenant. + pub tenant_id: TenantId, } -/// Stable reasons a run workflow is awakened. +/// Delivery of the exact watchdog owned by one active compensation attempt. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ExecutionCompensationAttemptWatchdogRequest { + /// Immutable durable-dispatch identity and workflow key. + pub dispatch_uid: Uuid, + /// Exact shared active-capacity receipt released by watchdog settlement. + pub capacity_reservation_uid: Uuid, + /// Trigger whose delivery caused this check. + pub watchdog_trigger_uid: Uuid, + /// Owning execution run. + pub run_uid: Uuid, + /// Stable compensation registration. + pub compensation_id: CompensationId, + /// Logical compensation generation selected in strict reverse order. + pub compensation_generation: u64, + /// Exact bounded compensation-attempt generation. + pub compensation_attempt_generation: u64, + /// Run-controller generation that admitted the slice. + pub controller_generation: u64, + /// Owning tenant. + pub tenant_id: TenantId, +} + +/// Durable reason one exact active attempt must checkpoint and relinquish ownership. #[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] #[serde(rename_all = "snake_case")] -pub enum ExecutionRunWakeReason { - /// A task persisted an outcome. - TaskOutcome, - /// The user confirmed the active plan and budget. - Confirmed, - /// Audience-bound input resumed a task. - InputDelivered, - /// An explicit review task was resolved. - ReviewDecided, - /// A named signal was delivered. - SignalDelivered, - /// An externally supplied amendment was accepted. - AmendmentAccepted, - /// The run was cancelled. - Cancelled, - /// A compensation registration or generation changed durably. - CompensationProgress, -} - -/// Internal request that dispatches one keyed task generation. +pub enum ExecutionAttemptCancelReason { + /// The run's immutable approved deadline elapsed. + DeadlineExceeded, + /// Another terminal intent fenced all remaining forward work. + RunTerminal, + /// An authorized pause fenced new work and is draining active slices. + PauseRequested, + /// Provider-owned asynchronous work was committed before the slice relinquished compute. + ExternalJobStarted, +} + +/// Durable reason one compensation slice relinquishes its active sandbox ownership. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ExecutionCompensationReleaseIntent { + /// A definitive compensation outcome is ready to settle. + Outcome, + /// A retryable compensation failure is ready to requeue. + Retry, + /// The governed action is parking for tenant review. + Review, + /// Provider-owned asynchronous work was durably started. + ExternalJob, + /// An authorized pause is draining the active slice. + Pause, + /// The exact attempt watchdog elapsed. + Watchdog, + /// The immutable run deadline elapsed. + Deadline, + /// Another terminal run intent fenced the active slice. + RunTerminal, +} + +/// Identity-free cancellation delivery for one exact active task attempt. #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] #[serde(deny_unknown_fields)] -pub struct ExecutionTaskWorkflowRequest { - /// Owning run. +pub struct ExecutionTaskAttemptCancelRequest { + /// Immutable outbox delivery identity and Restate idempotency key. + #[serde(rename = "dispatch_uid")] + pub cancellation_dispatch_uid: Uuid, + /// Owning tenant. + pub tenant_id: TenantId, + /// Owning execution run. pub run_uid: Uuid, - /// Stable workflow key. + /// Stable logical task. pub task_id: ExecutionTaskId, - /// Current generation fence. - pub generation: u64, + /// Exact controller generation that fenced the attempt. + pub controller_generation: u64, + /// Controller generation carried by the immutable attempt resources being released. + pub attempt_controller_generation: u64, + /// Exact logical task generation. + pub task_generation: u64, + /// Exact bounded attempt generation. + pub attempt_generation: u64, + /// Immutable dispatch identity that owns the active slice. + pub active_dispatch_uid: Uuid, + /// Exact active-capacity receipt released only after sandbox ownership is relinquished. + pub capacity_reservation_uid: Uuid, + /// Exact watchdog superseded by cancellation settlement. + pub watchdog_trigger_uid: Uuid, + /// Closed reason for the ownership transfer. + pub reason: ExecutionAttemptCancelReason, +} + +/// Identity-free cancellation delivery for one exact compensation attempt. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ExecutionCompensationAttemptCancelRequest { + /// Immutable outbox delivery identity and Restate idempotency key. + #[serde(rename = "dispatch_uid")] + pub cancellation_dispatch_uid: Uuid, /// Owning tenant. pub tenant_id: TenantId, - /// Optional owning contact. - pub contact_id: Option, - /// Parent session used for policy and model context. - pub session_id: SessionId, - /// Exact authenticated identity inherited from the owning run workflow. - pub identity: Identity, + /// Owning execution run. + pub run_uid: Uuid, + /// Stable compensation registration. + pub compensation_id: CompensationId, + /// Exact controller generation that fenced the attempt. + pub controller_generation: u64, + /// Controller generation carried by the immutable attempt resources being released. + pub attempt_controller_generation: u64, + /// Exact logical compensation generation. + pub compensation_generation: u64, + /// Exact bounded compensation-attempt generation. + pub compensation_attempt_generation: u64, + /// Immutable dispatch identity that owns the active slice. + pub active_dispatch_uid: Uuid, + /// Exact active-capacity receipt released only after sandbox ownership is relinquished. + pub capacity_reservation_uid: Uuid, + /// Exact watchdog superseded by cancellation settlement. + pub watchdog_trigger_uid: Uuid, + /// Closed reason for the compensation ownership transfer. + pub intent: ExecutionCompensationReleaseIntent, +} + +/// Immutable request to cancel one exact asynchronous provider-job generation. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ExecutionExternalJobCancelRequest { + /// Owning tenant. + pub tenant_id: TenantId, + /// Stable MOA external-job identity. + pub external_job_uid: Uuid, + /// Exact provider-job generation. + pub job_generation: u64, + /// Expected provider implementation name. + pub provider: String, + /// Expected provider-issued job identity. + pub provider_job_id: String, + /// Stable provider idempotency key reused for cancellation. + pub idempotency_key: String, +} + +/// Exact task or compensation owner duplicated by a start-recovery trigger. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(tag = "owner_kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum ExecutionExternalJobStartRecoveryOwner { + /// One forward-task attempt. + Task { + /// Stable logical task identity. + task_id: Uuid, + /// Exact task-attempt generation. + attempt_generation: u64, + }, + /// One compensation attempt. + Compensation { + /// Stable compensation identity. + compensation_id: Uuid, + /// Exact compensation logical generation. + compensation_generation: u64, + /// Exact compensation-attempt generation. + compensation_attempt_generation: u64, + }, } -/// Internal request that dispatches one keyed compensation generation. +/// Durable delivery request for crash-safe provider start recovery. #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] #[serde(deny_unknown_fields)] -pub struct ExecutionCompensationWorkflowRequest { +pub struct ExecutionExternalJobStartRecoveryRequest { + /// Owning tenant. + pub tenant_id: TenantId, /// Owning execution run. pub run_uid: Uuid, - /// Stable workflow key derived from the forward task. - pub compensation_id: CompensationId, - /// Current compensation generation fence. - pub generation: u64, + /// Exact task or compensation owner. + pub owner: ExecutionExternalJobStartRecoveryOwner, + /// Stable MOA external-job identity. + pub external_job_uid: Uuid, + /// Exact provider-job generation. + pub job_generation: u64, + /// Declared adapter/provider key reserved before dispatch. + pub provider: String, + /// Stable provider start idempotency key. + pub idempotency_key: String, + /// Exact temporal trigger being delivered. + pub trigger_uid: Uuid, +} + +/// Durable result of one provider start-recovery delivery. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ExecutionExternalJobStartRecoveryResponse { + /// Stable MOA external-job identity. + pub external_job_uid: Uuid, + /// Exact provider-job generation. + pub job_generation: u64, + /// Generation-fenced recovery disposition. + pub outcome: ExecutionExternalJobStartRecoveryResponseOutcome, +} + +/// Typed acknowledgement from a bounded task or compensation watchdog receiver. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ExecutionAttemptWatchdogResponse { + /// Whether trigger delivery may settle or must retry. + pub outcome: ExecutionAttemptWatchdogResponseOutcome, +} + +/// Durable watchdog receiver disposition. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ExecutionAttemptWatchdogResponseOutcome { + /// The exact active attempt and capacity were settled by this invocation. + Settled, + /// Exact work was already settled or became stale before this replay. + ReplayedOrStale, + /// The receiver could not safely settle; trigger delivery must remain retryable. + RetryDelivery, +} + +/// Result of recovering one pre-reserved provider start. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ExecutionExternalJobStartRecoveryResponseOutcome { + /// Provider proved that no work started and the intent was released. + NotStartedReleased, + /// Provider start was found and exact ownership was bound. + StartedBound, + /// Provider outcome remains ambiguous and recovery work was rearmed. + UnknownPreserved, + /// The delivery no longer names the current unbound intent. + StaleDelivery, + /// The intent was already bound or released by another delivery. + AlreadySettled, +} + +/// Durable result of one bounded provider cancellation invocation. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ExecutionExternalJobCancelResponse { + /// Stable MOA external-job identity. + pub external_job_uid: Uuid, + /// Exact provider-job generation. + pub job_generation: u64, + /// Typed settlement result, including generation-fenced no-op deliveries. + pub outcome: ExecutionExternalJobCancelResponseOutcome, +} + +/// Result of one generation-fenced external-job cancellation delivery. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[serde(tag = "disposition", rename_all = "snake_case", deny_unknown_fields)] +pub enum ExecutionExternalJobCancelResponseOutcome { + /// The exact current provider job was called and its result was persisted. + Applied { + /// Typed provider cancellation result. + provider_outcome: AsyncToolJobCancelOutcome, + }, + /// The delivery no longer names the current job generation or provider identity. + StaleDelivery, + /// The job had already reached a terminal state before this delivery. + AlreadyTerminal, + /// No visible job has the supplied MOA identity. + NotFound, +} + +/// Immutable request to reconcile one exact asynchronous provider-job generation. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ExecutionExternalJobReconcileRequest { /// Owning tenant. pub tenant_id: TenantId, - /// Optional owning contact. - pub contact_id: Option, - /// Parent session used for policy and action context. - pub session_id: SessionId, - /// Exact authenticated identity inherited from the owning run workflow. - pub identity: Identity, + /// Stable MOA external-job identity. + pub external_job_uid: Uuid, + /// Exact durable reconcile-trigger identity used as the synthetic provider event fence. + pub trigger_uid: Uuid, + /// Exact provider-job generation. + pub job_generation: u64, + /// Expected provider implementation name. + pub provider: String, + /// Expected provider-issued job identity. + pub provider_job_id: String, + /// Stable provider idempotency key reused for reconciliation. + pub idempotency_key: String, +} + +/// Typed result of one bounded sparse provider reconciliation. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ExecutionExternalJobReconcileResponse { + /// Stable MOA external-job identity. + pub external_job_uid: Uuid, + /// Exact provider-job generation. + pub job_generation: u64, + /// Generation-fenced durable reconciliation disposition. + pub outcome: ExecutionExternalJobReconcileResponseOutcome, +} + +/// Result of one generation-fenced sparse provider reconciliation delivery. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[serde(tag = "disposition", rename_all = "snake_case", deny_unknown_fields)] +pub enum ExecutionExternalJobReconcileResponseOutcome { + /// The exact provider job was observed and its result was persisted. + Applied { + /// Typed progress or terminal provider observation. + provider_outcome: AsyncToolJobCallbackOutcome, + }, + /// The delivery no longer names the current job generation or provider identity. + StaleDelivery, + /// The job had already reached a terminal state before this delivery. + AlreadyTerminal, + /// No visible job has the supplied MOA identity. + NotFound, } /// Encodes a cursor as canonical JSON in URL-safe unpadded base64. @@ -1158,6 +1537,137 @@ fn invalid_cursor(message: &str) -> Error { mod tests { use super::*; + #[test] + fn execution_progress_phase_exhaustively_maps_aggregate_wait_and_pause_states_offline() { + // Pins: run-only progress distinguishes every public storage-only wait and pause phase; + // task-only stale work has no aggregate run-status mapping. + let expected = [ + ( + ExecutionRunStatus::WaitingInput, + ExecutionProgressPhase::WaitingInput, + [true, false, false, false, false], + ), + ( + ExecutionRunStatus::WaitingReview, + ExecutionProgressPhase::WaitingReview, + [false, true, false, false, false], + ), + ( + ExecutionRunStatus::WaitingSignal, + ExecutionProgressPhase::WaitingSignal, + [false, false, true, false, false], + ), + ( + ExecutionRunStatus::WaitingTimer, + ExecutionProgressPhase::WaitingTimer, + [false, false, false, true, false], + ), + ( + ExecutionRunStatus::WaitingExternal, + ExecutionProgressPhase::WaitingExternal, + [false, false, false, false, true], + ), + ( + ExecutionRunStatus::PauseRequested, + ExecutionProgressPhase::PauseRequested, + [false; 5], + ), + ( + ExecutionRunStatus::Pausing, + ExecutionProgressPhase::Pausing, + [false; 5], + ), + ( + ExecutionRunStatus::Paused, + ExecutionProgressPhase::Paused, + [false; 5], + ), + ]; + for (status, phase, waits) in expected { + assert_eq!( + execution_progress_phase_from_flags( + status, waits[0], waits[1], waits[2], waits[3], waits[4] + ), + phase, + "status {status:?}" + ); + } + + let aggregate_running = [ + ExecutionRunStatus::AwaitingConfirmation, + ExecutionRunStatus::Queued, + ExecutionRunStatus::Running, + ExecutionRunStatus::WaitingReplan, + ExecutionRunStatus::Compensating, + ExecutionRunStatus::Completed, + ExecutionRunStatus::Partial, + ExecutionRunStatus::Blocked, + ExecutionRunStatus::Unsupported, + ExecutionRunStatus::Failed, + ExecutionRunStatus::Cancelled, + ]; + for status in aggregate_running { + assert_eq!( + execution_progress_phase_from_flags(status, false, false, false, false, false), + ExecutionProgressPhase::Running, + "status {status:?}" + ); + } + } + + #[test] + fn execution_blocker_audience_uses_exact_scalar_priority_not_reason_samples_offline() { + // Pins: truncated display samples cannot hide a higher-priority exact blocker; scalar + // audience counters always order User > TenantReviewer > External > Agent > System. + assert_eq!( + execution_blocker_audience_from_flags(true, true, true, true), + Some(ExecutionBlockerAudience::User) + ); + assert_eq!( + execution_blocker_audience_from_flags(false, true, true, true), + Some(ExecutionBlockerAudience::TenantReviewer) + ); + assert_eq!( + execution_blocker_audience_from_flags(false, false, true, true), + Some(ExecutionBlockerAudience::External) + ); + assert_eq!( + execution_blocker_audience_from_flags(false, false, false, true), + Some(ExecutionBlockerAudience::System) + ); + assert_eq!( + execution_blocker_audience_from_flags(false, false, false, false), + None + ); + } + + // Pins: cancel outbox payloads expose the SQL-validated `dispatch_uid` key while + // keeping the Rust field explicit about its cancellation-delivery ownership. + #[test] + fn attempt_cancel_payload_uses_dispatch_uid_wire_key_offline() { + let request = ExecutionTaskAttemptCancelRequest { + cancellation_dispatch_uid: Uuid::from_u128(1), + tenant_id: TenantId::from(Uuid::from_u128(2)), + run_uid: Uuid::from_u128(3), + task_id: ExecutionTaskId::from_uuid(Uuid::from_u128(4)), + controller_generation: 5, + attempt_controller_generation: 4, + task_generation: 6, + attempt_generation: 7, + active_dispatch_uid: Uuid::from_u128(8), + capacity_reservation_uid: Uuid::from_u128(9), + watchdog_trigger_uid: Uuid::from_u128(10), + reason: ExecutionAttemptCancelReason::DeadlineExceeded, + }; + + let value = serde_json::to_value(request).expect("serialize task cancel request"); + assert_eq!( + value.get("dispatch_uid"), + Some(&serde_json::json!(Uuid::from_u128(1))) + ); + assert!(value.get("cancellation_dispatch_uid").is_none()); + } + #[test] fn cursor_round_trip_is_url_safe_and_strict() { // Pins: public cursors are canonical URL-safe base64 and malformed data is rejected. diff --git a/crates/moa-execution/tests/compiler.rs b/crates/moa-execution/tests/compiler.rs index b3165580d..8e0bcdd43 100644 --- a/crates/moa-execution/tests/compiler.rs +++ b/crates/moa-execution/tests/compiler.rs @@ -10,8 +10,9 @@ use moa_artifacts::execution_plan::{ ExecutionBudgetLimit, ExecutionCancelPolicy, ExecutionCompensation, ExecutionCondition, ExecutionDeliverable, ExecutionGoalContract, ExecutionNode, ExecutionOperation, ExecutionPlanDefinition, ExecutionReducer, ExecutionReference, ExecutionRequirement, - ExecutionTaskOutcome, ExecutionTaskResult, ExecutionUsage, MapTask, PlanAmendment, - PlanAmendmentOperation, RetryPolicy, + ExecutionTaskOutcome, ExecutionTaskResult, ExecutionTemporalTarget, ExecutionUsage, + ExecutionWaitExpiryAction, ExecutionWaitPolicy, MapTask, PlanAmendment, PlanAmendmentOperation, + RetryPolicy, }; use moa_artifacts::reference::ArtifactRef; use moa_config::ExecutionConfig; @@ -31,8 +32,8 @@ use moa_execution::{ ExecutionValidationSeverity, ValidateAmendmentRequest, compile, validate_amendment, }, state::{ - ExecutionNodeStatus, ExecutionProjection, ExecutionTaskId, ExecutionTaskProjection, - ExecutionTaskStatus, + ExecutionAmendmentProjection, ExecutionNodeStatus, ExecutionProjection, ExecutionTaskId, + ExecutionTaskProjection, ExecutionTaskStatus, }, }; use proptest::{ @@ -213,6 +214,136 @@ fn compile_returns_canonical_hashes_and_exact_retry_estimate() { ); } +#[test] +fn compile_accepts_wait_until_strictly_between_validation_time_and_run_deadline() { + // Pins: a Durable plan may park on one exact timer only inside its approved horizon. + let mut request = valid_request(); + request.plan.nodes[0].operation = ExecutionOperation::WaitUntil { + wake: ExecutionTemporalTarget::At { + at: Utc + .with_ymd_and_hms(2026, 7, 15, 0, 0, 0) + .single() + .expect("wait timestamp"), + }, + result: json!({ "order_id": "ord-1" }), + }; + + let outcome = compile(request); + + assert!(outcome.compiled.is_some(), "{:?}", outcome.report.issues); + assert!(outcome.report.issues.is_empty()); +} + +#[test] +fn compile_accepts_wait_entry_relative_targets_inside_the_remaining_horizon() { + // Pins: reusable relative waits remain relative until their task enters storage-only waiting. + let mut request = valid_request(); + request.plan.input_wait_policy = ExecutionWaitPolicy { + expiry: ExecutionTemporalTarget::After { + delay_seconds: 7_200, + }, + on_expiry: ExecutionWaitExpiryAction::FailTask, + }; + request.plan.nodes[0].operation = ExecutionOperation::WaitUntil { + wake: ExecutionTemporalTarget::After { + delay_seconds: 3_600, + }, + result: json!({ "order_id": "ord-1" }), + }; + + let outcome = compile(request); + + let compiled = outcome + .compiled + .expect("relative waits inside the run horizon should compile"); + assert!(matches!( + compiled.plan.definition.nodes[0].operation, + ExecutionOperation::WaitUntil { + wake: ExecutionTemporalTarget::After { + delay_seconds: 3_600 + }, + .. + } + )); +} + +#[test] +fn compile_rejects_missing_expired_and_out_of_horizon_run_deadlines() { + // Pins: every Durable run has one future absolute deadline within maximum_horizon. + let cases = [ + (None, "missing_deadline"), + (Some(now()), "deadline_exceeded"), + ( + Some( + Utc.with_ymd_and_hms(2026, 8, 13, 12, 0, 1) + .single() + .expect("outside horizon"), + ), + "deadline_out_of_horizon", + ), + ]; + for (deadline_at, expected_code) in cases { + let mut request = valid_request(); + request.approved_budget.deadline_at = deadline_at; + + let outcome = compile(request); + + assert!( + outcome.compiled.is_none(), + "compiler accepted {expected_code}" + ); + assert!( + outcome + .report + .issues + .iter() + .any(|issue| issue.code == expected_code), + "missing {expected_code}: {:?}", + outcome.report.issues + ); + } +} + +#[test] +fn compile_rejects_timer_and_wait_expiry_at_or_after_run_deadline() { + // Pins: timers and wait fallbacks always settle before the enclosing run deadline. + let deadline = generous_budget() + .deadline_at + .expect("fixture deadline must exist"); + let mut timer = valid_request(); + timer.plan.nodes[0].operation = ExecutionOperation::WaitUntil { + wake: ExecutionTemporalTarget::At { at: deadline }, + result: json!({}), + }; + let timer_outcome = compile(timer); + assert!(timer_outcome.compiled.is_none()); + assert!( + timer_outcome + .report + .issues + .iter() + .any(|issue| issue.code == "temporal_target_after_deadline") + ); + + let mut review = valid_request(); + review.plan.nodes[0].operation = ExecutionOperation::Review { + prompt: "Approve the result".to_string(), + wait_policy: ExecutionWaitPolicy { + expiry: ExecutionTemporalTarget::At { at: deadline }, + on_expiry: ExecutionWaitExpiryAction::ContinueWith { output: json!({}) }, + }, + }; + let review_outcome = compile(review); + assert!(review_outcome.compiled.is_none()); + assert!( + review_outcome + .report + .issues + .iter() + .any(|issue| issue.code == "temporal_target_after_deadline") + ); +} + #[test] fn plan_hash_treats_node_declaration_order_as_nonsemantic() { // Pins: an amended DAG that returns to the same nodes cannot evade duplicate-plan detection @@ -535,6 +666,72 @@ fn compile_rejects_unpromised_read_non_idempotent_and_unauthorized_compensators( assert_issue_code(compile(unauthorized), "capability_not_authorized"); } +#[test] +fn compile_rejects_hand_and_skill_code_compensators_that_require_sandbox() { + // Pins: the compiler cannot admit a rollback contract that the durable compensation runtime + // will deterministically reject because no sandbox workspace belongs to compensation. + let sandbox_sources = [ + CapabilitySource::HandTool { + name: "orders.rollback".to_string(), + }, + CapabilitySource::SkillCode { + skill_ref: ArtifactRef::from_str("skill://orders") + .expect("valid skill reference fixture"), + revision_uid: Uuid::from_u128(401), + entrypoint: "rollback.py".to_string(), + }, + ]; + for source in sandbox_sources { + let mut request = compensated_request(); + let compensator = &mut request.catalog.capabilities[1]; + compensator.execution_class = ExecutionClass::Compute; + compensator.requires_sandbox = true; + compensator.source = source.clone(); + compensator.policy_context = CapabilityPolicyContext::registered(source); + rehash_catalog(&mut request); + + let outcome = compile(request); + + assert_issue_code(outcome.clone(), "sandbox_compensator_unsupported"); + assert!(outcome.report.issues.iter().any(|issue| { + issue.code == "sandbox_compensator_unsupported" + && issue.path == "plan.nodes[0].compensation.compensator" + && issue + .message + .contains("durable compensation does not support") + })); + } +} + +#[test] +fn compile_accepts_idempotent_async_non_sandbox_compensator() { + // Pins: asynchronous external rollback remains valid when it is idempotent and explicitly + // cataloged as not requiring a sandbox workspace. + let mut request = compensated_request(); + let source = CapabilitySource::McpTool { + server: "orders".to_string(), + tool_name: "orders.rollback".to_string(), + remote_name: "rollback".to_string(), + }; + let compensator = &mut request.catalog.capabilities[1]; + compensator.async_mode = moa_core::types::tools::ToolAsyncMode::MayReturnExternalJob { + provider: "orders-provider".to_string(), + }; + compensator.execution_class = ExecutionClass::External; + compensator.requires_sandbox = false; + compensator.source = source.clone(); + compensator.policy_context = CapabilityPolicyContext::registered(source); + rehash_catalog(&mut request); + + let outcome = compile(request); + + assert!( + outcome.compiled.is_some(), + "idempotent non-sandbox async compensator should compile: {:?}", + outcome.report.issues + ); +} + #[test] fn compile_rejects_compensation_mapping_outside_governed_schemas() { // Pins: rollback mappings can read only committed forward fields and write declared undo input. @@ -1153,11 +1350,11 @@ fn amendment_rejects_unknown_reference_path_before_persistence() { node: replacement, }], }, - projection: ExecutionProjection { + projection: amendment_projection(ExecutionProjection { plan_revision: 4, node_statuses: BTreeMap::new(), tasks: vec![], - }, + }), catalog: request.catalog, authorization: request.authorization, remaining_budget: generous_budget(), @@ -1182,11 +1379,11 @@ fn amendment_replaces_only_pending_work_with_a_distinct_identity() { let mut statuses = BTreeMap::new(); statuses.insert("lookup".to_string(), ExecutionNodeStatus::Completed); statuses.insert("output".to_string(), ExecutionNodeStatus::Pending); - let projection = ExecutionProjection { + let projection = amendment_projection(ExecutionProjection { plan_revision: 4, node_statuses: statuses, tasks: vec![], - }; + }); let replacement = ExecutionNode { id: "replacement_output".to_string(), requirement_ids: vec!["req_one".to_string()], @@ -1259,11 +1456,11 @@ fn amendment_cannot_remove_compensation_after_forward_work_starts() { node: replacement, }], }, - projection: ExecutionProjection { + projection: amendment_projection(ExecutionProjection { plan_revision: 3, node_statuses: BTreeMap::from([("lookup".to_string(), ExecutionNodeStatus::Running)]), tasks: vec![task_projection("lookup", ExecutionTaskStatus::Running)], - }, + }), catalog: request.catalog, authorization: request.authorization, remaining_budget: generous_budget(), @@ -1305,14 +1502,14 @@ fn amendment_validation_retains_remaining_estimate_without_completed_work() { node: replacement, }], }, - projection: ExecutionProjection { + projection: amendment_projection(ExecutionProjection { plan_revision: 4, node_statuses: BTreeMap::from([ ("lookup".to_string(), ExecutionNodeStatus::Completed), ("output".to_string(), ExecutionNodeStatus::Pending), ]), tasks: vec![task_projection("lookup", ExecutionTaskStatus::Completed)], - }, + }), catalog: request.catalog, authorization: request.authorization, remaining_budget: ExecutionBudgetLimit { @@ -1453,11 +1650,11 @@ fn amendment_rejects_references_unused_by_the_active_plan() { }, ], }, - projection: ExecutionProjection { + projection: amendment_projection(ExecutionProjection { plan_revision: 3, node_statuses: BTreeMap::new(), tasks: vec![], - }, + }), catalog: request.catalog, authorization: request.authorization, remaining_budget: generous_budget(), @@ -1572,11 +1769,11 @@ fn amendment_rejects_removed_or_increased_pending_node_budget() { }, ], }, - projection: ExecutionProjection { + projection: amendment_projection(ExecutionProjection { plan_revision: 5, node_statuses: BTreeMap::new(), tasks: vec![], - }, + }), catalog: request.catalog, authorization: request.authorization, remaining_budget: generous_budget(), @@ -1674,7 +1871,7 @@ fn waiting_replan_amendment_removes_origin_and_replaces_every_pending_dependent( let compiled = compile(request.clone()) .compiled .expect("compile replan fixture"); - let projection = ExecutionProjection { + let projection = amendment_projection(ExecutionProjection { plan_revision: 7, node_statuses: BTreeMap::from([ ("seed".to_string(), ExecutionNodeStatus::Completed), @@ -1682,7 +1879,7 @@ fn waiting_replan_amendment_removes_origin_and_replaces_every_pending_dependent( ("output".to_string(), ExecutionNodeStatus::Pending), ]), tasks: vec![waiting_replan_task("lookup")], - }; + }); let lookup = compiled .plan .definition @@ -1830,11 +2027,11 @@ fn map_replacement_accepts_literal_subset_and_rejects_scope_broadening() { }, ], }, - projection: ExecutionProjection { + projection: amendment_projection(ExecutionProjection { plan_revision: 2, node_statuses: BTreeMap::new(), tasks: vec![], - }, + }), catalog: request.catalog, authorization: request.authorization, remaining_budget: generous_budget(), @@ -1898,6 +2095,7 @@ fn valid_request() -> CompileExecutionRequest { }, plan: ExecutionPlanDefinition { cancel_policy: ExecutionCancelPolicy::RetainEffects, + input_wait_policy: default_wait_policy(), input_schema: json!({ "type": "object", "required": ["order_id"], @@ -2076,11 +2274,11 @@ fn amendment_validation_for_output( evidence: json!({}), operations: vec![operation], }, - projection: ExecutionProjection { + projection: amendment_projection(ExecutionProjection { plan_revision: 9, node_statuses, tasks: vec![task_projection("output", task_status)], - }, + }), catalog: request.catalog, authorization: request.authorization, remaining_budget: generous_budget(), @@ -2106,7 +2304,9 @@ fn capability(name: &str) -> ExecutionCapability { risk_level: RiskLevel::Low, default_effect: ActionPolicyEffect::Allow, idempotency_class: IdempotencyClass::Idempotent, + async_mode: moa_core::types::tools::ToolAsyncMode::SynchronousOnly, execution_class: ExecutionClass::Data, + requires_sandbox: false, policy_context: CapabilityPolicyContext::registered(source.clone()), source, estimate: ExecutionEstimate { @@ -2355,6 +2555,33 @@ fn task_projection(node_id: &str, status: ExecutionTaskStatus) -> ExecutionTaskP } } +fn amendment_projection(projection: ExecutionProjection) -> ExecutionAmendmentProjection { + let mut started_node_ids = projection + .node_statuses + .iter() + .filter(|(_, status)| **status != ExecutionNodeStatus::Pending) + .map(|(node_id, _)| node_id.clone()) + .collect::>(); + started_node_ids.extend( + projection + .tasks + .iter() + .filter(|task| task.status != ExecutionTaskStatus::Pending) + .map(|task| task.node_id.clone()), + ); + let replan_tasks = projection + .tasks + .into_iter() + .filter(|task| task.status == ExecutionTaskStatus::WaitingReplan) + .collect(); + ExecutionAmendmentProjection { + plan_revision: projection.plan_revision, + node_statuses: projection.node_statuses, + started_node_ids, + replan_tasks, + } +} + fn generous_budget() -> ExecutionBudgetLimit { ExecutionBudgetLimit { max_cost_microusd: Some(1_000_000), @@ -2363,13 +2590,25 @@ fn generous_budget() -> ExecutionBudgetLimit { max_tool_calls: Some(1_000), max_retrieved_bytes: Some(1_000_000), deadline_at: Some( - Utc.with_ymd_and_hms(2030, 1, 1, 0, 0, 0) + Utc.with_ymd_and_hms(2026, 8, 1, 0, 0, 0) .single() .expect("time"), ), } } +fn default_wait_policy() -> ExecutionWaitPolicy { + ExecutionWaitPolicy { + expiry: ExecutionTemporalTarget::At { + at: Utc + .with_ymd_and_hms(2026, 7, 20, 0, 0, 0) + .single() + .expect("wait expiry"), + }, + on_expiry: ExecutionWaitExpiryAction::FailTask, + } +} + fn now() -> chrono::DateTime { Utc.with_ymd_and_hms(2026, 7, 13, 12, 0, 0) .single() diff --git a/crates/moa-execution/tests/completion.rs b/crates/moa-execution/tests/completion.rs index fc108a8a3..b4e24a1a3 100644 --- a/crates/moa-execution/tests/completion.rs +++ b/crates/moa-execution/tests/completion.rs @@ -5,8 +5,8 @@ use moa_artifacts::execution_plan::{ CapabilityReference, CompletionCheck, CompletionCheckKind, CoverageRequirement, ExecutionBudgetLimit, ExecutionCancelPolicy, ExecutionCitation, ExecutionDeliverable, ExecutionGoalContract, ExecutionNode, ExecutionOperation, ExecutionPlanDefinition, - ExecutionRequirement, ExecutionTaskOutcome, ExecutionTaskResult, ExecutionUsage, MapTask, - RetryPolicy, + ExecutionRequirement, ExecutionTaskOutcome, ExecutionTaskResult, ExecutionTemporalTarget, + ExecutionUsage, ExecutionWaitExpiryAction, ExecutionWaitPolicy, MapTask, RetryPolicy, }; use moa_execution::{ budget::BudgetLedger, @@ -594,6 +594,12 @@ fn canonical(nodes: Vec) -> CanonicalExecutionPlan { CanonicalExecutionPlan { definition: ExecutionPlanDefinition { cancel_policy: ExecutionCancelPolicy::RetainEffects, + input_wait_policy: ExecutionWaitPolicy { + expiry: ExecutionTemporalTarget::After { + delay_seconds: 3_600, + }, + on_expiry: ExecutionWaitExpiryAction::FailTask, + }, input_schema: json!({ "type": "object" }), output_schema: json!({ "type": "object" }), nodes, diff --git a/crates/moa-execution/tests/execution_db.rs b/crates/moa-execution/tests/execution_db.rs index 5e41dcd76..31278b91f 100644 --- a/crates/moa-execution/tests/execution_db.rs +++ b/crates/moa-execution/tests/execution_db.rs @@ -1,14 +1,32 @@ //! Concurrent PostgreSQL contract coverage for durable execution-run persistence. +#[path = "execution_db/active_run_capacity_db.rs"] +mod active_run_capacity_db; +#[path = "execution_db/amendment_projection_db.rs"] +mod amendment_projection_db; #[path = "execution_db/budget_and_materialization_db.rs"] mod budget_and_materialization_db; +#[path = "execution_db/compensation_attempts_db.rs"] +mod compensation_attempts_db; #[path = "execution_db/compensation_db.rs"] mod compensation_db; +#[path = "execution_db/completion_projection_db.rs"] +mod completion_projection_db; +#[path = "execution_db/execution_capacity_db.rs"] +mod execution_capacity_db; +#[path = "execution_db/incremental_scheduler_db.rs"] +mod incremental_scheduler_db; +#[path = "execution_db/long_horizon_state_db.rs"] +mod long_horizon_state_db; #[path = "execution_db/outcomes_and_replan_db.rs"] mod outcomes_and_replan_db; #[path = "execution_db/planning_and_audit_db.rs"] mod planning_and_audit_db; +#[path = "execution_db/retention_db.rs"] +mod retention_db; #[path = "execution_db/scope_and_lifecycle_db.rs"] mod scope_and_lifecycle_db; #[path = "execution_db/support.rs"] mod support; +#[path = "execution_db/trigger_outbox_db.rs"] +mod trigger_outbox_db; diff --git a/crates/moa-execution/tests/execution_db/active_run_capacity_db.rs b/crates/moa-execution/tests/execution_db/active_run_capacity_db.rs new file mode 100644 index 000000000..7eaacdfc1 --- /dev/null +++ b/crates/moa-execution/tests/execution_db/active_run_capacity_db.rs @@ -0,0 +1,1366 @@ +//! Lifetime active-run admission and release contracts. + +use moa_artifacts::execution_plan::{ExecutionNode, ExecutionOperation}; +use moa_core::events::{ExecutionBlockerAudience, ExecutionProgressPhase}; +use moa_execution::repository::capacity::{ + ExecutionCapacityDimension, execution_capacity_reservation_uid, +}; +use moa_execution::repository::{ + RunDeadlineArmOutcome, + terminal::{RunTriggerDrainOutcome, RunTriggerDrainRequest}, + trigger::{ExecutionTriggerKind, NewExecutionTrigger}, +}; +use sqlx::Row; + +use super::support::*; + +fn output_node(id: &str, depends_on: &[&str]) -> ExecutionNode { + ExecutionNode { + id: id.to_string(), + requirement_ids: Vec::new(), + depends_on: depends_on + .iter() + .map(|value| (*value).to_string()) + .collect(), + when: None, + input: json!({}), + output_schema: json!({ "type": "object" }), + operation: ExecutionOperation::Output { value: json!({}) }, + compensation: None, + retry: RetryPolicy { + max_attempts: 1, + initial_backoff_ms: 1, + max_backoff_ms: 1, + }, + budget: None, + } +} + +fn two_node_run(tenant_id: TenantId, key: &str) -> NewExecutionRun { + let mut candidate = new_run(tenant_id, None, key, ExecutionRunStatus::Queued, budget(2)); + candidate.plan.definition.nodes = + vec![output_node("first", &[]), output_node("second", &["first"])]; + candidate.plan.estimate.tasks = 2; + candidate +} + +fn successful_request( + run: &moa_execution::repository::ExecutionRunRecord, +) -> Result { + let evaluation = CompletionEvaluation { + status: CompletionStatus::Completed, + limit_stop: None, + checks: Vec::new(), + satisfied_requirement_ids: Vec::new(), + unsatisfied_requirement_ids: Vec::new(), + gaps: Vec::new(), + }; + let cause = ExecutionTerminalCause::Completion { limit_stop: None }; + let terminal = TerminalProjection::Completed { output: json!({}) }; + let evidence = terminal_evidence_from_evaluation(cause.clone(), &evaluation)?; + let reason = execution_terminal_reason(&cause, &terminal, &evaluation)?; + Ok(RunFinalizationRequest { + run_uid: run.run_uid, + expected_revision: run.plan_revision, + expected_wake_epoch: run.wake_epoch, + terminal_projection: terminal, + completion_evaluation: evaluation, + terminal_evidence: evidence, + terminal_reason: reason, + }) +} + +#[tokio::test] +async fn run_admission_reserves_exact_capacity_seeds_nodes_and_rolls_back_saturation_db() +-> TestResult { + // Pins: one tenant-scoped transaction inserts the run, set-seeds every canonical node, + // reserves the deterministic fleet+tenant ActiveRuns receipt, and can see only its own + // tenant bucket plus the shared fleet bucket; saturation leaves no partial run. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let config = ExecutionConfig { + max_fleet_active_runs: 1, + max_tenant_active_runs: 1, + ..ExecutionConfig::default() + }; + let candidate = two_node_run(tenant_id, "active-run-first"); + let replay_candidate = candidate.clone(); + + let RunAdmissionOutcome::Admitted(run) = + create_run_with_config(&repository, scope, &config, candidate).await? + else { + panic!("first run must own the only active-run slot"); + }; + let expected_receipt = execution_capacity_reservation_uid( + ExecutionCapacityDimension::ActiveRuns, + run.run_uid, + None, + ); + let receipt = sqlx::query( + "SELECT reservation_uid, controller_generation, state \ + FROM moa.execution_capacity_reservation \ + WHERE run_uid = $1 AND resource_dimension = 'active_runs'", + ) + .bind(run.run_uid) + .fetch_one(&pool) + .await?; + assert_eq!( + receipt.try_get::("reservation_uid")?, + expected_receipt + ); + assert_eq!(receipt.try_get::("controller_generation")?, 1); + assert_eq!(receipt.try_get::("state")?, "reserved"); + + let deadline_boundary: (String, String, String, chrono::DateTime) = sqlx::query_as( + "SELECT trigger.state, dispatch.state, capacity.state, trigger.due_at \ + FROM moa.execution_trigger AS trigger \ + JOIN moa.execution_dispatch_outbox AS dispatch USING (trigger_uid) \ + JOIN moa.execution_capacity_reservation AS capacity USING (trigger_uid) \ + WHERE trigger.run_uid = $1 AND trigger.trigger_kind = 'run_deadline'", + ) + .bind(run.run_uid) + .fetch_one(&pool) + .await?; + assert_eq!( + deadline_boundary, + ( + "pending".to_string(), + "pending".to_string(), + "reserved".to_string(), + run.approved_budget.deadline_at.expect("fixture deadline"), + ), + "admission must durably own its deadline before a controller can activate" + ); + let initial_activation: (String, i64, i64) = sqlx::query_as( + "SELECT state, controller_generation, wake_epoch \ + FROM moa.execution_dispatch_outbox \ + WHERE run_uid = $1 AND dispatch_kind = 'run_activation'", + ) + .bind(run.run_uid) + .fetch_one(&pool) + .await?; + assert_eq!(initial_activation, ("pending".to_string(), 1, 1)); + assert_eq!(run.wake_epoch, 1); + assert_eq!(run.next_wake_at, run.approved_budget.deadline_at); + + let nodes = sqlx::query( + "SELECT node_state_uid, node_id, node_order, dependency_count, \ + remaining_dependency_count \ + FROM moa.execution_node_state WHERE run_uid = $1 ORDER BY node_order", + ) + .bind(run.run_uid) + .fetch_all(&pool) + .await?; + assert_eq!(nodes.len(), 2); + assert_eq!(nodes[0].try_get::("node_id")?, "first"); + assert_eq!(nodes[0].try_get::("node_order")?, 0); + assert_eq!(nodes[0].try_get::("dependency_count")?, 0); + assert_eq!(nodes[1].try_get::("node_id")?, "second"); + assert_eq!(nodes[1].try_get::("node_order")?, 1); + assert_eq!(nodes[1].try_get::("dependency_count")?, 1); + assert_eq!(nodes[1].try_get::("remaining_dependency_count")?, 1); + assert_eq!( + nodes[0].try_get::("node_state_uid")?, + Uuid::new_v5(&run.run_uid, b"first") + ); + + let RunAdmissionOutcome::Replayed(replayed) = + create_run_with_config(&repository, scope, &config, replay_candidate).await? + else { + panic!("the exact idempotency replay must reuse its admitted run"); + }; + assert_eq!(replayed.run_uid, run.run_uid); + + let saturated_key = "active-run-saturated"; + assert!(matches!( + create_run_with_config( + &repository, + scope, + &config, + two_node_run(tenant_id, saturated_key), + ) + .await?, + RunAdmissionOutcome::CapacitySaturated { + dimension: ExecutionCapacityDimension::ActiveRuns + } + )); + assert!( + repository + .load_run_by_idempotency_key(scope, tenant_id, None, saturated_key) + .await? + .is_none(), + "capacity saturation must roll back the inserted run" + ); + let fleet_reserved: i64 = sqlx::query_scalar( + "SELECT reserved_quantity FROM moa.execution_capacity_bucket \ + WHERE scope_kind = 'fleet' AND resource_dimension = 'active_runs'", + ) + .fetch_one(&pool) + .await?; + assert_eq!(fleet_reserved, 1); + let tenant_reserved: i64 = sqlx::query_scalar( + "SELECT reserved_quantity FROM moa.execution_capacity_bucket \ + WHERE scope_kind = 'tenant' AND tenant_id = $1 \ + AND resource_dimension = 'active_runs'", + ) + .bind(tenant_id.0) + .fetch_one(&pool) + .await?; + assert_eq!(tenant_reserved, 1); + + let mut owner = moa_db::ScopedConn::begin_tenant(&pool, tenant_id).await?; + owner.assume_app_role().await?; + let owner_buckets: Vec<(String, Option, i64)> = sqlx::query_as( + "SELECT scope_kind, tenant_id, reserved_quantity \ + FROM moa.execution_capacity_bucket \ + WHERE resource_dimension = 'active_runs' \ + ORDER BY scope_kind", + ) + .fetch_all(owner.as_mut()) + .await?; + owner.commit().await?; + assert_eq!( + owner_buckets, + vec![ + ("fleet".to_string(), None, 1), + ("tenant".to_string(), Some(tenant_id.0), 1), + ] + ); + + let other_tenant_id = TenantId::new(); + let mut other = moa_db::ScopedConn::begin_tenant(&pool, other_tenant_id).await?; + other.assume_app_role().await?; + let other_buckets: Vec<(String, Option, i64)> = sqlx::query_as( + "SELECT scope_kind, tenant_id, reserved_quantity \ + FROM moa.execution_capacity_bucket \ + WHERE resource_dimension = 'active_runs' \ + ORDER BY scope_kind", + ) + .fetch_all(other.as_mut()) + .await?; + other.commit().await?; + assert_eq!( + other_buckets, + vec![("fleet".to_string(), None, 1)], + "another tenant may observe the shared fleet ceiling but not the owner's bucket" + ); + Ok(()) +} + +#[tokio::test] +async fn resident_run_entitlement_caps_active_plus_parked_at_one_db() -> TestResult { + // Pins: ParkedRuns is the resident-run entitlement ceiling, so a cap of one rejects a + // second admission with the typed ParkedRuns dimension whether the resident is active or + // storage-only parked, and neither rejection leaves a partial run. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let repository = ExecutionRepository::new(test_db.store().pool().clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let config = ExecutionConfig { + max_fleet_active_runs: 1, + max_tenant_active_runs: 1, + max_fleet_parked_runs: 1, + max_tenant_parked_runs: 1, + ..ExecutionConfig::default() + }; + let RunAdmissionOutcome::Admitted(first) = create_run_with_config( + &repository, + scope, + &config, + new_run( + tenant_id, + None, + "resident-entitlement-first", + ExecutionRunStatus::Queued, + budget(1), + ), + ) + .await? + else { + panic!("first resident run must be admitted"); + }; + let second = || { + new_run( + tenant_id, + None, + "resident-entitlement-second", + ExecutionRunStatus::Queued, + budget(1), + ) + }; + assert!(matches!( + create_run_with_config(&repository, scope, &config, second()).await?, + RunAdmissionOutcome::CapacitySaturated { + dimension: ExecutionCapacityDimension::ParkedRuns + } + )); + assert!(matches!( + repository + .claim_controller_wake( + scope, + first.run_uid, + first.controller_generation, + first.wake_epoch, + ) + .await?, + RunControllerClaimOutcome::Claimed(_) + )); + assert!(matches!( + repository + .complete_controller_wake( + scope, + &config, + first.run_uid, + RunControllerCompletionRequest { + controller_generation: first.controller_generation, + wake_epoch: first.wake_epoch, + checkpoint: ExecutionRunActivationCheckpoint { + status: ExecutionRunStatus::WaitingInput, + activation_state: ExecutionActivationState::Idle, + next_wake_at: first.approved_budget.deadline_at, + waiting_since: Some(Utc::now()), + ready_task_count: 0, + active_task_count: 0, + }, + continuation_payload: None, + continuation_not_before_at: Utc::now(), + }, + ) + .await?, + RunControllerCompletionOutcome::Applied { .. } + )); + assert!(matches!( + create_run_with_config(&repository, scope, &config, second()).await?, + RunAdmissionOutcome::CapacitySaturated { + dimension: ExecutionCapacityDimension::ParkedRuns + } + )); + assert!( + repository + .load_run_by_idempotency_key(scope, tenant_id, None, "resident-entitlement-second",) + .await? + .is_none(), + "joint entitlement saturation must roll back the run row" + ); + Ok(()) +} + +#[tokio::test] +async fn scheduled_trigger_saturation_rolls_back_the_whole_run_admission_db() -> TestResult { + // Pins: ActiveRuns, node rows, the immutable deadline, and initial activation are one + // admission transaction; ScheduledTriggers saturation cannot leave any partial run evidence. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let config = ExecutionConfig { + max_fleet_active_runs: 2, + max_tenant_active_runs: 2, + max_fleet_scheduled_triggers: 1, + max_tenant_scheduled_triggers: 1, + ..ExecutionConfig::default() + }; + + assert!(matches!( + create_run_with_config( + &repository, + scope, + &config, + two_node_run(tenant_id, "scheduled-trigger-first"), + ) + .await?, + RunAdmissionOutcome::Admitted(_) + )); + let saturated_key = "scheduled-trigger-saturated"; + assert!(matches!( + create_run_with_config( + &repository, + scope, + &config, + two_node_run(tenant_id, saturated_key), + ) + .await?, + RunAdmissionOutcome::CapacitySaturated { + dimension: ExecutionCapacityDimension::ScheduledTriggers + } + )); + assert!( + repository + .load_run_by_idempotency_key(scope, tenant_id, None, saturated_key) + .await? + .is_none() + ); + let counts: (i64, i64, i64, i64) = sqlx::query_as( + "SELECT \ + (SELECT count(*) FROM moa.execution_run WHERE tenant_id = $1), \ + (SELECT count(*) FROM moa.execution_node_state WHERE tenant_id = $1), \ + (SELECT count(*) FROM moa.execution_trigger WHERE tenant_id = $1), \ + (SELECT count(*) FROM moa.execution_dispatch_outbox WHERE tenant_id = $1)", + ) + .bind(tenant_id.0) + .fetch_one(&pool) + .await?; + assert_eq!(counts, (1, 2, 1, 2)); + let reserved: Vec<(String, i64)> = sqlx::query_as( + "SELECT resource_dimension, sum(reserved_quantity)::BIGINT \ + FROM moa.execution_capacity_bucket \ + WHERE resource_dimension IN ('active_runs', 'scheduled_triggers') \ + GROUP BY resource_dimension ORDER BY resource_dimension", + ) + .fetch_all(&pool) + .await?; + assert_eq!( + reserved, + vec![ + ("active_runs".to_string(), 2), + ("scheduled_triggers".to_string(), 2), + ], + "fleet and tenant buckets each retain only the first run's receipt" + ); + Ok(()) +} + +#[tokio::test] +async fn terminal_finalization_releases_active_run_once_and_capacity_is_reusable_db() -> TestResult +{ + // Pins: successful terminal settlement releases the lifetime receipt and clears any stale + // run-budget reservation before the terminal constraint fires; replay cannot underflow it. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let config = ExecutionConfig { + max_fleet_active_runs: 1, + max_tenant_active_runs: 1, + ..ExecutionConfig::default() + }; + let RunAdmissionOutcome::Admitted(run) = create_run_with_config( + &repository, + scope, + &config, + new_run( + tenant_id, + None, + "active-run-terminal", + ExecutionRunStatus::Queued, + budget(1), + ), + ) + .await? + else { + panic!("terminal fixture must be admitted"); + }; + let running = claim_running_controller(&repository, scope, &config, &run).await?; + let RunDeadlineArmOutcome::Armed(deadline) = repository + .arm_run_deadline(scope, run.run_uid, running.controller_generation, &config) + .await? + else { + panic!("terminal fixture must arm a deadline trigger"); + }; + assert!(matches!( + repository + .claim_controller_wake( + scope, + running.run_uid, + running.controller_generation, + running.wake_epoch, + ) + .await?, + RunControllerClaimOutcome::Claimed(_) + )); + assert!(matches!( + repository + .drain_run_triggers_page( + scope, + &config, + RunTriggerDrainRequest { + run_uid: running.run_uid, + controller_generation: running.controller_generation, + wake_epoch: running.wake_epoch, + page_limit: 1, + now: Utc::now(), + }, + ) + .await?, + RunTriggerDrainOutcome::ReadyToFinalize { + drained_trigger_count: 1, + .. + } + )); + let stale_reservation = estimate(7); + sqlx::query( + "UPDATE moa.execution_run SET waiting_task_count = 1, waiting_timer_task_count = 1, \ + waiting_reasons_truncated = TRUE, waiting_since = NOW(), next_wake_at = $2, \ + reserved_cost_microusd = $3, reserved_tokens = $4, reserved_tasks = $5, \ + reserved_tool_calls = $6, reserved_retrieved_bytes = $7 \ + WHERE run_uid = $1", + ) + .bind(running.run_uid) + .bind(running.approved_budget.deadline_at) + .bind(i64::try_from(stale_reservation.cost_microusd)?) + .bind(i64::try_from(stale_reservation.tokens)?) + .bind(i64::try_from(stale_reservation.tasks)?) + .bind(i64::try_from(stale_reservation.tool_calls)?) + .bind(i64::try_from(stale_reservation.retrieved_bytes)?) + .execute(&pool) + .await?; + let prefinal = repository + .load_run(scope, running.run_uid) + .await? + .expect("prefinal run"); + let prefinal_progress = execution_progress_from_run(&prefinal)?; + assert_eq!(prefinal.reserved, stale_reservation); + assert_eq!(prefinal_progress.parked_tasks, 1); + assert_eq!( + prefinal_progress.phase, + ExecutionProgressPhase::WaitingTimer + ); + assert_eq!( + prefinal_progress.blocker_audience, + Some(ExecutionBlockerAudience::System) + ); + let request = successful_request(&prefinal)?; + let FinalizationOutcome::Finalized(finalized) = + repository.finalize_run(scope, request.clone()).await? + else { + panic!("drained run must finalize"); + }; + let terminal_progress = execution_progress_from_run(&finalized)?; + assert_eq!(terminal_progress.parked_tasks, 0); + assert_eq!(terminal_progress.blocker_audience, None); + assert_eq!( + finalized.reserved, + ExecutionEstimate::default(), + "terminal finalization must clear the stale ledger before its terminal constraint fires" + ); + assert_eq!(finalized.waiting_task_count, 0); + assert_eq!(finalized.waiting_timer_task_count, 0); + assert!(!finalized.waiting_reasons_truncated); + assert_eq!(finalized.waiting_since, None); + assert_eq!(finalized.next_wake_at, None); + assert_eq!( + finalized.activation_state, + ExecutionActivationState::Terminal + ); + assert_eq!(finalized.processed_wake_epoch, prefinal.wake_epoch); + assert_eq!(finalized.wake_epoch, prefinal.wake_epoch + 1); + assert!(matches!( + repository.finalize_run(scope, request).await?, + FinalizationOutcome::Replayed(_) + )); + let bucket: (i64, i64) = sqlx::query_as( + "SELECT reserved_quantity, limit_value FROM moa.execution_capacity_bucket \ + WHERE scope_kind = 'fleet' AND resource_dimension = 'active_runs'", + ) + .fetch_one(&pool) + .await?; + assert_eq!( + bucket, + (0, 1), + "terminal replay must not underflow capacity" + ); + let tenant_bucket: (i64, i64) = sqlx::query_as( + "SELECT reserved_quantity, limit_value FROM moa.execution_capacity_bucket \ + WHERE scope_kind = 'tenant' AND tenant_id = $1 \ + AND resource_dimension = 'active_runs'", + ) + .bind(tenant_id.0) + .fetch_one(&pool) + .await?; + assert_eq!( + tenant_bucket, + (0, 1), + "terminal replay must not underflow tenant capacity" + ); + let receipt_state: String = sqlx::query_scalar( + "SELECT state FROM moa.execution_capacity_reservation \ + WHERE run_uid = $1 AND resource_dimension = 'active_runs'", + ) + .bind(run.run_uid) + .fetch_one(&pool) + .await?; + assert_eq!(receipt_state, "released"); + let trigger_boundary: (String, String, String) = sqlx::query_as( + "SELECT trigger.state, dispatch.state, capacity.state \ + FROM moa.execution_trigger AS trigger \ + JOIN moa.execution_dispatch_outbox AS dispatch USING (trigger_uid) \ + JOIN moa.execution_capacity_reservation AS capacity USING (trigger_uid) \ + WHERE trigger.trigger_uid = $1", + ) + .bind(deadline.trigger.trigger_uid) + .fetch_one(&pool) + .await?; + assert_eq!( + trigger_boundary, + ( + "superseded".to_string(), + "cancelled".to_string(), + "released".to_string(), + ), + "terminal settlement must retire its delayed deadline and capacity receipt" + ); + let scheduled_buckets: Vec<(String, i64)> = sqlx::query_as( + "SELECT scope_kind, reserved_quantity FROM moa.execution_capacity_bucket \ + WHERE resource_dimension = 'scheduled_triggers' ORDER BY scope_kind", + ) + .fetch_all(&pool) + .await?; + assert_eq!( + scheduled_buckets, + vec![("fleet".to_string(), 0), ("tenant".to_string(), 0)] + ); + + assert!(matches!( + create_run_with_config( + &repository, + scope, + &config, + new_run( + tenant_id, + None, + "active-run-after-terminal", + ExecutionRunStatus::Queued, + budget(1), + ), + ) + .await?, + RunAdmissionOutcome::Admitted(_) + )); + Ok(()) +} + +#[tokio::test] +async fn concurrent_deadline_arm_and_terminal_finalization_use_scheduled_before_run_lock_db() +-> TestResult { + // Pins: public deadline reconciliation and terminal settlement both acquire + // ScheduledTriggers before the run row. Their race completes without deadlock and commits + // exactly one coherent winner: either terminal state with no trigger or a live rearm that + // keeps finalization not-ready. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let config = ExecutionConfig::default(); + let RunAdmissionOutcome::Admitted(run) = create_run_with_config( + &repository, + scope, + &config, + new_run( + tenant_id, + None, + "deadline-arm-terminal-lock-order", + ExecutionRunStatus::Queued, + budget(1), + ), + ) + .await? + else { + panic!("deadline race fixture must be admitted"); + }; + let running = claim_running_controller(&repository, scope, &config, &run).await?; + assert!(matches!( + repository + .arm_run_deadline( + scope, + running.run_uid, + running.controller_generation, + &config, + ) + .await?, + RunDeadlineArmOutcome::Armed(_) + )); + assert!(matches!( + repository + .claim_controller_wake( + scope, + running.run_uid, + running.controller_generation, + running.wake_epoch, + ) + .await?, + RunControllerClaimOutcome::Claimed(_) + )); + assert!(matches!( + repository + .drain_run_triggers_page( + scope, + &config, + RunTriggerDrainRequest { + run_uid: running.run_uid, + controller_generation: running.controller_generation, + wake_epoch: running.wake_epoch, + page_limit: 1, + now: Utc::now(), + }, + ) + .await?, + RunTriggerDrainOutcome::ReadyToFinalize { .. } + )); + let prefinal = repository + .load_run(scope, running.run_uid) + .await? + .expect("drained race fixture remains visible"); + let finalization = successful_request(&prefinal)?; + let arm_repository = repository.clone(); + let terminal_repository = repository.clone(); + let arm_config = config.clone(); + let (arm, terminal) = tokio::time::timeout(std::time::Duration::from_secs(10), async move { + tokio::join!( + arm_repository.arm_run_deadline( + scope, + prefinal.run_uid, + prefinal.controller_generation, + &arm_config, + ), + terminal_repository.finalize_run(scope, finalization), + ) + }) + .await + .expect("ScheduledTriggers-before-run ordering must not deadlock"); + match (arm?, terminal?) { + (RunDeadlineArmOutcome::Terminal, FinalizationOutcome::Finalized(_)) + | (RunDeadlineArmOutcome::Armed(_), FinalizationOutcome::Conflict) => {} + outcomes => { + panic!("deadline arm/terminal race committed incoherent outcomes: {outcomes:?}") + } + } + let persisted = repository + .load_run(scope, running.run_uid) + .await? + .expect("race fixture remains queryable"); + let active_trigger_count: i64 = sqlx::query_scalar( + "SELECT count(*) FROM moa.execution_trigger \ + WHERE run_uid=$1 AND trigger_kind='run_deadline' \ + AND state IN ('pending','dispatching')", + ) + .bind(running.run_uid) + .fetch_one(&pool) + .await?; + assert_eq!( + active_trigger_count, + if persisted.status.is_terminal() { 0 } else { 1 } + ); + Ok(()) +} + +#[tokio::test] +async fn concurrent_resume_and_terminal_release_preserve_capacity_lock_order_db() -> TestResult { + // Pins: canonical parked-run resume transfers ParkedRuns to ActiveRuns before concurrent + // terminal settlement releases the current owner and ScheduledTriggers, without deadlock, + // leaked receipts, or counter underflow. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let config = ExecutionConfig { + max_fleet_active_runs: 2, + max_tenant_active_runs: 1, + max_fleet_parked_runs: 2, + max_tenant_parked_runs: 1, + max_fleet_scheduled_triggers: 2, + max_tenant_scheduled_triggers: 1, + ..ExecutionConfig::default() + }; + let mut settlements = Vec::new(); + for (tenant_id, key) in [ + (TenantId::new(), "terminal-lock-order-left"), + (TenantId::new(), "terminal-lock-order-right"), + ] { + let scope = ExecutionScope::Tenant { tenant_id }; + let RunAdmissionOutcome::Admitted(run) = create_run_with_config( + &repository, + scope, + &config, + new_run(tenant_id, None, key, ExecutionRunStatus::Queued, budget(1)), + ) + .await? + else { + panic!("concurrency fixture must be admitted"); + }; + assert!(matches!( + repository + .claim_controller_wake( + scope, + run.run_uid, + run.controller_generation, + run.wake_epoch, + ) + .await?, + RunControllerClaimOutcome::Claimed(_) + )); + let next_wake_at = run + .approved_budget + .deadline_at + .expect("test budget has a deadline"); + let RunControllerCompletionOutcome::Applied { run: parked, .. } = repository + .complete_controller_wake( + scope, + &config, + run.run_uid, + RunControllerCompletionRequest { + controller_generation: run.controller_generation, + wake_epoch: run.wake_epoch, + checkpoint: ExecutionRunActivationCheckpoint { + status: ExecutionRunStatus::WaitingInput, + activation_state: ExecutionActivationState::Idle, + next_wake_at: Some(next_wake_at), + waiting_since: Some(Utc::now()), + ready_task_count: 0, + active_task_count: 0, + }, + continuation_payload: None, + continuation_not_before_at: Utc::now(), + }, + ) + .await? + else { + panic!("storage-only wait must reserve ParkedRuns"); + }; + let TransitionOutcome::RunApplied(paused) = repository + .pause_run(scope, &config, parked.run_uid, parked.controller_generation) + .await? + else { + panic!("storage-only fixture must enter a canonical paused state"); + }; + let TransitionOutcome::RunApplied(resumed) = repository + .resume_run(scope, &config, paused.run_uid, paused.controller_generation) + .await? + else { + panic!("paused fixture must enqueue one canonical resume activation"); + }; + let running = match repository + .claim_controller_wake( + scope, + resumed.run_uid, + resumed.controller_generation, + resumed.wake_epoch, + ) + .await? + { + RunControllerClaimOutcome::Claimed(running) => running, + outcome => panic!("resumed fixture wake must be claimable: {outcome:?}"), + }; + assert!(matches!( + repository + .arm_run_deadline( + scope, + running.run_uid, + running.controller_generation, + &config, + ) + .await?, + RunDeadlineArmOutcome::Armed(_) + )); + assert!(matches!( + repository + .drain_run_triggers_page( + scope, + &config, + RunTriggerDrainRequest { + run_uid: running.run_uid, + controller_generation: running.controller_generation, + wake_epoch: running.wake_epoch, + page_limit: 1, + now: Utc::now(), + }, + ) + .await?, + RunTriggerDrainOutcome::ReadyToFinalize { .. } + )); + settlements.push((scope, successful_request(&running)?)); + } + let left_repository = repository.clone(); + let right_repository = repository.clone(); + let (left_scope, left_request) = settlements.remove(0); + let (right_scope, right_request) = settlements.remove(0); + let (left, right) = tokio::time::timeout(std::time::Duration::from_secs(10), async move { + tokio::join!( + left_repository.finalize_run(left_scope, left_request), + right_repository.finalize_run(right_scope, right_request), + ) + }) + .await + .expect("canonical capacity locking must not deadlock"); + assert!(matches!(left?, FinalizationOutcome::Finalized(_))); + assert!(matches!(right?, FinalizationOutcome::Finalized(_))); + let leaked: i64 = sqlx::query_scalar( + "SELECT count(*) FROM moa.execution_capacity_reservation \ + WHERE state IN ('reserved', 'reconciling') AND resource_dimension IN \ + ('active_runs', 'parked_runs', 'scheduled_triggers')", + ) + .fetch_one(&pool) + .await?; + assert_eq!(leaked, 0); + let nonzero_buckets: i64 = sqlx::query_scalar( + "SELECT count(*) FROM moa.execution_capacity_bucket \ + WHERE resource_dimension IN ('active_runs', 'parked_runs', 'scheduled_triggers') \ + AND reserved_quantity <> 0", + ) + .fetch_one(&pool) + .await?; + assert_eq!(nonzero_buckets, 0); + Ok(()) +} + +#[tokio::test] +async fn parked_to_active_transfer_saturates_atomically_and_replays_without_dual_ownership_db() +-> TestResult { + // Pins: storage-only waits and pause do not retain ActiveRuns; a saturated resume rolls back + // its generation/wake/capacity changes, and the eventual exact replay cannot reserve both + // ActiveRuns and ParkedRuns or underflow either bucket. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let config = ExecutionConfig { + max_fleet_active_runs: 1, + max_tenant_active_runs: 1, + max_fleet_parked_runs: 2, + max_tenant_parked_runs: 2, + ..ExecutionConfig::default() + }; + + let RunAdmissionOutcome::Admitted(first) = create_run_with_config( + &repository, + scope, + &config, + new_run( + tenant_id, + None, + "parked-active-transfer-first", + ExecutionRunStatus::Queued, + budget(1), + ), + ) + .await? + else { + panic!("first run must consume the only ActiveRuns slot"); + }; + assert!(matches!( + repository + .claim_controller_wake( + scope, + first.run_uid, + first.controller_generation, + first.wake_epoch, + ) + .await?, + RunControllerClaimOutcome::Claimed(_) + )); + let RunControllerCompletionOutcome::Applied { + run: storage_parked, + .. + } = repository + .complete_controller_wake( + scope, + &config, + first.run_uid, + RunControllerCompletionRequest { + controller_generation: first.controller_generation, + wake_epoch: first.wake_epoch, + checkpoint: ExecutionRunActivationCheckpoint { + status: ExecutionRunStatus::WaitingInput, + activation_state: ExecutionActivationState::Idle, + next_wake_at: first.approved_budget.deadline_at, + waiting_since: Some(Utc::now()), + ready_task_count: 0, + active_task_count: 0, + }, + continuation_payload: None, + continuation_not_before_at: Utc::now(), + }, + ) + .await? + else { + panic!("storage-only checkpoint must park the first run"); + }; + let TransitionOutcome::RunApplied(paused_first) = repository + .pause_run( + scope, + &config, + storage_parked.run_uid, + storage_parked.controller_generation, + ) + .await? + else { + panic!("parked run must enter paused state"); + }; + + let RunAdmissionOutcome::Admitted(second) = create_run_with_config( + &repository, + scope, + &config, + new_run( + tenant_id, + None, + "parked-active-transfer-second", + ExecutionRunStatus::Queued, + budget(1), + ), + ) + .await? + else { + panic!("parking the first run must free the only ActiveRuns slot"); + }; + let saturation = repository + .resume_run( + scope, + &config, + paused_first.run_uid, + paused_first.controller_generation, + ) + .await + .expect_err("resume must defer while the only ActiveRuns slot is occupied"); + assert!(matches!( + saturation, + moa_execution::Error::CapacitySaturated { + dimension: "active_runs" + } + )); + let unchanged = repository + .load_run(scope, paused_first.run_uid) + .await? + .expect("saturated run remains visible"); + assert_eq!(unchanged.status, ExecutionRunStatus::Paused); + assert_eq!( + (unchanged.controller_generation, unchanged.wake_epoch), + (paused_first.controller_generation, paused_first.wake_epoch), + "capacity saturation must roll back the exact resume fence" + ); + let before_retry: Vec<(String, i64)> = sqlx::query_as( + "SELECT resource_dimension,reserved_quantity FROM moa.execution_capacity_bucket \ + WHERE scope_kind='tenant' AND tenant_id=$1 \ + AND resource_dimension IN ('active_runs','parked_runs') \ + ORDER BY resource_dimension", + ) + .bind(tenant_id.0) + .fetch_all(&pool) + .await?; + assert_eq!( + before_retry, + vec![ + ("active_runs".to_string(), 1), + ("parked_runs".to_string(), 1) + ] + ); + + let TransitionOutcome::RunApplied(paused_second) = repository + .pause_run(scope, &config, second.run_uid, second.controller_generation) + .await? + else { + panic!("second run must transfer its ActiveRuns slot to ParkedRuns"); + }; + let TransitionOutcome::RunApplied(resumed_first) = repository + .resume_run( + scope, + &config, + paused_first.run_uid, + paused_first.controller_generation, + ) + .await? + else { + panic!("first run must resume after the ActiveRuns slot is released"); + }; + assert!(matches!( + repository + .resume_run( + scope, + &config, + paused_first.run_uid, + paused_first.controller_generation, + ) + .await?, + TransitionOutcome::RunAlreadyApplied(ref replayed) + if replayed.controller_generation == resumed_first.controller_generation + )); + let active_receipts: Vec<(Uuid, String)> = sqlx::query_as( + "SELECT run_uid,resource_dimension FROM moa.execution_capacity_reservation \ + WHERE tenant_id=$1 AND run_uid IN ($2,$3) AND state IN ('reserved','reconciling') \ + AND resource_dimension IN ('active_runs','parked_runs') ORDER BY run_uid", + ) + .bind(tenant_id.0) + .bind(resumed_first.run_uid) + .bind(paused_second.run_uid) + .fetch_all(&pool) + .await?; + assert_eq!(active_receipts.len(), 2); + assert!(active_receipts.contains(&(resumed_first.run_uid, "active_runs".to_string()))); + assert!(active_receipts.contains(&(paused_second.run_uid, "parked_runs".to_string()))); + let after_retry: Vec<(String, i64)> = sqlx::query_as( + "SELECT resource_dimension,reserved_quantity FROM moa.execution_capacity_bucket \ + WHERE scope_kind='tenant' AND tenant_id=$1 \ + AND resource_dimension IN ('active_runs','parked_runs') \ + ORDER BY resource_dimension", + ) + .bind(tenant_id.0) + .fetch_all(&pool) + .await?; + assert_eq!(after_retry, before_retry); + Ok(()) +} + +#[tokio::test] +async fn deadline_rearm_releases_superseded_trigger_capacity_before_reserving_replacement_db() +-> TestResult { + // Pins: rearming a run deadline for a new controller generation settles the old trigger + // through its owning path, so a one-slot ScheduledTriggers budget admits the replacement. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let config = ExecutionConfig { + max_fleet_scheduled_triggers: 1, + max_tenant_scheduled_triggers: 1, + ..ExecutionConfig::default() + }; + let RunAdmissionOutcome::Admitted(run) = create_run_with_config( + &repository, + scope, + &config, + new_run( + tenant_id, + None, + "deadline-rearm-capacity", + ExecutionRunStatus::Queued, + budget(1), + ), + ) + .await? + else { + panic!("deadline fixture must be admitted"); + }; + let RunDeadlineArmOutcome::Armed(first) = repository + .arm_run_deadline(scope, run.run_uid, 1, &config) + .await? + else { + panic!("first generation must arm its deadline"); + }; + sqlx::query( + "UPDATE moa.execution_run \ + SET controller_generation = 2, updated_at = NOW() WHERE run_uid = $1", + ) + .bind(run.run_uid) + .execute(&pool) + .await?; + let RunDeadlineArmOutcome::Armed(second) = repository + .arm_run_deadline(scope, run.run_uid, 2, &config) + .await? + else { + panic!("replacement generation must reuse the released trigger slot"); + }; + assert_ne!(first.trigger.trigger_uid, second.trigger.trigger_uid); + let states: Vec<(Uuid, String)> = sqlx::query_as( + "SELECT trigger_uid, state FROM moa.execution_trigger \ + WHERE run_uid = $1 AND trigger_kind = 'run_deadline' ORDER BY controller_generation", + ) + .bind(run.run_uid) + .fetch_all(&pool) + .await?; + assert_eq!( + states, + vec![ + (first.trigger.trigger_uid, "superseded".to_string()), + (second.trigger.trigger_uid, "pending".to_string()), + ] + ); + let dispatch_states: Vec<(Uuid, String)> = sqlx::query_as( + "SELECT trigger_uid, state FROM moa.execution_dispatch_outbox \ + WHERE run_uid = $1 AND trigger_uid IS NOT NULL ORDER BY controller_generation", + ) + .bind(run.run_uid) + .fetch_all(&pool) + .await?; + assert_eq!( + dispatch_states, + vec![ + (first.trigger.trigger_uid, "cancelled".to_string()), + (second.trigger.trigger_uid, "pending".to_string()), + ] + ); + let receipts: Vec<(Uuid, String)> = sqlx::query_as( + "SELECT trigger_uid, state FROM moa.execution_capacity_reservation \ + WHERE run_uid = $1 AND resource_dimension = 'scheduled_triggers' \ + ORDER BY controller_generation", + ) + .bind(run.run_uid) + .fetch_all(&pool) + .await?; + assert_eq!( + receipts, + vec![ + (first.trigger.trigger_uid, "released".to_string()), + (second.trigger.trigger_uid, "reserved".to_string()), + ] + ); + let buckets: Vec<(String, i64)> = sqlx::query_as( + "SELECT scope_kind, reserved_quantity FROM moa.execution_capacity_bucket \ + WHERE resource_dimension = 'scheduled_triggers' ORDER BY scope_kind", + ) + .fetch_all(&pool) + .await?; + assert_eq!( + buckets, + vec![("fleet".to_string(), 1), ("tenant".to_string(), 1)] + ); + Ok(()) +} + +#[tokio::test] +async fn high_fanout_terminal_trigger_cleanup_is_strictly_activation_bounded_db() -> TestResult { + // Pins: terminal trigger cancellation settles no more than the requested page in one + // activation, durably queues exactly one continuation while work remains, and releases every + // delayed outbox/capacity receipt without an unbounded finalizer scan. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let config = ExecutionConfig { + max_fleet_scheduled_triggers: 16, + max_tenant_scheduled_triggers: 16, + ..ExecutionConfig::default() + }; + let RunAdmissionOutcome::Admitted(run) = create_run_with_config( + &repository, + scope, + &config, + new_run( + tenant_id, + None, + "bounded-terminal-trigger-drain", + ExecutionRunStatus::Queued, + budget(1), + ), + ) + .await? + else { + panic!("trigger-drain fixture must be admitted"); + }; + let mut run = *run; + let due_at = run.approved_budget.deadline_at.expect("fixture deadline"); + for controller_generation in 2..=6 { + repository + .create_trigger( + scope, + &config, + NewExecutionTrigger { + trigger_uid: Uuid::now_v7(), + tenant_id, + run_uid: Some(run.run_uid), + task_id: None, + compensation_id: None, + schedule_uid: None, + kind: ExecutionTriggerKind::RunDeadline, + controller_generation: Some(controller_generation), + attempt_generation: None, + compensation_generation: None, + compensation_attempt_generation: None, + schedule_incarnation: None, + occurrence_sequence: None, + due_at, + payload: json!({"run_uid": run.run_uid, "deadline_at": due_at}), + }, + ) + .await?; + } + + for expected_remaining in [4_i64, 2] { + assert!(matches!( + repository + .claim_controller_wake( + scope, + run.run_uid, + run.controller_generation, + run.wake_epoch, + ) + .await?, + RunControllerClaimOutcome::Claimed(_) + )); + let RunTriggerDrainOutcome::PageDrained(commit) = repository + .drain_run_triggers_page( + scope, + &config, + RunTriggerDrainRequest { + run_uid: run.run_uid, + controller_generation: run.controller_generation, + wake_epoch: run.wake_epoch, + page_limit: 2, + now: Utc::now(), + }, + ) + .await? + else { + panic!("non-final trigger page must durably enqueue one continuation"); + }; + assert_eq!(commit.drained_trigger_count, 2); + assert_eq!(commit.continuation.wake_epoch, Some(commit.run.wake_epoch)); + run = commit.run; + let active: i64 = sqlx::query_scalar( + "SELECT count(*) FROM moa.execution_trigger \ + WHERE run_uid = $1 AND state IN ('pending', 'dispatching')", + ) + .bind(run.run_uid) + .fetch_one(&pool) + .await?; + assert_eq!(active, expected_remaining); + } + assert!(matches!( + repository + .claim_controller_wake( + scope, + run.run_uid, + run.controller_generation, + run.wake_epoch, + ) + .await?, + RunControllerClaimOutcome::Claimed(_) + )); + assert!(matches!( + repository + .drain_run_triggers_page( + scope, + &config, + RunTriggerDrainRequest { + run_uid: run.run_uid, + controller_generation: run.controller_generation, + wake_epoch: run.wake_epoch, + page_limit: 2, + now: Utc::now(), + }, + ) + .await?, + RunTriggerDrainOutcome::ReadyToFinalize { + drained_trigger_count: 2, + .. + } + )); + let terminal_boundary: (i64, i64, i64) = sqlx::query_as( + "SELECT \ + (SELECT count(*) FROM moa.execution_trigger \ + WHERE run_uid = $1 AND state <> 'superseded'), \ + (SELECT count(*) FROM moa.execution_dispatch_outbox \ + WHERE run_uid = $1 AND trigger_uid IS NOT NULL AND state <> 'cancelled'), \ + (SELECT count(*) FROM moa.execution_capacity_reservation \ + WHERE run_uid = $1 AND resource_dimension = 'scheduled_triggers' \ + AND state <> 'released')", + ) + .bind(run.run_uid) + .fetch_one(&pool) + .await?; + assert_eq!(terminal_boundary, (0, 0, 0)); + let scheduled_reserved: i64 = sqlx::query_scalar( + "SELECT sum(reserved_quantity)::BIGINT FROM moa.execution_capacity_bucket \ + WHERE resource_dimension = 'scheduled_triggers'", + ) + .fetch_one(&pool) + .await?; + assert_eq!(scheduled_reserved, 0); + Ok(()) +} diff --git a/crates/moa-execution/tests/execution_db/amendment_projection_db.rs b/crates/moa-execution/tests/execution_db/amendment_projection_db.rs new file mode 100644 index 000000000..db9387c2c --- /dev/null +++ b/crates/moa-execution/tests/execution_db/amendment_projection_db.rs @@ -0,0 +1,172 @@ +//! Bounded persisted amendment-evidence contracts. + +use moa_artifacts::execution_plan::{ExecutionNode, ExecutionOperation}; +use moa_execution::repository::{ + amendment::{AmendmentProjectionOutcome, AmendmentProjectionRequest}, + ready::{ReadyMaterializationOutcome, ReadyMaterializationRequest}, +}; + +use super::support::*; + +fn output_node() -> ExecutionNode { + ExecutionNode { + id: "output".to_string(), + requirement_ids: vec!["req".to_string()], + depends_on: Vec::new(), + when: None, + input: json!({}), + output_schema: json!({ "type": "object" }), + operation: ExecutionOperation::Output { value: json!({}) }, + compensation: None, + retry: RetryPolicy { + max_attempts: 1, + initial_backoff_ms: 0, + max_backoff_ms: 0, + }, + budget: None, + } +} + +#[tokio::test] +async fn amendment_projection_counts_twenty_five_hundred_prior_failures_in_one_bounded_call_db() +-> TestResult { + // Pins: amendment validation sees one exact WaitingReplan origin and one indexed scalar count; + // it never reloads or pages across the full 2,501-task history. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let mut candidate = new_run( + tenant_id, + None, + "bounded-amendment-2501", + ExecutionRunStatus::Queued, + budget(5_000), + ); + candidate.plan.definition.nodes = vec![output_node()]; + let run = create_run(&repository, scope, candidate).await?; + let config = ExecutionConfig::default(); + + let mut all_tasks = (0_u64..2_501) + .map(|index| logical_task(run.run_uid, "output", &format!("{index:04}"), estimate(1))) + .collect::>(); + let replan_task = logical_task(run.run_uid, "output", "replan", estimate(1)); + let replan_task_id = replan_task.task_id; + all_tasks.push(replan_task); + let mut cursor = 0_u64; + for (page_index, page) in all_tasks.chunks(1_000).enumerate() { + let ReadyMaterializationOutcome::Applied { next_cursor, .. } = repository + .materialize_ready_page( + scope, + &config, + ReadyMaterializationRequest { + run_uid: run.run_uid, + plan_revision: run.plan_revision, + node_id: "output".to_string(), + expected_cursor: cursor, + reduce_cursor: None, + source_exhausted: page_index == 2, + terminal_output: None, + tasks: page.to_vec(), + }, + ) + .await? + else { + panic!("fresh amendment setup page must apply"); + }; + cursor = next_cursor; + } + + for (status, attempt_state) in [("dispatching", "dispatching"), ("running", "running")] { + sqlx::query( + "UPDATE moa.execution_task SET status=$2,attempt_state=$3,updated_at=NOW() \ + WHERE run_uid=$1", + ) + .bind(run.run_uid) + .bind(status) + .bind(attempt_state) + .execute(&pool) + .await?; + } + let repeated_failure = ExecutionTaskOutcome { + schema_version: 1, + usage: usage(0), + result: ExecutionTaskResult::Failed { + class: ExecutionFailureClass::Terminal, + message: "source unavailable".to_string(), + }, + }; + let repeated_fingerprint = failure_fingerprint(&FailureFingerprintInput { + class: ExecutionFailureClass::Terminal, + node_id: "output".to_string(), + capability_ref: None, + message: "source unavailable".to_string(), + })?; + sqlx::query( + "UPDATE moa.execution_task SET status='failed',attempt_state='terminal',current_outcome=$2, \ + failure_fingerprint=$3,completed_at=NOW(),updated_at=NOW() \ + WHERE run_uid=$1 AND task_id<>$4", + ) + .bind(run.run_uid) + .bind(serde_json::to_value(repeated_failure)?) + .bind(repeated_fingerprint.to_string()) + .bind(replan_task_id.as_uuid()) + .execute(&pool) + .await?; + sqlx::query("UPDATE moa.execution_run SET status='running',updated_at=NOW() WHERE run_uid=$1") + .bind(run.run_uid) + .execute(&pool) + .await?; + sqlx::query( + "UPDATE moa.execution_task SET status='waiting_replan',attempt_state='waiting', \ + current_outcome=$2,failure_fingerprint=$3,waiting_since=NOW(),updated_at=NOW() \ + WHERE run_uid=$1 AND task_id=$4", + ) + .bind(run.run_uid) + .bind(serde_json::to_value(needs_replan(1))?) + .bind(repeated_fingerprint.to_string()) + .bind(replan_task_id.as_uuid()) + .execute(&pool) + .await?; + sqlx::query( + "UPDATE moa.execution_node_state SET node_status='waiting',ready_task_count=0, \ + waiting_task_count=1,terminal_task_count=total_task_count-1,failed_task_count=total_task_count-1, \ + updated_at=NOW() WHERE run_uid=$1 AND node_id='output'", + ) + .bind(run.run_uid) + .execute(&pool) + .await?; + sqlx::query( + "UPDATE moa.execution_run SET status='waiting_replan',ready_task_count=0,active_task_count=0, \ + waiting_task_count=1,waiting_replan_task_count=1,waiting_reasons_truncated=TRUE, \ + waiting_since=NOW(),updated_at=NOW() \ + WHERE run_uid=$1", + ) + .bind(run.run_uid) + .execute(&pool) + .await?; + + let request = AmendmentProjectionRequest { + run_uid: run.run_uid, + session_id: run.session_id, + expected_plan_revision: run.plan_revision, + }; + let AmendmentProjectionOutcome::Ready(snapshot) = repository + .load_amendment_projection_for_session(scope, &config, request) + .await? + else { + panic!("current bounded amendment projection must be ready in one call"); + }; + assert_eq!(snapshot.projection.replan_tasks.len(), 1); + assert_eq!(snapshot.projection.replan_tasks[0].task_id, replan_task_id); + assert_eq!(snapshot.projection.node_statuses.len(), 1); + assert!(snapshot.projection.started_node_ids.contains("output")); + assert_eq!( + snapshot + .prior_failure_fingerprint_counts + .get(&repeated_fingerprint), + Some(&2_501) + ); + Ok(()) +} diff --git a/crates/moa-execution/tests/execution_db/compensation_attempts_db.rs b/crates/moa-execution/tests/execution_db/compensation_attempts_db.rs new file mode 100644 index 000000000..f93548ef8 --- /dev/null +++ b/crates/moa-execution/tests/execution_db/compensation_attempts_db.rs @@ -0,0 +1,1962 @@ +//! Durable compensation-attempt lifecycle behavior. + +use moa_artifacts::execution_plan::{ + CapabilityReference, CompensationInputBinding, CompensationInputMapping, + CompensationValueSource, ExecutionCompensation, +}; +use moa_config::ExecutionConfig; +use moa_core::types::{ + action_policy::{ActionClass, ActionPolicyEffect, RiskLevel}, + tools::{AsyncToolJob, IdempotencyClass}, +}; +use moa_execution::{ + capability::{ + CapabilityPolicyContext, CapabilityRollbackContract, CapabilitySource, ExecutionCapability, + ExecutionClass, + }, + repository::{ + compensation::{ + CompensationAttemptAdmission, CompensationAttemptAdmissionOutcome, + CompensationAttemptFence, CompensationAttemptReleaseClaimOutcome, + CompensationAttemptState, CompensationAttemptWriteOutcome, + CompensationReviewResolutionOutcome, + }, + external_job::{ + ExecutionExternalJobBinding, ExecutionExternalJobCallback, + ExecutionExternalJobCallbackOutcome, ExecutionExternalJobCallbackUpdate, + ExecutionExternalJobStartRecoveryAdoptionOutcome, NewExecutionExternalJobIntent, + }, + external_job::{ExecutionExternalJobOwner, ExecutionExternalJobState}, + terminal::PendingTerminalAdvanceOutcome, + }, + state::{ExecutionCompensationOutcome, ExecutionTerminalEvidence}, + wire::{ + ExecutionCompensationAttemptCancelRequest, ExecutionCompensationReleaseIntent, + ExecutionExternalJobStartRecoveryOwner, ExecutionExternalJobStartRecoveryRequest, + }, +}; + +use super::support::*; +use std::time::Duration as StdDuration; + +#[tokio::test] +async fn nonterminal_guard_uses_partial_index_for_more_than_2500_tasks_db() -> TestResult { + // Pins: compensation admission's forward-work guard remains an indexed existence probe even + // when a run retains thousands of terminal task rows and only one nonterminal task remains. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let repository = ExecutionRepository::new(test_db.store().pool().clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::ControlPlane; + let run = create_run( + &repository, + scope, + new_run( + tenant_id, + None, + "compensation-nonterminal-index", + ExecutionRunStatus::Queued, + budget(3_000), + ), + ) + .await?; + const TASK_COUNT: usize = 2_501; + const PAGE_SIZE: usize = 1_000; + const LIVE_ITEM_KEY: &str = "item-02500"; + for page_start in (0..TASK_COUNT).step_by(PAGE_SIZE) { + let page_end = (page_start + PAGE_SIZE).min(TASK_COUNT); + let tasks = (page_start..page_end) + .map(|index| { + logical_task( + run.run_uid, + "large-terminal-history", + &format!("item-{index:05}"), + estimate(1), + ) + }) + .collect::>(); + repository + .materialize_tasks(scope, run.run_uid, run.plan_revision, tasks) + .await?; + } + let settled = sqlx::query( + "UPDATE moa.execution_task SET status='skipped',attempt_state='terminal', \ + completed_at=NOW(),updated_at=NOW() \ + WHERE run_uid=$1 AND item_key<>$2", + ) + .bind(run.run_uid) + .bind(LIVE_ITEM_KEY) + .execute(test_db.store().pool()) + .await?; + assert_eq!(settled.rows_affected(), 2_500); + + let mut transaction = test_db.store().pool().begin().await?; + sqlx::query("SET LOCAL enable_seqscan=off") + .execute(transaction.as_mut()) + .await?; + let exists: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM moa.execution_task WHERE run_uid=$1 \ + AND status NOT IN ('completed','skipped','failed','cancelled','unknown_outcome'))", + ) + .bind(run.run_uid) + .fetch_one(transaction.as_mut()) + .await?; + assert!( + exists, + "the one pending task must block compensation admission" + ); + let explain: serde_json::Value = sqlx::query_scalar( + "EXPLAIN (ANALYZE, COSTS OFF, FORMAT JSON) \ + SELECT EXISTS (SELECT 1 FROM moa.execution_task WHERE run_uid=$1 \ + AND status NOT IN ('completed','skipped','failed','cancelled','unknown_outcome'))", + ) + .bind(run.run_uid) + .fetch_one(transaction.as_mut()) + .await?; + let scan = explain_index_scan(&explain, "execution_task_nonterminal_run_idx") + .expect("nonterminal existence probe must use its partial run index"); + assert_eq!( + scan.get("Actual Loops").and_then(serde_json::Value::as_u64), + Some(1) + ); + assert_eq!( + scan.get("Actual Rows").and_then(serde_json::Value::as_u64), + Some(1) + ); + transaction.rollback().await?; + Ok(()) +} + +#[tokio::test] +async fn concurrent_admission_replays_only_highest_reverse_order_slice_db() -> TestResult { + // Pins: after the bounded pending-terminal page admits the highest reverse registration, + // competing retries replay that exact slice without dispatching a lower registration. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let repository = ExecutionRepository::new(test_db.store().pool().clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let (run, registrations) = + compensating_run(&repository, scope, tenant_id, &["first", "second"]).await?; + let now = moa_test_support::fixtures::pg_now(); + let config = ExecutionConfig::default(); + + let (left, right) = tokio::join!( + repository.admit_next_compensation_attempt(scope, &config, run.run_uid, now), + repository.admit_next_compensation_attempt(scope, &config, run.run_uid, now), + ); + let outcomes = [left?, right?]; + assert_eq!( + outcomes + .iter() + .filter(|outcome| matches!(outcome, CompensationAttemptAdmissionOutcome::Replayed(_))) + .count(), + 2 + ); + for outcome in outcomes { + let admission = match outcome { + CompensationAttemptAdmissionOutcome::Admitted(admission) + | CompensationAttemptAdmissionOutcome::Replayed(admission) => admission, + other => panic!("concurrent admission returned {other:?}"), + }; + assert_eq!( + admission.attempt.registration.compensation_id, registrations[0].compensation_id, + "only the highest registered sequence may dispatch" + ); + assert_eq!( + admission.attempt.attempt_state, + CompensationAttemptState::Dispatching + ); + assert_eq!(admission.attempt.run, run); + assert_eq!( + admission.attempt.active_dispatch_uid, + Some(admission.dispatch.dispatch_uid) + ); + } + let dispatch_rows: Vec = sqlx::query_scalar( + "SELECT payload FROM moa.execution_dispatch_outbox WHERE run_uid=$1 \ + AND compensation_id=$2 AND dispatch_kind='compensation_attempt'", + ) + .bind(run.run_uid) + .bind(registrations[0].compensation_id.as_uuid()) + .fetch_all(test_db.store().pool()) + .await?; + assert_eq!(dispatch_rows.len(), 1); + let payload = dispatch_rows.first().expect("one compensation payload"); + assert!(payload.get("identity").is_none()); + assert!(payload.get("contact_id").is_none()); + assert!(payload.get("session_id").is_none()); + let expected_compensation_id = registrations[0].compensation_id.to_string(); + assert_eq!( + payload + .get("compensation_id") + .and_then(serde_json::Value::as_str), + Some(expected_compensation_id.as_str()) + ); + Ok(()) +} + +#[tokio::test] +async fn paused_slice_releases_capacity_only_after_verified_teardown_db() -> TestResult { + // Pins: pause first makes the slice non-dispatchable while retaining capacity, then the + // verified release finalizer returns the logical effect to idle without changing its generation. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let repository = ExecutionRepository::new(test_db.store().pool().clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let (run, _) = compensating_run(&repository, scope, tenant_id, &["effect"]).await?; + let now = moa_test_support::fixtures::pg_now(); + let config = ExecutionConfig::default(); + let admission = + active_compensation_admission(&repository, scope, &config, run.run_uid, now).await?; + let fence = fence(&admission); + let CompensationAttemptWriteOutcome::Applied(started) = repository + .start_compensation_attempt(scope, fence, now + Duration::milliseconds(1)) + .await? + else { + panic!("admitted compensation slice must start"); + }; + assert_eq!(started.attempt_state, CompensationAttemptState::Running); + let release_request = cancel_request( + &admission, + tenant_id, + ExecutionCompensationReleaseIntent::Pause, + ); + let CompensationAttemptReleaseClaimOutcome::Applied(claimed) = repository + .begin_compensation_attempt_release(&release_request, now + Duration::milliseconds(2)) + .await? + else { + panic!("pause must claim the active compensation before provider release"); + }; + assert_eq!(claimed.attempt_state, CompensationAttemptState::Cancelling); + assert_eq!( + claimed.release_intent, + Some(ExecutionCompensationReleaseIntent::Pause) + ); + let reservation_state: String = sqlx::query_scalar( + "SELECT state FROM moa.execution_capacity_reservation WHERE reservation_uid=$1", + ) + .bind(admission.capacity_reservation_uid) + .fetch_one(test_db.store().pool()) + .await?; + assert_eq!(reservation_state, "reserved"); + assert!(matches!( + repository + .yield_released_compensation_attempt( + &release_request, + now + Duration::milliseconds(3), + None, + ) + .await?, + CompensationAttemptWriteOutcome::Conflict + )); + let release_receipt = persist_compensation_release_receipt( + test_db.store().pool(), + &release_request, + now + Duration::milliseconds(3), + ) + .await?; + let CompensationAttemptWriteOutcome::Applied(yielded) = repository + .yield_released_compensation_attempt( + &release_request, + now + Duration::milliseconds(3), + Some(release_receipt), + ) + .await? + else { + panic!("active compensation slice must yield"); + }; + assert_eq!( + yielded.registration.generation, + fence.compensation_generation + ); + assert_eq!(yielded.attempt_generation, fence.attempt_generation + 1); + assert_eq!(yielded.attempt_state, CompensationAttemptState::Idle); + assert_eq!(yielded.active_dispatch_uid, None); + assert_eq!(yielded.release_intent, None); + let reservation_state: String = sqlx::query_scalar( + "SELECT state FROM moa.execution_capacity_reservation WHERE reservation_uid=$1", + ) + .bind(admission.capacity_reservation_uid) + .fetch_one(test_db.store().pool()) + .await?; + assert_eq!(reservation_state, "released"); + let next = active_compensation_admission( + &repository, + scope, + &ExecutionConfig::default(), + run.run_uid, + now + Duration::milliseconds(4), + ) + .await?; + assert_eq!( + next.attempt.registration.generation, + fence.compensation_generation + ); + assert_eq!( + next.attempt.attempt_generation, + fence.attempt_generation + 1 + ); + assert_ne!(next.dispatch.dispatch_uid, fence.dispatch_uid); + Ok(()) +} + +#[tokio::test] +async fn journaled_compensation_release_time_cannot_regress_progress_db() -> TestResult { + // Pins: a Restate-journaled release time may predate a later database progress write; the + // exact fenced teardown still applies while the durable progress watermark stays monotonic. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let repository = ExecutionRepository::new(test_db.store().pool().clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let (run, _) = + compensating_run(&repository, scope, tenant_id, &["stale-journal-clock"]).await?; + let claim_owner = "compensation-clock-pin"; + let claimed = repository + .claim_due_dispatches(scope, claim_owner, 100, StdDuration::from_secs(30)) + .await?; + let controller_wakes = claimed + .iter() + .filter(|dispatch| { + dispatch.kind == moa_execution::repository::outbox::ExecutionDispatchKind::RunActivation + && dispatch.run_uid == Some(run.run_uid) + && dispatch.controller_generation == Some(run.controller_generation) + && dispatch.wake_epoch == Some(run.wake_epoch) + }) + .collect::>(); + assert_eq!(controller_wakes.len(), 1); + assert_eq!( + repository + .mark_dispatches_delivered(scope, &[controller_wakes[0].dispatch_uid], claim_owner,) + .await?, + vec![controller_wakes[0].dispatch_uid] + ); + let journaled_at = moa_test_support::fixtures::pg_now(); + let admission = active_compensation_admission( + &repository, + scope, + &ExecutionConfig::default(), + run.run_uid, + journaled_at, + ) + .await?; + let fence = fence(&admission); + let CompensationAttemptWriteOutcome::Applied(started) = repository + .start_compensation_attempt(scope, fence, journaled_at + Duration::milliseconds(1)) + .await? + else { + panic!("admitted compensation slice must start"); + }; + let database_progress_at = journaled_at + Duration::seconds(1); + sqlx::query( + "UPDATE moa.execution_compensation SET last_progress_at=$3,updated_at=NOW() \ + WHERE run_uid=$1 AND compensation_id=$2", + ) + .bind(run.run_uid) + .bind(fence.compensation_id.as_uuid()) + .bind(database_progress_at) + .execute(test_db.store().pool()) + .await?; + sqlx::query( + "UPDATE moa.execution_run SET last_progress_at=$2,updated_at=NOW() WHERE run_uid=$1", + ) + .bind(run.run_uid) + .bind(database_progress_at) + .execute(test_db.store().pool()) + .await?; + + let release_request = cancel_request( + &admission, + tenant_id, + ExecutionCompensationReleaseIntent::Pause, + ); + let CompensationAttemptReleaseClaimOutcome::Applied(claimed) = repository + .begin_compensation_attempt_release( + &release_request, + journaled_at + Duration::milliseconds(2), + ) + .await? + else { + panic!("stale journal clock must not reject the exact release claim"); + }; + assert_eq!(claimed.attempt_state, CompensationAttemptState::Cancelling); + assert_eq!(claimed.last_progress_at, database_progress_at); + assert!(claimed.last_progress_at > started.last_progress_at); + + let receipt = persist_compensation_release_receipt( + test_db.store().pool(), + &release_request, + journaled_at + Duration::milliseconds(3), + ) + .await?; + let CompensationAttemptWriteOutcome::Applied(released) = repository + .yield_released_compensation_attempt( + &release_request, + journaled_at + Duration::milliseconds(3), + Some(receipt), + ) + .await? + else { + panic!("stale journal clock must not reject the exact release finalizer"); + }; + assert_eq!(released.attempt_state, CompensationAttemptState::Idle); + assert_eq!(released.last_progress_at, database_progress_at); + let released_run = repository + .load_run(scope, run.run_uid) + .await? + .expect("released compensation run remains visible"); + assert_eq!(released_run.last_progress_at, database_progress_at); + assert_eq!(released_run.wake_epoch, run.wake_epoch + 1); + Ok(()) +} + +#[tokio::test] +async fn running_compensation_pause_drains_exact_attempt_then_resumes_once_db() -> TestResult { + // Pins: pausing a compensating run fences the run at G+1 while cancellation retains the + // attempt's G resource owner; only verified teardown reaches Paused, and resume restores the + // original pending terminal intent with exactly one activation. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let config = ExecutionConfig::default(); + let (run, _) = compensating_run(&repository, scope, tenant_id, &["running-pause"]).await?; + let pending_terminal = run + .pending_terminal + .clone() + .expect("compensating fixture must preserve its terminal intent"); + let now = moa_test_support::fixtures::pg_now(); + let admission = + active_compensation_admission(&repository, scope, &config, run.run_uid, now).await?; + let fence = fence(&admission); + assert!(matches!( + repository + .start_compensation_attempt(scope, fence, now + Duration::milliseconds(1)) + .await?, + CompensationAttemptWriteOutcome::Applied(_) + )); + + let TransitionOutcome::RunApplied(pausing) = repository + .pause_run(scope, &config, run.run_uid, run.controller_generation) + .await? + else { + panic!("running compensation pause must install its cancellation fence"); + }; + assert_eq!(pausing.status, ExecutionRunStatus::Pausing); + assert_eq!(pausing.controller_generation, run.controller_generation + 1); + assert_eq!(pausing.active_task_count, 1); + assert_eq!(pausing.pending_terminal, Some(pending_terminal.clone())); + let cancellation_payload: serde_json::Value = sqlx::query_scalar( + "SELECT payload FROM moa.execution_dispatch_outbox WHERE run_uid=$1 \ + AND compensation_id=$2 AND dispatch_kind='compensation_attempt_cancel'", + ) + .bind(run.run_uid) + .bind(fence.compensation_id.as_uuid()) + .fetch_one(&pool) + .await?; + let cancellation: ExecutionCompensationAttemptCancelRequest = + serde_json::from_value(cancellation_payload)?; + assert_eq!( + cancellation.controller_generation, + pausing.controller_generation + ); + assert_eq!( + cancellation.attempt_controller_generation, + run.controller_generation + ); + assert_eq!( + cancellation.intent, + ExecutionCompensationReleaseIntent::Pause + ); + let cancelling_state: String = sqlx::query_scalar( + "SELECT attempt_state FROM moa.execution_compensation \ + WHERE run_uid=$1 AND compensation_id=$2", + ) + .bind(run.run_uid) + .bind(fence.compensation_id.as_uuid()) + .fetch_one(&pool) + .await?; + assert_eq!(cancelling_state, "cancelling"); + + let release_receipt = + persist_compensation_release_receipt(&pool, &cancellation, now + Duration::milliseconds(2)) + .await?; + assert!(matches!( + repository + .yield_released_compensation_attempt( + &cancellation, + now + Duration::milliseconds(3), + Some(release_receipt), + ) + .await?, + CompensationAttemptWriteOutcome::Applied(_) + )); + let paused = repository + .load_run(scope, run.run_uid) + .await? + .expect("drained compensation run remains visible"); + assert_eq!(paused.status, ExecutionRunStatus::Paused); + assert_eq!(paused.activation_state, ExecutionActivationState::Paused); + assert_eq!(paused.active_task_count, 0); + assert_eq!(paused.pending_terminal, Some(pending_terminal.clone())); + assert_eq!( + run_activation_count(&pool, paused.run_uid, paused.controller_generation).await?, + 0, + "provider teardown must not wake a Pausing or Paused run" + ); + + let TransitionOutcome::RunApplied(resumed) = repository + .resume_run(scope, &config, paused.run_uid, paused.controller_generation) + .await? + else { + panic!("fully drained compensation run must resume"); + }; + assert_eq!(resumed.status, ExecutionRunStatus::Compensating); + assert_eq!(resumed.pending_terminal, Some(pending_terminal)); + assert_eq!( + run_activation_count(&pool, resumed.run_uid, resumed.controller_generation).await?, + 1 + ); + assert!(matches!( + repository + .resume_run(scope, &config, paused.run_uid, paused.controller_generation,) + .await?, + TransitionOutcome::RunAlreadyApplied(_) + )); + assert_eq!( + run_activation_count(&pool, resumed.run_uid, resumed.controller_generation).await?, + 1, + "resume replay must not enqueue a second activation" + ); + Ok(()) +} + +#[tokio::test] +async fn dispatch_delivery_loss_releases_never_started_compensation_for_retry_db() -> TestResult { + // Pins: a dispatching compensation owns capacity/watchdog but no provider hand; its exact + // verified no-hand receipt releases both and creates one fresh admissible attempt generation. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let repository = ExecutionRepository::new(test_db.store().pool().clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let (run, _) = compensating_run(&repository, scope, tenant_id, &["delivery-lost"]).await?; + let now = moa_test_support::fixtures::pg_now(); + let config = ExecutionConfig::default(); + let admission = + active_compensation_admission(&repository, scope, &config, run.run_uid, now).await?; + assert_eq!( + admission.attempt.attempt_state, + CompensationAttemptState::Dispatching + ); + let request = cancel_request( + &admission, + tenant_id, + ExecutionCompensationReleaseIntent::Watchdog, + ); + assert!(matches!( + repository + .begin_compensation_attempt_release(&request, now + Duration::milliseconds(1)) + .await?, + CompensationAttemptReleaseClaimOutcome::Applied(_) + )); + let receipt = persist_compensation_release_receipt( + test_db.store().pool(), + &request, + now + Duration::milliseconds(2), + ) + .await?; + let CompensationAttemptWriteOutcome::Applied(retry) = repository + .settle_released_compensation_attempt( + &request, + ExecutionCompensationOutcome::Failed { + message: "dispatch delivery lost before provider start".to_string(), + retryable: true, + usage: usage(0), + }, + now + Duration::milliseconds(2), + Some(receipt), + ) + .await? + else { + panic!("verified never-started attempt must requeue"); + }; + assert_eq!(retry.attempt_state, CompensationAttemptState::Idle); + assert_eq!( + retry.attempt_generation, + admission.attempt.attempt_generation + 1 + ); + let reservation_state: String = sqlx::query_scalar( + "SELECT state FROM moa.execution_capacity_reservation WHERE reservation_uid=$1", + ) + .bind(admission.capacity_reservation_uid) + .fetch_one(test_db.store().pool()) + .await?; + assert_eq!(reservation_state, "released"); + let active_watchdogs: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM moa.execution_trigger WHERE trigger_uid=$1 \ + AND state IN ('pending','dispatching')", + ) + .bind(admission.watchdog.trigger.trigger_uid) + .fetch_one(test_db.store().pool()) + .await?; + assert_eq!(active_watchdogs, 0); + assert!(matches!( + repository + .admit_next_compensation_attempt( + scope, + &config, + run.run_uid, + now + Duration::milliseconds(3), + ) + .await?, + CompensationAttemptAdmissionOutcome::Admitted(_) + )); + Ok(()) +} + +#[tokio::test] +async fn recovered_not_started_compensation_requires_verified_release_before_retry_db() -> TestResult +{ + // Pins: provider NotStarted proof releases the exact external intent and atomically fences + // the running compensation as Retry; ActiveTasks/watchdog remain owned until a persisted + // verified hand receipt advances one fresh Idle attempt generation. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let repository = ExecutionRepository::new(test_db.store().pool().clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let (run, _) = compensating_run(&repository, scope, tenant_id, &["not-started"]).await?; + let now = moa_test_support::fixtures::pg_now(); + let config = ExecutionConfig::default(); + let admission = + active_compensation_admission(&repository, scope, &config, run.run_uid, now).await?; + let fence = fence(&admission); + assert!(matches!( + repository + .start_compensation_attempt(scope, fence, now + Duration::milliseconds(1)) + .await?, + CompensationAttemptWriteOutcome::Applied(_) + )); + let external_job_uid = Uuid::now_v7(); + let provider = "recovery-provider".to_string(); + let idempotency_key = format!("recovery-not-started-{}", Uuid::now_v7()); + let owner = ExecutionExternalJobOwner::Compensation { + compensation_id: fence.compensation_id.as_uuid(), + compensation_generation: fence.compensation_generation, + compensation_attempt_generation: fence.attempt_generation, + }; + repository + .reserve_external_job_intent( + scope, + &config, + NewExecutionExternalJobIntent { + external_job_uid, + tenant_id, + run_uid: run.run_uid, + owner, + job_generation: 1, + provider: provider.clone(), + idempotency_key: idempotency_key.clone(), + expires_at: now + Duration::minutes(1), + }, + ) + .await?; + let recovery = ExecutionExternalJobStartRecoveryRequest { + tenant_id, + run_uid: run.run_uid, + owner: ExecutionExternalJobStartRecoveryOwner::Compensation { + compensation_id: fence.compensation_id.as_uuid(), + compensation_generation: fence.compensation_generation, + compensation_attempt_generation: fence.attempt_generation, + }, + external_job_uid, + job_generation: 1, + provider, + idempotency_key, + trigger_uid: Uuid::now_v7(), + }; + let ExecutionExternalJobStartRecoveryAdoptionOutcome::Applied { + compensation_release: Some(release_request), + } = repository + .recover_external_job_start_not_started(&recovery, now + Duration::milliseconds(2)) + .await? + else { + panic!("NotStarted recovery must fence exact compensation teardown"); + }; + assert_eq!( + release_request.intent, + ExecutionCompensationReleaseIntent::Retry + ); + assert!( + repository + .load_external_job(scope, external_job_uid) + .await? + .is_none() + ); + assert!(matches!( + repository + .yield_released_compensation_attempt_after_external_not_started( + &release_request, + now + Duration::milliseconds(3), + None, + ) + .await?, + CompensationAttemptWriteOutcome::Conflict + )); + let receipt = persist_compensation_release_receipt( + test_db.store().pool(), + &release_request, + now + Duration::milliseconds(3), + ) + .await?; + let CompensationAttemptWriteOutcome::Applied(requeued) = repository + .yield_released_compensation_attempt_after_external_not_started( + &release_request, + now + Duration::milliseconds(3), + Some(receipt.clone()), + ) + .await? + else { + panic!("verified NotStarted teardown must return the slice to Idle"); + }; + assert_eq!(requeued.attempt_state, CompensationAttemptState::Idle); + assert_eq!(requeued.attempt_generation, fence.attempt_generation + 1); + assert!(matches!( + repository + .yield_released_compensation_attempt_after_external_not_started( + &release_request, + now + Duration::milliseconds(4), + Some(receipt), + ) + .await?, + CompensationAttemptWriteOutcome::Replayed(_) + )); + Ok(()) +} + +#[tokio::test] +async fn recovered_started_compensation_adopts_canonical_resources_before_external_wait_db() +-> TestResult { + // Pins: provider Started recovery binds the job and derives the exact immutable compensation + // dispatch/capacity/watchdog request from locked storage; it cannot park WaitingExternal until + // the persisted verified hand-release receipt releases active ownership. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let repository = ExecutionRepository::new(test_db.store().pool().clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let (run, _) = compensating_run(&repository, scope, tenant_id, &["started"]).await?; + let now = moa_test_support::fixtures::pg_now(); + let config = ExecutionConfig::default(); + let admission = + active_compensation_admission(&repository, scope, &config, run.run_uid, now).await?; + let fence = fence(&admission); + assert!(matches!( + repository + .start_compensation_attempt(scope, fence, now + Duration::milliseconds(1)) + .await?, + CompensationAttemptWriteOutcome::Applied(_) + )); + let external_job_uid = Uuid::now_v7(); + let provider = "recovered-start-provider".to_string(); + let idempotency_key = format!("recovery-started-{}", Uuid::now_v7()); + let owner = ExecutionExternalJobOwner::Compensation { + compensation_id: fence.compensation_id.as_uuid(), + compensation_generation: fence.compensation_generation, + compensation_attempt_generation: fence.attempt_generation, + }; + repository + .reserve_external_job_intent( + scope, + &config, + NewExecutionExternalJobIntent { + external_job_uid, + tenant_id, + run_uid: run.run_uid, + owner, + job_generation: 1, + provider: provider.clone(), + idempotency_key: idempotency_key.clone(), + expires_at: now + Duration::minutes(1), + }, + ) + .await?; + let recovery = ExecutionExternalJobStartRecoveryRequest { + tenant_id, + run_uid: run.run_uid, + owner: ExecutionExternalJobStartRecoveryOwner::Compensation { + compensation_id: fence.compensation_id.as_uuid(), + compensation_generation: fence.compensation_generation, + compensation_attempt_generation: fence.attempt_generation, + }, + external_job_uid, + job_generation: 1, + provider: provider.clone(), + idempotency_key: idempotency_key.clone(), + trigger_uid: Uuid::now_v7(), + }; + let binding = ExecutionExternalJobBinding { + external_job_uid, + tenant_id, + run_uid: run.run_uid, + owner, + job_generation: 1, + idempotency_key, + provider, + provider_job_id: format!("provider-job-{}", Uuid::now_v7()), + callback_auth_reference: "vault://recovered-start".to_string(), + state: ExecutionExternalJobState::Running, + progress_phase: Some("running".to_string()), + cancel_supported: true, + next_reconcile_at: Some(now + Duration::minutes(2)), + provider_contract_violation: None, + }; + let ExecutionExternalJobStartRecoveryAdoptionOutcome::Applied { + compensation_release: Some(release_request), + } = repository + .recover_external_job_start_started( + &config, + &recovery, + binding, + now + Duration::milliseconds(2), + ) + .await? + else { + panic!("Started recovery must adopt exact compensation ownership"); + }; + assert_eq!( + release_request.capacity_reservation_uid, + admission.capacity_reservation_uid + ); + assert_eq!( + release_request.watchdog_trigger_uid, + admission.watchdog.trigger.trigger_uid + ); + assert_eq!( + release_request.intent, + ExecutionCompensationReleaseIntent::ExternalJob + ); + let receipt = persist_compensation_release_receipt( + test_db.store().pool(), + &release_request, + now + Duration::milliseconds(3), + ) + .await?; + let waiting = repository + .yield_released_compensation_attempt_to_external_job( + &release_request, + external_job_uid, + Some(receipt), + now + Duration::milliseconds(3), + ) + .await?; + assert!(matches!( + waiting, + moa_execution::repository::compensation::CompensationAttemptExternalOutcome::Applied { + ref attempt, + .. + } if attempt.attempt_state == CompensationAttemptState::WaitingExternal + )); + Ok(()) +} + +#[tokio::test] +async fn review_resolution_requires_exact_uid_and_slice_generation_db() -> TestResult { + // Pins: a stale or mismatched review callback cannot settle a newer + // compensation slice, while the exact review can resolve once. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let repository = ExecutionRepository::new(test_db.store().pool().clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let (run, _) = compensating_run(&repository, scope, tenant_id, &["reviewed"]).await?; + let now = moa_test_support::fixtures::pg_now(); + let config = ExecutionConfig::default(); + let admission = + active_compensation_admission(&repository, scope, &config, run.run_uid, now).await?; + let fence = fence(&admission); + assert!(matches!( + repository + .start_compensation_attempt(scope, fence, now + Duration::milliseconds(1)) + .await?, + CompensationAttemptWriteOutcome::Applied(_) + )); + let review_uid = Uuid::now_v7(); + let cancel_request = cancel_request( + &admission, + tenant_id, + ExecutionCompensationReleaseIntent::Review, + ); + assert!(matches!( + repository + .begin_compensation_attempt_release(&cancel_request, now + Duration::milliseconds(2),) + .await?, + CompensationAttemptReleaseClaimOutcome::Applied(_) + )); + let release_receipt = persist_compensation_release_receipt( + test_db.store().pool(), + &cancel_request, + now + Duration::milliseconds(3), + ) + .await?; + assert!(matches!( + repository + .park_released_compensation_review( + &cancel_request, + review_uid, + now + Duration::minutes(5), + now + Duration::milliseconds(3), + Some(release_receipt), + ) + .await?, + CompensationAttemptWriteOutcome::Applied(_) + )); + let resolution = ExecutionActionReviewResolution::Completed { + tool_output: json!({"undone": true}), + }; + assert!(matches!( + repository + .resolve_current_compensation_review( + scope, + run.run_uid, + fence.compensation_id, + fence.compensation_generation, + Uuid::now_v7(), + &resolution, + now + Duration::milliseconds(4), + ) + .await?, + CompensationReviewResolutionOutcome::Stale + )); + let CompensationReviewResolutionOutcome::Applied(settled) = repository + .resolve_current_compensation_review( + scope, + run.run_uid, + fence.compensation_id, + fence.compensation_generation, + review_uid, + &resolution, + now + Duration::milliseconds(5), + ) + .await? + else { + panic!("exact review must settle the parked compensation"); + }; + assert_eq!(settled.attempt_state, CompensationAttemptState::Terminal); + assert_eq!( + settled.registration.status, + moa_execution::state::CompensationStatus::Completed + ); + assert!(matches!( + repository + .resolve_current_compensation_review( + scope, + run.run_uid, + fence.compensation_id, + fence.compensation_generation, + review_uid, + &resolution, + now + Duration::milliseconds(6), + ) + .await?, + CompensationReviewResolutionOutcome::Replayed(_) + )); + Ok(()) +} + +#[tokio::test] +async fn paused_compensation_review_decision_waits_for_resume_activation_db() -> TestResult { + // Pins: pausing after a compensation review parks must not stale its pre-pause attempt owner; + // the exact decision persists while paused and only resume creates a controller activation. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let repository = ExecutionRepository::new(test_db.store().pool().clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let config = ExecutionConfig::default(); + let (paused, fence, review_uid) = park_compensation_review_then_pause( + &repository, + test_db.store().pool(), + scope, + tenant_id, + &config, + ) + .await?; + let resolved_at = moa_test_support::fixtures::pg_now() + Duration::seconds(1); + let resolution = ExecutionActionReviewResolution::Completed { + tool_output: json!({"undone": true}), + }; + + let CompensationReviewResolutionOutcome::Applied(settled) = repository + .resolve_current_compensation_review( + scope, + paused.run_uid, + fence.compensation_id, + fence.compensation_generation, + review_uid, + &resolution, + resolved_at, + ) + .await? + else { + panic!("the exact pre-pause compensation review owner must remain resolvable"); + }; + assert_eq!(settled.attempt_state, CompensationAttemptState::Terminal); + assert_eq!( + settled.registration.status, + moa_execution::state::CompensationStatus::Completed + ); + let still_paused = repository + .load_run(scope, paused.run_uid) + .await? + .expect("paused compensation run remains visible"); + assert_eq!(still_paused.status, ExecutionRunStatus::Paused); + assert_eq!( + still_paused.controller_generation, + paused.controller_generation + ); + assert_eq!( + run_activation_count( + test_db.store().pool(), + paused.run_uid, + paused.controller_generation, + ) + .await?, + 0, + "storage-only review resolution must not wake a paused run" + ); + assert!(matches!( + repository + .resolve_current_compensation_review( + scope, + paused.run_uid, + fence.compensation_id, + fence.compensation_generation, + review_uid, + &resolution, + resolved_at + Duration::milliseconds(1), + ) + .await?, + CompensationReviewResolutionOutcome::Replayed(_) + )); + + let TransitionOutcome::RunApplied(resumed) = repository + .resume_run(scope, &config, paused.run_uid, paused.controller_generation) + .await? + else { + panic!("paused compensation must resume after its decision is persisted"); + }; + assert_eq!(resumed.status, ExecutionRunStatus::Compensating); + assert_eq!( + run_activation_count( + test_db.store().pool(), + resumed.run_uid, + resumed.controller_generation, + ) + .await?, + 1, + "resume must enqueue exactly one controller activation" + ); + Ok(()) +} + +#[tokio::test] +async fn paused_compensation_review_timeout_waits_for_resume_activation_db() -> TestResult { + // Pins: an authoritative timeout received while paused settles the exact parked review once, + // remains storage-only, and is observed by the single activation created on resume. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let repository = ExecutionRepository::new(test_db.store().pool().clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let config = ExecutionConfig::default(); + let (paused, fence, review_uid) = park_compensation_review_then_pause( + &repository, + test_db.store().pool(), + scope, + tenant_id, + &config, + ) + .await?; + let timed_out_at = moa_test_support::fixtures::pg_now() + Duration::seconds(1); + let resolution = ExecutionActionReviewResolution::TimedOut { + reason: "review expired".to_string(), + }; + + let CompensationReviewResolutionOutcome::Applied(settled) = repository + .resolve_current_compensation_review( + scope, + paused.run_uid, + fence.compensation_id, + fence.compensation_generation, + review_uid, + &resolution, + timed_out_at, + ) + .await? + else { + panic!("the paused compensation review timeout must consume its exact parked owner"); + }; + assert_eq!(settled.attempt_state, CompensationAttemptState::Terminal); + assert_eq!( + settled.registration.status, + moa_execution::state::CompensationStatus::Failed + ); + assert_eq!( + run_activation_count( + test_db.store().pool(), + paused.run_uid, + paused.controller_generation, + ) + .await?, + 0, + "timeout settlement must not activate a paused run" + ); + assert!(matches!( + repository + .resolve_current_compensation_review( + scope, + paused.run_uid, + fence.compensation_id, + fence.compensation_generation, + review_uid, + &resolution, + timed_out_at + Duration::milliseconds(1), + ) + .await?, + CompensationReviewResolutionOutcome::Replayed(_) + )); + + let TransitionOutcome::RunApplied(resumed) = repository + .resume_run(scope, &config, paused.run_uid, paused.controller_generation) + .await? + else { + panic!("paused compensation must resume after its timeout is persisted"); + }; + assert_eq!(resumed.status, ExecutionRunStatus::Compensating); + assert_eq!( + run_activation_count( + test_db.store().pool(), + resumed.run_uid, + resumed.controller_generation, + ) + .await?, + 1, + "resume must enqueue exactly one controller activation" + ); + Ok(()) +} + +#[tokio::test] +async fn reviewed_external_job_is_parked_atomically_after_review_db() -> TestResult { + // Pins: a decision arriving before the compensation is parked remains retryable, while an + // exact approved async result atomically accepts the review and installs its durable job owner. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let repository = ExecutionRepository::new(test_db.store().pool().clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let (run, _) = compensating_run(&repository, scope, tenant_id, &["reviewed-external"]).await?; + let now = moa_test_support::fixtures::pg_now(); + let config = ExecutionConfig::default(); + let admission = + active_compensation_admission(&repository, scope, &config, run.run_uid, now).await?; + let fence = fence(&admission); + assert!(matches!( + repository + .start_compensation_attempt(scope, fence, now + Duration::milliseconds(1)) + .await?, + CompensationAttemptWriteOutcome::Applied(_) + )); + let review_uid = Uuid::now_v7(); + let async_job = AsyncToolJob { + provider: "review-provider".to_string(), + provider_job_id: format!("provider-job-{}", Uuid::now_v7()), + idempotency_key: format!("idempotency-{}", Uuid::now_v7()), + callback_auth_reference: "vault://review-callback".to_string(), + progress_phase: "queued".to_string(), + cancel_supported: true, + next_reconcile_at: now + Duration::minutes(2), + }; + let external_job_uid = Uuid::now_v7(); + let resolution = ExecutionActionReviewResolution::ExternalJob { + external_job_uid, + job: async_job.clone(), + }; + assert!(matches!( + repository + .resolve_current_compensation_review( + scope, + run.run_uid, + fence.compensation_id, + fence.compensation_generation, + review_uid, + &resolution, + now + Duration::milliseconds(2), + ) + .await?, + CompensationReviewResolutionOutcome::NotReady + )); + let release_request = cancel_request( + &admission, + tenant_id, + ExecutionCompensationReleaseIntent::Review, + ); + assert!(matches!( + repository + .begin_compensation_attempt_release(&release_request, now + Duration::milliseconds(3),) + .await?, + CompensationAttemptReleaseClaimOutcome::Applied(_) + )); + let release_receipt = persist_compensation_release_receipt( + test_db.store().pool(), + &release_request, + now + Duration::milliseconds(4), + ) + .await?; + assert!(matches!( + repository + .park_released_compensation_review( + &release_request, + review_uid, + now + Duration::minutes(5), + now + Duration::milliseconds(4), + Some(release_receipt), + ) + .await?, + CompensationAttemptWriteOutcome::Applied(_) + )); + let owner = ExecutionExternalJobOwner::Compensation { + compensation_id: fence.compensation_id.as_uuid(), + compensation_generation: fence.compensation_generation, + compensation_attempt_generation: fence.attempt_generation, + }; + repository + .reserve_external_job_intent( + scope, + &config, + NewExecutionExternalJobIntent { + external_job_uid, + tenant_id, + run_uid: run.run_uid, + owner, + job_generation: 1, + provider: async_job.provider.clone(), + idempotency_key: async_job.idempotency_key.clone(), + expires_at: now + Duration::minutes(1), + }, + ) + .await?; + repository + .bind_external_job( + scope, + &config, + ExecutionExternalJobBinding { + external_job_uid, + tenant_id, + run_uid: run.run_uid, + owner, + job_generation: 1, + idempotency_key: async_job.idempotency_key.clone(), + provider: async_job.provider.clone(), + provider_job_id: async_job.provider_job_id.clone(), + callback_auth_reference: async_job.callback_auth_reference.clone(), + state: ExecutionExternalJobState::Running, + progress_phase: Some(async_job.progress_phase.clone()), + cancel_supported: async_job.cancel_supported, + provider_contract_violation: None, + next_reconcile_at: Some(async_job.next_reconcile_at), + }, + ) + .await?; + let CompensationReviewResolutionOutcome::Applied(waiting) = repository + .resolve_current_compensation_review( + scope, + run.run_uid, + fence.compensation_id, + fence.compensation_generation, + review_uid, + &resolution, + now + Duration::milliseconds(5), + ) + .await? + else { + panic!("exact reviewed async job must be durably parked"); + }; + assert_eq!( + waiting.attempt_state, + CompensationAttemptState::WaitingExternal + ); + let persisted_external_job_uid = waiting + .external_job_uid + .expect("waiting external compensation must name its exact job"); + assert_eq!(persisted_external_job_uid, external_job_uid); + let external_job = repository + .load_external_job(scope, persisted_external_job_uid) + .await? + .expect("review resolution must persist the external job atomically"); + assert_eq!(external_job.state, ExecutionExternalJobState::Running); + assert_eq!( + external_job.owner, + ExecutionExternalJobOwner::Compensation { + compensation_id: fence.compensation_id.as_uuid(), + compensation_generation: fence.compensation_generation, + compensation_attempt_generation: fence.attempt_generation, + } + ); + assert!(matches!( + repository + .resolve_current_compensation_review( + scope, + run.run_uid, + fence.compensation_id, + fence.compensation_generation, + review_uid, + &resolution, + now + Duration::milliseconds(6), + ) + .await?, + CompensationReviewResolutionOutcome::Replayed(_) + )); + Ok(()) +} + +#[tokio::test] +async fn direct_external_callback_waits_for_compensation_hand_release_db() -> TestResult { + // Pins: a terminal provider callback received during Cancelling is persisted without settling + // the compensation; the verified hand-release finalizer then consumes it exactly once. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let repository = ExecutionRepository::new(test_db.store().pool().clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let (run, _) = compensating_run(&repository, scope, tenant_id, &["direct-external"]).await?; + let now = moa_test_support::fixtures::pg_now(); + let config = ExecutionConfig::default(); + let admission = + active_compensation_admission(&repository, scope, &config, run.run_uid, now).await?; + let fence = fence(&admission); + assert!(matches!( + repository + .start_compensation_attempt(scope, fence, now + Duration::milliseconds(1)) + .await?, + CompensationAttemptWriteOutcome::Applied(_) + )); + let release_request = cancel_request( + &admission, + tenant_id, + ExecutionCompensationReleaseIntent::ExternalJob, + ); + let external_job_uid = Uuid::now_v7(); + let provider_job_id = format!("provider-job-{}", Uuid::now_v7()); + let idempotency_key = format!("idempotency-{}", Uuid::now_v7()); + let owner = ExecutionExternalJobOwner::Compensation { + compensation_id: fence.compensation_id.as_uuid(), + compensation_generation: fence.compensation_generation, + compensation_attempt_generation: fence.attempt_generation, + }; + repository + .reserve_external_job_intent( + scope, + &config, + NewExecutionExternalJobIntent { + external_job_uid, + tenant_id, + run_uid: run.run_uid, + owner, + job_generation: 1, + provider: "direct-provider".to_string(), + idempotency_key: idempotency_key.clone(), + expires_at: now + Duration::minutes(1), + }, + ) + .await?; + repository + .bind_external_job( + scope, + &config, + ExecutionExternalJobBinding { + external_job_uid, + tenant_id, + run_uid: run.run_uid, + owner, + job_generation: 1, + idempotency_key, + provider: "direct-provider".to_string(), + provider_job_id: provider_job_id.clone(), + callback_auth_reference: "vault://direct-callback".to_string(), + state: ExecutionExternalJobState::Running, + progress_phase: Some("running".to_string()), + cancel_supported: true, + provider_contract_violation: None, + next_reconcile_at: Some(now + Duration::minutes(2)), + }, + ) + .await?; + let started = repository + .begin_compensation_external_release( + &release_request, + external_job_uid, + now + Duration::milliseconds(2), + ) + .await?; + assert!(matches!( + started, + moa_execution::repository::compensation::CompensationAttemptExternalOutcome::Applied { + ref attempt, + .. + } if attempt.attempt_state == CompensationAttemptState::Cancelling + && attempt.release_intent == Some(ExecutionCompensationReleaseIntent::ExternalJob) + )); + let callback = repository + .apply_external_job_callback_and_activate( + scope, + &config, + ExecutionExternalJobCallback { + external_job_uid, + job_generation: 1, + provider: "direct-provider".to_string(), + provider_job_id, + provider_event_id: format!("event-{}", Uuid::now_v7()), + update: ExecutionExternalJobCallbackUpdate::Terminal { + state: ExecutionExternalJobState::Completed, + progress_phase: Some("completed".to_string()), + output: Some(json!({"undone": true})), + error: None, + }, + }, + ) + .await?; + assert!(matches!( + callback.outcome, + ExecutionExternalJobCallbackOutcome::Applied(ref job) + if job.state == ExecutionExternalJobState::Completed + )); + assert_eq!(callback.activation, None); + let waiting_before_release: String = sqlx::query_scalar( + "SELECT attempt_state FROM moa.execution_compensation \ + WHERE run_uid=$1 AND compensation_id=$2", + ) + .bind(run.run_uid) + .bind(fence.compensation_id.as_uuid()) + .fetch_one(test_db.store().pool()) + .await?; + assert_eq!(waiting_before_release, "cancelling"); + let release_receipt = persist_compensation_release_receipt( + test_db.store().pool(), + &release_request, + now + Duration::milliseconds(3), + ) + .await?; + let finalized = repository + .yield_released_compensation_attempt_to_external_job( + &release_request, + external_job_uid, + Some(release_receipt), + now + Duration::milliseconds(3), + ) + .await?; + assert!(matches!( + finalized, + moa_execution::repository::compensation::CompensationAttemptExternalOutcome::Applied { + ref attempt, + .. + } if attempt.attempt_state == CompensationAttemptState::Terminal + )); + Ok(()) +} + +#[tokio::test] +async fn compensation_cancel_releases_capacity_only_after_verified_finalize_db() -> TestResult { + // Pins: claiming compensation teardown makes the attempt non-dispatchable but preserves its + // active capacity until the exact cancellation receiver reports provider release. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let repository = ExecutionRepository::new(test_db.store().pool().clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let (run, _) = compensating_run(&repository, scope, tenant_id, &["cancelled"]).await?; + let now = moa_test_support::fixtures::pg_now(); + let config = ExecutionConfig::default(); + let admission = + active_compensation_admission(&repository, scope, &config, run.run_uid, now).await?; + let fence = fence(&admission); + assert!(matches!( + repository + .start_compensation_attempt(scope, fence, now + Duration::milliseconds(1)) + .await?, + CompensationAttemptWriteOutcome::Applied(_) + )); + let request = cancel_request( + &admission, + tenant_id, + ExecutionCompensationReleaseIntent::RunTerminal, + ); + let CompensationAttemptReleaseClaimOutcome::Applied(claimed) = repository + .begin_compensation_attempt_release(&request, now + Duration::milliseconds(2)) + .await? + else { + panic!("exact cancellation must claim the active compensation"); + }; + assert_eq!(claimed.attempt_state, CompensationAttemptState::Cancelling); + assert_eq!( + claimed.release_intent, + Some(ExecutionCompensationReleaseIntent::RunTerminal) + ); + assert!(matches!( + repository + .begin_compensation_attempt_release(&request, now + Duration::milliseconds(2)) + .await?, + CompensationAttemptReleaseClaimOutcome::Replayed(_) + )); + let mut conflicting_intent = request.clone(); + conflicting_intent.intent = ExecutionCompensationReleaseIntent::Outcome; + assert!(matches!( + repository + .begin_compensation_attempt_release( + &conflicting_intent, + now + Duration::milliseconds(2), + ) + .await?, + CompensationAttemptReleaseClaimOutcome::Stale + )); + let reservation_state: String = sqlx::query_scalar( + "SELECT state FROM moa.execution_capacity_reservation WHERE reservation_uid=$1", + ) + .bind(admission.capacity_reservation_uid) + .fetch_one(test_db.store().pool()) + .await?; + assert_eq!(reservation_state, "reserved"); + + assert!(matches!( + repository + .settle_released_compensation_attempt( + &request, + ExecutionCompensationOutcome::Failed { + message: "run terminal fence".to_string(), + retryable: false, + usage: usage(0), + }, + now + Duration::milliseconds(3), + None, + ) + .await?, + CompensationAttemptWriteOutcome::Conflict + )); + let release_receipt = persist_compensation_release_receipt( + test_db.store().pool(), + &request, + now + Duration::milliseconds(3), + ) + .await?; + + let CompensationAttemptWriteOutcome::Applied(settled) = repository + .settle_released_compensation_attempt( + &request, + ExecutionCompensationOutcome::Failed { + message: "run terminal fence".to_string(), + retryable: false, + usage: usage(0), + }, + now + Duration::milliseconds(3), + Some(release_receipt), + ) + .await? + else { + panic!("verified cancellation must settle the compensation"); + }; + assert_eq!(settled.attempt_state, CompensationAttemptState::Terminal); + let reservation_state: String = sqlx::query_scalar( + "SELECT state FROM moa.execution_capacity_reservation WHERE reservation_uid=$1", + ) + .bind(admission.capacity_reservation_uid) + .fetch_one(test_db.store().pool()) + .await?; + assert_eq!(reservation_state, "released"); + Ok(()) +} + +async fn persist_compensation_release_receipt( + pool: &sqlx::PgPool, + request: &ExecutionCompensationAttemptCancelRequest, + released_at: chrono::DateTime, +) -> Result { + use moa_core::types::{ + identifiers::{ExecutionCompensationScopeId, ExecutionRunScopeId}, + sandbox_workspace::{ExecutionHandReleaseOwner, ExecutionHandReleaseReceipt}, + }; + + let receipt = ExecutionHandReleaseReceipt { + receipt_id: Uuid::now_v7(), + tenant_id: request.tenant_id, + run_id: ExecutionRunScopeId(request.run_uid), + owner: ExecutionHandReleaseOwner::Compensation { + compensation_id: ExecutionCompensationScopeId(request.compensation_id.as_uuid()), + logical_generation: request.compensation_generation, + }, + attempt_generation: request.compensation_attempt_generation, + workspace_id: None, + writer_epoch: None, + instance_generation: None, + hand_provisioning_operation_id: None, + hand_lease_generation: None, + checkpoint_id: None, + checkpoint_generation: None, + checkpoint_manifest_digest: None, + checkpoint_logical_bytes: None, + requested_at: released_at, + released_at, + }; + sqlx::query( + "INSERT INTO moa.sandbox_execution_hand_release_receipts \ + (receipt_id,tenant_id,run_uid,owner_kind,task_id,compensation_id, \ + logical_generation,attempt_generation,workspace_id,writer_epoch,instance_generation, \ + hand_provisioning_operation_id,hand_lease_generation,checkpoint_id, \ + checkpoint_generation,checkpoint_manifest_digest,checkpoint_logical_bytes, \ + receipt_state,destroy_outcome,claim_token,claim_expires_at,requested_at,deadline_at, \ + released_at) VALUES ($1,$2,$3,'compensation',NULL,$4,$5,$6,NULL,NULL,NULL,NULL,NULL, \ + NULL,NULL,NULL,NULL,'released','verified_absent',NULL,NULL,$7,$7,$7)", + ) + .bind(receipt.receipt_id) + .bind(request.tenant_id.0) + .bind(request.run_uid) + .bind(request.compensation_id.as_uuid()) + .bind(i64::try_from(request.compensation_generation).expect("fixture generation fits i64")) + .bind( + i64::try_from(request.compensation_attempt_generation) + .expect("fixture attempt generation fits i64"), + ) + .bind(released_at) + .execute(pool) + .await?; + Ok(receipt) +} + +async fn park_compensation_review_then_pause( + repository: &ExecutionRepository, + pool: &sqlx::PgPool, + scope: ExecutionScope, + tenant_id: TenantId, + config: &ExecutionConfig, +) -> Result< + (ExecutionRunRecord, CompensationAttemptFence, Uuid), + Box, +> { + let (run, _) = compensating_run(repository, scope, tenant_id, &["paused-review"]).await?; + let now = moa_test_support::fixtures::pg_now(); + let admission = + active_compensation_admission(repository, scope, config, run.run_uid, now).await?; + let fence = fence(&admission); + assert!(matches!( + repository + .start_compensation_attempt(scope, fence, now + Duration::milliseconds(1)) + .await?, + CompensationAttemptWriteOutcome::Applied(_) + )); + let review_uid = Uuid::now_v7(); + let release_request = cancel_request( + &admission, + tenant_id, + ExecutionCompensationReleaseIntent::Review, + ); + assert!(matches!( + repository + .begin_compensation_attempt_release(&release_request, now + Duration::milliseconds(2),) + .await?, + CompensationAttemptReleaseClaimOutcome::Applied(_) + )); + let release_receipt = persist_compensation_release_receipt( + pool, + &release_request, + now + Duration::milliseconds(3), + ) + .await?; + assert!(matches!( + repository + .park_released_compensation_review( + &release_request, + review_uid, + now + Duration::minutes(5), + now + Duration::milliseconds(3), + Some(release_receipt), + ) + .await?, + CompensationAttemptWriteOutcome::Applied(_) + )); + let TransitionOutcome::RunApplied(paused) = repository + .pause_run(scope, config, run.run_uid, run.controller_generation) + .await? + else { + panic!("a storage-only compensation review must permit an exact run pause"); + }; + assert_eq!(paused.status, ExecutionRunStatus::Paused); + assert_eq!( + paused.controller_generation, + run.controller_generation + 1, + "pause must advance only the run controller generation" + ); + Ok((paused, fence, review_uid)) +} + +async fn run_activation_count( + pool: &sqlx::PgPool, + run_uid: Uuid, + controller_generation: u64, +) -> Result { + sqlx::query_scalar( + "SELECT COUNT(*) FROM moa.execution_dispatch_outbox WHERE run_uid=$1 \ + AND controller_generation=$2 AND dispatch_kind='run_activation' \ + AND state IN ('pending','dispatching')", + ) + .bind(run_uid) + .bind(i64::try_from(controller_generation).expect("fixture controller generation fits i64")) + .fetch_one(pool) + .await +} + +fn fence( + admission: &moa_execution::repository::compensation::CompensationAttemptAdmission, +) -> CompensationAttemptFence { + CompensationAttemptFence { + run_uid: admission.attempt.registration.run_uid, + compensation_id: admission.attempt.registration.compensation_id, + controller_generation: admission.attempt.controller_generation, + compensation_generation: admission.attempt.registration.generation, + attempt_generation: admission.attempt.attempt_generation, + dispatch_uid: admission.dispatch.dispatch_uid, + } +} + +fn cancel_request( + admission: &moa_execution::repository::compensation::CompensationAttemptAdmission, + tenant_id: TenantId, + intent: ExecutionCompensationReleaseIntent, +) -> ExecutionCompensationAttemptCancelRequest { + let fence = fence(admission); + ExecutionCompensationAttemptCancelRequest { + cancellation_dispatch_uid: Uuid::now_v7(), + tenant_id, + run_uid: fence.run_uid, + compensation_id: fence.compensation_id, + controller_generation: fence.controller_generation, + attempt_controller_generation: fence.controller_generation, + compensation_generation: fence.compensation_generation, + compensation_attempt_generation: fence.attempt_generation, + active_dispatch_uid: fence.dispatch_uid, + capacity_reservation_uid: admission.capacity_reservation_uid, + watchdog_trigger_uid: admission.watchdog.trigger.trigger_uid, + intent, + } +} + +async fn active_compensation_admission( + repository: &ExecutionRepository, + scope: ExecutionScope, + config: &ExecutionConfig, + run_uid: Uuid, + now: chrono::DateTime, +) -> Result { + match repository + .admit_next_compensation_attempt(scope, config, run_uid, now) + .await? + { + CompensationAttemptAdmissionOutcome::Admitted(admission) + | CompensationAttemptAdmissionOutcome::Replayed(admission) => Ok(*admission), + other => panic!("compensation fixture must own one active slice, got {other:?}"), + } +} + +fn explain_index_scan<'a>( + value: &'a serde_json::Value, + expected_index: &str, +) -> Option<&'a serde_json::Map> { + match value { + serde_json::Value::Object(object) => { + if object.get("Index Name").and_then(serde_json::Value::as_str) == Some(expected_index) + { + return Some(object); + } + object + .values() + .find_map(|child| explain_index_scan(child, expected_index)) + } + serde_json::Value::Array(values) => values + .iter() + .find_map(|child| explain_index_scan(child, expected_index)), + _ => None, + } +} + +async fn compensating_run( + repository: &ExecutionRepository, + scope: ExecutionScope, + tenant_id: TenantId, + node_ids: &[&str], +) -> Result< + ( + moa_execution::repository::ExecutionRunRecord, + Vec, + ), + Box, +> { + let (catalog, forward_reference, compensation) = compensated_catalog(); + let mut new = new_run( + tenant_id, + None, + &format!("compensation-attempt-{}", Uuid::now_v7()), + ExecutionRunStatus::Queued, + budget(20), + ); + new.plan.definition.cancel_policy = ExecutionCancelPolicy::CompensateCommitted; + new.plan.catalog_hash = catalog.catalog_hash; + new.authorization.capability_refs = catalog + .capabilities + .iter() + .map(|capability| capability.reference.clone()) + .collect(); + new.catalog = catalog; + let run = create_run(repository, scope, new).await?; + let tasks = node_ids + .iter() + .map(|node_id| { + compensated_task( + run.run_uid, + node_id, + forward_reference.clone(), + compensation.clone(), + ) + }) + .collect::>(); + repository + .materialize_tasks(scope, run.run_uid, 1, tasks.clone()) + .await?; + for task in &tasks { + reserve_and_start(repository, scope, run.run_uid, task.task_id).await?; + assert!(matches!( + repository + .record_task_outcome(scope, run.run_uid, task.task_id, 1, completed(1)) + .await?, + TaskOutcomeWrite::Applied { .. } + )); + } + let run = repository + .load_run(scope, run.run_uid) + .await? + .expect("fixture run exists"); + let config = ExecutionConfig::default(); + let PendingTerminalAdvanceOutcome::Applied(commit) = repository + .fence_completion_terminal_and_enqueue_settlement( + &config, + scope, + run.run_uid, + run.controller_generation, + run.wake_epoch, + PendingExecutionTerminal { + status: ExecutionRunStatus::Failed, + reason: ExecutionTerminalReason::InternalFailure, + terminal_evidence: ExecutionTerminalEvidence { + cause: ExecutionTerminalCause::InternalFailure, + satisfied_requirement_count: 0, + requirement_count: 0, + }, + completion_check_results: Vec::new(), + terminal_gaps: vec!["forward failure".to_string()], + output: None, + cancellation_reason: None, + }, + moa_test_support::fixtures::pg_now(), + 1, + ) + .await? + else { + panic!("fixture bounded terminal page must apply"); + }; + let admission = if let Some(admission) = commit.compensation_admission { + admission + } else { + assert_eq!(commit.run.status, ExecutionRunStatus::Compensating); + let PendingTerminalAdvanceOutcome::Applied(continuation) = repository + .advance_pending_terminal_settlement( + &config, + scope, + commit.run.run_uid, + commit.run.controller_generation, + commit.run.wake_epoch, + moa_test_support::fixtures::pg_now(), + 1, + ) + .await? + else { + panic!("fixture compensation continuation must apply"); + }; + continuation + .compensation_admission + .expect("compensation continuation must admit one reverse-order slice") + }; + Ok((commit.run, vec![admission.attempt.registration.clone()])) +} + +fn compensated_catalog() -> ( + ExecutionCapabilityCatalog, + CapabilityReference, + ExecutionCompensation, +) { + let mut forward = capability("effects.commit"); + let compensator = capability("effects.undo"); + let compensation = ExecutionCompensation { + compensator: compensator.reference.clone(), + input_mapping: token_mapping(), + }; + forward.rollback = Some(CapabilityRollbackContract { + compensator: compensation.compensator.clone(), + input_mapping: compensation.input_mapping.clone(), + }); + let forward_reference = forward.reference.clone(); + let catalog = ExecutionCapabilityCatalog::build(vec![forward, compensator]) + .expect("compensated test catalog must be valid"); + (catalog, forward_reference, compensation) +} + +fn capability(name: &str) -> ExecutionCapability { + let source = CapabilitySource::BuiltInTool { + name: name.to_string(), + }; + ExecutionCapability { + reference: CapabilityReference { + name: name.to_string(), + version: "v1".to_string(), + }, + contract_revision: "contract-v1".to_string(), + description: format!("test capability {name}"), + input_schema: json!({"type":"object","required":["tokens"],"properties":{"tokens":{"type":"integer"}},"additionalProperties":false}), + output_schema: json!({"type":"object","required":["tokens"],"properties":{"tokens":{"type":"integer"}},"additionalProperties":false}), + action_class: ActionClass::ExternalWrite, + risk_level: RiskLevel::Medium, + default_effect: ActionPolicyEffect::Allow, + idempotency_class: IdempotencyClass::Idempotent, + async_mode: moa_core::types::tools::ToolAsyncMode::SynchronousOnly, + execution_class: ExecutionClass::External, + requires_sandbox: false, + policy_context: CapabilityPolicyContext::registered(source.clone()), + source, + estimate: estimate(1), + rollback: None, + } +} + +fn token_mapping() -> CompensationInputMapping { + CompensationInputMapping { + bindings: vec![CompensationInputBinding { + target_pointer: "/tokens".to_string(), + source: CompensationValueSource::OriginalOutput { + pointer: "/tokens".to_string(), + }, + }], + } +} + +fn compensated_task( + run_uid: Uuid, + node_id: &str, + forward_reference: CapabilityReference, + compensation: ExecutionCompensation, +) -> LogicalTask { + let mut task = logical_task(run_uid, node_id, "", estimate(1)); + task.input = json!({"tokens": 1}); + task.kind = LogicalTaskKind::Capability { + reference: forward_reference, + }; + task.compensation = Some(compensation); + task +} diff --git a/crates/moa-execution/tests/execution_db/compensation_db.rs b/crates/moa-execution/tests/execution_db/compensation_db.rs index 4b7201dfa..318d04a24 100644 --- a/crates/moa-execution/tests/execution_db/compensation_db.rs +++ b/crates/moa-execution/tests/execution_db/compensation_db.rs @@ -13,16 +13,7 @@ use moa_execution::{ CapabilityPolicyContext, CapabilityRollbackContract, CapabilitySource, ExecutionCapability, ExecutionClass, }, - repository::{ - BeginCompensationOutcome, CompensationClaimOutcome, CompensationFinalizationOutcome, - CompensationOutcomeWrite, ExecutionEffectAdmissionOutcome, ExecutionEffectOwner, - FencedTerminalFinalizationOutcome, TerminalFenceOutcome, - }, - state::{ - CompensationId, CompensationStatus, ExecutionCompensationOutcome, - ExecutionTerminalEvidence, PendingExecutionTerminal, - }, - wire::ExecutionToolDispatchRejection, + state::{CompensationId, ExecutionCompensationOutcome}, }; use super::support::*; @@ -74,21 +65,22 @@ async fn concurrent_forward_commits_register_one_unique_monotonic_sequence_each_ for outcome in [first?, second?, third?] { assert!(matches!(outcome, TaskOutcomeWrite::Applied { .. })); } - let snapshot = repository - .load_compensation_snapshot(scope, run.run_uid) - .await? - .expect("concurrent forward run must remain visible"); - let mut sequences = snapshot - .registrations + let registrations: Vec<(i64, Uuid)> = sqlx::query_as( + "SELECT registered_sequence,forward_task_id FROM moa.execution_compensation \ + WHERE run_uid=$1 ORDER BY registered_sequence", + ) + .bind(run.run_uid) + .fetch_all(test_db.store().pool()) + .await?; + let mut sequences = registrations .iter() - .map(|registration| registration.registered_sequence) + .map(|(sequence, _)| u64::try_from(*sequence).expect("fixture sequence fits u64")) .collect::>(); sequences.sort_unstable(); assert_eq!(sequences, vec![1, 2, 3]); - let mut owners = snapshot - .registrations + let mut owners = registrations .iter() - .map(|registration| registration.forward_task_id) + .map(|(_, forward_task_id)| *forward_task_id) .collect::>(); owners.sort_unstable(); owners.dedup(); @@ -100,1091 +92,6 @@ async fn concurrent_forward_commits_register_one_unique_monotonic_sequence_each_ Ok(()) } -#[tokio::test] -async fn third_forward_failure_compensates_only_the_first_two_committed_effects_db() -> TestResult { - // Pins: the acceptance failure is a real third task outcome, not an - // artificial fence after all three effects have already committed. - let test_db = moa_test_support::postgres::bootstrap_test_db().await?; - let repository = ExecutionRepository::new(test_db.store().pool().clone()); - let tenant_id = TenantId::new(); - let scope = ExecutionScope::Tenant { tenant_id }; - let (catalog, forward_reference, compensation) = compensated_catalog(); - let mut new = new_run( - tenant_id, - None, - "third-forward-real-failure", - ExecutionRunStatus::Queued, - budget(20), - ); - new.plan.definition.cancel_policy = ExecutionCancelPolicy::CompensateCommitted; - new.plan.catalog_hash = catalog.catalog_hash; - new.authorization.capability_refs = catalog - .capabilities - .iter() - .map(|capability| capability.reference.clone()) - .collect(); - new.catalog = catalog; - let run = create_run(&repository, scope, new).await?; - let tasks = ["first_effect", "second_effect", "third_effect"].map(|node_id| { - compensated_task( - run.run_uid, - node_id, - forward_reference.clone(), - compensation.clone(), - ) - }); - repository - .materialize_tasks(scope, run.run_uid, 1, tasks.to_vec()) - .await?; - for task in &tasks { - reserve_and_start(&repository, scope, run.run_uid, task.task_id).await?; - } - for task in &tasks[..2] { - assert!(matches!( - repository - .record_task_outcome(scope, run.run_uid, task.task_id, 1, completed(1)) - .await?, - TaskOutcomeWrite::Applied { .. } - )); - } - let failed = ExecutionTaskOutcome { - schema_version: 1, - usage: usage(1), - result: ExecutionTaskResult::Failed { - class: ExecutionFailureClass::Terminal, - message: "third forward effect failed before commit".to_string(), - }, - }; - assert!(matches!( - repository - .record_task_outcome(scope, run.run_uid, tasks[2].task_id, 1, failed) - .await?, - TaskOutcomeWrite::Applied { .. } - )); - let TerminalFenceOutcome::Applied(fence) = - fence_failed_run(&repository, scope, run.run_uid).await? - else { - panic!("real third-task failure must install a terminal fence"); - }; - let BeginCompensationOutcome::Applied(begin) = repository - .begin_compensation( - scope, - run.run_uid, - fence.run.plan_revision, - fence.run.wake_epoch, - ) - .await? - else { - panic!("two committed effects must enter compensation"); - }; - assert_eq!( - begin - .registrations - .iter() - .map(|registration| registration.forward_task_id) - .collect::>(), - vec![tasks[1].task_id, tasks[0].task_id] - ); - assert!( - begin - .registrations - .iter() - .all(|registration| registration.forward_task_id != tasks[2].task_id), - "the failed third effect must not invent a rollback registration" - ); - Ok(()) -} - -#[tokio::test] -async fn reverse_order_failure_finalizes_without_settling_lower_compensations_db() -> TestResult { - // Pins: after the highest compensation completes and the next one fails, - // finalization records manual repair without trying to claim or settle the - // lower pending registration. - let test_db = moa_test_support::postgres::bootstrap_test_db().await?; - let repository = ExecutionRepository::new(test_db.store().pool().clone()); - let tenant_id = TenantId::new(); - let scope = ExecutionScope::Tenant { tenant_id }; - let (catalog, forward_reference, compensation) = compensated_catalog(); - let mut new = new_run( - tenant_id, - None, - "reverse-order-compensation-failure", - ExecutionRunStatus::Queued, - budget(20), - ); - new.plan.definition.cancel_policy = ExecutionCancelPolicy::CompensateCommitted; - new.plan.catalog_hash = catalog.catalog_hash; - new.authorization.capability_refs = catalog - .capabilities - .iter() - .map(|capability| capability.reference.clone()) - .collect(); - new.catalog = catalog; - let run = create_run(&repository, scope, new).await?; - let tasks = ["effect_lowest", "effect_middle", "effect_highest"].map(|node_id| { - compensated_task( - run.run_uid, - node_id, - forward_reference.clone(), - compensation.clone(), - ) - }); - repository - .materialize_tasks(scope, run.run_uid, 1, tasks.to_vec()) - .await?; - for task in &tasks { - reserve_and_start(&repository, scope, run.run_uid, task.task_id).await?; - assert!(matches!( - repository - .record_task_outcome(scope, run.run_uid, task.task_id, 1, completed(1)) - .await?, - TaskOutcomeWrite::Applied { .. } - )); - } - - let fenced = fence_failed_run(&repository, scope, run.run_uid).await?; - let TerminalFenceOutcome::Applied(fence) = fenced else { - panic!("first terminal fence must be applied"); - }; - assert!(fence.tasks_to_settle.is_empty()); - let BeginCompensationOutcome::Applied(begin) = repository - .begin_compensation( - scope, - run.run_uid, - fence.run.plan_revision, - fence.run.wake_epoch, - ) - .await? - else { - panic!("fenced run with three registrations must begin compensation"); - }; - assert_eq!( - begin - .registrations - .iter() - .map(|registration| registration.forward_task_id) - .collect::>(), - vec![tasks[2].task_id, tasks[1].task_id, tasks[0].task_id] - ); - - let highest = &begin.registrations[0]; - let CompensationClaimOutcome::Claimed(claimed_highest) = repository - .claim_next_compensation( - scope, - run.run_uid, - highest.compensation_id, - highest.generation, - ) - .await? - else { - panic!("highest reverse sequence must be claimable first"); - }; - assert_eq!(claimed_highest.status, CompensationStatus::Running); - let completed_outcome = ExecutionCompensationOutcome::Completed { - output: json!({"tokens": 0}), - usage: usage(1), - }; - let CompensationOutcomeWrite::Completed(completed_highest) = repository - .record_compensation_outcome( - scope, - run.run_uid, - highest.compensation_id, - highest.generation, - completed_outcome.clone(), - ) - .await? - else { - panic!("highest compensation must settle completed"); - }; - assert_eq!(completed_highest.outcome, Some(completed_outcome)); - - let middle = &begin.registrations[1]; - let CompensationClaimOutcome::Claimed(claimed_middle) = repository - .claim_next_compensation( - scope, - run.run_uid, - middle.compensation_id, - middle.generation, - ) - .await? - else { - panic!("middle reverse sequence must be claimable after the highest settles"); - }; - assert_eq!(claimed_middle.status, CompensationStatus::Running); - let failed_outcome = ExecutionCompensationOutcome::Failed { - message: "undo was rejected permanently".to_string(), - retryable: false, - usage: usage(1), - }; - let CompensationOutcomeWrite::Failed(failed_middle) = repository - .record_compensation_outcome( - scope, - run.run_uid, - middle.compensation_id, - middle.generation, - failed_outcome.clone(), - ) - .await? - else { - panic!("terminal undo failure must persist as failed"); - }; - assert_eq!(failed_middle.outcome, Some(failed_outcome.clone())); - - let snapshot = repository - .load_compensation_snapshot(scope, run.run_uid) - .await? - .expect("compensating run must remain visible"); - assert_eq!( - snapshot - .registrations - .iter() - .map(|registration| registration.status) - .collect::>(), - vec![ - CompensationStatus::Completed, - CompensationStatus::Failed, - CompensationStatus::Pending, - ] - ); - assert!(snapshot.manual_repair_required); - let current = repository - .load_run(scope, run.run_uid) - .await? - .expect("manual-repair run must remain visible"); - let CompensationFinalizationOutcome::ManualRepairRequired(finalized) = repository - .finalize_compensation(scope, run.run_uid, current.wake_epoch) - .await? - else { - panic!("failed compensation must finalize as manual repair without waiting on lower work"); - }; - assert_eq!(finalized.status, ExecutionRunStatus::Failed); - assert_eq!( - finalized.terminal_reason, - Some(ExecutionTerminalReason::CompensationFailed) - ); - assert!(finalized.manual_repair_required); - assert!(finalized.pending_terminal.is_none()); - assert!(matches!( - finalized.terminal_evidence.as_ref().map(|evidence| &evidence.cause), - Some(ExecutionTerminalCause::CompensationFailure { - compensation_id, - outcome, - .. - }) if *compensation_id == middle.compensation_id && outcome == &failed_outcome - )); - let finalized_snapshot = repository - .load_compensation_snapshot(scope, run.run_uid) - .await? - .expect("finalized run must retain compensation audit rows"); - assert_eq!( - finalized_snapshot - .registrations - .iter() - .map(|registration| registration.status) - .collect::>(), - vec![ - CompensationStatus::Completed, - CompensationStatus::Failed, - CompensationStatus::Pending, - ] - ); - Ok(()) -} - -#[tokio::test] -async fn external_effect_admission_is_generation_and_terminal_fence_linearized_db() -> TestResult { - // Pins: ToolExecutor can start an effect only for the exact current running - // owner before its terminal fence; pending, stale, manual-repair, and - // terminal lifecycle states fail closed. - let test_db = moa_test_support::postgres::bootstrap_test_db().await?; - let repository = ExecutionRepository::new(test_db.store().pool().clone()); - let tenant_id = TenantId::new(); - let scope = ExecutionScope::Tenant { tenant_id }; - - let (task_catalog, task_forward_reference, task_compensation) = compensated_catalog(); - let mut task_new = new_run( - tenant_id, - None, - "forward-effect-admission", - ExecutionRunStatus::Queued, - budget(5), - ); - task_new.plan.definition.cancel_policy = ExecutionCancelPolicy::CompensateCommitted; - task_new.plan.catalog_hash = task_catalog.catalog_hash; - task_new.authorization.capability_refs = task_catalog - .capabilities - .iter() - .map(|capability| capability.reference.clone()) - .collect(); - task_new.catalog = task_catalog; - let task_run = create_run(&repository, scope, task_new).await?; - let task = compensated_task( - task_run.run_uid, - "forward_effect", - task_forward_reference, - task_compensation, - ); - repository - .materialize_tasks(scope, task_run.run_uid, 1, vec![task.clone()]) - .await?; - reserve_and_start(&repository, scope, task_run.run_uid, task.task_id).await?; - let task_owner = ExecutionEffectOwner::Task { - task_id: task.task_id, - generation: 1, - }; - assert_eq!( - repository - .admit_execution_effect(scope, task_run.run_uid, task_run.session_id, task_owner) - .await?, - ExecutionEffectAdmissionOutcome::Admitted - ); - assert_eq!( - repository - .admit_execution_effect(scope, task_run.run_uid, SessionId::new(), task_owner,) - .await?, - ExecutionEffectAdmissionOutcome::Rejected(ExecutionToolDispatchRejection::OriginNotFound) - ); - assert_eq!( - repository - .admit_execution_effect( - scope, - task_run.run_uid, - task_run.session_id, - ExecutionEffectOwner::Task { - task_id: task.task_id, - generation: 2, - }, - ) - .await?, - ExecutionEffectAdmissionOutcome::Rejected(ExecutionToolDispatchRejection::StaleGeneration) - ); - let TerminalFenceOutcome::Applied(task_fence) = - fence_failed_run(&repository, scope, task_run.run_uid).await? - else { - panic!("running forward task must accept an admission fence"); - }; - assert_eq!( - task_fence - .tasks_to_settle - .iter() - .map(|task| task.task_id) - .collect::>(), - vec![task.task_id] - ); - assert_eq!( - repository - .admit_execution_effect(scope, task_run.run_uid, task_run.session_id, task_owner) - .await?, - ExecutionEffectAdmissionOutcome::Rejected( - ExecutionToolDispatchRejection::RunNotDispatchable - ) - ); - - let (catalog, forward_reference, compensation) = compensated_catalog(); - let mut new = new_run( - tenant_id, - None, - "compensation-effect-admission", - ExecutionRunStatus::Queued, - budget(10), - ); - new.plan.definition.cancel_policy = ExecutionCancelPolicy::CompensateCommitted; - new.plan.catalog_hash = catalog.catalog_hash; - new.authorization.capability_refs = catalog - .capabilities - .iter() - .map(|capability| capability.reference.clone()) - .collect(); - new.catalog = catalog; - let compensation_run = create_run(&repository, scope, new).await?; - let forward_task = compensated_task( - compensation_run.run_uid, - "compensated_effect", - forward_reference, - compensation, - ); - repository - .materialize_tasks( - scope, - compensation_run.run_uid, - 1, - vec![forward_task.clone()], - ) - .await?; - reserve_and_start( - &repository, - scope, - compensation_run.run_uid, - forward_task.task_id, - ) - .await?; - assert!(matches!( - repository - .record_task_outcome( - scope, - compensation_run.run_uid, - forward_task.task_id, - 1, - completed(1), - ) - .await?, - TaskOutcomeWrite::Applied { .. } - )); - let TerminalFenceOutcome::Applied(compensation_fence) = - fence_failed_run(&repository, scope, compensation_run.run_uid).await? - else { - panic!("completed forward effect must accept a compensation fence"); - }; - let BeginCompensationOutcome::Applied(begin) = repository - .begin_compensation( - scope, - compensation_run.run_uid, - compensation_fence.run.plan_revision, - compensation_fence.run.wake_epoch, - ) - .await? - else { - panic!("registered effect must begin compensation"); - }; - let registration = &begin.registrations[0]; - let compensation_owner = ExecutionEffectOwner::Compensation { - compensation_id: registration.compensation_id, - generation: registration.generation, - }; - assert_eq!(registration.status, CompensationStatus::Pending); - assert_eq!( - repository - .admit_execution_effect( - scope, - compensation_run.run_uid, - compensation_run.session_id, - compensation_owner, - ) - .await?, - ExecutionEffectAdmissionOutcome::Rejected( - ExecutionToolDispatchRejection::OperationNotRunning - ) - ); - assert!(matches!( - repository - .claim_next_compensation( - scope, - compensation_run.run_uid, - registration.compensation_id, - registration.generation, - ) - .await?, - CompensationClaimOutcome::Claimed(_) - )); - assert_eq!( - repository - .admit_execution_effect( - scope, - compensation_run.run_uid, - compensation_run.session_id, - compensation_owner, - ) - .await?, - ExecutionEffectAdmissionOutcome::Admitted - ); - assert_eq!( - repository - .admit_execution_effect( - scope, - compensation_run.run_uid, - compensation_run.session_id, - ExecutionEffectOwner::Compensation { - compensation_id: registration.compensation_id, - generation: registration.generation + 1, - }, - ) - .await?, - ExecutionEffectAdmissionOutcome::Rejected(ExecutionToolDispatchRejection::StaleGeneration) - ); - assert!(matches!( - repository - .record_compensation_outcome( - scope, - compensation_run.run_uid, - registration.compensation_id, - registration.generation, - ExecutionCompensationOutcome::Failed { - message: "manual repair required".to_string(), - retryable: false, - usage: usage(1), - }, - ) - .await?, - CompensationOutcomeWrite::Failed(_) - )); - assert_eq!( - repository - .admit_execution_effect( - scope, - compensation_run.run_uid, - compensation_run.session_id, - compensation_owner, - ) - .await?, - ExecutionEffectAdmissionOutcome::Rejected( - ExecutionToolDispatchRejection::RunNotDispatchable - ) - ); - let repair_run = repository - .load_run(scope, compensation_run.run_uid) - .await? - .expect("manual-repair run must remain visible"); - assert!(repair_run.manual_repair_required); - assert!(matches!( - repository - .finalize_compensation(scope, compensation_run.run_uid, repair_run.wake_epoch) - .await?, - CompensationFinalizationOutcome::ManualRepairRequired(_) - )); - assert_eq!( - repository - .admit_execution_effect( - scope, - compensation_run.run_uid, - compensation_run.session_id, - compensation_owner, - ) - .await?, - ExecutionEffectAdmissionOutcome::Rejected( - ExecutionToolDispatchRejection::RunNotDispatchable - ) - ); - Ok(()) -} - -#[tokio::test] -async fn clean_compensations_claim_strict_reverse_order_and_restore_terminal_intent_db() --> TestResult { - // Pins: clean rollback dispatches every registration in descending commit - // order, rejects an early lower claim, replays a settled generation, and - // installs the exact held terminal intent only after the full drain. - let test_db = moa_test_support::postgres::bootstrap_test_db().await?; - let repository = ExecutionRepository::new(test_db.store().pool().clone()); - let tenant_id = TenantId::new(); - let scope = ExecutionScope::Tenant { tenant_id }; - let (catalog, forward_reference, compensation) = compensated_catalog(); - let mut new = new_run( - tenant_id, - None, - "clean-reverse-compensation-drain", - ExecutionRunStatus::Queued, - budget(20), - ); - new.plan.definition.cancel_policy = ExecutionCancelPolicy::CompensateCommitted; - new.plan.catalog_hash = catalog.catalog_hash; - new.authorization.capability_refs = catalog - .capabilities - .iter() - .map(|capability| capability.reference.clone()) - .collect(); - new.catalog = catalog; - let run = create_run(&repository, scope, new).await?; - let tasks = ["clean_lowest", "clean_middle", "clean_highest"].map(|node_id| { - compensated_task( - run.run_uid, - node_id, - forward_reference.clone(), - compensation.clone(), - ) - }); - repository - .materialize_tasks(scope, run.run_uid, 1, tasks.to_vec()) - .await?; - for task in &tasks { - reserve_and_start(&repository, scope, run.run_uid, task.task_id).await?; - assert!(matches!( - repository - .record_task_outcome(scope, run.run_uid, task.task_id, 1, completed(1)) - .await?, - TaskOutcomeWrite::Applied { .. } - )); - } - let TerminalFenceOutcome::Applied(fence) = - fence_failed_run(&repository, scope, run.run_uid).await? - else { - panic!("clean rollback fixture must accept its terminal fence"); - }; - let BeginCompensationOutcome::Applied(begin) = repository - .begin_compensation( - scope, - run.run_uid, - fence.run.plan_revision, - fence.run.wake_epoch, - ) - .await? - else { - panic!("clean rollback fixture must begin compensation"); - }; - assert_eq!( - begin - .registrations - .iter() - .map(|registration| registration.forward_task_id) - .collect::>(), - vec![tasks[2].task_id, tasks[1].task_id, tasks[0].task_id] - ); - let highest = &begin.registrations[0]; - let middle = &begin.registrations[1]; - let lowest = &begin.registrations[2]; - assert_eq!( - repository - .claim_next_compensation( - scope, - run.run_uid, - lowest.compensation_id, - lowest.generation, - ) - .await?, - CompensationClaimOutcome::Conflict - ); - - let highest_outcome = ExecutionCompensationOutcome::Completed { - output: json!({"tokens": 0}), - usage: usage(1), - }; - assert!(matches!( - repository - .claim_next_compensation( - scope, - run.run_uid, - highest.compensation_id, - highest.generation, - ) - .await?, - CompensationClaimOutcome::Claimed(_) - )); - let CompensationOutcomeWrite::Completed(completed_highest) = repository - .record_compensation_outcome( - scope, - run.run_uid, - highest.compensation_id, - highest.generation, - highest_outcome.clone(), - ) - .await? - else { - panic!("highest clean compensation must complete"); - }; - let CompensationOutcomeWrite::Replayed(replayed_highest) = repository - .record_compensation_outcome( - scope, - run.run_uid, - highest.compensation_id, - highest.generation, - highest_outcome, - ) - .await? - else { - panic!("settled compensation outcome must replay exactly"); - }; - assert_eq!(replayed_highest, completed_highest); - - for registration in [middle, lowest] { - assert!(matches!( - repository - .claim_next_compensation( - scope, - run.run_uid, - registration.compensation_id, - registration.generation, - ) - .await?, - CompensationClaimOutcome::Claimed(_) - )); - assert!(matches!( - repository - .record_compensation_outcome( - scope, - run.run_uid, - registration.compensation_id, - registration.generation, - ExecutionCompensationOutcome::Completed { - output: json!({"tokens": 0}), - usage: usage(1), - }, - ) - .await?, - CompensationOutcomeWrite::Completed(_) - )); - } - - let before_finalization = repository - .load_run(scope, run.run_uid) - .await? - .expect("drained compensation run must remain visible"); - let CompensationFinalizationOutcome::Finalized(finalized) = repository - .finalize_compensation(scope, run.run_uid, before_finalization.wake_epoch) - .await? - else { - panic!("clean reverse drain must restore its held terminal intent"); - }; - assert_eq!(finalized.status, ExecutionRunStatus::Failed); - assert_eq!( - finalized.terminal_reason, - Some(ExecutionTerminalReason::InternalFailure) - ); - assert_eq!( - finalized.terminal_evidence, - Some(ExecutionTerminalEvidence { - cause: ExecutionTerminalCause::InternalFailure, - satisfied_requirement_count: 0, - requirement_count: 0, - }) - ); - assert_eq!( - finalized.completion_check_results, - vec![json!({ - "check_id": "pre-compensation", - "passed": false, - "evidence": {"reason": "forward execution failed"}, - })] - ); - assert_eq!( - finalized.terminal_gaps, - vec!["forward execution failed".to_string()] - ); - assert_eq!(finalized.output, Some(json!({"forward": "evidence"}))); - assert!(!finalized.manual_repair_required); - assert!(finalized.pending_terminal.is_none()); - Ok(()) -} - -#[tokio::test] -async fn ambiguous_forward_effect_fences_automatic_compensation_and_finalizes_manual_repair_db() --> TestResult { - // Pins: an ambiguous forward effect never invents an undo registration or - // dispatch; once fenced and settled, the general terminal path preserves - // the ambiguity as compensation_failed manual-repair evidence. - let test_db = moa_test_support::postgres::bootstrap_test_db().await?; - let repository = ExecutionRepository::new(test_db.store().pool().clone()); - let tenant_id = TenantId::new(); - let scope = ExecutionScope::Tenant { tenant_id }; - let (catalog, forward_reference, compensation) = compensated_catalog(); - let mut new = new_run( - tenant_id, - None, - "ambiguous-forward-effect", - ExecutionRunStatus::Queued, - budget(10), - ); - new.plan.definition.cancel_policy = ExecutionCancelPolicy::CompensateCommitted; - new.plan.catalog_hash = catalog.catalog_hash; - new.authorization.capability_refs = catalog - .capabilities - .iter() - .map(|capability| capability.reference.clone()) - .collect(); - new.catalog = catalog; - let run = create_run(&repository, scope, new).await?; - let task = compensated_task( - run.run_uid, - "ambiguous_effect", - forward_reference, - compensation, - ); - repository - .materialize_tasks(scope, run.run_uid, 1, vec![task.clone()]) - .await?; - reserve_and_start(&repository, scope, run.run_uid, task.task_id).await?; - let ambiguous_outcome = ExecutionTaskOutcome { - schema_version: 1, - usage: usage(1), - result: ExecutionTaskResult::UnknownOutcome { - message: "forward effect may have committed".to_string(), - }, - }; - let TaskOutcomeWrite::Applied { - run: ambiguous_run, - task: ambiguous_task, - .. - } = repository - .record_task_outcome( - scope, - run.run_uid, - task.task_id, - 1, - ambiguous_outcome.clone(), - ) - .await? - else { - panic!("forward UnknownOutcome must commit as a durable terminal task outcome"); - }; - assert_eq!(ambiguous_task.status, ExecutionTaskStatus::Failed); - assert_eq!(ambiguous_task.current_outcome, Some(ambiguous_outcome)); - assert!(ambiguous_run.manual_repair_required); - let snapshot = repository - .load_compensation_snapshot(scope, run.run_uid) - .await? - .expect("ambiguous run must remain visible"); - assert!(snapshot.registrations.is_empty()); - assert!(snapshot.manual_repair_required); - let completion_evaluation = CompletionEvaluation { - status: CompletionStatus::Completed, - limit_stop: None, - checks: Vec::new(), - satisfied_requirement_ids: Vec::new(), - unsatisfied_requirement_ids: Vec::new(), - gaps: Vec::new(), - }; - let completion_cause = ExecutionTerminalCause::Completion { limit_stop: None }; - let completion_projection = TerminalProjection::Completed { - output: json!({"status": "complete"}), - }; - let completion_evidence = - terminal_evidence_from_evaluation(completion_cause.clone(), &completion_evaluation)?; - let completion_reason = execution_terminal_reason( - &completion_cause, - &completion_projection, - &completion_evaluation, - )?; - let mut ordinary_finalization = RunFinalizationRequest { - run_uid: run.run_uid, - expected_revision: ambiguous_run.plan_revision, - expected_wake_epoch: ambiguous_run.wake_epoch, - terminal_projection: completion_projection, - completion_evaluation, - terminal_evidence: completion_evidence, - terminal_reason: completion_reason, - }; - assert_eq!( - repository - .finalize_run(scope, ordinary_finalization.clone()) - .await?, - FinalizationOutcome::Conflict, - "ordinary finalization must reject manual-repair state" - ); - - let TerminalFenceOutcome::Applied(fence) = - fence_failed_run(&repository, scope, run.run_uid).await? - else { - panic!("ambiguous settled forward task must accept the terminal fence"); - }; - assert!(fence.tasks_to_settle.is_empty()); - ordinary_finalization.expected_wake_epoch = fence.run.wake_epoch; - assert_eq!( - repository - .finalize_run(scope, ordinary_finalization) - .await?, - FinalizationOutcome::Conflict, - "ordinary finalization must reject a pending terminal intent" - ); - assert_eq!( - repository - .claim_next_compensation(scope, run.run_uid, CompensationId::derive(task.task_id), 1,) - .await?, - CompensationClaimOutcome::Conflict - ); - let FencedTerminalFinalizationOutcome::ManualRepairRequired(finalized) = repository - .finalize_fenced_terminal( - scope, - run.run_uid, - fence.run.plan_revision, - fence.run.wake_epoch, - ) - .await? - else { - panic!("ambiguous forward effect must finalize through the manual-repair path"); - }; - let expected_compensation_outcome = ExecutionCompensationOutcome::UnknownOutcome { - message: "forward effect may have committed".to_string(), - usage: usage(1), - }; - assert_eq!(finalized.status, ExecutionRunStatus::Failed); - assert_eq!( - finalized.terminal_reason, - Some(ExecutionTerminalReason::CompensationFailed) - ); - assert!(finalized.manual_repair_required); - assert!(finalized.pending_terminal.is_none()); - assert!(matches!( - finalized.terminal_evidence.as_ref().map(|evidence| &evidence.cause), - Some(ExecutionTerminalCause::CompensationFailure { - original_status: ExecutionRunStatus::Failed, - original_reason: ExecutionTerminalReason::InternalFailure, - compensation_id, - outcome, - .. - }) if *compensation_id == CompensationId::derive(task.task_id) - && outcome == &expected_compensation_outcome - )); - Ok(()) -} - -#[tokio::test] -async fn retryable_compensation_replay_recovers_requeued_generation_db() -> TestResult { - // Pins: replaying the accepted generation-one retry outcome returns the - // already-requeued generation-two projection instead of conflicting, and - // the next generation remains claimable with review idempotency intact. - let test_db = moa_test_support::postgres::bootstrap_test_db().await?; - let repository = ExecutionRepository::new(test_db.store().pool().clone()); - let tenant_id = TenantId::new(); - let scope = ExecutionScope::Tenant { tenant_id }; - let (catalog, forward_reference, compensation) = compensated_catalog(); - let mut new = new_run( - tenant_id, - None, - "compensation-retry-replay", - ExecutionRunStatus::Queued, - budget(20), - ); - new.plan.definition.cancel_policy = ExecutionCancelPolicy::CompensateCommitted; - new.plan.catalog_hash = catalog.catalog_hash; - new.authorization.capability_refs = catalog - .capabilities - .iter() - .map(|capability| capability.reference.clone()) - .collect(); - new.catalog = catalog; - let run = create_run(&repository, scope, new).await?; - let task = compensated_task(run.run_uid, "retry", forward_reference, compensation); - repository - .materialize_tasks(scope, run.run_uid, 1, vec![task.clone()]) - .await?; - reserve_and_start(&repository, scope, run.run_uid, task.task_id).await?; - assert!(matches!( - repository - .record_task_outcome(scope, run.run_uid, task.task_id, 1, completed(1)) - .await?, - TaskOutcomeWrite::Applied { .. } - )); - let TerminalFenceOutcome::Applied(fence) = - fence_failed_run(&repository, scope, run.run_uid).await? - else { - panic!("first terminal fence must apply"); - }; - let BeginCompensationOutcome::Applied(begin) = repository - .begin_compensation( - scope, - run.run_uid, - fence.run.plan_revision, - fence.run.wake_epoch, - ) - .await? - else { - panic!("single registration must begin compensation"); - }; - let registration = &begin.registrations[0]; - assert!(matches!( - repository - .claim_next_compensation(scope, run.run_uid, registration.compensation_id, 1,) - .await?, - CompensationClaimOutcome::Claimed(_) - )); - assert!(matches!( - repository - .record_task_outcome(scope, run.run_uid, task.task_id, 1, completed(1)) - .await?, - TaskOutcomeWrite::Replayed { .. } - )); - let claimed_snapshot = repository - .load_compensation_snapshot(scope, run.run_uid) - .await? - .expect("claimed compensation must remain visible"); - assert_eq!( - claimed_snapshot.registrations[0].status, - CompensationStatus::Running - ); - let retryable_failure = ExecutionCompensationOutcome::Failed { - message: "retry undo".to_string(), - retryable: true, - usage: usage(1), - }; - let CompensationOutcomeWrite::Requeued(requeued) = repository - .record_compensation_outcome( - scope, - run.run_uid, - registration.compensation_id, - 1, - retryable_failure.clone(), - ) - .await? - else { - panic!("first retryable failure must requeue atomically"); - }; - assert_eq!(requeued.status, CompensationStatus::Pending); - assert_eq!(requeued.attempt, 2); - assert_eq!(requeued.generation, 2); - assert_eq!(requeued.outcome, Some(retryable_failure.clone())); - - let CompensationOutcomeWrite::Replayed(replayed) = repository - .record_compensation_outcome( - scope, - run.run_uid, - registration.compensation_id, - 1, - retryable_failure, - ) - .await? - else { - panic!("generation-one retry replay must recover the current generation-two row"); - }; - assert_eq!(replayed, requeued); - - let CompensationClaimOutcome::Claimed(claimed) = repository - .claim_next_compensation(scope, run.run_uid, registration.compensation_id, 2) - .await? - else { - panic!("replayed retry outcome must leave generation two claimable"); - }; - assert_eq!(claimed.status, CompensationStatus::Running); - assert_eq!(claimed.generation, 2); - - let review_uid = Uuid::new_v4(); - let resolution = ExecutionActionReviewResolution::Completed { - tool_output: json!({"reviewed": true}), - }; - assert_eq!( - repository - .record_compensation_action_review_resolution( - scope, - run.run_uid, - registration.compensation_id, - 2, - review_uid, - &resolution, - ) - .await?, - ActionReviewResolutionWrite::Applied - ); - assert_eq!( - repository - .record_compensation_action_review_resolution( - scope, - run.run_uid, - registration.compensation_id, - 2, - review_uid, - &resolution, - ) - .await?, - ActionReviewResolutionWrite::Replayed - ); - let conflicting_resolution = ExecutionActionReviewResolution::Denied { - reason: "different resolution".to_string(), - }; - let conflict = repository - .record_compensation_action_review_resolution( - scope, - run.run_uid, - registration.compensation_id, - 2, - review_uid, - &conflicting_resolution, - ) - .await - .expect_err("same review identity with a different resolution must fail closed"); - assert!( - matches!(conflict, moa_execution::Error::InvalidRepositoryData { .. }), - "review identity conflict returned the wrong error: {conflict:?}" - ); - Ok(()) -} - #[tokio::test] async fn invalid_compensation_mapping_registers_failed_without_rolling_back_forward_commit_db() -> TestResult { @@ -1235,41 +142,41 @@ async fn invalid_compensation_mapping_registers_failed_without_rolling_back_forw assert!(committed_run.manual_repair_required); assert_eq!(committed_run.next_compensation_sequence, 2); - let snapshot = repository - .load_compensation_snapshot(scope, run.run_uid) - .await? - .expect("forward commit must retain its failed compensation registration"); - assert!(snapshot.manual_repair_required); - assert_eq!(snapshot.registrations.len(), 1); - let registration = &snapshot.registrations[0]; + let registration: serde_json::Value = sqlx::query_scalar( + "SELECT to_jsonb(compensation) - 'updated_at' \ + FROM moa.execution_compensation AS compensation WHERE run_uid=$1", + ) + .bind(run.run_uid) + .fetch_one(test_db.store().pool()) + .await?; let expected_message = "invalid execution repository data: persisted compensation contract has no pinned compensator"; assert_eq!( - registration.compensation_id, - CompensationId::derive(task.task_id) + registration["compensation_id"], + json!(CompensationId::derive(task.task_id)) ); - assert_eq!(registration.forward_task_id, task.task_id); - assert_eq!(registration.registered_sequence, 1); - assert_eq!(registration.forward_generation, 1); - assert_eq!(registration.compensator, compensation); - assert_eq!(registration.mapped_input, serde_json::Value::Null); - assert_eq!(registration.status, CompensationStatus::Failed); + assert_eq!(registration["forward_task_id"], json!(task.task_id)); + assert_eq!(registration["registered_sequence"], json!(1)); + assert_eq!(registration["forward_generation"], json!(1)); + assert_eq!(registration["compensator"], json!(compensation)); + assert_eq!(registration["mapped_input"], serde_json::Value::Null); + assert_eq!(registration["status"], json!("failed")); assert_eq!( - registration.outcome, - Some(ExecutionCompensationOutcome::Failed { + registration["outcome"]["outcome"], + json!(ExecutionCompensationOutcome::Failed { message: expected_message.to_string(), retryable: false, usage: usage(0), }) ); assert_eq!( - registration.error, - Some(json!({ + registration["error"], + json!({ "class": "mapping_input_invalid", "message": expected_message, - })) + }) ); - assert!(registration.started_at.is_some()); - assert!(registration.completed_at.is_some()); + assert!(!registration["started_at"].is_null()); + assert!(!registration["completed_at"].is_null()); assert!(matches!( repository @@ -1277,11 +184,14 @@ async fn invalid_compensation_mapping_registers_failed_without_rolling_back_forw .await?, TaskOutcomeWrite::Replayed { .. } )); - let replayed_snapshot = repository - .load_compensation_snapshot(scope, run.run_uid) - .await? - .expect("replayed forward commit must retain compensation audit"); - assert_eq!(replayed_snapshot.registrations, snapshot.registrations); + let replayed_registration: serde_json::Value = sqlx::query_scalar( + "SELECT to_jsonb(compensation) - 'updated_at' \ + FROM moa.execution_compensation AS compensation WHERE run_uid=$1", + ) + .bind(run.run_uid) + .fetch_one(test_db.store().pool()) + .await?; + assert_eq!(replayed_registration, registration); Ok(()) } @@ -1330,7 +240,9 @@ fn capability(name: &str) -> ExecutionCapability { risk_level: RiskLevel::Medium, default_effect: ActionPolicyEffect::Allow, idempotency_class: IdempotencyClass::Idempotent, + async_mode: moa_core::types::tools::ToolAsyncMode::SynchronousOnly, execution_class: ExecutionClass::External, + requires_sandbox: false, policy_context: CapabilityPolicyContext::registered(source.clone()), source, estimate: estimate(1), @@ -1370,40 +282,3 @@ fn compensated_task( task.compensation = Some(compensation); task } - -async fn fence_failed_run( - repository: &ExecutionRepository, - scope: ExecutionScope, - run_uid: Uuid, -) -> Result { - let run = repository.load_run(scope, run_uid).await?.ok_or_else(|| { - moa_execution::Error::InvalidRepositoryInput { - message: "compensation fixture run is missing".to_string(), - } - })?; - repository - .fence_run_for_terminal( - scope, - run_uid, - run.plan_revision, - run.wake_epoch, - PendingExecutionTerminal { - status: ExecutionRunStatus::Failed, - reason: ExecutionTerminalReason::InternalFailure, - terminal_evidence: ExecutionTerminalEvidence { - cause: ExecutionTerminalCause::InternalFailure, - satisfied_requirement_count: 0, - requirement_count: 0, - }, - completion_check_results: vec![json!({ - "check_id": "pre-compensation", - "passed": false, - "evidence": {"reason": "forward execution failed"}, - })], - terminal_gaps: vec!["forward execution failed".to_string()], - output: Some(json!({"forward": "evidence"})), - cancellation_reason: None, - }, - ) - .await -} diff --git a/crates/moa-execution/tests/execution_db/completion_projection_db.rs b/crates/moa-execution/tests/execution_db/completion_projection_db.rs new file mode 100644 index 000000000..fb60f3a7b --- /dev/null +++ b/crates/moa-execution/tests/execution_db/completion_projection_db.rs @@ -0,0 +1,665 @@ +//! Bounded persisted completion projection and terminal-evidence contracts. + +use moa_artifacts::execution_plan::{ + CompletionCheck, CompletionCheckKind, ExecutionNode, ExecutionOperation, +}; +use moa_execution::{ + capability::node_output_hash, + repository::{ + completion::{CompletionAdvanceOutcome, CompletionAdvanceRequest}, + ready::{ReadyMaterializationOutcome, ReadyMaterializationRequest}, + replan_stop::{NewExecutionReplanStopIntent, ReplanStopIntentWriteOutcome}, + task::{ + TaskAttemptFence, TaskAttemptReleaseClaimOutcome, TaskAttemptSettlementOutcome, + TaskAttemptStartOutcome, + }, + }, +}; + +use super::support::*; + +fn output_node() -> ExecutionNode { + output_node_with_dependencies("output", &[]) +} + +fn output_node_with_dependencies(id: &str, depends_on: &[&str]) -> ExecutionNode { + ExecutionNode { + id: id.to_string(), + requirement_ids: vec!["req".to_string()], + depends_on: depends_on + .iter() + .map(|dependency| (*dependency).to_string()) + .collect(), + when: None, + input: json!({}), + output_schema: json!({ "type": "object" }), + operation: ExecutionOperation::Output { value: json!({}) }, + compensation: None, + retry: RetryPolicy { + max_attempts: 1, + initial_backoff_ms: 0, + max_backoff_ms: 0, + }, + budget: None, + } +} + +#[tokio::test] +async fn failed_and_unknown_outcome_tasks_cancel_transitive_unmaterialized_dependents_db() +-> TestResult { + // Pins: both Failed and UnknownOutcome source tasks terminalize every transitive dependent + // that never materialized, allowing the bounded completion projection to reach terminal intent. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let config = ExecutionConfig::default(); + + for (case, outcome, expected_task_status) in [ + ( + "failed", + ExecutionTaskOutcome { + schema_version: 1, + usage: usage(1), + result: ExecutionTaskResult::Failed { + class: ExecutionFailureClass::Terminal, + message: "source failed".to_string(), + }, + }, + ExecutionTaskStatus::Failed, + ), + ( + "unknown-outcome", + ExecutionTaskOutcome { + schema_version: 1, + usage: usage(1), + result: ExecutionTaskResult::UnknownOutcome { + message: "source outcome is unknowable".to_string(), + }, + }, + ExecutionTaskStatus::UnknownOutcome, + ), + ] { + let mut candidate = new_run( + tenant_id, + None, + &format!("transitive-terminal-{case}"), + ExecutionRunStatus::Queued, + budget(3), + ); + candidate.plan.definition.nodes = vec![ + output_node_with_dependencies("source", &[]), + output_node_with_dependencies("middle", &["source"]), + output_node_with_dependencies("leaf", &["middle"]), + ]; + candidate.plan.estimate.tasks = 3; + let run = create_run(&repository, scope, candidate).await?; + let source = logical_task(run.run_uid, "source", case, estimate(1)); + assert!( + repository + .initialize_scheduler_state(scope, run.run_uid) + .await? + ); + assert!(matches!( + repository + .materialize_ready_page( + scope, + &config, + ReadyMaterializationRequest { + run_uid: run.run_uid, + plan_revision: 1, + node_id: "source".to_string(), + expected_cursor: 0, + reduce_cursor: None, + source_exhausted: true, + terminal_output: None, + tasks: vec![source], + }, + ) + .await?, + ReadyMaterializationOutcome::Applied { .. } + )); + let admission = repository + .admit_ready_attempts(&config, 1, Utc::now()) + .await? + .admitted + .into_iter() + .next() + .expect("one terminal-source task must be admitted"); + let fence = TaskAttemptFence { + tenant_id: admission.tenant_id, + run_uid: admission.run_uid, + task_id: admission.task_id, + controller_generation: admission.controller_generation, + attempt_generation: admission.attempt_generation, + dispatch_uid: admission.dispatch_uid, + capacity_reservation_uid: admission.capacity_reservation_uid, + watchdog_trigger_uid: admission.watchdog_trigger_uid, + attempt_deadline_at: admission.attempt_deadline_at, + }; + let TaskAttemptStartOutcome::Started(started) = + repository.start_task_attempt(fence).await? + else { + panic!("{case} source attempt must start"); + }; + let settled_at = Utc::now(); + assert!(matches!( + repository + .begin_task_attempt_release( + fence, + started.task.generation, + "terminal_source", + settled_at, + ) + .await?, + TaskAttemptReleaseClaimOutcome::Applied(_) + )); + let TaskAttemptSettlementOutcome::Applied { task, .. } = repository + .settle_released_task_attempt(&config, fence, outcome, None, settled_at, None) + .await? + else { + panic!("{case} source attempt must settle"); + }; + assert_eq!(task.status, expected_task_status, "{case}"); + let nodes: Vec<(String, String, i64, i64, bool, bool)> = sqlx::query_as( + "SELECT node_id,node_status,total_task_count,remaining_dependency_count, \ + materialization_complete,aggregate_complete \ + FROM moa.execution_node_state WHERE run_uid=$1 ORDER BY node_order", + ) + .bind(run.run_uid) + .fetch_all(&pool) + .await?; + assert_eq!( + nodes, + vec![ + ( + "source".to_string(), + "failed".to_string(), + 1, + 0, + true, + false + ), + ( + "middle".to_string(), + "cancelled".to_string(), + 0, + 0, + true, + true + ), + ( + "leaf".to_string(), + "cancelled".to_string(), + 0, + 0, + true, + true + ), + ], + "{case} must close every never-materialized transitive dependent" + ); + + let current = repository + .load_run(scope, run.run_uid) + .await? + .expect("terminal-source run remains visible"); + let request = CompletionAdvanceRequest { + run_uid: run.run_uid, + controller_generation: current.controller_generation, + wake_epoch: current.wake_epoch, + page_size: 10, + now: Utc::now(), + }; + let mut pages = Vec::new(); + loop { + match repository + .advance_completion_projection(scope, &config, request) + .await? + { + CompletionAdvanceOutcome::Continue { + scanned_tasks, + scanned_nodes, + } => pages.push((scanned_tasks, scanned_nodes)), + CompletionAdvanceOutcome::NonSuccessTerminal { pending_terminal } => { + assert_eq!( + pending_terminal.status, + ExecutionRunStatus::Failed, + "{case}" + ); + break; + } + other => panic!("{case} completion projection did not advance: {other:?}"), + } + assert!( + pages.len() <= 2, + "{case} completion scan must remain bounded" + ); + } + assert_eq!(pages, vec![(1, 0), (0, 3)], "{case}"); + } + Ok(()) +} + +#[tokio::test] +async fn completion_projection_pages_twenty_five_hundred_tasks_and_nodes_db() -> TestResult { + // Pins: terminal evaluation advances an exact task cursor and a separate node cursor; the + // finalization activation never reloads the full task or node projection. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let mut candidate = new_run( + tenant_id, + None, + "bounded-completion-2501", + ExecutionRunStatus::Queued, + budget(5_000), + ); + candidate.plan.definition.nodes = vec![output_node()]; + let run = create_run(&repository, scope, candidate).await?; + let config = ExecutionConfig::default(); + + let all_tasks = (0_u64..2_501) + .map(|index| logical_task(run.run_uid, "output", &format!("{index:04}"), estimate(1))) + .collect::>(); + let mut cursor = 0_u64; + for (page_index, page) in all_tasks.chunks(1_000).enumerate() { + let source_exhausted = page_index == 2; + let ReadyMaterializationOutcome::Applied { next_cursor, .. } = repository + .materialize_ready_page( + scope, + &config, + ReadyMaterializationRequest { + run_uid: run.run_uid, + plan_revision: run.plan_revision, + node_id: "output".to_string(), + expected_cursor: cursor, + reduce_cursor: None, + source_exhausted, + terminal_output: None, + tasks: page.to_vec(), + }, + ) + .await? + else { + panic!("fresh completion setup page must apply"); + }; + cursor = next_cursor; + } + for (status, attempt_state) in [ + ("dispatching", "dispatching"), + ("running", "running"), + ("completed", "terminal"), + ] { + sqlx::query( + "UPDATE moa.execution_task SET status=$2,attempt_state=$3, \ + attempt_started_at=CASE WHEN $2='running' THEN NOW() ELSE attempt_started_at END, \ + output=CASE WHEN $2='completed' THEN '{}'::JSONB ELSE output END, \ + completed_at=CASE WHEN $2='completed' THEN NOW() ELSE completed_at END, \ + updated_at=NOW() WHERE run_uid=$1", + ) + .bind(run.run_uid) + .bind(status) + .bind(attempt_state) + .execute(&pool) + .await?; + } + let output = json!({}); + sqlx::query( + "UPDATE moa.execution_node_state SET node_status='completed', \ + ready_task_count=0,terminal_task_count=total_task_count, \ + succeeded_task_count=total_task_count,aggregate_output=$2,aggregate_output_hash=$3, \ + updated_at=NOW() WHERE run_uid=$1 AND node_id='output'", + ) + .bind(run.run_uid) + .bind(&output) + .bind(node_output_hash(&output)?.to_string()) + .execute(&pool) + .await?; + sqlx::query( + "UPDATE moa.execution_run SET status='running',ready_task_count=0,active_task_count=0, \ + updated_at=NOW() WHERE run_uid=$1", + ) + .bind(run.run_uid) + .execute(&pool) + .await?; + let current = repository + .load_run(scope, run.run_uid) + .await? + .expect("completion run remains visible"); + let RunControllerClaimOutcome::Claimed(mut current) = repository + .claim_controller_wake( + scope, + current.run_uid, + current.controller_generation, + current.wake_epoch, + ) + .await? + else { + panic!("initial completion wake must be claimable"); + }; + + let mut pages = Vec::new(); + let mut source_progress_at = None; + loop { + match repository + .advance_completion_projection( + scope, + &config, + CompletionAdvanceRequest { + run_uid: run.run_uid, + controller_generation: current.controller_generation, + wake_epoch: current.wake_epoch, + page_size: 1_000, + now: Utc::now(), + }, + ) + .await? + { + CompletionAdvanceOutcome::Continue { + scanned_tasks, + scanned_nodes, + } => { + pages.push((scanned_tasks, scanned_nodes)); + let persisted_source = sqlx::query_scalar::<_, chrono::DateTime>( + "SELECT source_progress_at FROM moa.execution_completion_scan WHERE run_uid=$1", + ) + .bind(run.run_uid) + .fetch_one(&pool) + .await?; + if let Some(expected) = source_progress_at { + assert_eq!(persisted_source, expected); + } else { + source_progress_at = Some(persisted_source); + } + let RunControllerCompletionOutcome::Applied { + run: continued, + continuation: Some(continuation), + } = repository + .complete_controller_wake( + scope, + &config, + run.run_uid, + RunControllerCompletionRequest { + controller_generation: current.controller_generation, + wake_epoch: current.wake_epoch, + checkpoint: ExecutionRunActivationCheckpoint { + status: current.status, + activation_state: ExecutionActivationState::Idle, + next_wake_at: current.next_wake_at, + waiting_since: current.waiting_since, + ready_task_count: current.ready_task_count, + active_task_count: current.active_task_count, + }, + continuation_payload: Some(json!({ + "reason": "completion_projection_test_continue" + })), + continuation_not_before_at: Utc::now(), + }, + ) + .await? + else { + panic!("completion page must atomically enqueue one continuation"); + }; + assert_eq!(continuation.wake_epoch, Some(continued.wake_epoch)); + let RunControllerClaimOutcome::Claimed(claimed) = repository + .claim_controller_wake( + scope, + continued.run_uid, + continued.controller_generation, + continued.wake_epoch, + ) + .await? + else { + panic!("completion continuation must be claimable"); + }; + assert_eq!(claimed.last_progress_at, source_progress_at.unwrap()); + current = claimed; + } + CompletionAdvanceOutcome::FinalizationReady(request) => { + assert_eq!(request.run_uid, run.run_uid); + break; + } + other => panic!("unexpected completion outcome: {other:?}"), + } + assert!( + pages.len() <= 4, + "completion cursor failed to make progress" + ); + } + assert_eq!(pages, vec![(1_000, 0), (1_000, 0), (501, 0), (0, 1)]); + let scan = sqlx::query_as::<_, (i64, bool, bool, chrono::DateTime)>( + "SELECT scanned_task_count,scan_complete,node_scan_complete,source_progress_at \ + FROM moa.execution_completion_scan WHERE run_uid=$1", + ) + .bind(run.run_uid) + .fetch_one(&pool) + .await?; + assert_eq!(scan, (2_501, true, true, source_progress_at.unwrap())); + Ok(()) +} + +#[tokio::test] +async fn replan_stop_completion_pages_rebind_exact_wake_without_duplicate_verifiers_db() +-> TestResult { + // Pins: every bounded ReplanStop page, old-wake ACK, single continuation, and intent-wake + // rebind commit atomically; the excluded WaitingReplan origin becomes blocked evidence and + // never causes an unbounded scan or verifier materialization. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let mut candidate = new_run( + tenant_id, + None, + "bounded-replan-stop", + ExecutionRunStatus::Queued, + budget(20), + ); + candidate.plan.definition.nodes = vec![output_node()]; + candidate.goal.completion_checks = vec![CompletionCheck { + id: "semantic".to_string(), + description: "Verifier must not be materialized after ReplanStop".to_string(), + requirement_ids: Vec::new(), + constraint_ids: Vec::new(), + kind: CompletionCheckKind::AgentVerifier { + instructions: "verify terminal output".to_string(), + max_turns: 1, + }, + }]; + let run = create_run(&repository, scope, candidate).await?; + let config = ExecutionConfig::default(); + let tasks = (0_u64..3) + .map(|index| logical_task(run.run_uid, "output", &format!("{index:04}"), estimate(1))) + .collect::>(); + let origin = tasks[2].task_id; + let ReadyMaterializationOutcome::Applied { .. } = repository + .materialize_ready_page( + scope, + &config, + ReadyMaterializationRequest { + run_uid: run.run_uid, + plan_revision: run.plan_revision, + node_id: "output".to_string(), + expected_cursor: 0, + reduce_cursor: None, + source_exhausted: true, + terminal_output: None, + tasks, + }, + ) + .await? + else { + panic!("fresh ReplanStop setup page must apply"); + }; + let failure = ExecutionTaskOutcome { + schema_version: 1, + usage: usage(0), + result: ExecutionTaskResult::Failed { + class: ExecutionFailureClass::Terminal, + message: "source unavailable".to_string(), + }, + }; + sqlx::query( + "UPDATE moa.execution_task SET status='failed',attempt_state='terminal', \ + current_outcome=$2,completed_at=NOW(),updated_at=NOW() \ + WHERE run_uid=$1 AND task_id<>$3", + ) + .bind(run.run_uid) + .bind(serde_json::to_value(failure)?) + .bind(origin.as_uuid()) + .execute(&pool) + .await?; + sqlx::query( + "UPDATE moa.execution_task SET status='waiting_replan',attempt_state='waiting', \ + current_outcome=$2,waiting_since=NOW(),updated_at=NOW() \ + WHERE run_uid=$1 AND task_id=$3", + ) + .bind(run.run_uid) + .bind(serde_json::to_value(needs_replan(1))?) + .bind(origin.as_uuid()) + .execute(&pool) + .await?; + sqlx::query( + "UPDATE moa.execution_node_state SET node_status='waiting',ready_task_count=0, \ + waiting_task_count=1,terminal_task_count=2,failed_task_count=2,updated_at=NOW() \ + WHERE run_uid=$1 AND node_id='output'", + ) + .bind(run.run_uid) + .execute(&pool) + .await?; + sqlx::query( + "UPDATE moa.execution_run SET status='waiting_replan',ready_task_count=0, \ + active_task_count=0,waiting_task_count=1,waiting_replan_task_count=1, \ + waiting_reasons_truncated=TRUE,waiting_since=NOW(),last_progress_at=NOW(), \ + updated_at=NOW() WHERE run_uid=$1", + ) + .bind(run.run_uid) + .execute(&pool) + .await?; + let amendment_hash: ExecutionHash = "a".repeat(64).parse()?; + let ReplanStopIntentWriteOutcome::Applied(queued) = repository + .request_replan_stop( + scope, + &ExecutionConfig::default(), + NewExecutionReplanStopIntent { + run_uid: run.run_uid, + session_id: run.session_id, + base_plan_revision: run.plan_revision, + origin_task_id: origin, + task_generation: 1, + amendment_hash, + stop_reason: ReplanStopReason::RepeatedFailure, + detail: Some("same failure exhausted replan policy".to_string()), + }, + ) + .await? + else { + panic!("fresh ReplanStop intent must persist with one activation"); + }; + let mut wake_epoch = queued.wake_epoch; + let mut source_progress_at = None; + let mut page_count = 0_u32; + loop { + let RunControllerClaimOutcome::Claimed(claimed) = repository + .claim_controller_wake(scope, run.run_uid, run.controller_generation, wake_epoch) + .await? + else { + panic!("exact rebound ReplanStop wake must be claimable"); + }; + let intent = repository + .load_replan_stop_intent(scope, run.run_uid, run.controller_generation, wake_epoch) + .await? + .expect("intent must follow its exact current wake"); + match repository + .advance_replan_stop_completion_projection( + scope, + &config, + CompletionAdvanceRequest { + run_uid: run.run_uid, + controller_generation: run.controller_generation, + wake_epoch, + page_size: 1, + now: Utc::now(), + }, + &intent, + ) + .await? + { + CompletionAdvanceOutcome::ReplanStopContinue { + scanned_tasks, + scanned_nodes, + continuation, + } => { + assert_eq!(u64::from(scanned_tasks) + u64::from(scanned_nodes), 1); + let next_wake = continuation + .wake_epoch + .expect("ReplanStop continuation has an exact wake"); + assert!(next_wake > wake_epoch); + let persisted_source = sqlx::query_scalar::<_, chrono::DateTime>( + "SELECT source_progress_at FROM moa.execution_completion_scan WHERE run_uid=$1", + ) + .bind(run.run_uid) + .fetch_one(&pool) + .await?; + if let Some(expected) = source_progress_at { + assert_eq!(persisted_source, expected); + } else { + source_progress_at = Some(persisted_source); + } + assert_eq!(claimed.last_progress_at, persisted_source); + assert!( + repository + .load_replan_stop_intent( + scope, + run.run_uid, + run.controller_generation, + wake_epoch, + ) + .await? + .is_none(), + "old wake must stop owning the intent after commit" + ); + wake_epoch = next_wake; + page_count += 1; + } + CompletionAdvanceOutcome::ReplanStopReady { + pending_terminal, + receipt, + } => { + assert_eq!(pending_terminal.status, ExecutionRunStatus::Blocked); + assert_eq!( + pending_terminal.reason, + ExecutionTerminalReason::RepeatedFailure + ); + assert!(pending_terminal.output.is_none()); + assert!( + pending_terminal + .terminal_gaps + .iter() + .any(|gap| { gap == "replan stop reason: repeated_failure" }) + ); + assert_eq!(receipt.task_id, origin); + assert_eq!(receipt.task_generation, 1); + assert_eq!(receipt.base_plan_revision, run.plan_revision); + assert_eq!(receipt.amendment_hash, amendment_hash); + break; + } + other => panic!("unexpected ReplanStop completion outcome: {other:?}"), + } + assert!(page_count <= 4, "ReplanStop cursor failed to make progress"); + } + assert_eq!(page_count, 4, "three task pages plus one node page"); + let verifier_count = sqlx::query_scalar::<_, i64>( + "SELECT COUNT(*) FROM moa.execution_task WHERE run_uid=$1 AND node_id LIKE '@check/%'", + ) + .bind(run.run_uid) + .fetch_one(&pool) + .await?; + assert_eq!(verifier_count, 0); + Ok(()) +} diff --git a/crates/moa-execution/tests/execution_db/execution_capacity_db.rs b/crates/moa-execution/tests/execution_db/execution_capacity_db.rs new file mode 100644 index 000000000..78c75d21d --- /dev/null +++ b/crates/moa-execution/tests/execution_db/execution_capacity_db.rs @@ -0,0 +1,463 @@ +//! Fleet-owned weighted-fair task admission contracts. + +use chrono::DateTime; +use moa_artifacts::execution_plan::{ExecutionNode, ExecutionOperation}; +use moa_config::ExecutionConfig; +use moa_db::ScopedConn; +use moa_execution::repository::ready::ReadyMaterializationRequest; +use moa_execution::repository::{ + capacity::ExecutionAdmissionBatch, ready::ReadyMaterializationOutcome, +}; + +use super::support::*; + +fn output_node() -> ExecutionNode { + ExecutionNode { + id: "work".to_string(), + requirement_ids: vec!["req".to_string()], + depends_on: Vec::new(), + when: None, + input: json!({}), + output_schema: json!({ "type": "object" }), + operation: ExecutionOperation::Output { value: json!({}) }, + compensation: None, + retry: RetryPolicy { + max_attempts: 1, + initial_backoff_ms: 1, + max_backoff_ms: 1, + }, + budget: None, + } +} + +async fn ready_run( + repository: &ExecutionRepository, + tenant_id: TenantId, + key: &str, + task_count: u64, +) -> Result { + let scope = ExecutionScope::Tenant { tenant_id }; + let mut candidate = new_run( + tenant_id, + None, + key, + ExecutionRunStatus::Queued, + budget(task_count.saturating_mul(2)), + ); + candidate.plan.definition.nodes = vec![output_node()]; + let run = create_run(repository, scope, candidate).await?; + repository + .initialize_scheduler_state(scope, run.run_uid) + .await?; + let tasks = (0..task_count) + .map(|index| logical_task(run.run_uid, "work", &format!("{index:04}"), estimate(1))) + .collect(); + assert!(matches!( + repository + .materialize_ready_page( + scope, + &ExecutionConfig::default(), + ReadyMaterializationRequest { + run_uid: run.run_uid, + plan_revision: 1, + node_id: "work".to_string(), + expected_cursor: 0, + reduce_cursor: None, + source_exhausted: true, + terminal_output: None, + tasks, + }, + ) + .await?, + ReadyMaterializationOutcome::Applied { .. } + )); + Ok(run.run_uid) +} + +#[tokio::test] +async fn tenant_capacity_scope_shares_one_fleet_bucket_without_cross_tenant_access_db() -> TestResult +{ + // Pins: tenant-scoped admission can create, read, and update its own bucket plus the one + // shared fleet sentinel; another tenant reuses that fleet row but cannot observe or mutate + // the first tenant's bucket, and fleet owner coordinates remain immutable. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let tenant_a = TenantId::new(); + let tenant_b = TenantId::new(); + let fleet_uid = Uuid::now_v7(); + let tenant_a_uid = Uuid::now_v7(); + let tenant_b_uid = Uuid::now_v7(); + + let mut tenant_a_conn = ScopedConn::begin_tenant(&pool, tenant_a).await?; + tenant_a_conn.assume_app_role().await?; + sqlx::query( + "INSERT INTO moa.execution_capacity_bucket ( \ + capacity_bucket_uid,scope_kind,tenant_id,resource_dimension,limit_value \ + ) VALUES ($1,'fleet',NULL,'active_runs',100), \ + ($2,'tenant',$3,'active_runs',10)", + ) + .bind(fleet_uid) + .bind(tenant_a_uid) + .bind(tenant_a.0) + .execute(tenant_a_conn.as_mut()) + .await?; + sqlx::query( + "UPDATE moa.execution_capacity_bucket SET reserved_quantity=reserved_quantity+1 \ + WHERE resource_dimension='active_runs'", + ) + .execute(tenant_a_conn.as_mut()) + .await?; + let tenant_a_rows: Vec<(String, Option, i64)> = sqlx::query_as( + "SELECT scope_kind,tenant_id,reserved_quantity \ + FROM moa.execution_capacity_bucket ORDER BY scope_kind", + ) + .fetch_all(tenant_a_conn.as_mut()) + .await?; + assert_eq!( + tenant_a_rows, + vec![ + ("fleet".to_string(), None, 1), + ("tenant".to_string(), Some(tenant_a.0), 1), + ] + ); + sqlx::query("SAVEPOINT cross_tenant_insert") + .execute(tenant_a_conn.as_mut()) + .await?; + let cross_tenant_insert = sqlx::query( + "INSERT INTO moa.execution_capacity_bucket ( \ + capacity_bucket_uid,scope_kind,tenant_id,resource_dimension,limit_value \ + ) VALUES ($1,'tenant',$2,'active_tasks',10)", + ) + .bind(Uuid::now_v7()) + .bind(tenant_b.0) + .execute(tenant_a_conn.as_mut()) + .await; + assert!(cross_tenant_insert.is_err()); + sqlx::query("ROLLBACK TO SAVEPOINT cross_tenant_insert") + .execute(tenant_a_conn.as_mut()) + .await?; + sqlx::query("SAVEPOINT fleet_owner_mutation") + .execute(tenant_a_conn.as_mut()) + .await?; + let fleet_owner_mutation = sqlx::query( + "UPDATE moa.execution_capacity_bucket SET resource_dimension='active_tasks' \ + WHERE capacity_bucket_uid=$1", + ) + .bind(fleet_uid) + .execute(tenant_a_conn.as_mut()) + .await; + assert!(fleet_owner_mutation.is_err()); + sqlx::query("ROLLBACK TO SAVEPOINT fleet_owner_mutation") + .execute(tenant_a_conn.as_mut()) + .await?; + tenant_a_conn.commit().await?; + + let mut tenant_b_conn = ScopedConn::begin_tenant(&pool, tenant_b).await?; + tenant_b_conn.assume_app_role().await?; + sqlx::query( + "INSERT INTO moa.execution_capacity_bucket ( \ + capacity_bucket_uid,scope_kind,tenant_id,resource_dimension,limit_value \ + ) VALUES ($1,'fleet',NULL,'active_runs',100) ON CONFLICT DO NOTHING", + ) + .bind(Uuid::now_v7()) + .execute(tenant_b_conn.as_mut()) + .await?; + sqlx::query( + "INSERT INTO moa.execution_capacity_bucket ( \ + capacity_bucket_uid,scope_kind,tenant_id,resource_dimension,limit_value \ + ) VALUES ($1,'tenant',$2,'active_runs',10)", + ) + .bind(tenant_b_uid) + .bind(tenant_b.0) + .execute(tenant_b_conn.as_mut()) + .await?; + sqlx::query( + "UPDATE moa.execution_capacity_bucket SET reserved_quantity=reserved_quantity+1 \ + WHERE scope_kind='fleet' AND resource_dimension='active_runs'", + ) + .execute(tenant_b_conn.as_mut()) + .await?; + let tenant_a_visible: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM moa.execution_capacity_bucket WHERE tenant_id=$1)", + ) + .bind(tenant_a.0) + .fetch_one(tenant_b_conn.as_mut()) + .await?; + assert!(!tenant_a_visible); + let cross_tenant_update = + sqlx::query("UPDATE moa.execution_capacity_bucket SET limit_value=11 WHERE tenant_id=$1") + .bind(tenant_a.0) + .execute(tenant_b_conn.as_mut()) + .await?; + assert_eq!(cross_tenant_update.rows_affected(), 0); + let cross_tenant_delete = + sqlx::query("DELETE FROM moa.execution_capacity_bucket WHERE tenant_id=$1") + .bind(tenant_a.0) + .execute(tenant_b_conn.as_mut()) + .await?; + assert_eq!(cross_tenant_delete.rows_affected(), 0); + tenant_b_conn.commit().await?; + + let mut control = ScopedConn::begin_control_plane(&pool).await?; + control.assume_app_role().await?; + let fleet: (i64, i64) = sqlx::query_as( + "SELECT count(*),max(reserved_quantity) \ + FROM moa.execution_capacity_bucket \ + WHERE scope_kind='fleet' AND resource_dimension='active_runs'", + ) + .fetch_one(control.as_mut()) + .await?; + assert_eq!(fleet, (1, 2)); + let tenant_ids: Vec = sqlx::query_scalar( + "SELECT tenant_id FROM moa.execution_capacity_bucket \ + WHERE scope_kind='tenant' AND resource_dimension='active_runs' ORDER BY tenant_id", + ) + .fetch_all(control.as_mut()) + .await?; + let mut expected_tenants = vec![tenant_a.0, tenant_b.0]; + expected_tenants.sort_unstable(); + assert_eq!(tenant_ids, expected_tenants); + control.commit().await?; + Ok(()) +} + +#[tokio::test] +async fn weighted_admission_is_atomic_bounded_and_fleet_owned_db() -> TestResult { + // Pins: one globally serialized admission transaction preserves the fleet ceiling, + // persists 2:1 weighted fairness, and commits task/outbox/watchdog/counters together. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let heavy_tenant = TenantId::new(); + let normal_tenant = TenantId::new(); + let heavy_run = ready_run(&repository, heavy_tenant, "capacity-heavy", 6).await?; + let normal_run = ready_run(&repository, normal_tenant, "capacity-normal", 6).await?; + sqlx::query( + "UPDATE moa.execution_tenant_dispatch_state SET weight = 2 \ + WHERE tenant_id = $1", + ) + .bind(heavy_tenant.0) + .execute(&pool) + .await?; + + let config = ExecutionConfig { + max_fleet_active_tasks: 6, + max_tenant_active_tasks: 6, + max_in_flight_tasks: 12, + ..ExecutionConfig::default() + }; + let ExecutionAdmissionBatch { admitted, .. } = repository + .admit_ready_attempts(&config, 6, Utc::now()) + .await?; + assert_eq!(admitted.len(), 6); + let heavy_count = admitted + .iter() + .filter(|item| item.tenant_id == heavy_tenant) + .count(); + let normal_count = admitted + .iter() + .filter(|item| item.tenant_id == normal_tenant) + .count(); + assert_eq!((heavy_count, normal_count), (4, 2)); + assert!(admitted.iter().all(|item| { + !item.dispatch_uid.is_nil() + && !item.capacity_reservation_uid.is_nil() + && !item.watchdog_trigger_uid.is_nil() + && !item.watchdog_dispatch_uid.is_nil() + })); + + let fleet_reserved: i64 = sqlx::query_scalar( + "SELECT reserved_quantity FROM moa.execution_capacity_bucket \ + WHERE scope_kind = 'fleet' AND resource_dimension = 'active_tasks'", + ) + .fetch_one(&pool) + .await?; + assert_eq!(fleet_reserved, 6); + let active_rows: i64 = sqlx::query_scalar( + "SELECT count(*) FROM moa.execution_task WHERE run_uid = ANY($1::UUID[]) \ + AND status = 'dispatching'", + ) + .bind(vec![heavy_run, normal_run]) + .fetch_one(&pool) + .await?; + assert_eq!(active_rows, 6); + let outbox_rows: i64 = sqlx::query_scalar( + "SELECT count(*) FROM moa.execution_dispatch_outbox \ + WHERE dispatch_kind = 'task_attempt' AND state = 'pending' \ + AND run_uid = ANY($1::UUID[])", + ) + .bind(vec![heavy_run, normal_run]) + .fetch_one(&pool) + .await?; + assert_eq!(outbox_rows, 6); + + let second = repository + .admit_ready_attempts(&config, 1, Utc::now()) + .await?; + assert!(second.admitted.is_empty(), "fleet ceiling must be exact"); + Ok(()) +} + +#[tokio::test] +async fn requested_admission_limit_is_a_hard_bound_independent_of_vec_capacity_db() -> TestResult { + // Pins: a dispatcher request for one attempt admits exactly one durable task/outbox/watchdog + // tuple even when fleet, tenant, run, and allocator capacity could accept a larger batch. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let run_uid = ready_run(&repository, tenant_id, "capacity-request-bound", 3).await?; + let config = ExecutionConfig { + max_fleet_active_tasks: 10, + max_tenant_active_tasks: 10, + max_in_flight_tasks: 10, + ..ExecutionConfig::default() + }; + + let batch = repository + .admit_ready_attempts(&config, 1, Utc::now()) + .await?; + assert_eq!(batch.admitted.len(), 1); + let state_counts: (i64, i64, i64) = sqlx::query_as( + "SELECT \ + count(*) FILTER (WHERE status='dispatching'), \ + count(*) FILTER (WHERE status='ready'), \ + (SELECT count(*) FROM moa.execution_dispatch_outbox \ + WHERE run_uid=$1 AND dispatch_kind='task_attempt' AND state='pending') \ + FROM moa.execution_task WHERE run_uid=$1", + ) + .bind(run_uid) + .fetch_one(&pool) + .await?; + assert_eq!(state_counts, (1, 2, 1)); + Ok(()) +} + +#[tokio::test] +async fn future_ready_task_is_admitted_only_at_its_persisted_due_time_db() -> TestResult { + // Pins: retry backoff remains storage-owned even when a dispatcher runs early; the exact + // task stays Ready without an outbox or capacity receipt until the supplied clock reaches it. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let run_uid = ready_run(&repository, tenant_id, "capacity-future-ready", 1).await?; + let observed_at = pg_deadline(Duration::zero()); + let ready_at = observed_at + Duration::seconds(10); + sqlx::query( + "UPDATE moa.execution_task SET ready_at=$2, last_progress_at=$2, updated_at=NOW() \ + WHERE run_uid=$1 AND status='ready'", + ) + .bind(run_uid) + .bind(ready_at) + .execute(&pool) + .await?; + + let config = ExecutionConfig::default(); + let early = repository + .admit_ready_attempts(&config, 1, observed_at) + .await?; + assert!( + early.admitted.is_empty(), + "future-ready task must not consume attempt capacity" + ); + let early_state: (String, DateTime, i64, i64) = sqlx::query_as( + "SELECT task.status, task.ready_at, \ + (SELECT count(*) FROM moa.execution_dispatch_outbox AS dispatch \ + WHERE dispatch.run_uid=task.run_uid AND dispatch.dispatch_kind='task_attempt'), \ + (SELECT count(*) FROM moa.execution_capacity_reservation AS capacity \ + WHERE capacity.run_uid=task.run_uid AND capacity.resource_dimension='active_tasks') \ + FROM moa.execution_task AS task WHERE task.run_uid=$1", + ) + .bind(run_uid) + .fetch_one(&pool) + .await?; + assert_eq!( + early_state, + ("ready".to_string(), ready_at, 0, 0), + "early admission must leave the storage-owned backoff intact" + ); + let due = repository + .admit_ready_attempts(&config, 1, ready_at) + .await?; + assert_eq!(due.admitted.len(), 1); + assert_eq!(due.admitted[0].run_uid, run_uid); + let due_state: (String, Option>, i64, i64) = sqlx::query_as( + "SELECT task.status, task.ready_at, \ + (SELECT count(*) FROM moa.execution_dispatch_outbox AS dispatch \ + WHERE dispatch.run_uid=task.run_uid AND dispatch.dispatch_kind='task_attempt'), \ + (SELECT count(*) FROM moa.execution_capacity_reservation AS capacity \ + WHERE capacity.run_uid=task.run_uid AND capacity.resource_dimension='active_tasks' \ + AND capacity.state='reserved') \ + FROM moa.execution_task AS task WHERE task.run_uid=$1", + ) + .bind(run_uid) + .fetch_one(&pool) + .await?; + assert_eq!( + due_state, + ("dispatching".to_string(), None, 1, 1), + "due admission must create one exact task attempt" + ); + Ok(()) +} + +#[tokio::test] +async fn no_work_admission_does_not_rewrite_unchanged_capacity_bucket_db() -> TestResult { + // Pins: polling an empty ready queue keeps the fleet admission lock and configured limit, + // but does not create a new bucket version when no durable capacity state changes. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let config = ExecutionConfig::default(); + + let first = repository + .admit_ready_attempts(&config, 1, Utc::now()) + .await?; + assert!(first.admitted.is_empty()); + let version_after_first_poll: i64 = sqlx::query_scalar( + "SELECT version FROM moa.execution_capacity_bucket \ + WHERE scope_kind='fleet' AND resource_dimension='active_tasks'", + ) + .fetch_one(&pool) + .await?; + + let second = repository + .admit_ready_attempts(&config, 1, Utc::now()) + .await?; + assert!(second.admitted.is_empty()); + let version_after_second_poll: i64 = sqlx::query_scalar( + "SELECT version FROM moa.execution_capacity_bucket \ + WHERE scope_kind='fleet' AND resource_dimension='active_tasks'", + ) + .fetch_one(&pool) + .await?; + assert_eq!( + version_after_second_poll, version_after_first_poll, + "a no-work prelock must not rewrite an unchanged capacity bucket" + ); + let changed_limit = config.max_fleet_active_tasks.saturating_add(1); + let changed_config = ExecutionConfig { + max_fleet_active_tasks: changed_limit, + ..config + }; + let changed = repository + .admit_ready_attempts(&changed_config, 1, Utc::now()) + .await?; + assert!(changed.admitted.is_empty()); + let changed_bucket: (i64, i64) = sqlx::query_as( + "SELECT limit_value, version FROM moa.execution_capacity_bucket \ + WHERE scope_kind='fleet' AND resource_dimension='active_tasks'", + ) + .fetch_one(&pool) + .await?; + assert_eq!( + changed_bucket, + ( + i64::from(changed_limit), + version_after_second_poll.saturating_add(1) + ), + "a real configured-limit change must remain durable and versioned" + ); + Ok(()) +} diff --git a/crates/moa-execution/tests/execution_db/incremental_scheduler_db.rs b/crates/moa-execution/tests/execution_db/incremental_scheduler_db.rs new file mode 100644 index 000000000..4b2a78ebd --- /dev/null +++ b/crates/moa-execution/tests/execution_db/incremental_scheduler_db.rs @@ -0,0 +1,1378 @@ +//! Bounded incremental scheduler projection and materialization contracts. + +use chrono::DateTime; +use moa_artifacts::execution_plan::{ + CapabilityReference, ExecutionNode, ExecutionOperation, ExecutionReducer, MapTask, +}; +use moa_config::ExecutionConfig; +use moa_execution::repository::capacity::ExecutionAdmissionItem; +use moa_execution::repository::ready::{ + ExecutionNodeQueueStatus, ExecutionReduceMaterializationCursor, MapAggregatePageOutcome, + MapAggregatePageRequest, ReadyMaterializationOutcome, ReadyMaterializationRequest, +}; +use moa_execution::repository::task::{TaskAttemptFence, TaskAttemptStartOutcome}; +use moa_execution::repository::{TransitionOutcome, TransitionRejection}; +use moa_execution::state::{WaitSettlement, completed_task_outcome}; +use serde_json::Value; + +use super::support::*; + +fn output_node(id: &str) -> ExecutionNode { + ExecutionNode { + id: id.to_string(), + requirement_ids: vec!["req".to_string()], + depends_on: Vec::new(), + when: None, + input: json!({}), + output_schema: json!({ "type": "object" }), + operation: ExecutionOperation::Output { value: json!({}) }, + compensation: None, + retry: RetryPolicy { + max_attempts: 1, + initial_backoff_ms: 1, + max_backoff_ms: 1, + }, + budget: None, + } +} + +fn reduce_node(id: &str) -> ExecutionNode { + let mut node = output_node(id); + node.operation = ExecutionOperation::Reduce { + items: Value::Array((0_u64..5_001).map(|item| json!(item)).collect()), + max_items: 5_001, + reducer: ExecutionReducer::Capability { + reference: CapabilityReference { + name: "test.reduce".to_string(), + version: "v1".to_string(), + }, + }, + batch_size: 2, + }; + node +} + +fn empty_map_node(id: &str) -> ExecutionNode { + let mut node = output_node(id); + node.operation = ExecutionOperation::Map { + items: json!([]), + item_key: String::new(), + max_items: 1, + item_output_schema: json!({}), + task: MapTask::Capability { + reference: CapabilityReference { + name: "test.map".to_string(), + version: "v1".to_string(), + }, + }, + }; + node +} + +fn map_node(id: &str, max_items: u32) -> ExecutionNode { + let mut node = empty_map_node(id); + let ExecutionOperation::Map { + max_items: stored_max_items, + .. + } = &mut node.operation + else { + unreachable!("empty-map fixture must remain a map") + }; + *stored_max_items = u64::from(max_items); + node +} + +async fn force_map_tasks_terminal( + pool: &sqlx::PgPool, + run_uid: Uuid, + node_id: &str, + output: Option<&Value>, +) -> TestResult { + for (status, attempt_state) in [("dispatching", "dispatching"), ("running", "running")] { + sqlx::query( + "UPDATE moa.execution_task SET status=$3,attempt_state=$4,updated_at=NOW() \ + WHERE run_uid=$1 AND node_id=$2", + ) + .bind(run_uid) + .bind(node_id) + .bind(status) + .bind(attempt_state) + .execute(pool) + .await?; + } + sqlx::query( + "UPDATE moa.execution_task SET status='completed',attempt_state='terminal', \ + output=COALESCE($3::JSONB,to_jsonb(item_key)),completed_at=NOW(),updated_at=NOW() \ + WHERE run_uid=$1 AND node_id=$2", + ) + .bind(run_uid) + .bind(node_id) + .bind(output) + .execute(pool) + .await?; + sqlx::query( + "UPDATE moa.execution_node_state SET node_status='pending',ready_task_count=0, \ + terminal_task_count=total_task_count,succeeded_task_count=total_task_count, \ + aggregate_output=NULL,aggregate_output_hash=NULL,aggregate_cursor_item_key=NULL, \ + aggregate_complete=FALSE,updated_at=NOW() WHERE run_uid=$1 AND node_id=$2", + ) + .bind(run_uid) + .bind(node_id) + .execute(pool) + .await?; + sqlx::query( + "UPDATE moa.execution_run SET status='running',ready_task_count=0,active_task_count=0, \ + updated_at=NOW() WHERE run_uid=$1", + ) + .bind(run_uid) + .execute(pool) + .await?; + Ok(()) +} + +#[tokio::test] +async fn ten_thousand_tasks_materialize_in_cursor_fenced_pages_db() -> TestResult { + // Pins: a large map-sized logical node never requires one unbounded task vector or + // full-task scheduling snapshot; each committed page is at most 1,000 tasks. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let repository = ExecutionRepository::new(test_db.store().pool().clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let mut candidate = new_run( + tenant_id, + None, + "incremental-10k", + ExecutionRunStatus::Queued, + budget(20_000), + ); + candidate.plan.definition.nodes = vec![output_node("collect")]; + let run = create_run(&repository, scope, candidate).await?; + assert!( + repository + .initialize_scheduler_state(scope, run.run_uid) + .await? + ); + + let mut cursor = 0_u64; + for page in 0_u64..10 { + let tasks = (0_u64..1_000) + .map(|offset| { + logical_task( + run.run_uid, + "collect", + &format!("item-{:05}", page * 1_000 + offset), + estimate(1), + ) + }) + .collect::>(); + let ReadyMaterializationOutcome::Applied { + tasks, + next_cursor, + triggers, + } = repository + .materialize_ready_page( + scope, + &ExecutionConfig::default(), + ReadyMaterializationRequest { + run_uid: run.run_uid, + plan_revision: 1, + node_id: "collect".to_string(), + expected_cursor: cursor, + reduce_cursor: None, + source_exhausted: false, + terminal_output: None, + tasks, + }, + ) + .await? + else { + panic!("a fresh cursor page must apply exactly once"); + }; + assert_eq!(tasks.len(), 1_000); + assert!(triggers.is_empty()); + assert_eq!(next_cursor, cursor + 1_000); + cursor = next_cursor; + } + + let projection = repository + .load_activation_projection(scope, run.run_uid, 32) + .await? + .expect("run must remain visible"); + assert_eq!(projection.nodes.len(), 1); + let node = &projection.nodes[0]; + assert_eq!(node.status, ExecutionNodeQueueStatus::Ready); + assert_eq!(node.materialization_cursor, 10_000); + assert_eq!(node.total_task_count, 10_000); + assert_eq!(node.ready_task_count, 10_000); + assert_eq!(projection.run.ready_task_count, 10_000); + + let verification = repository + .load_terminal_verification_page(scope, run.run_uid, None, 64) + .await?; + assert_eq!(verification.nonterminal_tasks.len(), 64); + assert!(verification.next_cursor.is_some()); + Ok(()) +} + +#[tokio::test] +async fn ten_thousand_map_outputs_aggregate_in_sixteen_row_crash_replay_pages_db() -> TestResult { + // Pins: a 10,000-item map builds its deterministic output through <=16-row persisted pages; + // replaying a committed cursor cannot duplicate values and dependencies release only once. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let mut candidate = new_run( + tenant_id, + None, + "map-aggregate-10k", + ExecutionRunStatus::Queued, + budget(20_000), + ); + let mut dependent = output_node("dependent"); + dependent.depends_on = vec!["map".to_string()]; + candidate.plan.definition.nodes = vec![map_node("map", 10_000), dependent]; + let run = create_run(&repository, scope, candidate).await?; + + let mut cursor = 0_u64; + for page in 0_u64..10 { + let tasks = (0_u64..1_000) + .map(|offset| { + logical_task( + run.run_uid, + "map", + &format!("item-{:05}", page * 1_000 + offset), + estimate(1), + ) + }) + .collect::>(); + let ReadyMaterializationOutcome::Applied { next_cursor, .. } = repository + .materialize_ready_page( + scope, + &ExecutionConfig::default(), + ReadyMaterializationRequest { + run_uid: run.run_uid, + plan_revision: run.plan_revision, + node_id: "map".to_string(), + expected_cursor: cursor, + reduce_cursor: None, + source_exhausted: page == 9, + terminal_output: None, + tasks, + }, + ) + .await? + else { + panic!("fresh map page must apply"); + }; + cursor = next_cursor; + } + force_map_tasks_terminal(&pool, run.run_uid, "map", None).await?; + let current = repository + .load_run(scope, run.run_uid) + .await? + .expect("map run"); + assert!(matches!( + repository + .claim_controller_wake( + scope, + run.run_uid, + current.controller_generation, + current.wake_epoch, + ) + .await?, + RunControllerClaimOutcome::Claimed(_) + )); + + let first = repository + .load_map_aggregate_candidate( + scope, + run.run_uid, + current.controller_generation, + current.wake_epoch, + ) + .await? + .expect("first aggregate page"); + let first_request = MapAggregatePageRequest { + run_uid: run.run_uid, + plan_revision: run.plan_revision, + controller_generation: current.controller_generation, + wake_epoch: current.wake_epoch, + node_id: first.node_id, + expected_cursor_item_key: first.cursor_item_key, + }; + let MapAggregatePageOutcome::Applied { + aggregated_tasks: 16, + aggregate_complete: false, + .. + } = repository + .advance_map_aggregate_page(scope, first_request.clone()) + .await? + else { + panic!("first aggregate page must append sixteen outputs"); + }; + assert!(matches!( + repository + .advance_map_aggregate_page(scope, first_request) + .await?, + MapAggregatePageOutcome::Replayed { + aggregate_complete: false, + .. + } + )); + + let mut committed_pages = 1_u32; + loop { + let Some(candidate) = repository + .load_map_aggregate_candidate( + scope, + run.run_uid, + current.controller_generation, + current.wake_epoch, + ) + .await? + else { + break; + }; + match repository + .advance_map_aggregate_page( + scope, + MapAggregatePageRequest { + run_uid: run.run_uid, + plan_revision: run.plan_revision, + controller_generation: current.controller_generation, + wake_epoch: current.wake_epoch, + node_id: candidate.node_id, + expected_cursor_item_key: candidate.cursor_item_key, + }, + ) + .await? + { + MapAggregatePageOutcome::Applied { + aggregated_tasks, + aggregate_complete, + .. + } => { + assert!((1..=16).contains(&aggregated_tasks)); + committed_pages += 1; + if aggregate_complete { + break; + } + } + other => panic!("unexpected map aggregate outcome: {other:?}"), + } + assert!(committed_pages <= 625, "aggregate cursor failed to advance"); + } + assert_eq!(committed_pages, 625); + let aggregate = sqlx::query_as::<_, (String, bool, Value, i64)>( + "SELECT node_status,aggregate_complete,aggregate_output, \ + (SELECT remaining_dependency_count FROM moa.execution_node_state \ + WHERE run_uid=$1 AND node_id='dependent') \ + FROM moa.execution_node_state WHERE run_uid=$1 AND node_id='map'", + ) + .bind(run.run_uid) + .fetch_one(&pool) + .await?; + assert_eq!(aggregate.0, "completed"); + assert!(aggregate.1); + assert_eq!(aggregate.2.as_array().map(Vec::len), Some(10_000)); + assert_eq!(aggregate.3, 0); + Ok(()) +} + +#[tokio::test] +async fn seventeenth_near_sixty_four_kib_map_output_fails_before_unbounded_aggregate_db() +-> TestResult { + // Pins: sixteen near-limit inline outputs fit one bounded page, while the seventeenth crosses + // the cumulative one-MiB ceiling, fails the node, and never releases its dependency. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let mut candidate = new_run( + tenant_id, + None, + "map-aggregate-overflow", + ExecutionRunStatus::Queued, + budget(100), + ); + let mut dependent = output_node("dependent"); + dependent.depends_on = vec!["map".to_string()]; + candidate.plan.definition.nodes = vec![map_node("map", 17), dependent]; + let run = create_run(&repository, scope, candidate).await?; + let tasks = (0_u64..17) + .map(|index| logical_task(run.run_uid, "map", &format!("item-{index:02}"), estimate(1))) + .collect::>(); + assert!(matches!( + repository + .materialize_ready_page( + scope, + &ExecutionConfig::default(), + ReadyMaterializationRequest { + run_uid: run.run_uid, + plan_revision: run.plan_revision, + node_id: "map".to_string(), + expected_cursor: 0, + reduce_cursor: None, + source_exhausted: true, + terminal_output: None, + tasks, + }, + ) + .await?, + ReadyMaterializationOutcome::Applied { .. } + )); + let near_limit = Value::String("x".repeat(64_000)); + force_map_tasks_terminal(&pool, run.run_uid, "map", Some(&near_limit)).await?; + let current = repository + .load_run(scope, run.run_uid) + .await? + .expect("overflow run"); + assert!(matches!( + repository + .claim_controller_wake( + scope, + run.run_uid, + current.controller_generation, + current.wake_epoch, + ) + .await?, + RunControllerClaimOutcome::Claimed(_) + )); + let first = repository + .load_map_aggregate_candidate( + scope, + run.run_uid, + current.controller_generation, + current.wake_epoch, + ) + .await? + .expect("overflow first page"); + assert!(matches!( + repository + .advance_map_aggregate_page( + scope, + MapAggregatePageRequest { + run_uid: run.run_uid, + plan_revision: run.plan_revision, + controller_generation: current.controller_generation, + wake_epoch: current.wake_epoch, + node_id: first.node_id, + expected_cursor_item_key: first.cursor_item_key, + }, + ) + .await?, + MapAggregatePageOutcome::Applied { + aggregated_tasks: 16, + aggregate_complete: false, + .. + } + )); + let second = repository + .load_map_aggregate_candidate( + scope, + run.run_uid, + current.controller_generation, + current.wake_epoch, + ) + .await? + .expect("overflow second page"); + assert_eq!( + repository + .advance_map_aggregate_page( + scope, + MapAggregatePageRequest { + run_uid: run.run_uid, + plan_revision: run.plan_revision, + controller_generation: current.controller_generation, + wake_epoch: current.wake_epoch, + node_id: second.node_id, + expected_cursor_item_key: second.cursor_item_key, + }, + ) + .await?, + MapAggregatePageOutcome::Overflow + ); + let state = sqlx::query_as::<_, (String, bool, Option, i64)>( + "SELECT node_status,aggregate_complete,aggregate_output, \ + (SELECT remaining_dependency_count FROM moa.execution_node_state \ + WHERE run_uid=$1 AND node_id='dependent') \ + FROM moa.execution_node_state WHERE run_uid=$1 AND node_id='map'", + ) + .bind(run.run_uid) + .fetch_one(&pool) + .await?; + assert_eq!(state, ("failed".to_string(), true, None, 1)); + Ok(()) +} + +#[tokio::test] +async fn twenty_five_hundred_reduce_batches_persist_exact_round_cursor_db() -> TestResult { + // Pins: three bounded commits persist 2,501 distinct first-round reducer batches, replay the + // last page exactly, and reject an off-by-one round cursor without duplicating task rows. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let repository = ExecutionRepository::new(test_db.store().pool().clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let mut candidate = new_run( + tenant_id, + None, + "reduce-2501-batches", + ExecutionRunStatus::Queued, + budget(10_000), + ); + candidate.plan.definition.nodes = vec![reduce_node("reduce")]; + let run = create_run(&repository, scope, candidate).await?; + assert!( + repository + .initialize_scheduler_state(scope, run.run_uid) + .await? + ); + + let mut total_cursor = 0_u64; + let mut replay_request = None; + for (batch_cursor, page_count) in [(0_u64, 1_000_u64), (1_000, 1_000), (2_000, 501)] { + let tasks = (batch_cursor..batch_cursor + page_count) + .map(|batch| logical_task(run.run_uid, "reduce", &format!("r1:b{batch}"), estimate(1))) + .collect::>(); + let request = ReadyMaterializationRequest { + run_uid: run.run_uid, + plan_revision: 1, + node_id: "reduce".to_string(), + expected_cursor: total_cursor, + reduce_cursor: Some(ExecutionReduceMaterializationCursor { + round: 1, + batch_cursor, + round_input_count: 5_001, + }), + source_exhausted: batch_cursor + page_count == 2_501, + terminal_output: None, + tasks, + }; + let ReadyMaterializationOutcome::Applied { next_cursor, .. } = repository + .materialize_ready_page(scope, &ExecutionConfig::default(), request.clone()) + .await? + else { + panic!("fresh reduce page must apply"); + }; + total_cursor = next_cursor; + replay_request = Some(request); + } + assert_eq!(total_cursor, 2_501); + assert!(matches!( + repository + .materialize_ready_page( + scope, + &ExecutionConfig::default(), + replay_request.expect("last page"), + ) + .await?, + ReadyMaterializationOutcome::Replayed { + next_cursor: 2_501, + .. + } + )); + + let off_by_one = ReadyMaterializationRequest { + run_uid: run.run_uid, + plan_revision: 1, + node_id: "reduce".to_string(), + expected_cursor: 2_501, + reduce_cursor: Some(ExecutionReduceMaterializationCursor { + round: 1, + batch_cursor: 2_500, + round_input_count: 5_001, + }), + source_exhausted: true, + terminal_output: None, + tasks: vec![logical_task(run.run_uid, "reduce", "r1:b2500", estimate(1))], + }; + assert_eq!( + repository + .materialize_ready_page(scope, &ExecutionConfig::default(), off_by_one) + .await?, + ReadyMaterializationOutcome::Conflict + ); + + let projection = repository + .load_activation_projection(scope, run.run_uid, 1) + .await? + .expect("reduce run projection"); + let node = &projection.nodes[0]; + assert_eq!(node.materialization_cursor, 2_501); + assert_eq!(node.reduce_round, 1); + assert_eq!(node.reduce_batch_cursor, 2_501); + assert_eq!(node.reduce_round_input_count, Some(5_001)); + assert_eq!(node.reduce_round_task_count, 2_501); + assert_eq!(node.reduce_round_terminal_task_count, 0); + assert!(node.reduce_ready); + Ok(()) +} + +#[tokio::test] +async fn actionable_projection_skips_completed_prefix_larger_than_activation_bound_db() -> TestResult +{ + // Pins: 129 completed source-fenced nodes cannot hide the next actionable node from a + // controller whose ordinary maximum_activation_steps bound is 128. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let mut candidate = new_run( + tenant_id, + None, + "completed-prefix", + ExecutionRunStatus::Queued, + budget(1_000), + ); + candidate.plan.definition.nodes = (0_u32..130) + .map(|index| output_node(&format!("node-{index:03}"))) + .collect(); + let run = create_run(&repository, scope, candidate).await?; + repository + .initialize_scheduler_state(scope, run.run_uid) + .await?; + sqlx::query( + "UPDATE moa.execution_node_state SET node_status = 'completed', \ + materialization_complete = TRUE, aggregate_output = 'null'::JSONB, \ + aggregate_output_hash = $2, updated_at = NOW() - INTERVAL '1 hour' \ + WHERE run_uid = $1 AND node_order < 129", + ) + .bind(run.run_uid) + .bind(moa_execution::capability::node_output_hash(&Value::Null)?.to_string()) + .execute(&pool) + .await?; + + let projection = repository + .load_activation_projection(scope, run.run_uid, 1) + .await? + .expect("actionable projection"); + assert_eq!(projection.nodes.len(), 1); + assert_eq!(projection.nodes[0].node_id, "node-129"); + assert!(!projection.has_more_actionable); + Ok(()) +} + +#[tokio::test] +async fn empty_map_completion_cas_releases_its_dependent_without_repeating_db() -> TestResult { + // Pins: an exhausted empty source commits a terminal aggregate even with zero task rows, and + // the next controller projection advances to the newly released dependent exactly once. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let repository = ExecutionRepository::new(test_db.store().pool().clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let mut candidate = new_run( + tenant_id, + None, + "empty-map", + ExecutionRunStatus::Queued, + budget(10), + ); + let mut dependent = output_node("after"); + dependent.depends_on = vec!["empty".to_string()]; + candidate.plan.definition.nodes = vec![empty_map_node("empty"), dependent]; + let run = create_run(&repository, scope, candidate).await?; + repository + .initialize_scheduler_state(scope, run.run_uid) + .await?; + + let request = ReadyMaterializationRequest { + run_uid: run.run_uid, + plan_revision: 1, + node_id: "empty".to_string(), + expected_cursor: 0, + reduce_cursor: None, + source_exhausted: true, + terminal_output: Some(json!({ "items": [] })), + tasks: Vec::new(), + }; + assert!(matches!( + repository + .materialize_ready_page(scope, &ExecutionConfig::default(), request.clone()) + .await?, + ReadyMaterializationOutcome::Applied { tasks, next_cursor: 0, .. } + if tasks.is_empty() + )); + assert!(matches!( + repository + .materialize_ready_page(scope, &ExecutionConfig::default(), request) + .await?, + ReadyMaterializationOutcome::Replayed { tasks, next_cursor: 0, .. } + if tasks.is_empty() + )); + + let projection = repository + .load_activation_projection(scope, run.run_uid, 8) + .await? + .expect("dependent projection"); + assert_eq!(projection.nodes.len(), 1); + assert_eq!(projection.nodes[0].node_id, "after"); + assert_eq!(projection.nodes[0].remaining_dependency_count, 0); + let readiness = repository + .load_activation_readiness(scope, run.run_uid) + .await? + .expect("readiness summary"); + assert!(readiness.has_actionable_nodes); + assert!(readiness.has_unfinished_nodes); + assert!(!readiness.has_nonterminal_tasks); + assert!(!readiness.terminal_ready()); + Ok(()) +} + +#[tokio::test] +async fn terminal_partial_map_page_cannot_complete_node_before_source_exhaustion_db() -> TestResult +{ + // Pins: settling every task in a committed map prefix cannot complete the node or release its + // dependent until the exact source-exhausted page has also committed and settled. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let repository = ExecutionRepository::new(test_db.store().pool().clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let mut candidate = new_run( + tenant_id, + None, + "partial-map-terminal-fence", + ExecutionRunStatus::Queued, + budget(10), + ); + let mut dependent = output_node("after"); + dependent.depends_on = vec!["map".to_string()]; + candidate.plan.definition.nodes = vec![output_node("map"), dependent]; + let run = create_run(&repository, scope, candidate).await?; + repository + .initialize_scheduler_state(scope, run.run_uid) + .await?; + let config = ExecutionConfig::default(); + + let first = logical_task(run.run_uid, "map", "0000", estimate(1)); + assert!(matches!( + repository + .materialize_ready_page( + scope, + &config, + ReadyMaterializationRequest { + run_uid: run.run_uid, + plan_revision: 1, + node_id: "map".to_string(), + expected_cursor: 0, + reduce_cursor: None, + source_exhausted: false, + terminal_output: None, + tasks: vec![first], + }, + ) + .await?, + ReadyMaterializationOutcome::Applied { next_cursor: 1, .. } + )); + settle_one_ready_task(&repository, &config, run.run_uid, "0000").await?; + + let partial = repository + .load_activation_projection(scope, run.run_uid, 8) + .await? + .expect("partial map projection"); + assert_eq!(partial.nodes.len(), 1); + assert_eq!(partial.nodes[0].node_id, "map"); + assert_eq!(partial.nodes[0].status, ExecutionNodeQueueStatus::Pending); + assert!(!partial.nodes[0].materialization_complete); + assert_eq!(partial.nodes[0].terminal_task_count, 1); + assert_eq!(partial.nodes[0].total_task_count, 1); + + let second = logical_task(run.run_uid, "map", "0001", estimate(1)); + assert!(matches!( + repository + .materialize_ready_page( + scope, + &config, + ReadyMaterializationRequest { + run_uid: run.run_uid, + plan_revision: 1, + node_id: "map".to_string(), + expected_cursor: 1, + reduce_cursor: None, + source_exhausted: true, + terminal_output: None, + tasks: vec![second], + }, + ) + .await?, + ReadyMaterializationOutcome::Applied { next_cursor: 2, .. } + )); + settle_one_ready_task(&repository, &config, run.run_uid, "0001").await?; + + let released = repository + .load_activation_projection(scope, run.run_uid, 8) + .await? + .expect("released dependent projection"); + assert_eq!(released.nodes.len(), 1); + assert_eq!(released.nodes[0].node_id, "after"); + assert_eq!(released.nodes[0].remaining_dependency_count, 0); + Ok(()) +} + +async fn settle_one_ready_task( + repository: &ExecutionRepository, + config: &ExecutionConfig, + run_uid: Uuid, + item_key: &str, +) -> TestResult { + let admission = repository + .admit_ready_attempts(config, 1, Utc::now()) + .await?; + let admitted = admission + .admitted + .into_iter() + .find(|item| item.run_uid == run_uid) + .expect("the only ready task must be admitted"); + let fence = TaskAttemptFence { + tenant_id: admitted.tenant_id, + run_uid: admitted.run_uid, + task_id: admitted.task_id, + controller_generation: admitted.controller_generation, + attempt_generation: admitted.attempt_generation, + dispatch_uid: admitted.dispatch_uid, + capacity_reservation_uid: admitted.capacity_reservation_uid, + watchdog_trigger_uid: admitted.watchdog_trigger_uid, + attempt_deadline_at: admitted.attempt_deadline_at, + }; + assert!(matches!( + repository.start_task_attempt(fence).await?, + TaskAttemptStartOutcome::Started(_) + )); + let outcome = completed_task_outcome( + json!({ "item_key": item_key }), + ExecutionUsage { + cost_microusd: 0, + tokens: 0, + tool_calls: 0, + retrieved_bytes: 0, + }, + ); + assert!(matches!( + repository + .settle_task_attempt(config, fence, outcome, None, Utc::now()) + .await?, + moa_execution::repository::task::TaskAttemptSettlementOutcome::Applied { .. } + )); + Ok(()) +} + +#[tokio::test] +async fn failed_root_cancels_join_without_rolling_back_later_sibling_settlement_db() -> TestResult { + // Pins: once one root failure canonically cancels an unmaterialized join, the independent + // sibling may still complete or fail without reopening the join or rolling back settlement. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let config = ExecutionConfig::default(); + + for sibling_fails in [false, true] { + let key = if sibling_fails { + "failed-root-then-failed-sibling" + } else { + "failed-root-then-completed-sibling" + }; + let mut candidate = new_run(tenant_id, None, key, ExecutionRunStatus::Queued, budget(3)); + let mut join = output_node("join"); + join.depends_on = vec!["root-a".to_string(), "root-b".to_string()]; + candidate.plan.definition.nodes = vec![output_node("root-a"), output_node("root-b"), join]; + candidate.plan.estimate.tasks = 3; + let run = create_run(&repository, scope, candidate).await?; + repository + .initialize_scheduler_state(scope, run.run_uid) + .await?; + let root_tasks = [ + logical_task(run.run_uid, "root-a", "", estimate(1)), + logical_task(run.run_uid, "root-b", "", estimate(1)), + ]; + for task in &root_tasks { + assert!(matches!( + repository + .materialize_ready_page( + scope, + &config, + ReadyMaterializationRequest { + run_uid: run.run_uid, + plan_revision: 1, + node_id: task.node_id.clone(), + expected_cursor: 0, + reduce_cursor: None, + source_exhausted: true, + terminal_output: None, + tasks: vec![task.clone()], + }, + ) + .await?, + ReadyMaterializationOutcome::Applied { next_cursor: 1, .. } + )); + } + + let admitted = repository + .admit_ready_attempts(&config, 2, Utc::now()) + .await? + .admitted; + assert_eq!(admitted.len(), 2, "both independent roots must be admitted"); + let fence_for = |item: &ExecutionAdmissionItem| TaskAttemptFence { + tenant_id: item.tenant_id, + run_uid: item.run_uid, + task_id: item.task_id, + controller_generation: item.controller_generation, + attempt_generation: item.attempt_generation, + dispatch_uid: item.dispatch_uid, + capacity_reservation_uid: item.capacity_reservation_uid, + watchdog_trigger_uid: item.watchdog_trigger_uid, + attempt_deadline_at: item.attempt_deadline_at, + }; + let failed_fence = fence_for( + admitted + .iter() + .find(|item| item.task_id == root_tasks[0].task_id) + .expect("root-a admission"), + ); + let sibling_fence = fence_for( + admitted + .iter() + .find(|item| item.task_id == root_tasks[1].task_id) + .expect("root-b admission"), + ); + for fence in [failed_fence, sibling_fence] { + assert!(matches!( + repository.start_task_attempt(fence).await?, + TaskAttemptStartOutcome::Started(_) + )); + } + + let failed = ExecutionTaskOutcome { + schema_version: 1, + usage: usage(0), + result: ExecutionTaskResult::Failed { + class: ExecutionFailureClass::Terminal, + message: "root failed".to_string(), + }, + }; + assert!(matches!( + repository + .settle_task_attempt(&config, failed_fence, failed.clone(), None, Utc::now()) + .await?, + moa_execution::repository::task::TaskAttemptSettlementOutcome::Applied { .. } + )); + + let sibling_outcome = if sibling_fails { + failed + } else { + completed_task_outcome(json!({ "root": "b" }), usage(0)) + }; + assert!(matches!( + repository + .settle_task_attempt(&config, sibling_fence, sibling_outcome, None, Utc::now(),) + .await?, + moa_execution::repository::task::TaskAttemptSettlementOutcome::Applied { .. } + )); + + let join_projection = sqlx::query_as::<_, (String, i64, bool, bool, i64)>( + "SELECT node_status, remaining_dependency_count, materialization_complete, \ + aggregate_complete, total_task_count \ + FROM moa.execution_node_state WHERE run_uid=$1 AND node_id='join'", + ) + .bind(run.run_uid) + .fetch_one(&pool) + .await?; + assert_eq!( + join_projection, + ("cancelled".to_string(), 0, true, true, 0), + "the join remains one canonical unmaterialized cancellation" + ); + let sibling = repository + .load_task(scope, run.run_uid, root_tasks[1].task_id) + .await? + .expect("settled sibling task"); + assert_eq!( + sibling.status, + if sibling_fails { + ExecutionTaskStatus::Failed + } else { + ExecutionTaskStatus::Completed + }, + "the sibling settlement must commit independently" + ); + } + Ok(()) +} + +#[tokio::test] +async fn external_signal_settlement_preserves_newer_task_progress_db() -> TestResult { + // Pins: resolving a storage-only signal must preserve a task projection timestamp that is + // newer than the process-observed settlement time while still committing the exact outcome. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let mut candidate = new_run( + tenant_id, + None, + "external-signal-monotonic-progress", + ExecutionRunStatus::Queued, + budget(1), + ); + candidate.plan.definition.nodes = vec![output_node("signal")]; + let run = create_run(&repository, scope, candidate).await?; + repository + .initialize_scheduler_state(scope, run.run_uid) + .await?; + let mut signal = logical_task(run.run_uid, "signal", "", estimate(1)); + signal.kind = LogicalTaskKind::WaitSignal { + signal_name: "upstream-ready".to_string(), + wait_policy: ExecutionWaitPolicy { + expiry: ExecutionTemporalTarget::After { delay_seconds: 180 }, + on_expiry: ExecutionWaitExpiryAction::FailTask, + }, + }; + let ReadyMaterializationOutcome::Applied { tasks, .. } = repository + .materialize_ready_page( + scope, + &ExecutionConfig::default(), + ReadyMaterializationRequest { + run_uid: run.run_uid, + plan_revision: 1, + node_id: "signal".to_string(), + expected_cursor: 0, + reduce_cursor: None, + source_exhausted: true, + terminal_output: None, + tasks: vec![signal], + }, + ) + .await? + else { + panic!("fresh signal wait must materialize"); + }; + let task = &tasks[0]; + let future_progress_at: chrono::DateTime = sqlx::query_scalar( + "UPDATE moa.execution_task SET last_progress_at=NOW() + INTERVAL '1 minute' \ + WHERE run_uid=$1 AND task_id=$2 RETURNING last_progress_at", + ) + .bind(run.run_uid) + .bind(task.task_id.as_uuid()) + .fetch_one(&pool) + .await?; + sqlx::query( + "UPDATE moa.execution_run SET last_progress_at=$2 \ + WHERE run_uid=$1", + ) + .bind(run.run_uid) + .bind(future_progress_at) + .execute(&pool) + .await?; + + let TaskOutcomeWrite::Applied { task: settled, .. } = repository + .complete_external_wait( + scope, + &ExecutionConfig::default(), + run.run_uid, + task.task_id, + task.generation, + ExecutionTaskOutcome { + schema_version: 1, + usage: usage(0), + result: ExecutionTaskResult::Completed { + output: json!({"signal": "accepted"}), + citations: Vec::new(), + }, + }, + ) + .await? + else { + panic!("exact external signal settlement must apply"); + }; + assert_eq!(settled.status, ExecutionTaskStatus::Completed); + assert_eq!(settled.output, Some(json!({"signal": "accepted"}))); + assert_eq!(settled.last_progress_at, future_progress_at); + let settled_run = repository + .load_run(scope, run.run_uid) + .await? + .expect("settled signal run must remain visible"); + assert_eq!(settled_run.last_progress_at, future_progress_at); + Ok(()) +} + +#[tokio::test] +async fn relative_timer_is_parked_once_and_stale_delivery_is_fenced_db() -> TestResult { + // Pins: timer, review, and signal waits atomically persist exact absolute reasons and delayed + // deliveries; their compact run projection keeps deterministic phase precedence and earliest + // wake without scanning task rows, and stale delivery remains generation-fenced. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let mut candidate = new_run( + tenant_id, + None, + "relative-wait", + ExecutionRunStatus::Queued, + budget(10), + ); + candidate.plan.definition.nodes = vec![ + output_node("timer"), + output_node("review"), + output_node("signal"), + ]; + let run = create_run(&repository, scope, candidate).await?; + repository + .initialize_scheduler_state(scope, run.run_uid) + .await?; + let mut task = logical_task(run.run_uid, "timer", "", estimate(1)); + task.kind = LogicalTaskKind::WaitUntil { + wake: ExecutionTemporalTarget::After { delay_seconds: 60 }, + result: json!({ "elapsed": true }), + }; + let ReadyMaterializationOutcome::Applied { + tasks, + triggers, + next_cursor, + } = repository + .materialize_ready_page( + scope, + &ExecutionConfig::default(), + ReadyMaterializationRequest { + run_uid: run.run_uid, + plan_revision: 1, + node_id: "timer".to_string(), + expected_cursor: 0, + reduce_cursor: None, + source_exhausted: true, + terminal_output: None, + tasks: vec![task], + }, + ) + .await? + else { + panic!("fresh timer materialization must apply"); + }; + assert_eq!(next_cursor, 1); + assert_eq!(tasks[0].status, ExecutionTaskStatus::WaitingTimer); + let waiting_since = tasks[0] + .waiting_since + .expect("timer must persist its wait-entry anchor"); + assert_eq!(triggers.len(), 1); + assert_eq!(triggers[0].due_at, waiting_since + Duration::seconds(60)); + let timer_task_id = tasks[0].task_id; + let timer_trigger = triggers[0].clone(); + + let waits = [ + ( + "review", + LogicalTaskKind::Review { + prompt: "approve durable work".to_string(), + wait_policy: ExecutionWaitPolicy { + expiry: ExecutionTemporalTarget::After { delay_seconds: 120 }, + on_expiry: ExecutionWaitExpiryAction::FailTask, + }, + }, + ), + ( + "signal", + LogicalTaskKind::WaitSignal { + signal_name: "upstream-ready".to_string(), + wait_policy: ExecutionWaitPolicy { + expiry: ExecutionTemporalTarget::After { delay_seconds: 180 }, + on_expiry: ExecutionWaitExpiryAction::FailTask, + }, + }, + ), + ]; + let mut exact_waits = Vec::new(); + for (node_id, kind) in waits { + let mut task = logical_task(run.run_uid, node_id, "", estimate(1)); + task.kind = kind; + let ReadyMaterializationOutcome::Applied { + tasks, triggers, .. + } = repository + .materialize_ready_page( + scope, + &ExecutionConfig::default(), + ReadyMaterializationRequest { + run_uid: run.run_uid, + plan_revision: 1, + node_id: node_id.to_string(), + expected_cursor: 0, + reduce_cursor: None, + source_exhausted: true, + terminal_output: None, + tasks: vec![task], + }, + ) + .await? + else { + panic!("fresh {node_id} wait must materialize"); + }; + assert_eq!(tasks.len(), 1); + assert_eq!(triggers.len(), 1); + exact_waits.push((node_id, tasks[0].task_id, triggers[0].due_at)); + } + let run_state = repository + .load_run(scope, run.run_uid) + .await? + .expect("run must remain visible"); + assert_eq!(run_state.ready_task_count, 0); + assert_eq!(run_state.status, ExecutionRunStatus::WaitingReview); + assert_eq!(run_state.waiting_since, Some(waiting_since)); + assert_eq!(run_state.next_wake_at, Some(timer_trigger.due_at)); + assert_eq!(run_state.waiting_task_count, 3); + assert_eq!(run_state.waiting_review_task_count, 1); + assert_eq!(run_state.waiting_signal_task_count, 1); + assert_eq!(run_state.waiting_timer_task_count, 1); + assert!(!run_state.waiting_reasons_truncated); + assert_eq!(run_state.waiting_reasons.len(), 3); + assert!(run_state.waiting_reasons.iter().any(|reason| matches!( + reason, + moa_execution::state::WaitingReason::Timer { + task_id, + wake: ExecutionTemporalTarget::At { at }, + } if *task_id == timer_task_id && *at == timer_trigger.due_at + ))); + assert!(run_state.waiting_reasons.iter().any(|reason| matches!( + reason, + moa_execution::state::WaitingReason::Review { + task_id, + wait_policy: ExecutionWaitPolicy { + expiry: ExecutionTemporalTarget::At { at }, + .. + }, + .. + } if *task_id == exact_waits[0].1 && *at == exact_waits[0].2 + ))); + assert!(run_state.waiting_reasons.iter().any(|reason| matches!( + reason, + moa_execution::state::WaitingReason::Signal { + task_id, + wait_policy: ExecutionWaitPolicy { + expiry: ExecutionTemporalTarget::At { at }, + .. + }, + .. + } if *task_id == exact_waits[1].1 && *at == exact_waits[1].2 + ))); + + sqlx::query( + "UPDATE moa.execution_run SET controller_generation = controller_generation + 1 \ + WHERE run_uid = $1", + ) + .bind(run.run_uid) + .execute(&pool) + .await?; + let (outcome, activation) = repository + .fire_wait_trigger( + scope, + &ExecutionConfig::default(), + timer_trigger.trigger_uid, + ) + .await?; + assert_eq!( + outcome, + TransitionOutcome::Rejected(TransitionRejection::InvalidTaskStatus) + ); + assert!(activation.is_none()); + Ok(()) +} + +#[tokio::test] +async fn storage_wait_settlement_preserves_newer_database_progress_db() -> TestResult { + // Pins: a process-observed settlement may predate progress already committed by Postgres; + // settlement preserves that newer task clock while retaining the observation in audit history. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let mut candidate = new_run( + tenant_id, + None, + "stale-process-wait-settlement", + ExecutionRunStatus::Queued, + budget(1), + ); + candidate.plan.definition.nodes = vec![output_node("timer")]; + let run = create_run(&repository, scope, candidate).await?; + repository + .initialize_scheduler_state(scope, run.run_uid) + .await?; + + let settled_at = moa_test_support::fixtures::pg_now() - Duration::seconds(30); + let mut task = logical_task(run.run_uid, "timer", "", estimate(1)); + task.kind = LogicalTaskKind::WaitUntil { + wake: ExecutionTemporalTarget::At { + at: settled_at - Duration::seconds(1), + }, + result: json!({ "elapsed": true }), + }; + let ReadyMaterializationOutcome::Applied { tasks, .. } = repository + .materialize_ready_page( + scope, + &ExecutionConfig::default(), + ReadyMaterializationRequest { + run_uid: run.run_uid, + plan_revision: 1, + node_id: "timer".to_string(), + expected_cursor: 0, + reduce_cursor: None, + source_exhausted: true, + terminal_output: None, + tasks: vec![task], + }, + ) + .await? + else { + panic!("fresh timer wait must materialize"); + }; + let waiting_task = tasks + .into_iter() + .next() + .expect("timer materialization must return exactly one task"); + let waiting_since = waiting_task + .waiting_since + .expect("timer materialization must persist its wait anchor"); + let database_progress_at: DateTime = sqlx::query_scalar( + "UPDATE moa.execution_task SET last_progress_at=NOW(), updated_at=NOW() \ + WHERE run_uid=$1 AND task_id=$2 RETURNING last_progress_at", + ) + .bind(run.run_uid) + .bind(waiting_task.task_id.as_uuid()) + .fetch_one(&pool) + .await?; + assert!( + database_progress_at > settled_at, + "fixture must establish database progress newer than the supplied observation" + ); + + let outcome = repository + .settle_wait( + scope, + run.run_uid, + waiting_task.generation, + waiting_since, + WaitSettlement::TimerElapsed { + task_id: waiting_task.task_id, + output: json!({ "elapsed": true }), + }, + settled_at, + ) + .await?; + let TransitionOutcome::Applied(settled_task) = outcome else { + panic!("due timer settlement must apply: {outcome:?}"); + }; + assert!( + settled_task.last_progress_at >= database_progress_at, + "settlement must not move canonical task progress backward" + ); + assert!(settled_task.generation_history.iter().any(|entry| { + entry.get("kind").and_then(Value::as_str) == Some("storage_wait_settlement") + && entry.get("settled_at") == Some(&json!(settled_at)) + })); + Ok(()) +} diff --git a/crates/moa-execution/tests/execution_db/long_horizon_state_db.rs b/crates/moa-execution/tests/execution_db/long_horizon_state_db.rs new file mode 100644 index 000000000..1e7ffecb2 --- /dev/null +++ b/crates/moa-execution/tests/execution_db/long_horizon_state_db.rs @@ -0,0 +1,257 @@ +//! Long-horizon execution state, identity, RLS, and generation-fence contracts. + +use super::support::*; + +#[tokio::test] +async fn admitted_identity_and_activation_checkpoint_round_trip_exactly_db() -> TestResult { + // Pins: the authenticated admission principal and bounded-controller checkpoint are + // canonical Postgres state, and exact activation replays do not mutate them twice. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let repository = ExecutionRepository::new(test_db.store().pool().clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let mut candidate = new_run( + tenant_id, + None, + "long-horizon-identity", + ExecutionRunStatus::Queued, + budget(10), + ); + candidate.admitted_identity = Identity { + identity_type: IdentityType::Service, + id: Uuid::now_v7(), + tenant_id, + api_key_id: Some(Uuid::now_v7()), + acting_on_behalf_of: Some(Uuid::now_v7()), + }; + let expected_identity = candidate.admitted_identity.clone(); + + let created = create_run(&repository, scope, candidate).await?; + assert_eq!(created.admitted_identity, expected_identity); + let stored_identity: serde_json::Value = + sqlx::query_scalar("SELECT admitted_identity FROM moa.execution_run WHERE run_uid = $1") + .bind(created.run_uid) + .fetch_one(test_db.store().pool()) + .await?; + assert_eq!(stored_identity, serde_json::to_value(&expected_identity)?); + assert_eq!(created.controller_generation, 1); + assert_eq!(created.activation_state, ExecutionActivationState::Queued); + assert_eq!(created.ready_task_count, 0); + assert_eq!(created.active_task_count, 0); + assert_eq!(created.last_progress_at, created.created_at); + + let RunActivationWriteOutcome::Applied(claimed) = repository + .claim_run_activation(scope, created.run_uid, 1) + .await? + else { + panic!("current queued generation must be claimed"); + }; + assert_eq!( + claimed.activation_state, + ExecutionActivationState::Advancing + ); + assert!(claimed.last_progress_at >= created.last_progress_at); + assert_eq!( + repository + .claim_run_activation(scope, created.run_uid, 1) + .await?, + RunActivationWriteOutcome::AlreadyApplied(claimed.clone()) + ); + assert_eq!( + repository + .claim_run_activation(scope, created.run_uid, 2) + .await?, + RunActivationWriteOutcome::GenerationMismatch + ); + + let next_wake_at = pg_deadline(Duration::minutes(5)); + let checkpoint = ExecutionRunActivationCheckpoint { + status: ExecutionRunStatus::Running, + activation_state: ExecutionActivationState::Idle, + next_wake_at: Some(next_wake_at), + waiting_since: None, + ready_task_count: 2, + active_task_count: 1, + }; + let RunActivationWriteOutcome::Applied(parked) = repository + .checkpoint_run_activation(scope, created.run_uid, 1, checkpoint.clone()) + .await? + else { + panic!("claimed generation must persist its checkpoint"); + }; + assert_eq!(parked.status, ExecutionRunStatus::Running); + assert_eq!(parked.activation_state, ExecutionActivationState::Idle); + assert_eq!(parked.next_wake_at, Some(next_wake_at)); + assert_eq!(parked.ready_task_count, 2); + assert_eq!(parked.active_task_count, 1); + assert_eq!( + repository + .checkpoint_run_activation(scope, created.run_uid, 1, checkpoint) + .await?, + RunActivationWriteOutcome::AlreadyApplied(parked) + ); + Ok(()) +} + +#[tokio::test] +async fn long_horizon_children_are_tenant_isolated_and_cross_tenant_fenced_db() -> TestResult { + // Pins: every V59 child row is visible only to its tenant, and composite foreign keys + // prevent a child from attaching to a run owned by a different tenant. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let owner_tenant = TenantId::new(); + let other_tenant = TenantId::new(); + let run = create_run( + &repository, + ExecutionScope::Tenant { + tenant_id: owner_tenant, + }, + new_run( + owner_tenant, + None, + "long-horizon-rls", + ExecutionRunStatus::Queued, + budget(10), + ), + ) + .await?; + let node_state_uid = Uuid::now_v7(); + sqlx::query( + "INSERT INTO moa.execution_node_state \ + (node_state_uid, tenant_id, run_uid, node_id, node_order) \ + VALUES ($1, $2, $3, 'collect', 0)", + ) + .bind(node_state_uid) + .bind(owner_tenant.0) + .bind(run.run_uid) + .execute(&pool) + .await?; + + let mut owner = moa_db::ScopedConn::begin_tenant(&pool, owner_tenant).await?; + owner.assume_app_role().await?; + let owner_count: i64 = sqlx::query_scalar( + "SELECT count(*) FROM moa.execution_node_state WHERE node_state_uid = $1", + ) + .bind(node_state_uid) + .fetch_one(owner.as_mut()) + .await?; + owner.commit().await?; + assert_eq!(owner_count, 1); + + let mut other = moa_db::ScopedConn::begin_tenant(&pool, other_tenant).await?; + other.assume_app_role().await?; + let other_count: i64 = sqlx::query_scalar( + "SELECT count(*) FROM moa.execution_node_state WHERE node_state_uid = $1", + ) + .bind(node_state_uid) + .fetch_one(other.as_mut()) + .await?; + other.commit().await?; + assert_eq!(other_count, 0); + assert_eq!( + repository + .claim_run_activation( + ExecutionScope::Tenant { + tenant_id: other_tenant, + }, + run.run_uid, + 1, + ) + .await?, + RunActivationWriteOutcome::NotFound + ); + + let cross_tenant = sqlx::query( + "INSERT INTO moa.execution_node_state \ + (node_state_uid, tenant_id, run_uid, node_id, node_order) \ + VALUES ($1, $2, $3, 'cross-tenant', 1)", + ) + .bind(Uuid::now_v7()) + .bind(other_tenant.0) + .bind(run.run_uid) + .execute(&pool) + .await; + assert_db_error_contains(cross_tenant, "execution_node_state_run_tenant_fk"); + Ok(()) +} + +#[tokio::test] +async fn attempt_generation_and_long_horizon_guards_reject_stale_or_invalid_state_db() -> TestResult +{ + // Pins: repository task mutations fence both logical and attempt generations, while + // immutable identity, monotonic generations, and nonnegative counters fail closed. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let run = create_run( + &repository, + scope, + new_run( + tenant_id, + None, + "long-horizon-generation", + ExecutionRunStatus::Queued, + budget(10), + ), + ) + .await?; + let tasks = repository + .materialize_tasks( + scope, + run.run_uid, + 1, + vec![logical_task(run.run_uid, "collect", "one", estimate(1))], + ) + .await?; + let task = &tasks[0]; + assert_eq!(task.attempt_generation, task.generation); + assert_eq!(task.active_dispatch_uid, None); + assert_eq!(task.dispatch_sequence, 0); + assert_eq!(task.attempt_state, ExecutionAttemptState::Idle); + assert!(task.attempt_started_at.is_none()); + assert!(task.attempt_deadline_at.is_none()); + assert!(task.waiting_since.is_none()); + assert!(task.ready_at.is_none()); + assert!(task.external_job_uid.is_none()); + + sqlx::query("UPDATE moa.execution_task SET attempt_generation = 2 WHERE task_id = $1") + .bind(task.task_id.as_uuid()) + .execute(&pool) + .await?; + assert_eq!( + repository + .reserve_task(scope, run.run_uid, task.task_id, 1) + .await?, + ReservationOutcome::Rejected(ReservationRejection::GenerationMismatch) + ); + + assert_db_error_contains( + sqlx::query("UPDATE moa.execution_task SET attempt_generation = 1 WHERE task_id = $1") + .bind(task.task_id.as_uuid()) + .execute(&pool) + .await, + "attempt generation must be monotonic", + ); + assert_db_error_contains( + sqlx::query("UPDATE moa.execution_run SET ready_task_count = -1 WHERE run_uid = $1") + .bind(run.run_uid) + .execute(&pool) + .await, + "execution_run_ready_task_count_check", + ); + assert_db_error_contains( + sqlx::query( + "UPDATE moa.execution_run SET admitted_identity = \ + jsonb_set(admitted_identity, '{id}', to_jsonb($2::TEXT)) WHERE run_uid = $1", + ) + .bind(run.run_uid) + .bind(Uuid::now_v7().to_string()) + .execute(&pool) + .await, + "execution run admitted identity is immutable", + ); + Ok(()) +} diff --git a/crates/moa-execution/tests/execution_db/outcomes_and_replan_db.rs b/crates/moa-execution/tests/execution_db/outcomes_and_replan_db.rs index d86db3885..0c3a95876 100644 --- a/crates/moa-execution/tests/execution_db/outcomes_and_replan_db.rs +++ b/crates/moa-execution/tests/execution_db/outcomes_and_replan_db.rs @@ -49,6 +49,7 @@ async fn input_resume_starts_a_schedulable_generation_without_a_prior_outcome_db let TransitionOutcome::Applied(resumed) = repository .resume_task_with_input( scope, + &ExecutionConfig::default(), run.run_uid, task.task_id, 1, @@ -66,27 +67,15 @@ async fn input_resume_starts_a_schedulable_generation_without_a_prior_outcome_db ); assert_eq!(resumed.outcome_audit.len(), 1); - let snapshot = repository - .load_scheduling_snapshot(scope, run.run_uid) + let persisted_run = repository + .load_run(scope, run.run_uid) .await? - .expect("resumed run should remain schedulable"); - let scheduled = moa_execution::schedule(moa_execution::ScheduleRequest { - run_uid: snapshot.run.run_uid, - goal: snapshot.run.goal.clone(), - plan: snapshot.run.active_plan.clone(), - catalog: snapshot.catalog.clone(), - run_input: snapshot.run.input.clone(), - projection: snapshot.projection, - config: moa_config::ExecutionConfig::default(), - budget_ledger: snapshot.budget_ledger, - now: moa_test_support::fixtures::pg_now(), - })?; - assert_eq!( - scheduled.decision, - moa_execution::state::ScheduleDecision::Waiting(vec![ - moa_execution::state::WaitingReason::RunningTasks, - ]) - ); + .expect("resumed run should remain visible"); + let persisted_task = listed_task(&repository, scope, run.run_uid, task.task_id).await?; + assert_eq!(persisted_run.status, ExecutionRunStatus::Running); + assert_eq!(persisted_task.status, ExecutionTaskStatus::Running); + assert_eq!(persisted_task.generation, 2); + assert!(persisted_task.current_outcome.is_none()); Ok(()) } @@ -149,7 +138,14 @@ async fn retry_and_input_resume_terminalize_elapsed_or_exhausted_run_envelope_db .expect("waiting run should remain queryable"); let transition = if kind == "input" { repository - .resume_task_with_input(scope, run.run_uid, task.task_id, 1, json!({"ok": true})) + .resume_task_with_input( + scope, + &ExecutionConfig::default(), + run.run_uid, + task.task_id, + 1, + json!({"ok": true}), + ) .await? } else { repository @@ -173,7 +169,14 @@ async fn retry_and_input_resume_terminalize_elapsed_or_exhausted_run_envelope_db assert_eq!(terminal_run.wake_epoch, before_terminal.wake_epoch + 1); let replay = if kind == "input" { repository - .resume_task_with_input(scope, run.run_uid, task.task_id, 1, json!({"ok": true})) + .resume_task_with_input( + scope, + &ExecutionConfig::default(), + run.run_uid, + task.task_id, + 1, + json!({"ok": true}), + ) .await? } else { repository @@ -186,7 +189,14 @@ async fn retry_and_input_resume_terminalize_elapsed_or_exhausted_run_envelope_db ); let stale = if kind == "input" { repository - .resume_task_with_input(scope, run.run_uid, task.task_id, 0, json!({"ok": true})) + .resume_task_with_input( + scope, + &ExecutionConfig::default(), + run.run_uid, + task.task_id, + 0, + json!({"ok": true}), + ) .await? } else { repository @@ -238,7 +248,14 @@ async fn retry_and_input_resume_terminalize_elapsed_or_exhausted_run_envelope_db .expect("waiting run should remain queryable"); let transition = if kind == "input" { repository - .resume_task_with_input(scope, run.run_uid, task.task_id, 1, json!({"ok": true})) + .resume_task_with_input( + scope, + &ExecutionConfig::default(), + run.run_uid, + task.task_id, + 1, + json!({"ok": true}), + ) .await? } else { repository @@ -262,7 +279,14 @@ async fn retry_and_input_resume_terminalize_elapsed_or_exhausted_run_envelope_db assert_eq!(terminal_run.wake_epoch, before_terminal.wake_epoch + 1); let replay = if kind == "input" { repository - .resume_task_with_input(scope, run.run_uid, task.task_id, 1, json!({"ok": true})) + .resume_task_with_input( + scope, + &ExecutionConfig::default(), + run.run_uid, + task.task_id, + 1, + json!({"ok": true}), + ) .await? } else { repository @@ -305,6 +329,7 @@ async fn exact_external_wait_outcome_replay_recovers_committed_handoff_db() -> T repository .complete_external_wait( scope, + &ExecutionConfig::default(), run.run_uid, task.task_id, task.generation, @@ -317,6 +342,7 @@ async fn exact_external_wait_outcome_replay_recovers_committed_handoff_db() -> T let replay = repository .complete_external_wait( scope, + &ExecutionConfig::default(), run.run_uid, task.task_id, task.generation, @@ -457,6 +483,7 @@ async fn stale_and_terminal_outcomes_are_audited_without_projection_mutation_db( let TransitionOutcome::Applied(resumed) = repository .resume_task_with_input( scope, + &ExecutionConfig::default(), run.run_uid, task.task_id, 1, @@ -476,6 +503,7 @@ async fn stale_and_terminal_outcomes_are_audited_without_projection_mutation_db( repository .resume_task_with_input( scope, + &ExecutionConfig::default(), run.run_uid, task.task_id, 1, @@ -489,6 +517,7 @@ async fn stale_and_terminal_outcomes_are_audited_without_projection_mutation_db( repository .resume_task_with_input( scope, + &ExecutionConfig::default(), run.run_uid, task.task_id, 1, @@ -589,32 +618,52 @@ async fn task_outcomes_update_review_state_and_failure_accounting_exactly_db() - new_run(tenant_id, None, key, ExecutionRunStatus::Queued, budget(1)), ) .await?; - assert!(matches!( - repository - .transition_run_wait( - scope, - run.run_uid, - ExecutionRunStatus::Queued, - ExecutionRunStatus::Running, - ) - .await?, - TransitionOutcome::RunApplied(_) - )); + let _running = + claim_running_controller(&repository, scope, &ExecutionConfig::default(), &run).await?; let task = logical_task(run.run_uid, "outcome", key, estimate(1)); repository .materialize_tasks(scope, run.run_uid, 1, vec![task.clone()]) .await?; reserve_and_start(&repository, scope, run.run_uid, task.task_id).await?; + let current = repository + .load_run(scope, run.run_uid) + .await? + .expect("outcome fixture run remains visible"); + let claimed = match repository + .claim_controller_wake( + scope, + current.run_uid, + current.controller_generation, + current.wake_epoch, + ) + .await? + { + RunControllerClaimOutcome::Claimed(claimed) => claimed, + outcome => panic!("task wake must be claimable: {outcome:?}"), + }; assert!(matches!( repository - .transition_run_wait( + .complete_controller_wake( scope, - run.run_uid, - ExecutionRunStatus::Running, - waiting_status, + &ExecutionConfig::default(), + claimed.run_uid, + RunControllerCompletionRequest { + controller_generation: claimed.controller_generation, + wake_epoch: claimed.wake_epoch, + checkpoint: ExecutionRunActivationCheckpoint { + status: waiting_status, + activation_state: ExecutionActivationState::Idle, + next_wake_at: claimed.next_wake_at, + waiting_since: Some(Utc::now()), + ready_task_count: claimed.ready_task_count, + active_task_count: claimed.active_task_count, + }, + continuation_payload: None, + continuation_not_before_at: Utc::now(), + }, ) .await?, - TransitionOutcome::RunApplied(_) + RunControllerCompletionOutcome::Applied { .. } )); let TaskOutcomeWrite::Applied { @@ -633,105 +682,3 @@ async fn task_outcomes_update_review_state_and_failure_accounting_exactly_db() - } Ok(()) } - -#[tokio::test] -async fn action_review_resolution_is_review_uid_idempotent_and_generation_fenced_db() -> TestResult -{ - // Pins: outbox replay applies one review UID once, while stale generations - // remain auditable without resolving or mutating the current task projection; - // a reused identity cannot smuggle in different typed resolution semantics. - let test_db = moa_test_support::postgres::bootstrap_test_db().await?; - let repository = ExecutionRepository::new(test_db.store().pool().clone()); - let tenant_id = TenantId::new(); - let scope = ExecutionScope::Tenant { tenant_id }; - let run = create_run( - &repository, - scope, - new_run( - tenant_id, - None, - "review-resolution", - ExecutionRunStatus::Queued, - budget(10), - ), - ) - .await?; - let task = logical_task(run.run_uid, "review", "", estimate(1)); - repository - .materialize_tasks(scope, run.run_uid, 1, vec![task.clone()]) - .await?; - reserve_and_start(&repository, scope, run.run_uid, task.task_id).await?; - let stale_review = Uuid::new_v4(); - let current_review = Uuid::new_v4(); - let resolution = ExecutionActionReviewResolution::Denied { - reason: "operator denied".to_string(), - }; - - assert_eq!( - repository - .record_action_review_resolution( - scope, - run.run_uid, - task.task_id, - 2, - stale_review, - &resolution, - ) - .await?, - ActionReviewResolutionWrite::AuditedStale - ); - assert_eq!( - repository - .record_action_review_resolution( - scope, - run.run_uid, - task.task_id, - 1, - current_review, - &resolution, - ) - .await?, - ActionReviewResolutionWrite::Applied - ); - assert_eq!( - repository - .record_action_review_resolution( - scope, - run.run_uid, - task.task_id, - 1, - current_review, - &resolution, - ) - .await?, - ActionReviewResolutionWrite::Replayed - ); - let conflicting_resolution = ExecutionActionReviewResolution::Completed { - tool_output: json!({"unexpected": true}), - }; - let conflict = repository - .record_action_review_resolution( - scope, - run.run_uid, - task.task_id, - 1, - current_review, - &conflicting_resolution, - ) - .await - .expect_err("same task review identity with a different resolution must fail closed"); - assert!( - matches!(conflict, moa_execution::Error::InvalidRepositoryData { .. }), - "task review identity conflict returned the wrong error: {conflict:?}" - ); - let persisted = repository - .load_task(scope, run.run_uid, task.task_id) - .await? - .expect("task should remain visible"); - assert_eq!(persisted.status, ExecutionTaskStatus::Running); - assert_eq!(persisted.generation, 1); - assert_eq!(persisted.outcome_audit.len(), 2); - assert_eq!(persisted.outcome_audit[0]["accepted"], false); - assert_eq!(persisted.outcome_audit[1]["accepted"], true); - Ok(()) -} diff --git a/crates/moa-execution/tests/execution_db/planning_and_audit_db.rs b/crates/moa-execution/tests/execution_db/planning_and_audit_db.rs index 4641bacac..295934a96 100644 --- a/crates/moa-execution/tests/execution_db/planning_and_audit_db.rs +++ b/crates/moa-execution/tests/execution_db/planning_and_audit_db.rs @@ -480,6 +480,18 @@ async fn confirmation_is_plan_hash_bound_and_exact_replay_only_db() -> TestResul assert_eq!(confirmed.approved_budget, approved); assert!(confirmed.confirmed_at.is_some()); assert_eq!(confirmed.confirmed_plan_hash, Some(run.active_plan_hash)); + assert_eq!(confirmed.wake_epoch, 1); + let confirmation_dispatch: (i64, String, String) = sqlx::query_as( + "SELECT wake_epoch, dispatch_kind, state FROM moa.execution_dispatch_outbox \ + WHERE run_uid = $1", + ) + .bind(run.run_uid) + .fetch_one(test_db.store().pool()) + .await?; + assert_eq!( + confirmation_dispatch, + (1, "run_activation".into(), "pending".into()) + ); let queued_at = confirmed .queued_at .expect("successful confirmation sets the queue timestamp"); @@ -496,6 +508,12 @@ async fn confirmation_is_plan_hash_bound_and_exact_replay_only_db() -> TestResul replay, ConfirmationOutcome::AlreadyConfirmed(ref replay) if replay.queued_at == Some(queued_at) )); + let confirmation_dispatch_count: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM moa.execution_dispatch_outbox WHERE run_uid = $1") + .bind(run.run_uid) + .fetch_one(test_db.store().pool()) + .await?; + assert_eq!(confirmation_dispatch_count, 1); assert_eq!( repository .confirm_run(scope, run.run_uid, &run.active_plan_hash, budget(6)) @@ -589,22 +607,24 @@ async fn confirmation_is_plan_hash_bound_and_exact_replay_only_db() -> TestResul "execution run queued timestamp is immutable", ); - let snapshot = repository - .load_scheduling_snapshot(scope, run.run_uid) + let current = repository + .load_run(scope, run.run_uid) .await? .expect("confirmation test run remains visible before fencing"); - let terminal_evidence = moa_execution::completion::cancellation_terminal_evidence( - &snapshot.run.goal, - &snapshot.run.active_plan, - &snapshot.projection, - )?; + let terminal_evidence = + moa_execution::completion::cancellation_terminal_evidence_from_completed_nodes( + ¤t.goal, + ¤t.active_plan, + &std::collections::BTreeSet::::new(), + )?; assert!(matches!( repository - .fence_run_for_terminal( + .fence_completion_terminal_and_enqueue_settlement( + &ExecutionConfig::default(), scope, run.run_uid, - run.plan_revision, - snapshot.run.wake_epoch, + current.controller_generation, + current.wake_epoch, PendingExecutionTerminal { status: ExecutionRunStatus::Cancelled, reason: ExecutionTerminalReason::Cancelled, @@ -614,9 +634,11 @@ async fn confirmation_is_plan_hash_bound_and_exact_replay_only_db() -> TestResul terminal_gaps: Vec::new(), cancellation_reason: Some("confirmation test terminalization".to_string()), }, + moa_test_support::fixtures::pg_now(), + 1, ) .await?, - TerminalFenceOutcome::Applied(_) + moa_execution::repository::terminal::PendingTerminalAdvanceOutcome::Applied(_) )); assert_eq!( repository @@ -692,20 +714,50 @@ async fn amendment_append_is_revision_fenced_and_preserves_initial_plan_db() -> let amendment_digest = validated.amendment_hash; assert_eq!( repository - .append_amendment(scope, run.run_uid, 2, validated.clone()) + .append_amendment( + scope, + &ExecutionConfig::default(), + run.run_uid, + 2, + validated.clone(), + ) .await?, AmendmentWrite::Conflict ); let AmendmentWrite::Applied(amended) = repository - .append_amendment(scope, run.run_uid, 1, validated.clone()) + .append_amendment( + scope, + &ExecutionConfig::default(), + run.run_uid, + 1, + validated.clone(), + ) .await? else { panic!("expected applied amendment"); }; assert_eq!(amended.task_ids_to_release, vec![task.task_id]); let applied_wake_epoch = amended.run.wake_epoch; + let amendment_dispatch: (i64, String) = sqlx::query_as( + "SELECT wake_epoch, payload->>'reason' FROM moa.execution_dispatch_outbox \ + WHERE run_uid = $1 AND wake_epoch = $2", + ) + .bind(run.run_uid) + .bind(i64::try_from(applied_wake_epoch)?) + .fetch_one(test_db.store().pool()) + .await?; + assert_eq!( + amendment_dispatch, + (i64::try_from(applied_wake_epoch)?, "plan_amended".into()) + ); let AmendmentWrite::Replayed(replayed) = repository - .append_amendment(scope, run.run_uid, 1, validated) + .append_amendment( + scope, + &ExecutionConfig::default(), + run.run_uid, + 1, + validated, + ) .await? else { panic!("exact amendment replay must recover its committed handoff"); @@ -713,7 +765,7 @@ async fn amendment_append_is_revision_fenced_and_preserves_initial_plan_db() -> assert_eq!(replayed.run.wake_epoch, applied_wake_epoch); assert_eq!(replayed.task_ids_to_release, vec![task.task_id]); let AmendmentReplayOutcome::Replayed(recovered) = repository - .recover_amendment_handoff(scope, run.run_uid, 1, &amendment_digest) + .recover_amendment_handoff(scope, run.run_uid, run.session_id, 1, &amendment_digest) .await? else { panic!( @@ -724,7 +776,13 @@ async fn amendment_append_is_revision_fenced_and_preserves_initial_plan_db() -> assert_eq!(recovered.task_ids_to_release, vec![task.task_id]); assert_eq!( repository - .recover_amendment_handoff(scope, run.run_uid, 1, &ExecutionHash::from_bytes([99; 32]),) + .recover_amendment_handoff( + scope, + run.run_uid, + run.session_id, + 1, + &ExecutionHash::from_bytes([99; 32]), + ) .await?, AmendmentReplayOutcome::Conflict ); @@ -824,156 +882,3 @@ async fn amendment_append_is_revision_fenced_and_preserves_initial_plan_db() -> } Ok(()) } - -#[tokio::test] -async fn replan_stop_fence_recovers_exact_amendment_after_terminal_db() -> TestResult { - // Pins: an amendment-driven terminal fence records exact retry identity without releasing a - // task workflow that the compensation driver already owns and settles. - let test_db = moa_test_support::postgres::bootstrap_test_db().await?; - let repository = ExecutionRepository::new(test_db.store().pool().clone()); - let tenant_id = TenantId::new(); - let scope = ExecutionScope::Tenant { tenant_id }; - let created = create_run( - &repository, - scope, - new_run( - tenant_id, - None, - "replan-stop-receipt", - ExecutionRunStatus::AwaitingConfirmation, - budget(10), - ), - ) - .await?; - let ConfirmationOutcome::Confirmed(run) = repository - .confirm_run( - scope, - created.run_uid, - &created.active_plan_hash, - created.approved_budget, - ) - .await? - else { - panic!("replan-stop receipt fixture must begin from a confirmed plan"); - }; - let task = logical_task(run.run_uid, "replan", "", estimate(1)); - repository - .materialize_tasks(scope, run.run_uid, 1, vec![task.clone()]) - .await?; - reserve_and_start(&repository, scope, run.run_uid, task.task_id).await?; - assert!(matches!( - repository - .record_task_outcome(scope, run.run_uid, task.task_id, 1, needs_replan(1)) - .await?, - TaskOutcomeWrite::Applied { .. } - )); - let waiting = repository - .load_run(scope, run.run_uid) - .await? - .expect("waiting-replan run remains visible"); - let amendment_hash = ExecutionHash::from_bytes([42; 32]); - let pending_terminal = PendingExecutionTerminal { - status: ExecutionRunStatus::Blocked, - reason: ExecutionTerminalReason::DuplicateAmendment, - terminal_evidence: moa_execution::state::ExecutionTerminalEvidence { - cause: ExecutionTerminalCause::ReplanStop { - reason: ReplanStopReason::DuplicateAmendment, - }, - satisfied_requirement_count: 0, - requirement_count: 1, - }, - output: None, - completion_check_results: Vec::new(), - terminal_gaps: vec!["duplicate amendment".to_string()], - cancellation_reason: None, - }; - let receipt = ReplanStopReceipt { - task_id: task.task_id, - task_generation: 1, - base_plan_revision: 1, - amendment_hash, - }; - let TerminalFenceOutcome::Applied(fence) = repository - .fence_replan_stop( - scope, - run.run_uid, - 1, - waiting.wake_epoch, - pending_terminal.clone(), - receipt, - ) - .await? - else { - panic!("first replan-stop fence must apply"); - }; - assert!(matches!( - repository - .fence_replan_stop( - scope, - run.run_uid, - 1, - waiting.wake_epoch, - pending_terminal, - receipt, - ) - .await?, - TerminalFenceOutcome::Replayed(_) - )); - let AmendmentReplayOutcome::Replayed(replayed) = repository - .recover_amendment_handoff(scope, run.run_uid, 1, &amendment_hash) - .await? - else { - panic!("exact fenced replan stop must replay"); - }; - assert!(replayed.task_ids_to_release.is_empty()); - assert_eq!( - repository - .recover_amendment_handoff(scope, run.run_uid, 1, &ExecutionHash::from_bytes([43; 32])) - .await?, - AmendmentReplayOutcome::Conflict - ); - - assert!(matches!( - repository - .record_task_outcome( - scope, - run.run_uid, - task.task_id, - 1, - ExecutionTaskOutcome { - schema_version: 1, - usage: usage(1), - result: ExecutionTaskResult::Cancelled { - reason: "replan stop fenced".to_string(), - }, - }, - ) - .await?, - TaskOutcomeWrite::Applied { .. } - )); - let settled = repository - .load_run(scope, run.run_uid) - .await? - .expect("settled replan-stop run remains visible"); - assert!(matches!( - repository - .finalize_fenced_terminal(scope, run.run_uid, 1, settled.wake_epoch) - .await?, - FencedTerminalFinalizationOutcome::Finalized(_) - )); - let AmendmentReplayOutcome::Replayed(replayed) = repository - .recover_amendment_handoff(scope, run.run_uid, 1, &amendment_hash) - .await? - else { - panic!("exact replan-stop receipt must survive terminal finalization"); - }; - assert!(replayed.task_ids_to_release.is_empty()); - assert_eq!( - repository - .recover_amendment_handoff(scope, run.run_uid, 1, &ExecutionHash::from_bytes([43; 32])) - .await?, - AmendmentReplayOutcome::Conflict - ); - assert_eq!(fence.run.plan_revision, 1); - Ok(()) -} diff --git a/crates/moa-execution/tests/execution_db/retention_db.rs b/crates/moa-execution/tests/execution_db/retention_db.rs new file mode 100644 index 000000000..18c6d7366 --- /dev/null +++ b/crates/moa-execution/tests/execution_db/retention_db.rs @@ -0,0 +1,344 @@ +//! Terminal execution-detail retention contracts. + +use sqlx::Row; + +use super::support::*; +use moa_execution::repository::retention::{ + ExecutionRetentionClaimOutcome, ExecutionRetentionPageOutcome, +}; +use moa_execution::repository::terminal::{RunTriggerDrainOutcome, RunTriggerDrainRequest}; + +#[tokio::test] +async fn retention_self_schedule_is_singleton_and_generation_fenced_db() -> TestResult { + // Pins: a duplicate repair invocation cannot own an in-flight pass, and an + // old delayed generation cannot supersede the one persisted by its owner. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let scope = ExecutionScope::ControlPlane; + + let ExecutionRetentionClaimOutcome::Claimed { generation, .. } = + repository.claim_execution_retention(scope, None).await? + else { + panic!("first retention repair must claim the singleton generation"); + }; + assert_eq!(generation, 1); + assert!(matches!( + repository.claim_execution_retention(scope, None).await?, + ExecutionRetentionClaimOutcome::NotDue { .. } + )); + let receipt = repository + .schedule_execution_retention(scope, generation, 30, None) + .await?; + assert_eq!(receipt.scheduled_generation, 2); + assert_eq!( + repository + .schedule_execution_retention(scope, generation, 30, None) + .await?, + receipt, + "a retried journal step must replay its accepted schedule" + ); + assert!(matches!( + repository + .claim_execution_retention(scope, Some(generation)) + .await?, + ExecutionRetentionClaimOutcome::NotDue { + scheduled_generation: Some(2), + .. + } + )); + sqlx::query( + "UPDATE moa.execution_maintenance_checkpoint SET next_run_at = now() - interval '1 second' WHERE job_kind = 'execution_terminal_retention'", + ) + .execute(&pool) + .await?; + let ExecutionRetentionClaimOutcome::Claimed { generation, .. } = repository + .claim_execution_retention(scope, Some(receipt.scheduled_generation)) + .await? + else { + panic!("the exact due delayed generation must claim the next pass"); + }; + assert_eq!(generation, 2); + assert!( + repository + .schedule_execution_retention(scope, 1, 30, None) + .await + .is_err(), + "a claimed successor prevents an old generation from replacing its schedule" + ); + Ok(()) +} + +#[tokio::test] +async fn terminal_retention_honors_legal_hold_then_archives_before_bounded_deletion_db() +-> TestResult { + // Pins: a held terminal run remains untouched; after release, persisted keyset cursors + // resume multi-page sources without duplicate/missing evidence across crash-style calls, + // every immutable segment advances the rolling manifest root, and no bounded deletion starts + // before that exact root receipt is bound to execution_run. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let tenant_scope = ExecutionScope::Tenant { tenant_id }; + let run = create_run( + &repository, + tenant_scope, + new_run( + tenant_id, + None, + "terminal-retention", + ExecutionRunStatus::Queued, + budget(10), + ), + ) + .await?; + let _running = + claim_running_controller(&repository, tenant_scope, &ExecutionConfig::default(), &run) + .await?; + let logical_tasks = ["one", "two", "three"] + .into_iter() + .map(|item| logical_task(run.run_uid, "retained-output", item, estimate(1))) + .collect::>(); + repository + .materialize_tasks(tenant_scope, run.run_uid, 1, logical_tasks.clone()) + .await?; + for logical in logical_tasks { + reserve_and_start(&repository, tenant_scope, run.run_uid, logical.task_id).await?; + assert!(matches!( + repository + .record_task_outcome(tenant_scope, run.run_uid, logical.task_id, 1, completed(1),) + .await?, + TaskOutcomeWrite::Applied { .. } + )); + } + let predrain = repository + .load_run(tenant_scope, run.run_uid) + .await? + .expect("retention fixture run remains visible"); + assert!(matches!( + repository + .claim_controller_wake( + tenant_scope, + predrain.run_uid, + predrain.controller_generation, + predrain.wake_epoch, + ) + .await?, + RunControllerClaimOutcome::Claimed(_) + )); + assert!(matches!( + repository + .drain_run_triggers_page( + tenant_scope, + &ExecutionConfig::default(), + RunTriggerDrainRequest { + run_uid: predrain.run_uid, + controller_generation: predrain.controller_generation, + wake_epoch: predrain.wake_epoch, + page_limit: 1_000, + now: Utc::now(), + }, + ) + .await?, + RunTriggerDrainOutcome::ReadyToFinalize { .. } + )); + let before_finalization = repository + .load_run(tenant_scope, run.run_uid) + .await? + .expect("drained retention fixture remains visible"); + let evaluation = CompletionEvaluation { + status: CompletionStatus::Completed, + limit_stop: None, + checks: Vec::new(), + satisfied_requirement_ids: Vec::new(), + unsatisfied_requirement_ids: Vec::new(), + gaps: Vec::new(), + }; + let cause = ExecutionTerminalCause::Completion { limit_stop: None }; + let terminal = TerminalProjection::Completed { + output: json!({"retained": true}), + }; + let evidence = terminal_evidence_from_evaluation(cause.clone(), &evaluation)?; + let terminal_reason = execution_terminal_reason(&cause, &terminal, &evaluation)?; + assert!(matches!( + repository + .finalize_run( + tenant_scope, + RunFinalizationRequest { + run_uid: run.run_uid, + expected_revision: before_finalization.plan_revision, + expected_wake_epoch: before_finalization.wake_epoch, + terminal_projection: terminal, + completion_evaluation: evaluation, + terminal_evidence: evidence, + terminal_reason, + }, + ) + .await?, + FinalizationOutcome::Finalized(_) + )); + sqlx::query( + "UPDATE moa.execution_run SET completed_at = now() - interval '2 days' WHERE tenant_id = $1 AND run_uid = $2", + ) + .bind(tenant_id.0) + .bind(run.run_uid) + .execute(&pool) + .await?; + + sqlx::query( + "INSERT INTO moa.legal_hold (tenant_id, subject_id, reason, placed_by) VALUES ($1, NULL, 'execution retention test', 'retention-test')", + ) + .bind(tenant_id.0) + .execute(&pool) + .await?; + assert_eq!( + repository + .advance_execution_retention_page(ExecutionScope::ControlPlane, 1, 1) + .await?, + ExecutionRetentionPageOutcome::Idle + ); + let archive_count: i64 = sqlx::query_scalar( + "SELECT count(*) FROM moa.execution_terminal_archive WHERE tenant_id = $1 AND run_uid = $2", + ) + .bind(tenant_id.0) + .bind(run.run_uid) + .fetch_one(&pool) + .await?; + assert_eq!( + archive_count, 0, + "a legal hold must prevent even manifest creation" + ); + + sqlx::query( + "UPDATE moa.legal_hold SET released_at = now(), released_by = 'retention-test' WHERE tenant_id = $1 AND released_at IS NULL", + ) + .bind(tenant_id.0) + .execute(&pool) + .await?; + let mut saw_segment = false; + let mut saw_finalization = false; + let mut saw_deletion = false; + let mut completed = false; + for _ in 0..256 { + match repository + .advance_execution_retention_page(ExecutionScope::ControlPlane, 1, 1) + .await? + { + ExecutionRetentionPageOutcome::SegmentArchived { .. } => saw_segment = true, + ExecutionRetentionPageOutcome::ArchiveFinalized { .. } => saw_finalization = true, + ExecutionRetentionPageOutcome::DetailDeleted { .. } => saw_deletion = true, + ExecutionRetentionPageOutcome::Complete { run_uid } => { + assert_eq!(run_uid, run.run_uid); + completed = true; + break; + } + ExecutionRetentionPageOutcome::Idle => { + panic!("eligible retention work became idle before completion") + } + } + } + assert!(saw_segment && saw_finalization && saw_deletion && completed); + + let run_receipt: (Option, Option, Option>) = + sqlx::query_as( + "SELECT terminal_archive_uid, terminal_archive_hash, terminal_details_archived_at FROM moa.execution_run WHERE tenant_id = $1 AND run_uid = $2", + ) + .bind(tenant_id.0) + .bind(run.run_uid) + .fetch_one(&pool) + .await?; + let archive_uid = run_receipt + .0 + .expect("run retains the archive receipt identity"); + let run_root = run_receipt.1.expect("run retains the archive root digest"); + assert!(run_receipt.2.is_some()); + + let segments = sqlx::query( + "SELECT segment_kind, segment_sequence, record_count, payload, content_digest FROM moa.execution_terminal_archive_segment WHERE archive_uid = $1 ORDER BY segment_sequence", + ) + .bind(archive_uid) + .fetch_all(&pool) + .await?; + assert!(!segments.is_empty()); + let mut rolling: Option = None; + let mut expected_records = 0_i64; + let mut expected_bytes = 0_i64; + let mut expected_sequence = 1_i64; + let mut task_segment_count = 0_u32; + for segment in segments { + let kind: String = segment.try_get("segment_kind")?; + let sequence: i64 = segment.try_get("segment_sequence")?; + let records: i64 = segment.try_get("record_count")?; + let payload: Vec = segment.try_get("payload")?; + let stored_digest: Vec = segment.try_get("content_digest")?; + let computed_digest = blake3::hash(&payload); + assert_eq!(stored_digest.as_slice(), computed_digest.as_bytes()); + assert_eq!(sequence, expected_sequence); + expected_sequence += 1; + expected_records += records; + expected_bytes += i64::try_from(payload.len())?; + if kind == "execution_task" { + task_segment_count += 1; + } + let mut chain = blake3::Hasher::new(); + chain.update(b"moa.execution-terminal-archive.chain.v1\0"); + match rolling.as_deref() { + Some(previous) => chain.update(previous.as_bytes()), + None => chain.update(b"genesis"), + }; + chain.update(&(kind.len() as u64).to_be_bytes()); + chain.update(kind.as_bytes()); + chain.update(&sequence.to_be_bytes()); + chain.update(&records.to_be_bytes()); + chain.update(&i64::try_from(payload.len())?.to_be_bytes()); + chain.update(&stored_digest); + rolling = Some(chain.finalize().to_hex().to_string()); + } + assert_eq!(rolling.as_deref(), Some(run_root.as_str())); + assert_eq!( + task_segment_count, 3, + "page_size=1 must archive a three-record source across three durable keyset pages" + ); + let manifest_progress: (i64, i64, i64, serde_json::Value, Option) = sqlx::query_as( + "SELECT source_record_count, source_logical_bytes, segment_count, source_cursor, \ + rolling_chain_digest FROM moa.execution_terminal_archive WHERE archive_uid = $1", + ) + .bind(archive_uid) + .fetch_one(&pool) + .await?; + assert_eq!(manifest_progress.0, expected_records); + assert_eq!(manifest_progress.1, expected_bytes); + assert_eq!(manifest_progress.2, expected_sequence - 1); + assert_eq!(manifest_progress.3["kind"], "complete"); + assert_eq!(manifest_progress.4.as_deref(), Some(run_root.as_str())); + + assert!( + sqlx::query( + "UPDATE moa.execution_terminal_archive_segment SET payload = payload || '\\x00'::BYTEA \ + WHERE archive_uid = $1 AND segment_sequence = 1", + ) + .bind(archive_uid) + .execute(&pool) + .await + .is_err(), + "immutable segment evidence must reject digest tampering" + ); + + let live_task_count: i64 = sqlx::query_scalar( + "SELECT count(*) FROM moa.execution_task WHERE tenant_id = $1 AND run_uid = $2", + ) + .bind(tenant_id.0) + .bind(run.run_uid) + .fetch_one(&pool) + .await?; + assert_eq!(live_task_count, 0, "archived task detail must be removed"); + let details_deleted_at: Option> = sqlx::query_scalar( + "SELECT details_deleted_at FROM moa.execution_terminal_archive WHERE archive_uid = $1", + ) + .bind(archive_uid) + .fetch_one(&pool) + .await?; + assert!(details_deleted_at.is_some()); + Ok(()) +} diff --git a/crates/moa-execution/tests/execution_db/scope_and_lifecycle_db.rs b/crates/moa-execution/tests/execution_db/scope_and_lifecycle_db.rs index 9f2f19af8..be8f6e997 100644 --- a/crates/moa-execution/tests/execution_db/scope_and_lifecycle_db.rs +++ b/crates/moa-execution/tests/execution_db/scope_and_lifecycle_db.rs @@ -2,6 +2,149 @@ use super::support::*; +#[tokio::test] +async fn queued_run_admission_persists_initial_activation_before_returning_db() -> TestResult { + // Pins: the real create_run path accepts the current five-key plan contract, and a + // serving-process crash after admission cannot strand the queued run because its canonical + // generation-one, wake-one activation commits atomically, matching scheduled admission. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let run = create_run( + &repository, + scope, + new_run( + tenant_id, + None, + "initial-run-activation", + ExecutionRunStatus::Queued, + budget(1), + ), + ) + .await?; + + assert_eq!(run.activation_state, ExecutionActivationState::Queued); + assert_eq!(run.controller_generation, 1); + assert_eq!(run.wake_epoch, 1); + assert_eq!(run.processed_wake_epoch, 0); + let current_plan_shape: (bool, bool) = sqlx::query_as( + "SELECT moa.execution_plan_snapshot_is_current(initial_plan), \ + moa.execution_plan_snapshot_is_current(active_plan) \ + FROM moa.execution_run WHERE run_uid=$1", + ) + .bind(run.run_uid) + .fetch_one(&pool) + .await?; + assert_eq!(current_plan_shape, (true, true)); + let persisted: (Uuid, i64, i64, String, String, String) = sqlx::query_as( + "SELECT dispatch.dispatch_uid, dispatch.controller_generation, dispatch.wake_epoch, \ + dispatch.dispatch_kind, \ + dispatch.state, run.activation_state \ + FROM moa.execution_dispatch_outbox AS dispatch \ + JOIN moa.execution_run AS run USING (run_uid) \ + WHERE dispatch.run_uid = $1", + ) + .bind(run.run_uid) + .fetch_one(&pool) + .await?; + assert_ne!(persisted.0, Uuid::nil()); + assert_eq!(persisted.1, 1); + assert_eq!(persisted.2, 1); + assert_eq!(persisted.3, "run_activation"); + assert_eq!(persisted.4, "pending"); + assert_eq!(persisted.5, "queued"); + Ok(()) +} + +#[tokio::test] +async fn session_fenced_run_reads_hide_cross_session_rows_db() -> TestResult { + // Pins: after caller authorization, a guessed run UID from another parent session is + // excluded by the protected SQL read itself, including the scheduler snapshot path. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let repository = ExecutionRepository::new(test_db.store().pool().clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let owning_session_id = SessionId::new(); + let other_session_id = SessionId::new(); + let mut run = new_run( + tenant_id, + None, + "session-fenced-read", + ExecutionRunStatus::Queued, + budget(1), + ); + run.session_id = owning_session_id; + let run = create_run(&repository, scope, run).await?; + + assert_eq!( + repository + .load_run_for_session(scope, run.run_uid, owning_session_id) + .await?, + Some(run.clone()) + ); + assert_eq!( + repository + .load_run_for_session(scope, run.run_uid, other_session_id) + .await?, + None, + "a same-tenant session must not observe another session's run" + ); + assert!( + repository + .load_planning_context_for_session(scope, run.planning_context_uid, owning_session_id,) + .await? + .is_some() + ); + assert_eq!( + repository + .load_planning_context_for_session(scope, run.planning_context_uid, other_session_id,) + .await?, + None, + "execution start must not materialize another session's planning context" + ); + assert_eq!( + repository + .load_run_by_idempotency_key_for_session( + scope, + tenant_id, + None, + owning_session_id, + "session-fenced-read", + ) + .await?, + Some(run.clone()) + ); + assert_eq!( + repository + .load_run_by_idempotency_key_for_session( + scope, + tenant_id, + None, + other_session_id, + "session-fenced-read", + ) + .await?, + None, + "execution start replay must not materialize another session's idempotent run" + ); + assert!( + repository + .load_run_for_session(scope, run.run_uid, owning_session_id) + .await? + .is_some() + ); + assert_eq!( + repository + .load_run_for_session(scope, run.run_uid, other_session_id) + .await?, + None, + "the run read must enforce the same parent-session fence" + ); + Ok(()) +} + #[tokio::test] async fn originating_user_sequence_num_round_trips_to_execution_run_db() -> TestResult { // Pins: execution admission persists the exact user-event sequence as immutable run @@ -66,17 +209,8 @@ async fn execution_analytics_metadata_round_trips_normalized_source_and_terminal .fetch_one(&pool) .await?; - let TransitionOutcome::RunApplied(running) = repository - .transition_run_wait( - scope, - run.run_uid, - ExecutionRunStatus::Queued, - ExecutionRunStatus::Running, - ) - .await? - else { - panic!("analytics fixture must transition to running"); - }; + let running = + claim_running_controller(&repository, scope, &ExecutionConfig::default(), &run).await?; let evaluation = CompletionEvaluation { status: CompletionStatus::Completed, limit_stop: None, @@ -152,256 +286,6 @@ async fn execution_analytics_metadata_round_trips_normalized_source_and_terminal Ok(()) } -#[tokio::test] -async fn terminal_delivery_is_derived_from_durable_run_state_db() -> TestResult { - // Pins: terminal session delivery reuses the persisted origin and canonical full output, - // rather than accepting caller-supplied projection fields. - let test_db = moa_test_support::postgres::bootstrap_test_db().await?; - let repository = ExecutionRepository::new(test_db.store().pool().clone()); - let tenant_id = TenantId::new(); - let scope = ExecutionScope::Tenant { tenant_id }; - let mut new_run = new_run( - tenant_id, - None, - "terminal-delivery", - ExecutionRunStatus::Queued, - budget(10), - ); - new_run.originating_user_sequence_num = 57; - let run = create_run(&repository, scope, new_run).await?; - let TransitionOutcome::RunApplied(_running) = repository - .transition_run_wait( - scope, - run.run_uid, - ExecutionRunStatus::Queued, - ExecutionRunStatus::Running, - ) - .await? - else { - panic!("terminal delivery fixture must transition to running"); - }; - let tasks = repository - .materialize_tasks( - scope, - run.run_uid, - 1, - vec![ - logical_task(run.run_uid, "collect", "a", estimate(3)), - logical_task(run.run_uid, "collect", "b", estimate(3)), - ], - ) - .await?; - for task in &tasks { - reserve_and_start(&repository, scope, run.run_uid, task.task_id).await?; - } - assert!(matches!( - repository - .record_task_outcome( - scope, - run.run_uid, - tasks[0].task_id, - 1, - ExecutionTaskOutcome { - schema_version: 1, - usage: usage(1), - result: ExecutionTaskResult::Completed { - output: json!({ "part": "a" }), - citations: vec![ - ExecutionCitation { - source_id: "source-b".to_string(), - uri: None, - locator: None, - }, - ExecutionCitation { - source_id: "source-a".to_string(), - uri: None, - locator: None, - }, - ExecutionCitation { - source_id: "source-a".to_string(), - uri: None, - locator: None, - }, - ], - }, - }, - ) - .await?, - TaskOutcomeWrite::Applied { .. } - )); - assert!(matches!( - repository - .record_task_outcome( - scope, - run.run_uid, - tasks[1].task_id, - 1, - ExecutionTaskOutcome { - schema_version: 1, - usage: usage(1), - result: ExecutionTaskResult::Failed { - class: ExecutionFailureClass::Terminal, - message: "source b failed".to_string(), - }, - }, - ) - .await?, - TaskOutcomeWrite::Applied { .. } - )); - let prefinal = repository - .load_run(scope, run.run_uid) - .await? - .expect("run remains visible before finalization"); - let progress = execution_progress_from_run(&prefinal); - assert_eq!(progress.run_uid, run.run_uid); - assert_eq!(progress.originating_user_sequence_num, 57); - assert_eq!(progress.plan_revision, 1); - assert_eq!(progress.status, "running"); - assert_eq!(progress.total, 2); - assert_eq!(progress.completed, 1); - assert_eq!(progress.failed, 1); - assert_eq!(progress.cancelled, 0); - let output = json!({ "z": 1, "a": [2, 3] }); - let evaluation = CompletionEvaluation { - status: CompletionStatus::Partial, - limit_stop: None, - checks: Vec::new(), - satisfied_requirement_ids: Vec::new(), - unsatisfied_requirement_ids: Vec::new(), - gaps: vec!["source b missing".to_string()], - }; - let cause = ExecutionTerminalCause::Completion { limit_stop: None }; - let terminal = TerminalProjection::Partial { - output: Some(output.clone()), - gaps: vec!["source b missing".to_string()], - }; - let evidence = terminal_evidence_from_evaluation(cause.clone(), &evaluation)?; - let terminal_reason = execution_terminal_reason(&cause, &terminal, &evaluation)?; - let fence = repository - .fence_run_for_terminal( - scope, - run.run_uid, - 1, - prefinal.wake_epoch, - PendingExecutionTerminal { - status: ExecutionRunStatus::Partial, - reason: terminal_reason, - terminal_evidence: evidence, - output: Some(output.clone()), - completion_check_results: Vec::new(), - terminal_gaps: evaluation.gaps, - cancellation_reason: None, - }, - ) - .await?; - let TerminalFenceOutcome::Applied(fence) = fence else { - panic!("partial terminal intent must enter the compensation fence: {fence:?}"); - }; - assert!(matches!( - repository - .finalize_fenced_terminal(scope, run.run_uid, 1, fence.run.wake_epoch) - .await?, - FencedTerminalFinalizationOutcome::Finalized(_) - )); - - let delivery = repository - .load_terminal_delivery(scope, run.run_uid) - .await? - .expect("finalized run must have terminal delivery"); - let canonical = moa_core::canonical_json::canonical_json_bytes(&output)?; - assert_eq!(delivery.status, ExecutionRunStatus::Partial); - assert_eq!(delivery.summary.run_uid, run.run_uid); - assert_eq!(delivery.summary.originating_user_sequence_num, 57); - assert_eq!(delivery.summary.output, Some(output)); - assert_eq!( - delivery.summary.output_hash, - *blake3::hash(&canonical).as_bytes() - ); - assert_eq!( - delivery.summary.citation_ids, - vec!["source-a".to_string(), "source-b".to_string()] - ); - assert_eq!( - delivery.summary.failures, - vec!["source b failed".to_string()] - ); - assert_eq!(delivery.summary.gaps, vec!["source b missing".to_string()]); - assert_eq!( - delivery.summary.task_results, - ExecutionTaskResultsRef::ExecutionTaskTable { - run_uid: run.run_uid - } - ); - Ok(()) -} - -#[tokio::test] -async fn wake_epoch_acknowledgement_is_lossless_and_compare_and_set_db() -> TestResult { - // Pins: a scheduler can acknowledge only the exact persisted wake epoch, and a later wake remains pending. - let test_db = moa_test_support::postgres::bootstrap_test_db().await?; - let repository = ExecutionRepository::new(test_db.store().pool().clone()); - let tenant_id = TenantId::new(); - let scope = ExecutionScope::Tenant { tenant_id }; - let run = create_run( - &repository, - scope, - new_run( - tenant_id, - None, - "wake-epoch-cas", - ExecutionRunStatus::Queued, - budget(10), - ), - ) - .await?; - - let initial_epoch = run.wake_epoch; - assert_eq!(run.processed_wake_epoch, 0); - let task = logical_task(run.run_uid, "wake", "one", estimate(1)); - let materialized = repository - .materialize_tasks(scope, run.run_uid, 1, vec![task]) - .await?; - assert_eq!(materialized.len(), 1); - - assert_eq!( - repository - .ack_run_wake(scope, run.run_uid, initial_epoch) - .await?, - WakeAckOutcome::Changed { - current_wake_epoch: initial_epoch + 1, - } - ); - assert_eq!( - repository - .ack_run_wake(scope, run.run_uid, initial_epoch + 1) - .await?, - WakeAckOutcome::Acknowledged { - processed_wake_epoch: initial_epoch + 1, - } - ); - assert_eq!( - repository - .ack_run_wake(scope, run.run_uid, initial_epoch + 1) - .await?, - WakeAckOutcome::Replayed { - processed_wake_epoch: initial_epoch + 1, - } - ); - - let page = repository - .list_runs(scope, ExecutionRunPageRequest::default()) - .await?; - assert_eq!(page.runs.len(), 1); - assert_eq!(page.runs[0].processed_wake_epoch, initial_epoch + 1); - let snapshot = repository - .load_scheduling_snapshot(scope, run.run_uid) - .await? - .expect("repeatable-read scheduling snapshot should load"); - assert_eq!(snapshot.run.run_uid, run.run_uid); - assert_eq!(snapshot.projection.tasks.len(), 1); - Ok(()) -} - #[tokio::test] async fn tenant_contact_and_control_plane_scopes_are_isolated_db() -> TestResult { // Pins: apply_contact_rls exposes exactly control-plane, tenant-null-contact, and matching-contact rows. @@ -716,285 +600,6 @@ async fn terminal_run_rows_require_typed_cause_and_requirement_counts_db() -> Te Ok(()) } -#[tokio::test] -async fn terminal_finalization_persists_every_runtime_cause_and_replays_exactly_db() -> TestResult { - // Pins: completion, typed task failure, zero-dispatch limits, scheduler - // no-progress, and internal failure persist one complete replay identity. - let test_db = moa_test_support::postgres::bootstrap_test_db().await?; - let pool = test_db.store().pool().clone(); - let repository = ExecutionRepository::new(pool.clone()); - let tenant_id = TenantId::new(); - let scope = ExecutionScope::Tenant { tenant_id }; - let mut cases = vec![ - ( - "completion", - ExecutionTerminalCause::Completion { limit_stop: None }, - CompletionStatus::Completed, - TerminalProjection::Completed { output: json!({}) }, - ), - ( - "completion-deadline-partial", - ExecutionTerminalCause::Completion { - limit_stop: Some(ExecutionLimitStop::DeadlineExceeded), - }, - CompletionStatus::Partial, - TerminalProjection::Partial { - output: Some(json!({"useful": true})), - gaps: vec!["deadline".to_string()], - }, - ), - ( - "completion-budget-no-result", - ExecutionTerminalCause::Completion { - limit_stop: Some(ExecutionLimitStop::BudgetExceeded), - }, - CompletionStatus::Failed, - terminal_failure_projection(ExecutionFailureClass::BudgetExceeded), - ), - ( - "limit-deadline-partial", - ExecutionTerminalCause::LimitStop { - reason: ExecutionLimitStop::DeadlineExceeded, - }, - CompletionStatus::Partial, - TerminalProjection::Partial { - output: Some(json!({"useful": true})), - gaps: vec!["deadline".to_string()], - }, - ), - ( - "limit-budget-no-result", - ExecutionTerminalCause::LimitStop { - reason: ExecutionLimitStop::BudgetExceeded, - }, - CompletionStatus::Failed, - terminal_failure_projection(ExecutionFailureClass::BudgetExceeded), - ), - ( - "scheduler-no-progress", - ExecutionTerminalCause::SchedulerNoProgress, - CompletionStatus::Failed, - terminal_failure_projection(ExecutionFailureClass::Terminal), - ), - ( - "internal-failure", - ExecutionTerminalCause::InternalFailure, - CompletionStatus::Failed, - terminal_failure_projection(ExecutionFailureClass::Terminal), - ), - ]; - for class in [ - ExecutionFailureClass::Retryable, - ExecutionFailureClass::DependencyFailed, - ExecutionFailureClass::InvalidInput, - ExecutionFailureClass::InvalidOutput, - ExecutionFailureClass::AuthorizationDenied, - ExecutionFailureClass::BudgetExceeded, - ExecutionFailureClass::DeadlineExceeded, - ExecutionFailureClass::Cancelled, - ExecutionFailureClass::Unsupported, - ExecutionFailureClass::Terminal, - ] { - cases.push(( - "task-failure", - ExecutionTerminalCause::TaskFailure { - class: class.clone(), - }, - CompletionStatus::Failed, - terminal_failure_projection(class), - )); - } - - for (index, (name, cause, status, terminal)) in cases.into_iter().enumerate() { - let run = create_run( - &repository, - scope, - new_run( - tenant_id, - None, - &format!("terminal-cause-{index}-{name}"), - ExecutionRunStatus::Queued, - budget(1), - ), - ) - .await?; - let TransitionOutcome::RunApplied(running) = repository - .transition_run_wait( - scope, - run.run_uid, - ExecutionRunStatus::Queued, - ExecutionRunStatus::Running, - ) - .await? - else { - panic!("terminal fixture must transition to running"); - }; - let expected_wake_epoch = running.wake_epoch; - let evaluation = CompletionEvaluation { - status, - limit_stop: match &cause { - ExecutionTerminalCause::Completion { limit_stop } => *limit_stop, - ExecutionTerminalCause::TaskFailure { .. } - | ExecutionTerminalCause::LimitStop { .. } - | ExecutionTerminalCause::SchedulerNoProgress - | ExecutionTerminalCause::ReplanStop { .. } - | ExecutionTerminalCause::Cancellation - | ExecutionTerminalCause::InternalFailure - | ExecutionTerminalCause::CompensationFailure { .. } => None, - }, - checks: Vec::new(), - satisfied_requirement_ids: Vec::new(), - unsatisfied_requirement_ids: Vec::new(), - gaps: match &terminal { - TerminalProjection::Partial { gaps, .. } - | TerminalProjection::Blocked { gaps, .. } - | TerminalProjection::Unsupported { gaps, .. } => gaps.clone(), - TerminalProjection::Completed { .. } - | TerminalProjection::Failed { .. } - | TerminalProjection::Cancelled { .. } => Vec::new(), - }, - }; - let evidence = terminal_evidence_from_evaluation(cause.clone(), &evaluation)?; - let terminal_reason = execution_terminal_reason(&cause, &terminal, &evaluation)?; - if status != CompletionStatus::Completed { - let direct = repository - .finalize_run( - scope, - RunFinalizationRequest { - run_uid: run.run_uid, - expected_revision: 1, - expected_wake_epoch, - terminal_projection: terminal.clone(), - completion_evaluation: evaluation.clone(), - terminal_evidence: evidence.clone(), - terminal_reason, - }, - ) - .await; - assert!( - matches!( - direct, - Err(moa_execution::Error::InvalidRepositoryInput { .. }) - ), - "{name} bypassed the compensation fence: {direct:?}" - ); - let fence = repository - .fence_run_for_terminal( - scope, - run.run_uid, - 1, - expected_wake_epoch, - PendingExecutionTerminal { - status: moa_execution::state::run_status_from_terminal_projection( - &terminal, - ), - reason: terminal_reason, - terminal_evidence: evidence.clone(), - output: match &terminal { - TerminalProjection::Completed { output } => Some(output.clone()), - TerminalProjection::Partial { output, .. } => output.clone(), - TerminalProjection::Cancelled { .. } - | TerminalProjection::Blocked { .. } - | TerminalProjection::Unsupported { .. } - | TerminalProjection::Failed { .. } => None, - }, - completion_check_results: Vec::new(), - terminal_gaps: evaluation.gaps.clone(), - cancellation_reason: match &terminal { - TerminalProjection::Cancelled { reason } => Some(reason.clone()), - _ => None, - }, - }, - ) - .await?; - let TerminalFenceOutcome::Applied(fence) = fence else { - panic!("{name} must fence before terminal settlement: {fence:?}"); - }; - let finalized = repository - .finalize_fenced_terminal(scope, run.run_uid, 1, fence.run.wake_epoch) - .await?; - let FencedTerminalFinalizationOutcome::Finalized(finalized) = finalized else { - panic!("{name} must finalize from the fence: {finalized:?}"); - }; - assert_eq!( - finalized.terminal_evidence, - Some(evidence.clone()), - "{name}" - ); - assert!(matches!( - ExecutionRepository::new(pool.clone()) - .finalize_fenced_terminal(scope, run.run_uid, 1, fence.run.wake_epoch) - .await?, - FencedTerminalFinalizationOutcome::Replayed(_) - )); - continue; - } - let finalized = repository - .finalize_run( - scope, - RunFinalizationRequest { - run_uid: run.run_uid, - expected_revision: 1, - expected_wake_epoch, - terminal_projection: terminal.clone(), - completion_evaluation: evaluation.clone(), - terminal_evidence: evidence.clone(), - terminal_reason, - }, - ) - .await?; - let FinalizationOutcome::Finalized(finalized) = finalized else { - panic!("{name} must finalize on first delivery: {finalized:?}"); - }; - assert_eq!( - finalized.terminal_evidence, - Some(evidence.clone()), - "{name}" - ); - - let restarted_repository = ExecutionRepository::new(pool.clone()); - assert!(matches!( - restarted_repository - .finalize_run( - scope, - RunFinalizationRequest { - run_uid: run.run_uid, - expected_revision: 1, - expected_wake_epoch, - terminal_projection: terminal.clone(), - completion_evaluation: evaluation.clone(), - terminal_evidence: evidence.clone(), - terminal_reason, - }, - ) - .await?, - FinalizationOutcome::Replayed(_) - )); - let mut conflicting = evidence; - conflicting.satisfied_requirement_count = 1; - conflicting.requirement_count = 1; - assert_eq!( - restarted_repository - .finalize_run( - scope, - RunFinalizationRequest { - run_uid: run.run_uid, - expected_revision: 1, - expected_wake_epoch, - terminal_projection: terminal, - completion_evaluation: evaluation, - terminal_evidence: conflicting, - terminal_reason, - }, - ) - .await?, - FinalizationOutcome::Conflict, - "{name} conflicting replay" - ); - } - Ok(()) -} - #[tokio::test] async fn terminal_finalization_rejects_a_projection_changed_after_evaluation_db() -> TestResult { // Pins: cause and counts computed before a scheduling-relevant task mutation @@ -1015,17 +620,8 @@ async fn terminal_finalization_rejects_a_projection_changed_after_evaluation_db( ), ) .await?; - let TransitionOutcome::RunApplied(running) = repository - .transition_run_wait( - scope, - run.run_uid, - ExecutionRunStatus::Queued, - ExecutionRunStatus::Running, - ) - .await? - else { - panic!("fixture must be running"); - }; + let running = + claim_running_controller(&repository, scope, &ExecutionConfig::default(), &run).await?; let evaluation = CompletionEvaluation { status: CompletionStatus::Completed, limit_stop: None, diff --git a/crates/moa-execution/tests/execution_db/support.rs b/crates/moa-execution/tests/execution_db/support.rs index bbae92bb8..2522a2811 100644 --- a/crates/moa-execution/tests/execution_db/support.rs +++ b/crates/moa-execution/tests/execution_db/support.rs @@ -3,11 +3,12 @@ pub(crate) use chrono::{Duration, Utc}; pub(crate) use moa_artifacts::execution_plan::{ ExecutionBudgetLimit, ExecutionCancelPolicy, ExecutionCitation, ExecutionFailureClass, - ExecutionGoalContract, ExecutionTaskOutcome, ExecutionTaskResult, ExecutionUsage, - PlanAmendment, RetryPolicy, + ExecutionGoalContract, ExecutionTaskOutcome, ExecutionTaskResult, ExecutionTemporalTarget, + ExecutionUsage, ExecutionWaitExpiryAction, ExecutionWaitPolicy, PlanAmendment, RetryPolicy, }; +pub(crate) use moa_config::ExecutionConfig; pub(crate) use moa_core::canonical_json::canonical_json_bytes; -pub(crate) use moa_core::events::ExecutionTaskResultsRef; +pub(crate) use moa_core::traits::{Identity, IdentityType}; pub(crate) use moa_core::types::{ contact::ContactId, execution_planning::{ @@ -30,22 +31,25 @@ pub(crate) use moa_execution::{ terminal_evidence_from_evaluation, }, replan::{ReplanStopReason, failure_fingerprint}, + repository::audit::{ + CompileAuditWriteOutcome, NewExecutionPlanningContext, PlannerCallAuditWriteOutcome, + PlanningContextWriteOutcome, RouteAuditWriteOutcome, + }, + repository::run::RunAdmissionOutcome, + repository::terminal::{FinalizationOutcome, RunFinalizationRequest}, repository::{ - ActionReviewResolutionWrite, AmendmentReplayOutcome, AmendmentWrite, - CompileAuditWriteOutcome, ConfirmationConflict, ConfirmationOutcome, - ExecutionNodeMaterialization, ExecutionRepository, ExecutionRunPageRequest, ExecutionScope, - ExecutionTaskPageRequest, ExecutionTaskRecord, FencedTerminalFinalizationOutcome, - FinalizationOutcome, MaterializationOutcome, NewExecutionPlanningContext, NewExecutionRun, - PlannerCallAuditWriteOutcome, PlanningContextWriteOutcome, ReplanStopReceipt, - ReservationOutcome, ReservationRejection, RouteAuditWriteOutcome, RunFinalizationRequest, - TaskOutcomeRejection, TaskOutcomeWrite, TerminalFenceOutcome, TransitionOutcome, - ValidatedAmendment, WakeAckOutcome, + AmendmentReplayOutcome, AmendmentWrite, ConfirmationConflict, ConfirmationOutcome, + ExecutionActivationState, ExecutionAttemptState, ExecutionNodeMaterialization, + ExecutionRepository, ExecutionRunActivationCheckpoint, ExecutionRunRecord, ExecutionScope, + ExecutionTaskPageRequest, ExecutionTaskRecord, MaterializationOutcome, NewExecutionRun, + ReservationOutcome, ReservationRejection, RunActivationWriteOutcome, + RunControllerClaimOutcome, RunControllerCompletionOutcome, RunControllerCompletionRequest, + TaskOutcomeRejection, TaskOutcomeWrite, TransitionOutcome, ValidatedAmendment, }, state::{ - ExecutionLimitStop, ExecutionRunStatus, ExecutionSourceKind, ExecutionTaskId, - ExecutionTaskStatus, ExecutionTerminalCause, ExecutionTerminalReason, - FailureFingerprintInput, LogicalTask, LogicalTaskKind, PendingExecutionTerminal, - TerminalProjection, + ExecutionRunStatus, ExecutionSourceKind, ExecutionTaskId, ExecutionTaskStatus, + ExecutionTerminalCause, ExecutionTerminalReason, FailureFingerprintInput, LogicalTask, + LogicalTaskKind, PendingExecutionTerminal, TerminalProjection, }, wire::{ ExecutionActionReviewResolution, ExecutionPlanningContextSnapshot, @@ -59,6 +63,74 @@ pub(crate) use uuid::Uuid; /// Shared fallible result for concurrent database contract tests. pub(crate) type TestResult = Result<(), Box>; +/// Advances a newly admitted run through one bounded controller continuation. +pub(crate) async fn claim_running_controller( + repository: &ExecutionRepository, + scope: ExecutionScope, + config: &ExecutionConfig, + run: &ExecutionRunRecord, +) -> Result { + let claimed = match repository + .claim_controller_wake( + scope, + run.run_uid, + run.controller_generation, + run.wake_epoch, + ) + .await? + { + RunControllerClaimOutcome::Claimed(claimed) => claimed, + outcome => { + return Err(moa_execution::Error::InvalidRepositoryData { + message: format!("initial controller wake was not claimable: {outcome:?}"), + }); + } + }; + let continued = match repository + .complete_controller_wake( + scope, + config, + claimed.run_uid, + RunControllerCompletionRequest { + controller_generation: claimed.controller_generation, + wake_epoch: claimed.wake_epoch, + checkpoint: ExecutionRunActivationCheckpoint { + status: ExecutionRunStatus::Running, + activation_state: ExecutionActivationState::Queued, + next_wake_at: claimed.next_wake_at, + waiting_since: None, + ready_task_count: claimed.ready_task_count, + active_task_count: claimed.active_task_count, + }, + continuation_payload: Some(json!({"reason": "test_controller_continuation"})), + continuation_not_before_at: Utc::now(), + }, + ) + .await? + { + RunControllerCompletionOutcome::Applied { run, .. } => *run, + outcome => { + return Err(moa_execution::Error::InvalidRepositoryData { + message: format!("initial controller continuation was not committed: {outcome:?}"), + }); + } + }; + match repository + .claim_controller_wake( + scope, + continued.run_uid, + continued.controller_generation, + continued.wake_epoch, + ) + .await? + { + RunControllerClaimOutcome::Claimed(running) => Ok(running), + outcome => Err(moa_execution::Error::InvalidRepositoryData { + message: format!("continued controller wake was not claimable: {outcome:?}"), + }), + } +} + /// Asserts complete, non-overlapping pagination for one run's expected tasks. pub(crate) async fn assert_task_pages( repository: &ExecutionRepository, @@ -137,13 +209,23 @@ pub(crate) fn run_transition_allowed(source: &str, target: &str) -> bool { "awaiting_confirmation" => matches!(target, "queued" | "cancelled"), "queued" => matches!( target, - "running" | "compensating" | "blocked" | "unsupported" | "failed" | "cancelled" + "running" + | "pause_requested" + | "compensating" + | "blocked" + | "unsupported" + | "failed" + | "cancelled" ), "running" => matches!( target, "waiting_input" | "waiting_review" + | "waiting_signal" + | "waiting_timer" + | "waiting_external" | "waiting_replan" + | "pause_requested" | "compensating" | "completed" | "partial" @@ -152,9 +234,11 @@ pub(crate) fn run_transition_allowed(source: &str, target: &str) -> bool { | "failed" | "cancelled" ), - "waiting_input" | "waiting_review" | "waiting_replan" => matches!( + "waiting_input" | "waiting_review" | "waiting_signal" | "waiting_timer" + | "waiting_external" | "waiting_replan" => matches!( target, "running" + | "pause_requested" | "compensating" | "partial" | "blocked" @@ -162,6 +246,9 @@ pub(crate) fn run_transition_allowed(source: &str, target: &str) -> bool { | "failed" | "cancelled" ), + "pause_requested" => matches!(target, "pausing" | "paused" | "running" | "cancelled"), + "pausing" => matches!(target, "paused" | "failed" | "cancelled"), + "paused" => matches!(target, "queued" | "cancelled"), "compensating" => matches!( target, "completed" | "partial" | "blocked" | "unsupported" | "failed" | "cancelled" @@ -178,7 +265,13 @@ pub(crate) fn run_setup_path(status: &str) -> &'static [&'static str] { "running" => &["running"], "waiting_input" => &["running", "waiting_input"], "waiting_review" => &["running", "waiting_review"], + "waiting_signal" => &["running", "waiting_signal"], + "waiting_timer" => &["running", "waiting_timer"], + "waiting_external" => &["running", "waiting_external"], "waiting_replan" => &["running", "waiting_replan"], + "pause_requested" => &["running", "pause_requested"], + "pausing" => &["running", "pause_requested", "pausing"], + "paused" => &["running", "pause_requested", "paused"], "compensating" => &["running", "compensating"], "completed" => &["running", "completed"], "partial" => &["running", "partial"], @@ -253,15 +346,33 @@ pub(crate) async fn set_run_status_path( /// Returns whether the durable task contract permits one status transition. pub(crate) fn task_transition_allowed(source: &str, target: &str) -> bool { match source { - "pending" => matches!(target, "reserved" | "skipped" | "cancelled"), - "reserved" => matches!(target, "running" | "cancelled"), + "pending" => matches!(target, "ready" | "reserved" | "skipped" | "cancelled"), + "ready" => matches!(target, "dispatching" | "reserved" | "cancelled"), + "reserved" => matches!(target, "dispatching" | "running" | "cancelled"), + "dispatching" => matches!(target, "running" | "ready" | "failed" | "cancelled"), "running" => matches!( target, - "waiting_input" | "waiting_replan" | "completed" | "failed" | "cancelled" + "ready" + | "waiting_input" + | "waiting_review" + | "waiting_signal" + | "waiting_timer" + | "waiting_external" + | "waiting_replan" + | "completed" + | "failed" + | "cancelled" + | "unknown_outcome" + ), + "waiting_input" | "waiting_review" | "waiting_signal" | "waiting_timer" => { + matches!(target, "ready" | "cancelled") + } + "waiting_external" => matches!( + target, + "ready" | "completed" | "failed" | "cancelled" | "unknown_outcome" ), - "waiting_input" => matches!(target, "running" | "cancelled"), - "waiting_replan" => target == "cancelled", - "completed" | "skipped" | "failed" | "cancelled" => false, + "waiting_replan" => matches!(target, "ready" | "cancelled"), + "completed" | "skipped" | "failed" | "cancelled" | "unknown_outcome" => false, other => panic!("unknown task status in contract table: {other}"), } } @@ -270,13 +381,20 @@ pub(crate) fn task_transition_allowed(source: &str, target: &str) -> bool { pub(crate) fn task_setup_path(status: &str) -> &'static [&'static str] { match status { "pending" => &[], + "ready" => &["ready"], "reserved" => &["reserved"], + "dispatching" => &["ready", "dispatching"], "running" => &["reserved", "running"], "waiting_input" => &["reserved", "running", "waiting_input"], + "waiting_review" => &["reserved", "running", "waiting_review"], + "waiting_signal" => &["reserved", "running", "waiting_signal"], + "waiting_timer" => &["reserved", "running", "waiting_timer"], + "waiting_external" => &["reserved", "running", "waiting_external"], "waiting_replan" => &["reserved", "running", "waiting_replan"], "completed" => &["reserved", "running", "completed"], "skipped" => &["skipped"], "failed" => &["reserved", "running", "failed"], + "unknown_outcome" => &["reserved", "running", "unknown_outcome"], "cancelled" => &["cancelled"], other => panic!("unknown task status setup: {other}"), } @@ -351,8 +469,25 @@ pub(crate) async fn count_route_audits_as_app_role( pub(crate) async fn create_run( repository: &ExecutionRepository, scope: ExecutionScope, - mut run: NewExecutionRun, + run: NewExecutionRun, ) -> Result { + match create_run_with_config(repository, scope, &ExecutionConfig::default(), run).await? { + RunAdmissionOutcome::Admitted(run) | RunAdmissionOutcome::Replayed(run) => Ok(*run), + RunAdmissionOutcome::CapacitySaturated { dimension } => { + Err(moa_execution::Error::CapacitySaturated { + dimension: dimension.as_str(), + }) + } + } +} + +/// Admits a run with explicit execution-capacity limits after seeding its planning context. +pub(crate) async fn create_run_with_config( + repository: &ExecutionRepository, + scope: ExecutionScope, + config: &ExecutionConfig, + mut run: NewExecutionRun, +) -> Result { if repository .load_planning_context(scope, run.planning_context_uid) .await? @@ -394,7 +529,7 @@ pub(crate) async fn create_run( run.planning_context_uid = context.planning_context_uid; run.planning_context_hash = context_hash; } - repository.create_run(scope, run).await + repository.create_run(scope, config, run).await } /// Builds a minimal durable execution-run fixture. @@ -416,6 +551,17 @@ pub(crate) fn new_run( planning_context_uid: Uuid::now_v7(), planning_context_hash: ExecutionHash::from_bytes([97; 32]), owner_user_id: UserId::new("researcher"), + admitted_identity: Identity { + identity_type: if contact_id.is_some() { + IdentityType::Contact + } else { + IdentityType::Operator + }, + id: contact_id.map_or_else(Uuid::now_v7, |value| value.0), + tenant_id, + api_key_id: None, + acting_on_behalf_of: None, + }, goal: ExecutionGoalContract { objective: "test durable execution".to_string(), requirements: Vec::new(), @@ -447,6 +593,12 @@ pub(crate) fn canonical_plan(seed: u8) -> CanonicalExecutionPlan { CanonicalExecutionPlan { definition: moa_artifacts::execution_plan::ExecutionPlanDefinition { cancel_policy: ExecutionCancelPolicy::RetainEffects, + input_wait_policy: ExecutionWaitPolicy { + expiry: ExecutionTemporalTarget::After { + delay_seconds: 3_600, + }, + on_expiry: ExecutionWaitExpiryAction::FailTask, + }, input_schema: json!({ "type": "object" }), output_schema: json!({ "type": "object" }), nodes: Vec::new(), @@ -529,17 +681,6 @@ pub(crate) fn logical_task( } } -/// Builds a terminal task-failure projection for the requested failure class. -pub(crate) fn terminal_failure_projection(class: ExecutionFailureClass) -> TerminalProjection { - TerminalProjection::Failed { - failure: moa_execution::state::ExecutionTaskFailure { - class, - message: "terminal test failure".to_string(), - capability_ref: None, - }, - } -} - /// Reserves and starts one materialized task. pub(crate) async fn reserve_and_start( repository: &ExecutionRepository, diff --git a/crates/moa-execution/tests/execution_db/trigger_outbox_db.rs b/crates/moa-execution/tests/execution_db/trigger_outbox_db.rs new file mode 100644 index 000000000..56715dad0 --- /dev/null +++ b/crates/moa-execution/tests/execution_db/trigger_outbox_db.rs @@ -0,0 +1,3441 @@ +//! Temporal-trigger, dispatch-outbox, and asynchronous-job PostgreSQL contracts. + +use std::{collections::HashSet, time::Duration as StdDuration}; + +use super::support::*; +use chrono::DateTime; +use moa_artifacts::execution_plan::{ + ExecutionFailureClass, ExecutionNode, ExecutionOperation, ExecutionTaskOutcome, + ExecutionTaskResult, +}; +use moa_config::ExecutionConfig; +use moa_core::{ + types::completion::ToolInvocation, + types::identifiers::{ExecutionRunScopeId, ExecutionTaskScopeId}, + types::sandbox_workspace::{ExecutionHandReleaseOwner, ExecutionHandReleaseReceipt}, + types::tools::IdempotencyClass, +}; +use moa_execution::repository::run::enqueue_run_activation_in_conn; +use moa_execution::repository::{ + external_job::{ + ExecutionExternalJobBinding, ExecutionExternalJobCallback, + ExecutionExternalJobCallbackOutcome, ExecutionExternalJobCallbackUpdate, + ExecutionExternalJobCancellation, ExecutionExternalJobCancellationOutcome, + ExecutionExternalJobOwner, ExecutionExternalJobStartRecoveryAdoptionOutcome, + ExecutionExternalJobState, NewExecutionExternalJobIntent, + }, + outbox::{ + ExecutionDeliveryState, ExecutionDispatchFailureOutcome, ExecutionDispatchKind, + ExecutionDispatchRetryPolicy, ExecutionMaintenanceJobKind, + ExecutionMaintenanceSettlementOutcome, NewExecutionDispatch, + }, + trigger::{ + ExecutionExternalStartRecoveryRearmOutcome, ExecutionExternalStartRecoveryTriggerOutcome, + ExecutionRunDeadlineTriggerOutcome, ExecutionTriggerFireOutcome, ExecutionTriggerKind, + ExecutionTriggerNoOp, ExecutionTriggerSupersedeOutcome, ExecutionWatchdogTriggerOutcome, + NewExecutionTrigger, create_trigger_with_dispatch_in_conn, supersede_trigger_in_conn, + }, +}; +use moa_execution::repository::{ + ready::{ReadyMaterializationOutcome, ReadyMaterializationRequest}, + task::{ + NewTaskAttemptCheckpoint, ReleasedTaskAttemptCapacityOutcome, + ResolveTaskAttemptReviewRequest, TaskAttemptCheckpointKind, + TaskAttemptCheckpointWriteOutcome, TaskAttemptExternalOutcome, TaskAttemptFence, + TaskAttemptReleaseClaimOutcome, TaskAttemptReviewParkOutcome, + TaskAttemptReviewResolutionOutcome, TaskAttemptSettlementOutcome, TaskAttemptStartOutcome, + }, + terminal::PendingTerminalAdvanceOutcome, +}; +use moa_execution::wire::{ + ExecutionActionReviewResolution, ExecutionExternalJobStartRecoveryOwner, + ExecutionExternalJobStartRecoveryRequest, +}; + +fn watchdog_output_node() -> ExecutionNode { + ExecutionNode { + id: "watchdog-work".to_string(), + requirement_ids: vec!["req".to_string()], + depends_on: Vec::new(), + when: None, + input: json!({}), + output_schema: json!({ "type": "object" }), + operation: ExecutionOperation::Output { value: json!({}) }, + compensation: None, + retry: RetryPolicy { + max_attempts: 1, + initial_backoff_ms: 1, + max_backoff_ms: 1, + }, + budget: None, + } +} + +#[tokio::test] +async fn trigger_creation_is_atomic_and_firing_is_due_generation_fenced_db() -> TestResult { + // Pins: a trigger and its fallback delivery commit together; early delivery does not + // advance state; one current due generation wakes once; stale and duplicate delivery no-op. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let execution_config = execution_capacity_config(); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let run = create_run( + &repository, + scope, + new_run( + tenant_id, + None, + "trigger-atomic-generation", + ExecutionRunStatus::Queued, + budget(10), + ), + ) + .await?; + + let rolled_back_uid = Uuid::now_v7(); + let rolled_back = run_deadline( + rolled_back_uid, + tenant_id, + run.run_uid, + 1, + pg_deadline(Duration::minutes(1)), + ); + let mut transaction = pool.begin().await?; + create_trigger_with_dispatch_in_conn(&mut transaction, &execution_config, &rolled_back).await?; + transaction.rollback().await?; + let (trigger_count, dispatch_count): (i64, i64) = sqlx::query_as( + "SELECT (SELECT count(*) FROM moa.execution_trigger WHERE trigger_uid = $1), \ + (SELECT count(*) FROM moa.execution_dispatch_outbox WHERE trigger_uid = $1)", + ) + .bind(rolled_back_uid) + .fetch_one(&pool) + .await?; + assert_eq!((trigger_count, dispatch_count), (0, 0)); + + sqlx::query( + "UPDATE moa.execution_dispatch_outbox SET state='delivered',delivered_at=NOW(),updated_at=NOW() \ + WHERE run_uid=$1 AND state='pending'", + ) + .bind(run.run_uid) + .execute(&pool) + .await?; + + let future_uid = Uuid::now_v7(); + let future = repository + .create_trigger( + scope, + &execution_config, + run_deadline( + future_uid, + tenant_id, + run.run_uid, + 1, + pg_deadline(Duration::minutes(5)), + ), + ) + .await?; + assert_eq!(future.trigger.state, ExecutionDeliveryState::Pending); + // Pins: creating a future trigger persists no process-local timer; the indexed outbox head is + // the sole normal timing authority, remains unclaimable early, and becomes the due delivery. + let wake = repository.next_pending_dispatch_wake(scope).await?; + assert_eq!(wake.dispatch_uid, Some(future.dispatch.dispatch_uid)); + assert_eq!(wake.next_due_at, Some(future.trigger.due_at)); + assert!( + repository + .claim_due_dispatches(scope, "future-trigger-owner", 1, StdDuration::from_secs(30)) + .await? + .is_empty() + ); + assert_eq!( + repository.fire_trigger(scope, future_uid).await?, + ExecutionTriggerFireOutcome::NoOp(ExecutionTriggerNoOp::NotDue) + ); + let future_dispatch_state: String = sqlx::query_scalar( + "SELECT state FROM moa.execution_dispatch_outbox WHERE dispatch_uid = $1", + ) + .bind(future.dispatch.dispatch_uid) + .fetch_one(&pool) + .await?; + assert_eq!(future_dispatch_state, "pending"); + let mut transaction = pool.begin().await?; + assert_eq!( + supersede_trigger_in_conn( + &mut transaction, + future_uid, + ExecutionTriggerKind::RunDeadline, + Some(1), + None, + None, + None, + ) + .await?, + ExecutionTriggerSupersedeOutcome::Superseded + ); + transaction.commit().await?; + + let due_uid = Uuid::now_v7(); + let due = repository + .create_trigger( + scope, + &execution_config, + run_deadline( + due_uid, + tenant_id, + run.run_uid, + 1, + pg_deadline(Duration::minutes(-1)), + ), + ) + .await?; + sqlx::query("DELETE FROM moa.execution_dispatch_outbox WHERE dispatch_uid = $1") + .bind(due.dispatch.dispatch_uid) + .execute(&pool) + .await?; + let repaired = repository + .reconcile_due_trigger_dispatches(scope, 10) + .await?; + assert_eq!(repaired.len(), 1); + assert_eq!(repaired[0].dispatch_uid, due.dispatch.dispatch_uid); + let ExecutionTriggerFireOutcome::Delivered { + activation: Some(activation), + } = repository.fire_trigger(scope, due_uid).await? + else { + panic!("current due run trigger must enqueue one activation"); + }; + assert_eq!(activation.kind, ExecutionDispatchKind::RunActivation); + assert_eq!(activation.controller_generation, Some(1)); + assert_eq!( + repository.fire_trigger(scope, due_uid).await?, + ExecutionTriggerFireOutcome::NoOp(ExecutionTriggerNoOp::Duplicate) + ); + let due_dispatch_state: String = sqlx::query_scalar( + "SELECT state FROM moa.execution_dispatch_outbox WHERE dispatch_uid = $1", + ) + .bind(due.dispatch.dispatch_uid) + .fetch_one(&pool) + .await?; + assert_eq!(due_dispatch_state, "delivered"); + + let stale_uid = Uuid::now_v7(); + let stale = repository + .create_trigger( + scope, + &execution_config, + run_deadline( + stale_uid, + tenant_id, + run.run_uid, + 1, + pg_deadline(Duration::seconds(-1)), + ), + ) + .await?; + sqlx::query("UPDATE moa.execution_run SET controller_generation = 2 WHERE run_uid = $1") + .bind(run.run_uid) + .execute(&pool) + .await?; + assert_eq!( + repository.fire_trigger(scope, stale_uid).await?, + ExecutionTriggerFireOutcome::NoOp(ExecutionTriggerNoOp::StaleGeneration) + ); + let (trigger_state, dispatch_state): (String, String) = sqlx::query_as( + "SELECT trigger.state, dispatch.state \ + FROM moa.execution_trigger AS trigger \ + JOIN moa.execution_dispatch_outbox AS dispatch USING (trigger_uid) \ + WHERE trigger.trigger_uid = $1", + ) + .bind(stale.trigger.trigger_uid) + .fetch_one(&pool) + .await?; + assert_eq!( + (trigger_state.as_str(), dispatch_state.as_str()), + ("superseded", "cancelled") + ); + + let activation_count: i64 = sqlx::query_scalar( + "SELECT count(*) FROM moa.execution_dispatch_outbox \ + WHERE run_uid = $1 AND dispatch_kind = 'run_activation'", + ) + .bind(run.run_uid) + .fetch_one(&pool) + .await?; + assert_eq!(activation_count, 1); + Ok(()) +} + +#[tokio::test] +async fn trigger_fire_missing_existing_capacity_bucket_rolls_back_db() -> TestResult { + // Pins: receipt-backed trigger fire fails closed when one exact canonical bucket is missing; + // neither trigger nor outbox state advances in the aborted transaction. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let mut request = new_run( + tenant_id, + None, + "trigger-missing-capacity-bucket", + ExecutionRunStatus::Queued, + budget(10), + ); + request.approved_budget.deadline_at = Some(pg_deadline(Duration::seconds(-1))); + let run = create_run(&repository, scope, request).await?; + let trigger_uid: Uuid = sqlx::query_scalar( + "SELECT trigger_uid FROM moa.execution_trigger \ + WHERE run_uid=$1 AND trigger_kind='run_deadline' AND state='pending'", + ) + .bind(run.run_uid) + .fetch_one(&pool) + .await?; + sqlx::query( + "DELETE FROM moa.execution_capacity_bucket \ + WHERE scope_kind='tenant' AND tenant_id=$1 AND resource_dimension='parked_runs'", + ) + .bind(tenant_id.0) + .execute(&pool) + .await?; + + let error = repository + .fire_trigger(scope, trigger_uid) + .await + .expect_err("missing canonical capacity must fail trigger fire"); + assert!( + error + .to_string() + .contains("missing canonical capacity buckets during existing-row prelock") + ); + let states: (String, String) = sqlx::query_as( + "SELECT trigger.state, dispatch.state \ + FROM moa.execution_trigger AS trigger \ + JOIN moa.execution_dispatch_outbox AS dispatch USING (trigger_uid) \ + WHERE trigger.trigger_uid=$1", + ) + .bind(trigger_uid) + .fetch_one(&pool) + .await?; + assert_eq!(states, ("pending".to_string(), "pending".to_string())); + Ok(()) +} + +#[tokio::test] +async fn trigger_fire_prelocks_existing_capacity_without_reconciling_limits_db() -> TestResult { + // Pins: firing a committed run trigger locks all six receipt-backed capacity keys in canonical + // order without rewriting persisted limits from mutable runtime configuration. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let mut request = new_run( + tenant_id, + None, + "trigger-existing-capacity-prelock", + ExecutionRunStatus::Queued, + budget(10), + ); + request.approved_budget.deadline_at = Some(pg_deadline(Duration::seconds(-1))); + let run = create_run(&repository, scope, request).await?; + let trigger_uid: Uuid = sqlx::query_scalar( + "SELECT trigger_uid FROM moa.execution_trigger \ + WHERE run_uid=$1 AND trigger_kind='run_deadline' AND state='pending'", + ) + .bind(run.run_uid) + .fetch_one(&pool) + .await?; + sqlx::query( + "UPDATE moa.execution_capacity_bucket SET limit_value=limit_value+100 \ + WHERE resource_dimension IN ('active_runs','parked_runs','scheduled_triggers') \ + AND (scope_kind='fleet' OR (scope_kind='tenant' AND tenant_id=$1))", + ) + .bind(tenant_id.0) + .execute(&pool) + .await?; + let limits = || async { + sqlx::query_as::<_, (String, Option, String, i64)>( + "SELECT scope_kind,tenant_id,resource_dimension,limit_value \ + FROM moa.execution_capacity_bucket \ + WHERE resource_dimension IN ('active_runs','parked_runs','scheduled_triggers') \ + AND (scope_kind='fleet' OR (scope_kind='tenant' AND tenant_id=$1)) \ + ORDER BY resource_dimension,scope_kind", + ) + .bind(tenant_id.0) + .fetch_all(&pool) + .await + }; + let before = limits().await?; + assert_eq!(before.len(), 6); + + assert!(matches!( + repository.fire_trigger(scope, trigger_uid).await?, + ExecutionTriggerFireOutcome::Delivered { .. } + )); + assert_eq!(limits().await?, before); + Ok(()) +} + +#[tokio::test] +async fn paused_run_deadline_race_reprepares_current_fences_before_trigger_settlement_db() +-> TestResult { + // Pins: a pause that wins after deadline preparation cannot consume the only absolute + // deadline trigger. The stale fence conflicts with the trigger/capacity still active; + // re-preparation adopts the paused run's current generation and enforces the elapsed deadline. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let config = execution_capacity_config(); + let mut request = new_run( + tenant_id, + None, + "paused-deadline-race", + ExecutionRunStatus::Queued, + budget(1), + ); + request.approved_budget.deadline_at = Some(pg_deadline(Duration::minutes(-1))); + let RunAdmissionOutcome::Admitted(run) = + create_run_with_config(&repository, scope, &config, request).await? + else { + panic!("deadline race fixture must be admitted"); + }; + let trigger_uid: Uuid = sqlx::query_scalar( + "SELECT trigger_uid FROM moa.execution_trigger \ + WHERE run_uid=$1 AND trigger_kind='run_deadline' AND state='pending'", + ) + .bind(run.run_uid) + .fetch_one(&pool) + .await?; + let ExecutionRunDeadlineTriggerOutcome::Ready { + run_uid, + controller_generation: stale_generation, + wake_epoch: stale_wake_epoch, + observed_at: stale_observed_at, + } = repository + .prepare_run_deadline_trigger(scope, trigger_uid) + .await? + else { + panic!("elapsed deadline must prepare against the admitted run fence"); + }; + + let TransitionOutcome::RunApplied(paused) = repository + .pause_run(scope, &config, run_uid, stale_generation) + .await? + else { + panic!("zero-active-task run must pause immediately"); + }; + assert_eq!(paused.status, ExecutionRunStatus::Paused); + assert!(paused.controller_generation > stale_generation); + sqlx::query( + "UPDATE moa.execution_dispatch_outbox \ + SET state='delivered', delivered_at=NOW()-INTERVAL '2 minutes', updated_at=NOW() \ + WHERE trigger_uid=$1 AND dispatch_kind='trigger_delivery'", + ) + .bind(trigger_uid) + .execute(&pool) + .await?; + let repaired = repository + .reconcile_due_trigger_dispatches(scope, 3) + .await?; + assert!( + repaired + .iter() + .any(|dispatch| dispatch.trigger_uid == Some(trigger_uid)), + "a paused run's immutable absolute deadline must survive Restate delivery-state loss" + ); + let repaired_boundary: (String, String, String) = sqlx::query_as( + "SELECT trigger.state, dispatch.state, capacity.state \ + FROM moa.execution_trigger AS trigger \ + JOIN moa.execution_dispatch_outbox AS dispatch USING (trigger_uid) \ + JOIN moa.execution_capacity_reservation AS capacity USING (trigger_uid) \ + WHERE trigger.trigger_uid=$1", + ) + .bind(trigger_uid) + .fetch_one(&pool) + .await?; + assert_eq!( + repaired_boundary, + ( + "pending".to_string(), + "pending".to_string(), + "reserved".to_string(), + ), + "reconciliation must redrive, not supersede, a paused run deadline" + ); + assert_eq!( + repository + .fence_deadline_and_enqueue_settlement( + &config, + scope, + run_uid, + stale_generation, + stale_wake_epoch, + stale_observed_at, + 1, + ) + .await?, + PendingTerminalAdvanceOutcome::Conflict + ); + let active_boundary: (String, String, String) = sqlx::query_as( + "SELECT trigger.state, dispatch.state, capacity.state \ + FROM moa.execution_trigger AS trigger \ + JOIN moa.execution_dispatch_outbox AS dispatch USING (trigger_uid) \ + JOIN moa.execution_capacity_reservation AS capacity USING (trigger_uid) \ + WHERE trigger.trigger_uid=$1", + ) + .bind(trigger_uid) + .fetch_one(&pool) + .await?; + assert_eq!( + active_boundary, + ( + "pending".to_string(), + "pending".to_string(), + "reserved".to_string(), + ), + "a stale deadline fence must not consume its sole recovery trigger" + ); + + let ExecutionRunDeadlineTriggerOutcome::Ready { + controller_generation, + wake_epoch, + observed_at, + .. + } = repository + .prepare_run_deadline_trigger(scope, trigger_uid) + .await? + else { + panic!("paused elapsed deadline must reprepare against the current run fence"); + }; + assert_eq!(controller_generation, paused.controller_generation); + assert_eq!(wake_epoch, paused.wake_epoch); + assert!(matches!( + repository + .fence_deadline_and_enqueue_settlement( + &config, + scope, + run_uid, + controller_generation, + wake_epoch, + observed_at, + 1, + ) + .await?, + PendingTerminalAdvanceOutcome::Applied(_) | PendingTerminalAdvanceOutcome::Replayed(_) + )); + assert!(matches!( + repository + .settle_run_deadline_trigger(scope, trigger_uid) + .await?, + ExecutionTriggerSupersedeOutcome::Superseded + | ExecutionTriggerSupersedeOutcome::AlreadySuperseded + | ExecutionTriggerSupersedeOutcome::AlreadyInactive + )); + let settled_boundary: (String, String, String) = sqlx::query_as( + "SELECT trigger.state, dispatch.state, capacity.state \ + FROM moa.execution_trigger AS trigger \ + JOIN moa.execution_dispatch_outbox AS dispatch USING (trigger_uid) \ + JOIN moa.execution_capacity_reservation AS capacity USING (trigger_uid) \ + WHERE trigger.trigger_uid=$1", + ) + .bind(trigger_uid) + .fetch_one(&pool) + .await?; + assert_eq!( + settled_boundary, + ( + "superseded".to_string(), + "cancelled".to_string(), + "released".to_string(), + ) + ); + Ok(()) +} + +#[tokio::test] +async fn run_deadline_settlement_locks_scheduled_capacity_before_trigger_db() -> TestResult { + // Pins: terminal draining locks capacity before trigger rows. A concurrent deadline + // settlement must wait without holding the trigger, preventing a capacity-trigger cycle. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let config = execution_capacity_config(); + let mut request = new_run( + tenant_id, + None, + "deadline-capacity-lock-order", + ExecutionRunStatus::Queued, + budget(1), + ); + request.approved_budget.deadline_at = Some(pg_deadline(Duration::minutes(5))); + let RunAdmissionOutcome::Admitted(run) = + create_run_with_config(&repository, scope, &config, request).await? + else { + panic!("deadline lock-order fixture must be admitted"); + }; + let trigger_uid: Uuid = sqlx::query_scalar( + "SELECT trigger_uid FROM moa.execution_trigger \ + WHERE tenant_id=$1 AND run_uid=$2 AND trigger_kind='run_deadline' AND state='pending'", + ) + .bind(tenant_id.0) + .bind(run.run_uid) + .fetch_one(&pool) + .await?; + + let mut capacity_holder = pool.begin().await?; + for (scope_kind, owner) in [("fleet", None), ("tenant", Some(tenant_id.0))] { + sqlx::query( + "SELECT capacity_bucket_uid FROM moa.execution_capacity_bucket \ + WHERE scope_kind=$1 AND tenant_id IS NOT DISTINCT FROM $2 \ + AND resource_dimension='scheduled_triggers' FOR UPDATE", + ) + .bind(scope_kind) + .bind(owner) + .fetch_one(&mut *capacity_holder) + .await?; + } + + let settlement_repository = repository.clone(); + let mut settlement = tokio::spawn(async move { + settlement_repository + .settle_run_deadline_trigger(scope, trigger_uid) + .await + }); + assert!( + tokio::time::timeout(StdDuration::from_millis(100), &mut settlement) + .await + .is_err(), + "settlement must wait on the prelocked ScheduledTriggers bucket" + ); + sqlx::query( + "SELECT trigger_uid FROM moa.execution_trigger \ + WHERE trigger_uid=$1 FOR UPDATE NOWAIT", + ) + .bind(trigger_uid) + .fetch_one(&mut *capacity_holder) + .await?; + capacity_holder.commit().await?; + + assert_eq!( + tokio::time::timeout(StdDuration::from_secs(5), settlement).await???, + ExecutionTriggerSupersedeOutcome::Superseded + ); + Ok(()) +} + +#[tokio::test] +async fn reconciliation_redrives_accepted_trigger_and_run_dispatches_after_restate_loss_db() +-> TestResult { + // Pins: after total Restate state loss, a sufficiently old accepted trigger delivery and + // run activation are requeued with the same immutable dispatch IDs only while their exact + // database generations remain current. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let execution_config = execution_capacity_config(); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + assert!( + repository + .reconcile_due_trigger_dispatches(scope, 2) + .await + .is_err(), + "a reconciliation batch must fund trigger, accepted-dispatch, and run lanes" + ); + let run = create_run( + &repository, + scope, + new_run( + tenant_id, + None, + "restate-loss-redrive", + ExecutionRunStatus::Queued, + budget(10), + ), + ) + .await?; + + let trigger = repository + .create_trigger( + scope, + &execution_config, + run_deadline( + Uuid::now_v7(), + tenant_id, + run.run_uid, + run.controller_generation, + pg_deadline(Duration::minutes(-2)), + ), + ) + .await?; + let mut transaction = pool.begin().await?; + let activation = enqueue_run_activation_in_conn( + &mut transaction, + tenant_id, + run.run_uid, + run.controller_generation, + pg_deadline(Duration::minutes(-2)), + json!({"source": "restate-loss-test"}), + ) + .await?; + transaction.commit().await?; + sqlx::query( + "UPDATE moa.execution_dispatch_outbox \ + SET state = 'delivered', delivered_at = now() - interval '2 minutes', \ + delivery_attempts = 4, updated_at = now() - interval '2 minutes' \ + WHERE dispatch_uid = ANY($1)", + ) + .bind(vec![trigger.dispatch.dispatch_uid, activation.dispatch_uid]) + .execute(&pool) + .await?; + + let repaired = repository + .reconcile_due_trigger_dispatches(scope, 10) + .await?; + let repaired_ids = repaired + .iter() + .map(|dispatch| dispatch.dispatch_uid) + .collect::>(); + assert_eq!( + repaired_ids, + HashSet::from([trigger.dispatch.dispatch_uid, activation.dispatch_uid]) + ); + let rows: Vec<(Uuid, String, Option>, i32)> = sqlx::query_as( + "SELECT dispatch_uid, state, delivered_at, delivery_attempts \ + FROM moa.execution_dispatch_outbox WHERE dispatch_uid = ANY($1) \ + ORDER BY dispatch_uid", + ) + .bind(vec![trigger.dispatch.dispatch_uid, activation.dispatch_uid]) + .fetch_all(&pool) + .await?; + assert_eq!(rows.len(), 2); + assert!(rows.iter().all(|(_, state, delivered_at, attempts)| { + state == "pending" && delivered_at.is_none() && *attempts == 0 + })); + + sqlx::query( + "UPDATE moa.execution_dispatch_outbox \ + SET state = 'delivered', delivered_at = now() - interval '2 minutes', \ + updated_at = now() - interval '2 minutes' \ + WHERE dispatch_uid = ANY($1)", + ) + .bind(vec![trigger.dispatch.dispatch_uid, activation.dispatch_uid]) + .execute(&pool) + .await?; + sqlx::query( + "UPDATE moa.execution_run SET controller_generation = controller_generation + 1, \ + updated_at = now() WHERE run_uid = $1", + ) + .bind(run.run_uid) + .execute(&pool) + .await?; + assert!( + repository + .reconcile_due_trigger_dispatches(scope, 10) + .await? + .is_empty() + ); + let stale_states: Vec<(Uuid, String)> = sqlx::query_as( + "SELECT dispatch_uid, state FROM moa.execution_dispatch_outbox \ + WHERE dispatch_uid = ANY($1) ORDER BY dispatch_uid", + ) + .bind(vec![trigger.dispatch.dispatch_uid, activation.dispatch_uid]) + .fetch_all(&pool) + .await?; + assert!(stale_states.iter().all(|(_, state)| state == "delivered")); + let trigger_state: String = + sqlx::query_scalar("SELECT state FROM moa.execution_trigger WHERE trigger_uid = $1") + .bind(trigger.trigger.trigger_uid) + .fetch_one(&pool) + .await?; + assert_eq!(trigger_state, "superseded"); + Ok(()) +} + +#[tokio::test] +async fn trigger_capacity_saturates_atomically_and_releases_once_db() -> TestResult { + // Pins: a pending trigger owns one fleet and tenant receipt; saturation rolls + // the new trigger back, and replayed supersession cannot decrement twice. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let mut execution_config = execution_capacity_config(); + execution_config.max_tenant_scheduled_triggers = 1; + execution_config.max_fleet_scheduled_triggers = 1; + execution_config.validate()?; + let first_run = create_run( + &repository, + scope, + new_run( + tenant_id, + None, + "trigger-capacity-first", + ExecutionRunStatus::Queued, + budget(10), + ), + ) + .await?; + let second_run = create_run( + &repository, + scope, + new_run( + tenant_id, + None, + "trigger-capacity-second", + ExecutionRunStatus::Queued, + budget(10), + ), + ) + .await?; + let first_uid = Uuid::now_v7(); + repository + .create_trigger( + scope, + &execution_config, + run_deadline( + first_uid, + tenant_id, + first_run.run_uid, + first_run.controller_generation, + pg_deadline(Duration::minutes(5)), + ), + ) + .await?; + let rejected_uid = Uuid::now_v7(); + assert!(matches!( + repository + .create_trigger( + scope, + &execution_config, + run_deadline( + rejected_uid, + tenant_id, + second_run.run_uid, + second_run.controller_generation, + pg_deadline(Duration::minutes(5)), + ), + ) + .await, + Err(moa_execution::Error::CapacitySaturated { + dimension: "scheduled_triggers" + }) + )); + let rejected_rows: (i64, i64) = sqlx::query_as( + "SELECT (SELECT count(*) FROM moa.execution_trigger WHERE trigger_uid = $1), \ + (SELECT count(*) FROM moa.execution_dispatch_outbox WHERE trigger_uid = $1)", + ) + .bind(rejected_uid) + .fetch_one(&pool) + .await?; + assert_eq!(rejected_rows, (0, 0)); + + let mut transaction = pool.begin().await?; + assert_eq!( + supersede_trigger_in_conn( + &mut transaction, + first_uid, + ExecutionTriggerKind::RunDeadline, + Some(first_run.controller_generation), + None, + None, + None, + ) + .await?, + ExecutionTriggerSupersedeOutcome::Superseded + ); + assert_eq!( + supersede_trigger_in_conn( + &mut transaction, + first_uid, + ExecutionTriggerKind::RunDeadline, + Some(first_run.controller_generation), + None, + None, + None, + ) + .await?, + ExecutionTriggerSupersedeOutcome::AlreadySuperseded + ); + transaction.commit().await?; + let counters: Vec<(String, i64)> = sqlx::query_as( + "SELECT scope_kind, reserved_quantity FROM moa.execution_capacity_bucket \ + WHERE resource_dimension = 'scheduled_triggers' ORDER BY scope_kind", + ) + .fetch_all(&pool) + .await?; + assert_eq!( + counters, + vec![("fleet".to_string(), 0), ("tenant".to_string(), 0)] + ); + Ok(()) +} + +#[tokio::test] +async fn outbox_claims_are_disjoint_expiry_recoverable_and_dead_lettered_db() -> TestResult { + // Pins: bounded SKIP LOCKED claimers never overlap; expired ownership can be stolen; + // stale owners cannot ack; bounded exponential retry ends in durable dead letter. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let run = create_run( + &repository, + scope, + new_run( + tenant_id, + None, + "outbox-claim-retry", + ExecutionRunStatus::Queued, + budget(10), + ), + ) + .await?; + for wake_epoch in 10..14 { + repository + .enqueue_dispatch( + scope, + run_activation( + tenant_id, + run.run_uid, + wake_epoch, + pg_deadline(Duration::seconds(-1)), + ), + ) + .await?; + } + let health = repository.sample_execution_queue_health(scope, 3).await?; + assert_eq!(health.claimable_dispatches.observed_count, 3); + assert!(health.claimable_dispatches.saturated); + assert!(health.claimable_dispatches.oldest_at.is_some()); + let other_health = repository + .sample_execution_queue_health( + ExecutionScope::Tenant { + tenant_id: TenantId::new(), + }, + 3, + ) + .await?; + assert_eq!(other_health.claimable_dispatches.observed_count, 0); + assert!(!other_health.claimable_dispatches.saturated); + + let (owner_a, owner_b) = tokio::join!( + repository.claim_due_dispatches(scope, "owner-a", 2, StdDuration::from_secs(30)), + repository.claim_due_dispatches(scope, "owner-b", 2, StdDuration::from_secs(30)), + ); + let owner_a = owner_a?; + let owner_b = owner_b?; + assert_eq!((owner_a.len(), owner_b.len()), (2, 2)); + let a_ids = owner_a + .iter() + .map(|dispatch| dispatch.dispatch_uid) + .collect::>(); + let b_ids = owner_b + .iter() + .map(|dispatch| dispatch.dispatch_uid) + .collect::>(); + assert!(a_ids.is_disjoint(&b_ids)); + + let abandoned = owner_a[0].dispatch_uid; + sqlx::query( + "UPDATE moa.execution_dispatch_outbox \ + SET claimed_at = now() - interval '2 seconds', \ + claim_expires_at = now() - interval '1 second' WHERE dispatch_uid = $1", + ) + .bind(abandoned) + .execute(&pool) + .await?; + // Pins: a totally lost drain leaves no pending row, but its expired dispatching claim is the + // indexed head at claim expiry and can be reclaimed without an unrelated producer kick. + let expired_head = repository.next_pending_dispatch_wake(scope).await?; + assert_eq!(expired_head.dispatch_uid, Some(abandoned)); + assert!(expired_head.next_due_at <= Some(expired_head.observed_at)); + assert!(expired_head.head_updated_at.is_some()); + let recovered = repository + .claim_due_dispatches(scope, "owner-c", 1, StdDuration::from_secs(30)) + .await?; + assert_eq!(recovered.len(), 1); + assert_eq!(recovered[0].dispatch_uid, abandoned); + assert_eq!(recovered[0].delivery_attempts, 2); + assert_eq!( + repository + .mark_dispatches_delivered(scope, &[abandoned], "owner-a") + .await?, + Vec::::new() + ); + + let retry_uid = owner_b[0].dispatch_uid; + let retry = ExecutionDispatchRetryPolicy { + max_attempts: 2, + base_delay: StdDuration::from_secs(1), + maximum_delay: StdDuration::from_secs(4), + }; + assert!(matches!( + repository + .record_dispatch_failure(scope, retry_uid, "owner-b", "transient", retry) + .await?, + ExecutionDispatchFailureOutcome::RetryScheduled { .. } + )); + sqlx::query( + "UPDATE moa.execution_dispatch_outbox \ + SET not_before_at = now() - interval '1 second' WHERE dispatch_uid = $1", + ) + .bind(retry_uid) + .execute(&pool) + .await?; + let reclaimed = repository + .claim_due_dispatches(scope, "owner-d", 1, StdDuration::from_secs(30)) + .await?; + assert_eq!(reclaimed[0].dispatch_uid, retry_uid); + assert_eq!(reclaimed[0].delivery_attempts, 2); + assert_eq!( + repository + .record_dispatch_failure(scope, retry_uid, "owner-d", "permanent", retry) + .await?, + ExecutionDispatchFailureOutcome::DeadLettered + ); + let state: String = sqlx::query_scalar( + "SELECT state FROM moa.execution_dispatch_outbox WHERE dispatch_uid = $1", + ) + .bind(retry_uid) + .fetch_one(&pool) + .await?; + assert_eq!(state, "dead_letter"); + let health = repository.sample_execution_queue_health(scope, 10).await?; + assert_eq!(health.dead_letter_dispatches.observed_count, 1); + assert!(!health.dead_letter_dispatches.saturated); + Ok(()) +} + +#[tokio::test] +async fn dispatch_batch_acknowledges_only_exact_owned_unique_claims_db() -> TestResult { + // Pins: one bounded success acknowledgement preserves request identity order, transitions + // every exact current claim in one repository call, and leaves already-delivered or differently + // owned rows untouched. Duplicate identities fail closed before any row can transition. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let run = create_run( + &repository, + scope, + new_run( + tenant_id, + None, + "dispatch-batch-ack", + ExecutionRunStatus::Queued, + budget(10), + ), + ) + .await?; + sqlx::query( + "UPDATE moa.execution_dispatch_outbox SET state='delivered', delivered_at=NOW(), \ + updated_at=NOW() WHERE run_uid=$1 AND state='pending'", + ) + .bind(run.run_uid) + .execute(&pool) + .await?; + for wake_epoch in 10..14 { + repository + .enqueue_dispatch( + scope, + run_activation( + tenant_id, + run.run_uid, + wake_epoch, + pg_deadline(Duration::seconds(-1)), + ), + ) + .await?; + } + let claimed = repository + .claim_due_dispatches(scope, "batch-owner", 4, StdDuration::from_secs(30)) + .await?; + assert_eq!(claimed.len(), 4); + let differently_owned = claimed[1].dispatch_uid; + sqlx::query( + "UPDATE moa.execution_dispatch_outbox SET claim_owner='other-owner' \ + WHERE dispatch_uid=$1 AND state='dispatching'", + ) + .bind(differently_owned) + .execute(&pool) + .await?; + let already_delivered = claimed[2].dispatch_uid; + assert_eq!( + repository + .mark_dispatches_delivered(scope, &[already_delivered], "batch-owner") + .await?, + vec![already_delivered] + ); + + let requested = vec![ + claimed[3].dispatch_uid, + claimed[0].dispatch_uid, + already_delivered, + differently_owned, + ]; + assert_eq!( + repository + .mark_dispatches_delivered(scope, &requested, "batch-owner") + .await?, + vec![claimed[3].dispatch_uid, claimed[0].dispatch_uid] + ); + let differently_owned_row: (String, Option) = sqlx::query_as( + "SELECT state, claim_owner FROM moa.execution_dispatch_outbox WHERE dispatch_uid=$1", + ) + .bind(differently_owned) + .fetch_one(&pool) + .await?; + assert_eq!( + differently_owned_row, + ("dispatching".to_string(), Some("other-owner".to_string())) + ); + + assert!(matches!( + repository + .mark_dispatches_delivered( + scope, + &[differently_owned, differently_owned], + "other-owner", + ) + .await, + Err(moa_execution::Error::InvalidRepositoryInput { message }) + if message == "execution dispatch acknowledgement identities must be unique" + )); + let state_after_rejection: String = + sqlx::query_scalar("SELECT state FROM moa.execution_dispatch_outbox WHERE dispatch_uid=$1") + .bind(differently_owned) + .fetch_one(&pool) + .await?; + assert_eq!(state_after_rejection, "dispatching"); + Ok(()) +} + +#[tokio::test] +async fn pending_dispatch_wake_reports_earliest_indexed_deadline_db() -> TestResult { + // Pins: an empty due claim pass still discovers the exact earliest future dispatch through + // the pending queue index, while tenant scope cannot observe another tenant's timer. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let repository = ExecutionRepository::new(test_db.store().pool().clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let run = create_run( + &repository, + scope, + new_run( + tenant_id, + None, + "pending-dispatch-wake", + ExecutionRunStatus::Queued, + budget(10), + ), + ) + .await?; + sqlx::query( + "UPDATE moa.execution_dispatch_outbox SET state='delivered', delivered_at=NOW(), \ + updated_at=NOW() WHERE run_uid=$1 AND state='pending'", + ) + .bind(run.run_uid) + .execute(test_db.store().pool()) + .await?; + let first_due_at = pg_deadline(Duration::seconds(10)); + let second_due_at = pg_deadline(Duration::seconds(20)); + let second = repository + .enqueue_dispatch( + scope, + run_activation(tenant_id, run.run_uid, 21, second_due_at), + ) + .await?; + let first = repository + .enqueue_dispatch( + scope, + run_activation(tenant_id, run.run_uid, 20, first_due_at), + ) + .await?; + + assert!( + repository + .claim_due_dispatches(scope, "future-owner", 10, StdDuration::from_secs(30)) + .await? + .is_empty() + ); + let wake = repository.next_pending_dispatch_wake(scope).await?; + assert_eq!(wake.dispatch_uid, Some(first.dispatch_uid)); + assert_eq!(wake.next_due_at, Some(first_due_at)); + assert!(wake.observed_at < first_due_at); + assert_ne!(wake.dispatch_uid, Some(second.dispatch_uid)); + assert_eq!( + repository + .next_pending_dispatch_wake(ExecutionScope::Tenant { + tenant_id: TenantId::new(), + }) + .await? + .dispatch_uid, + None + ); + Ok(()) +} + +#[tokio::test] +async fn task_watchdog_preparation_is_due_fenced_and_exact_owner_replay_safe_db() -> TestResult { + // Pins: watchdog delivery does not invoke an attempt before DB time is due, resolves the + // exact active dispatch/capacity owner when due, and terminal trigger settlement is replay-safe. + // Total Restate loss redrives the accepted start only while DB state remains Dispatching; + // once Running, watchdog recovery owns ambiguity and the start is never replayed. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let config = execution_capacity_config(); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let mut candidate = new_run( + tenant_id, + None, + "watchdog-exact-owner", + ExecutionRunStatus::Queued, + budget(10), + ); + candidate.plan.definition.nodes = vec![watchdog_output_node()]; + let run = create_run(&repository, scope, candidate).await?; + assert!( + repository + .initialize_scheduler_state(scope, run.run_uid) + .await? + ); + sqlx::query( + "UPDATE moa.execution_dispatch_outbox SET state='delivered', delivered_at=NOW(), \ + updated_at=NOW() WHERE run_uid=$1 AND dispatch_kind='run_activation' AND state='pending'", + ) + .bind(run.run_uid) + .execute(&pool) + .await?; + assert!(matches!( + repository + .materialize_ready_page( + scope, + &config, + ReadyMaterializationRequest { + run_uid: run.run_uid, + plan_revision: 1, + node_id: "watchdog-work".to_string(), + expected_cursor: 0, + reduce_cursor: None, + source_exhausted: true, + terminal_output: None, + tasks: vec![logical_task( + run.run_uid, + "watchdog-work", + "one", + estimate(1), + )], + }, + ) + .await?, + ReadyMaterializationOutcome::Applied { .. } + )); + let admitted = repository + .admit_ready_attempts(&config, 1, Utc::now()) + .await? + .admitted + .into_iter() + .next() + .expect("one task must be admitted"); + // Pins: the dispatch drain's post-delivery admission pass turns a task materialized by a + // synchronous RunActivation into the immediate indexed TaskAttempt head. The drain can thus + // self-chain without relying on a second producer kick. + let admitted_head = repository.next_pending_dispatch_wake(scope).await?; + assert_eq!(admitted_head.dispatch_uid, Some(admitted.dispatch_uid)); + assert!(admitted_head.next_due_at <= Some(admitted_head.observed_at)); + assert_eq!( + repository + .prepare_watchdog_trigger(scope, admitted.watchdog_trigger_uid) + .await?, + ExecutionWatchdogTriggerOutcome::NoOp(ExecutionTriggerNoOp::NotDue) + ); + let fence = TaskAttemptFence { + tenant_id: admitted.tenant_id, + run_uid: admitted.run_uid, + task_id: admitted.task_id, + controller_generation: admitted.controller_generation, + attempt_generation: admitted.attempt_generation, + dispatch_uid: admitted.dispatch_uid, + capacity_reservation_uid: admitted.capacity_reservation_uid, + watchdog_trigger_uid: admitted.watchdog_trigger_uid, + attempt_deadline_at: admitted.attempt_deadline_at, + }; + sqlx::query( + "UPDATE moa.execution_dispatch_outbox \ + SET state='delivered', delivered_at=NOW()-INTERVAL '2 minutes', updated_at=NOW() \ + WHERE dispatch_uid=$1", + ) + .bind(admitted.dispatch_uid) + .execute(&pool) + .await?; + let accepted_before_start = repository + .reconcile_due_trigger_dispatches(scope, 10) + .await?; + assert_eq!(accepted_before_start.len(), 1); + assert_eq!(accepted_before_start[0].dispatch_uid, admitted.dispatch_uid); + sqlx::query( + "UPDATE moa.execution_dispatch_outbox \ + SET state='delivered', delivered_at=NOW()-INTERVAL '2 minutes', updated_at=NOW() \ + WHERE dispatch_uid=$1", + ) + .bind(admitted.dispatch_uid) + .execute(&pool) + .await?; + assert!(matches!( + repository.start_task_attempt(fence).await?, + TaskAttemptStartOutcome::Started(_) + )); + assert!( + repository + .reconcile_due_trigger_dispatches(scope, 10) + .await? + .is_empty() + ); + let running_dispatch_state: String = + sqlx::query_scalar("SELECT state FROM moa.execution_dispatch_outbox WHERE dispatch_uid=$1") + .bind(admitted.dispatch_uid) + .fetch_one(&pool) + .await?; + assert_eq!(running_dispatch_state, "delivered"); + sqlx::query( + "UPDATE moa.execution_trigger SET due_at=NOW()-INTERVAL '1 second' \ + WHERE trigger_uid=$1", + ) + .bind(admitted.watchdog_trigger_uid) + .execute(&pool) + .await?; + let ExecutionWatchdogTriggerOutcome::Task(request) = repository + .prepare_watchdog_trigger(scope, admitted.watchdog_trigger_uid) + .await? + else { + panic!("the exact due active task watchdog must resolve its receiver"); + }; + assert_eq!(request.dispatch_uid, admitted.dispatch_uid); + assert_eq!( + request.capacity_reservation_uid, + admitted.capacity_reservation_uid + ); + assert_eq!(request.watchdog_trigger_uid, admitted.watchdog_trigger_uid); + assert_eq!(request.attempt_generation, admitted.attempt_generation); + assert_eq!( + repository + .settle_watchdog_trigger(scope, admitted.watchdog_trigger_uid) + .await?, + ExecutionTriggerSupersedeOutcome::Superseded + ); + assert_eq!( + repository + .prepare_watchdog_trigger(scope, admitted.watchdog_trigger_uid) + .await?, + ExecutionWatchdogTriggerOutcome::NoOp(ExecutionTriggerNoOp::Inactive) + ); + Ok(()) +} + +#[tokio::test] +async fn task_external_start_recovery_adopts_started_not_started_and_replay_atomically_db() +-> TestResult { + // Pins: task recovery consumes a provisional checkpoint under exact active-resource fences. + // NotStarted returns one fresh Ready attempt without losing the checkpoint; Started binds the + // provider job and parks WaitingExternal; exact replay is stable and a missing checkpoint + // rolls back intent/capacity release. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let config = execution_capacity_config(); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let mut candidate = new_run( + tenant_id, + None, + "task-external-start-recovery", + ExecutionRunStatus::Queued, + budget(20), + ); + candidate.plan.definition.nodes = vec![watchdog_output_node()]; + let run = create_run(&repository, scope, candidate).await?; + assert!( + repository + .initialize_scheduler_state(scope, run.run_uid) + .await? + ); + assert!(matches!( + repository + .materialize_ready_page( + scope, + &config, + ReadyMaterializationRequest { + run_uid: run.run_uid, + plan_revision: 1, + node_id: "watchdog-work".to_string(), + expected_cursor: 0, + reduce_cursor: None, + source_exhausted: true, + terminal_output: None, + tasks: ["not-started", "started", "missing-checkpoint"] + .into_iter() + .map(|item| { + logical_task(run.run_uid, "watchdog-work", item, estimate(1)) + }) + .collect(), + }, + ) + .await?, + ReadyMaterializationOutcome::Applied { .. } + )); + let admissions = repository + .admit_ready_attempts(&config, 3, Utc::now()) + .await? + .admitted; + assert_eq!(admissions.len(), 3); + + let mut started = Vec::new(); + for admission in admissions { + let fence = TaskAttemptFence { + tenant_id: admission.tenant_id, + run_uid: admission.run_uid, + task_id: admission.task_id, + controller_generation: admission.controller_generation, + attempt_generation: admission.attempt_generation, + dispatch_uid: admission.dispatch_uid, + capacity_reservation_uid: admission.capacity_reservation_uid, + watchdog_trigger_uid: admission.watchdog_trigger_uid, + attempt_deadline_at: admission.attempt_deadline_at, + }; + let TaskAttemptStartOutcome::Started(record) = repository.start_task_attempt(fence).await? + else { + panic!("each admitted recovery fixture task must start exactly once"); + }; + started.push((fence, record)); + } + for (fence, record) in &started[..2] { + assert!(matches!( + repository + .persist_running_task_external_start_checkpoint(NewTaskAttemptCheckpoint { + fence: *fence, + task_generation: record.task.generation, + kind: TaskAttemptCheckpointKind::CapabilityExternalStart, + schema_version: 1, + payload: json!({ + "state": { + "kind": "capability_external_start", + "tool_id": "fixture-async-tool", + "usage": {} + } + }), + workspace_release_receipt: None, + created_at: Utc::now(), + }) + .await?, + TaskAttemptCheckpointWriteOutcome::Applied(_) + )); + } + + let make_recovery = |fence: TaskAttemptFence, suffix: &str| { + let external_job_uid = Uuid::now_v7(); + let provider = "task-recovery-provider".to_string(); + let idempotency_key = format!("task-recovery-{suffix}-{external_job_uid}"); + let owner = ExecutionExternalJobOwner::Task { + task_id: fence.task_id.as_uuid(), + attempt_generation: fence.attempt_generation, + }; + ( + NewExecutionExternalJobIntent { + external_job_uid, + tenant_id, + run_uid: run.run_uid, + owner, + job_generation: 1, + provider: provider.clone(), + idempotency_key: idempotency_key.clone(), + expires_at: pg_deadline(Duration::minutes(1)), + }, + ExecutionExternalJobStartRecoveryRequest { + tenant_id, + run_uid: run.run_uid, + owner: ExecutionExternalJobStartRecoveryOwner::Task { + task_id: fence.task_id.as_uuid(), + attempt_generation: fence.attempt_generation, + }, + external_job_uid, + job_generation: 1, + provider, + idempotency_key, + trigger_uid: Uuid::now_v7(), + }, + ) + }; + + let (not_started_intent, not_started_recovery) = make_recovery(started[0].0, "none"); + repository + .reserve_external_job_intent(scope, &config, not_started_intent) + .await?; + assert!(matches!( + repository + .recover_external_job_start_not_started(¬_started_recovery, Utc::now()) + .await?, + ExecutionExternalJobStartRecoveryAdoptionOutcome::Applied { + compensation_release: None + } + )); + assert!(matches!( + repository + .recover_external_job_start_not_started(¬_started_recovery, Utc::now()) + .await?, + ExecutionExternalJobStartRecoveryAdoptionOutcome::Replayed { + compensation_release: None + } + )); + let not_started_state: (String, String, i64, Option) = sqlx::query_as( + "SELECT status,attempt_state,attempt_generation,active_dispatch_uid \ + FROM moa.execution_task WHERE run_uid=$1 AND task_id=$2", + ) + .bind(run.run_uid) + .bind(started[0].0.task_id.as_uuid()) + .fetch_one(&pool) + .await?; + assert_eq!( + not_started_state, + ("ready".to_string(), "idle".to_string(), 2, None) + ); + assert!( + repository + .load_task_attempt_checkpoint(scope, run.run_uid, started[0].0.task_id) + .await? + .is_some() + ); + + let (started_intent, started_recovery) = make_recovery(started[1].0, "started"); + let started_owner = started_intent.owner; + repository + .reserve_external_job_intent(scope, &config, started_intent.clone()) + .await?; + sqlx::query( + "UPDATE moa.execution_capacity_reservation \ + SET expires_at=NOW() - INTERVAL '1 second' \ + WHERE external_job_uid=$1 AND resource_dimension='external_jobs'", + ) + .bind(started_intent.external_job_uid) + .execute(&pool) + .await?; + let binding = ExecutionExternalJobBinding { + external_job_uid: started_intent.external_job_uid, + tenant_id, + run_uid: run.run_uid, + owner: started_owner, + job_generation: 1, + idempotency_key: started_intent.idempotency_key.clone(), + provider: started_intent.provider.clone(), + provider_job_id: format!("provider-job-{}", Uuid::now_v7()), + callback_auth_reference: "vault://task-recovery".to_string(), + state: ExecutionExternalJobState::Running, + progress_phase: Some("running".to_string()), + cancel_supported: true, + next_reconcile_at: Some(pg_deadline(Duration::minutes(2))), + provider_contract_violation: None, + }; + assert!(matches!( + repository + .recover_external_job_start_started( + &config, + &started_recovery, + binding.clone(), + Utc::now(), + ) + .await?, + ExecutionExternalJobStartRecoveryAdoptionOutcome::Applied { + compensation_release: None + } + )); + assert!(matches!( + repository + .recover_external_job_start_started(&config, &started_recovery, binding, Utc::now(),) + .await?, + ExecutionExternalJobStartRecoveryAdoptionOutcome::Replayed { + compensation_release: None + } + )); + let started_state: (String, String, Option) = sqlx::query_as( + "SELECT status,attempt_state,external_job_uid FROM moa.execution_task \ + WHERE run_uid=$1 AND task_id=$2", + ) + .bind(run.run_uid) + .bind(started[1].0.task_id.as_uuid()) + .fetch_one(&pool) + .await?; + assert_eq!( + started_state, + ( + "waiting_external".to_string(), + "waiting".to_string(), + Some(started_intent.external_job_uid) + ) + ); + let recovered_job: (String, bool) = sqlx::query_as( + "SELECT job.state, capacity.expires_at IS NULL \ + FROM moa.execution_external_job AS job \ + JOIN moa.execution_capacity_reservation AS capacity \ + ON capacity.external_job_uid=job.external_job_uid \ + AND capacity.resource_dimension='external_jobs' \ + WHERE job.external_job_uid=$1", + ) + .bind(started_intent.external_job_uid) + .fetch_one(&pool) + .await?; + assert_eq!(recovered_job, ("running".to_string(), true)); + + let (missing_intent, missing_recovery) = make_recovery(started[2].0, "missing"); + repository + .reserve_external_job_intent(scope, &config, missing_intent.clone()) + .await?; + assert!(!matches!( + repository + .recover_external_job_start_not_started(&missing_recovery, Utc::now()) + .await?, + ExecutionExternalJobStartRecoveryAdoptionOutcome::Applied { .. } + | ExecutionExternalJobStartRecoveryAdoptionOutcome::Replayed { .. } + )); + let preserved = repository + .load_external_job(scope, missing_intent.external_job_uid) + .await? + .expect("failed adoption must roll back unbound intent release"); + assert_eq!(preserved.state, ExecutionExternalJobState::Unbound); + Ok(()) +} + +#[tokio::test] +async fn external_start_rearm_replaces_completed_restate_delivery_identity_db() -> TestResult { + // Pins: Unknown provider-start recovery keeps one durable timer row but changes the Restate + // delivery identity, so the next due claim cannot replay the completed NotDue invocation. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let config = execution_capacity_config(); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let run = create_run( + &repository, + scope, + new_run( + tenant_id, + None, + "external-start-rearm-identity", + ExecutionRunStatus::Queued, + budget(10), + ), + ) + .await?; + let task = repository + .materialize_tasks( + scope, + run.run_uid, + 1, + vec![logical_task( + run.run_uid, + "provider-job", + "rearm", + estimate(1), + )], + ) + .await? + .into_iter() + .next() + .expect("one external-start recovery task"); + sqlx::query( + "UPDATE moa.execution_task SET status='reserved', updated_at=NOW() WHERE task_id=$1", + ) + .bind(task.task_id.as_uuid()) + .execute(&pool) + .await?; + sqlx::query( + "UPDATE moa.execution_task SET status='running', attempt_state='running', \ + last_progress_at=NOW(), updated_at=NOW() WHERE task_id=$1", + ) + .bind(task.task_id.as_uuid()) + .execute(&pool) + .await?; + let external_job_uid = Uuid::now_v7(); + repository + .reserve_external_job_intent( + scope, + &config, + NewExecutionExternalJobIntent { + external_job_uid, + tenant_id, + run_uid: run.run_uid, + owner: ExecutionExternalJobOwner::Task { + task_id: task.task_id.as_uuid(), + attempt_generation: 1, + }, + job_generation: 1, + provider: "batch-provider".to_string(), + idempotency_key: "external-start-rearm-identity".to_string(), + expires_at: pg_deadline(Duration::minutes(15)), + }, + ) + .await?; + let (trigger_uid, original_dispatch_uid): (Uuid, Uuid) = sqlx::query_as( + "SELECT trigger.trigger_uid, dispatch.dispatch_uid \ + FROM moa.execution_trigger AS trigger \ + JOIN moa.execution_dispatch_outbox AS dispatch USING (tenant_id, trigger_uid) \ + WHERE trigger.payload->>'external_job_uid'=$1 \ + AND trigger.trigger_kind='external_start_recovery'", + ) + .bind(external_job_uid.to_string()) + .fetch_one(&pool) + .await?; + sqlx::query( + "UPDATE moa.execution_trigger SET due_at=NOW()-interval '1 second' \ + WHERE trigger_uid=$1", + ) + .bind(trigger_uid) + .execute(&pool) + .await?; + let ExecutionExternalStartRecoveryTriggerOutcome::Ready(recovery) = repository + .prepare_external_start_recovery_trigger(scope, trigger_uid) + .await? + else { + panic!("the expired unbound intent must be recoverable"); + }; + let retry_at = pg_deadline(Duration::minutes(5)); + let ExecutionExternalStartRecoveryRearmOutcome::Rearmed(rearmed) = repository + .rearm_external_start_recovery( + ExecutionScope::ControlPlane, + &recovery, + retry_at, + "provider start remains ambiguous", + ) + .await? + else { + panic!("the current recovery must rearm a fresh delivery identity"); + }; + assert_ne!(rearmed.dispatch_uid, original_dispatch_uid); + assert_eq!(rearmed.delivery_attempts, 0); + let persisted: (i64, Uuid, String, DateTime) = sqlx::query_as( + "SELECT COUNT(*) OVER (), dispatch_uid, state, not_before_at \ + FROM moa.execution_dispatch_outbox \ + WHERE trigger_uid=$1 AND dispatch_kind='trigger_delivery'", + ) + .bind(trigger_uid) + .fetch_one(&pool) + .await?; + assert_eq!( + persisted, + (1, rearmed.dispatch_uid, "pending".to_string(), retry_at) + ); + assert!( + !sqlx::query_scalar::<_, bool>( + "SELECT EXISTS (SELECT 1 FROM moa.execution_dispatch_outbox WHERE dispatch_uid=$1)", + ) + .bind(original_dispatch_uid) + .fetch_one(&pool) + .await? + ); + + let elapsed_due_at = pg_deadline(Duration::minutes(-2)); + sqlx::query( + "UPDATE moa.execution_trigger SET due_at=$2, updated_at=NOW() WHERE trigger_uid=$1", + ) + .bind(trigger_uid) + .bind(elapsed_due_at) + .execute(&pool) + .await?; + sqlx::query( + "UPDATE moa.execution_dispatch_outbox \ + SET state='delivered', not_before_at=$2, \ + delivered_at=NOW()-INTERVAL '2 minutes', updated_at=NOW() \ + WHERE dispatch_uid=$1", + ) + .bind(rearmed.dispatch_uid) + .bind(elapsed_due_at) + .execute(&pool) + .await?; + let redriven = repository + .reconcile_due_trigger_dispatches(ExecutionScope::ControlPlane, 3) + .await?; + assert_eq!(redriven.len(), 1); + assert_eq!(redriven[0].dispatch_uid, rearmed.dispatch_uid); + assert_eq!(redriven[0].state, ExecutionDeliveryState::Pending); + let persisted_redrive: (i64, Uuid, String) = sqlx::query_as( + "SELECT COUNT(*) OVER (), dispatch_uid, state \ + FROM moa.execution_dispatch_outbox \ + WHERE trigger_uid=$1 AND dispatch_kind='trigger_delivery'", + ) + .bind(trigger_uid) + .fetch_one(&pool) + .await?; + assert_eq!( + persisted_redrive, + (1, rearmed.dispatch_uid, "pending".to_string()) + ); + + sqlx::query("DELETE FROM moa.execution_dispatch_outbox WHERE dispatch_uid=$1") + .bind(rearmed.dispatch_uid) + .execute(&pool) + .await?; + let repaired = repository + .reconcile_due_trigger_dispatches(ExecutionScope::ControlPlane, 3) + .await?; + assert_eq!(repaired.len(), 1); + assert_ne!(repaired[0].dispatch_uid, original_dispatch_uid); + assert_ne!(repaired[0].dispatch_uid, rearmed.dispatch_uid); + let persisted_repair: (i64, Uuid, String) = sqlx::query_as( + "SELECT COUNT(*) OVER (), dispatch_uid, state \ + FROM moa.execution_dispatch_outbox \ + WHERE trigger_uid=$1 AND dispatch_kind='trigger_delivery'", + ) + .bind(trigger_uid) + .fetch_one(&pool) + .await?; + assert_eq!( + persisted_repair, + (1, repaired[0].dispatch_uid, "pending".to_string()) + ); + Ok(()) +} + +#[tokio::test] +async fn external_callbacks_are_tenant_generation_deduped_and_reconciled_db() -> TestResult { + // Pins: async callbacks admit progress and terminal outcomes once for the exact provider + // generation, while reconciliation and callback lookup remain tenant-scoped under RLS. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let repository = ExecutionRepository::new(test_db.store().pool().clone()); + let mut execution_config = execution_capacity_config(); + execution_config.max_tenant_external_jobs = 1; + execution_config.max_fleet_external_jobs = 1; + execution_config.validate()?; + let tenant_id = TenantId::new(); + let other_tenant = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let run = create_run( + &repository, + scope, + new_run( + tenant_id, + None, + "external-callback-dedupe", + ExecutionRunStatus::Queued, + budget(10), + ), + ) + .await?; + let tasks = repository + .materialize_tasks( + scope, + run.run_uid, + 1, + vec![logical_task( + run.run_uid, + "provider-job", + "one", + estimate(1), + )], + ) + .await?; + let task = &tasks[0]; + sqlx::query( + "UPDATE moa.execution_task SET status='reserved', updated_at=NOW() WHERE task_id=$1", + ) + .bind(task.task_id.as_uuid()) + .execute(test_db.store().pool()) + .await?; + sqlx::query( + "UPDATE moa.execution_task SET status='running', attempt_state='running', \ + last_progress_at=NOW(), updated_at=NOW() WHERE task_id=$1", + ) + .bind(task.task_id.as_uuid()) + .execute(test_db.store().pool()) + .await?; + let external_job_uid = Uuid::now_v7(); + let first_intent = NewExecutionExternalJobIntent { + external_job_uid, + tenant_id, + run_uid: run.run_uid, + owner: ExecutionExternalJobOwner::Task { + task_id: task.task_id.as_uuid(), + attempt_generation: 1, + }, + job_generation: 1, + provider: "batch-provider".to_string(), + idempotency_key: "external-job-start-1".to_string(), + expires_at: pg_deadline(Duration::minutes(15)), + }; + repository + .reserve_external_job_intent(scope, &execution_config, first_intent.clone()) + .await?; + let (start_recovery_trigger_uid, start_recovery_dispatch_uid): (Uuid, Uuid) = sqlx::query_as( + "SELECT trigger.trigger_uid, dispatch.dispatch_uid \ + FROM moa.execution_trigger AS trigger \ + JOIN moa.execution_dispatch_outbox AS dispatch USING (tenant_id, trigger_uid) \ + WHERE trigger.payload->>'external_job_uid'=$1 \ + AND trigger.trigger_kind='external_start_recovery'", + ) + .bind(external_job_uid.to_string()) + .fetch_one(test_db.store().pool()) + .await?; + // Isolate this fixture's timing head from the run-activation row created during setup. The + // callback assertions below drive repository methods directly and do not consume that row. + sqlx::query( + "UPDATE moa.execution_dispatch_outbox \ + SET state='delivered',delivered_at=NOW(),updated_at=NOW() \ + WHERE state='pending' AND run_uid=$1 AND dispatch_kind='run_activation'", + ) + .bind(run.run_uid) + .execute(test_db.store().pool()) + .await?; + let wake_before_rearm = repository + .next_pending_dispatch_wake(ExecutionScope::ControlPlane) + .await?; + assert_eq!( + wake_before_rearm.dispatch_uid, + Some(start_recovery_dispatch_uid) + ); + sqlx::query( + "UPDATE moa.execution_trigger SET due_at=NOW()-interval '1 second' \ + WHERE trigger_uid=$1", + ) + .bind(start_recovery_trigger_uid) + .execute(test_db.store().pool()) + .await?; + let ExecutionExternalStartRecoveryTriggerOutcome::Ready(start_recovery) = repository + .prepare_external_start_recovery_trigger(scope, start_recovery_trigger_uid) + .await? + else { + panic!("the exact expired unbound intent must be recoverable"); + }; + let retry_at = pg_deadline(Duration::minutes(5)); + let ExecutionExternalStartRecoveryRearmOutcome::Rearmed(rearmed) = repository + .rearm_external_start_recovery( + ExecutionScope::ControlPlane, + &start_recovery, + retry_at, + "provider start remains ambiguous", + ) + .await? + else { + panic!("an ambiguous current provider start must rearm a fresh delivery"); + }; + assert_ne!(rearmed.dispatch_uid, start_recovery_dispatch_uid); + assert_eq!(rearmed.delivery_attempts, 0); + let rearmed_state: (String, String, DateTime, DateTime) = sqlx::query_as( + "SELECT trigger.state, dispatch.state, trigger.due_at, dispatch.not_before_at \ + FROM moa.execution_trigger AS trigger \ + JOIN moa.execution_dispatch_outbox AS dispatch USING (tenant_id, trigger_uid) \ + WHERE trigger.trigger_uid=$1", + ) + .bind(start_recovery_trigger_uid) + .fetch_one(test_db.store().pool()) + .await?; + assert_eq!( + (rearmed_state.0.as_str(), rearmed_state.1.as_str()), + ("pending", "pending") + ); + assert_eq!(rearmed_state.2, retry_at); + assert_eq!(rearmed_state.3, retry_at); + // Pins: an ambiguous provider result retains one outbox-backed timing authority but replaces + // its Restate idempotency identity, so the next due delivery cannot replay the completed + // NotDue invocation from the preceding provider lookup. + let delivery_rows: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM moa.execution_dispatch_outbox \ + WHERE trigger_uid=$1 AND dispatch_kind='trigger_delivery'", + ) + .bind(start_recovery_trigger_uid) + .fetch_one(test_db.store().pool()) + .await?; + assert_eq!(delivery_rows, 1); + let old_delivery_exists: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM moa.execution_dispatch_outbox WHERE dispatch_uid=$1)", + ) + .bind(start_recovery_dispatch_uid) + .fetch_one(test_db.store().pool()) + .await?; + assert!(!old_delivery_exists); + assert_ne!(wake_before_rearm.next_due_at, Some(retry_at)); + let rearmed_wake = repository + .next_pending_dispatch_wake(ExecutionScope::ControlPlane) + .await?; + assert_eq!(rearmed_wake.dispatch_uid, Some(rearmed.dispatch_uid)); + assert_eq!(rearmed_wake.next_due_at, Some(retry_at)); + assert_ne!( + rearmed_wake.head_updated_at, + wake_before_rearm.head_updated_at + ); + repository + .bind_external_job( + scope, + &execution_config, + ExecutionExternalJobBinding { + external_job_uid, + tenant_id, + run_uid: run.run_uid, + owner: first_intent.owner, + job_generation: 1, + idempotency_key: first_intent.idempotency_key.clone(), + provider: first_intent.provider.clone(), + provider_job_id: "provider-job-1".to_string(), + callback_auth_reference: "vault://callback/provider-1".to_string(), + state: ExecutionExternalJobState::Starting, + progress_phase: None, + cancel_supported: true, + next_reconcile_at: Some(pg_deadline(Duration::seconds(-1))), + provider_contract_violation: None, + }, + ) + .await?; + sqlx::query( + "UPDATE moa.execution_task SET status='waiting_external', attempt_state='waiting', \ + waiting_since=NOW(), external_job_uid=$3, updated_at=NOW() \ + WHERE run_uid=$1 AND task_id=$2", + ) + .bind(run.run_uid) + .bind(task.task_id.as_uuid()) + .bind(external_job_uid) + .execute(test_db.store().pool()) + .await?; + sqlx::query( + "UPDATE moa.execution_node_state SET node_status='waiting', waiting_task_count=1, \ + updated_at=NOW() WHERE run_uid=$1 AND node_id=$2", + ) + .bind(run.run_uid) + .bind(&task.node_id) + .execute(test_db.store().pool()) + .await?; + let second_run = create_run( + &repository, + scope, + new_run( + tenant_id, + None, + "external-callback-capacity-second", + ExecutionRunStatus::Queued, + budget(10), + ), + ) + .await?; + let second_tasks = repository + .materialize_tasks( + scope, + second_run.run_uid, + 1, + vec![logical_task( + second_run.run_uid, + "provider-job-second", + "one", + estimate(1), + )], + ) + .await?; + let second_external_job_uid = Uuid::now_v7(); + sqlx::query( + "UPDATE moa.execution_task SET status='reserved', updated_at=NOW() WHERE task_id=$1", + ) + .bind(second_tasks[0].task_id.as_uuid()) + .execute(test_db.store().pool()) + .await?; + sqlx::query( + "UPDATE moa.execution_task SET status='running', attempt_state='running', \ + last_progress_at=NOW(), updated_at=NOW() WHERE task_id=$1", + ) + .bind(second_tasks[0].task_id.as_uuid()) + .execute(test_db.store().pool()) + .await?; + let second_intent = NewExecutionExternalJobIntent { + external_job_uid: second_external_job_uid, + tenant_id, + run_uid: second_run.run_uid, + owner: ExecutionExternalJobOwner::Task { + task_id: second_tasks[0].task_id.as_uuid(), + attempt_generation: 1, + }, + job_generation: 1, + provider: "batch-provider".to_string(), + idempotency_key: "external-job-start-2".to_string(), + expires_at: pg_deadline(Duration::minutes(15)), + }; + assert!(matches!( + repository + .reserve_external_job_intent(scope, &execution_config, second_intent.clone()) + .await, + Err(moa_execution::Error::CapacitySaturated { + dimension: "external_jobs" + }) + )); + let rejected_job_count: i64 = sqlx::query_scalar( + "SELECT count(*) FROM moa.execution_external_job WHERE external_job_uid = $1", + ) + .bind(second_external_job_uid) + .fetch_one(test_db.store().pool()) + .await?; + assert_eq!(rejected_job_count, 0); + sqlx::query("UPDATE moa.execution_run SET wake_epoch = $2 WHERE run_uid = $1") + .bind(run.run_uid) + .bind(i64::MAX) + .execute(test_db.store().pool()) + .await?; + let rollback_event = callback( + external_job_uid, + 1, + "rollback-event", + ExecutionExternalJobCallbackUpdate::Terminal { + state: ExecutionExternalJobState::Completed, + progress_phase: Some("must-rollback".to_string()), + output: Some(json!({"must": "rollback"})), + error: None, + }, + ); + assert!( + repository + .apply_external_job_callback_and_activate( + ExecutionScope::ControlPlane, + &execution_config, + rollback_event, + ) + .await + .is_err(), + "wake-epoch overflow must fail after callback mutation" + ); + let rolled_back: (String, Option, i64) = sqlx::query_as( + "SELECT job.state, job.last_provider_event_id, \ + (SELECT count(*) FROM moa.execution_external_job_callback_receipt receipt \ + WHERE receipt.external_job_uid = job.external_job_uid \ + AND receipt.provider_event_id = 'rollback-event') \ + FROM moa.execution_external_job job WHERE job.external_job_uid = $1", + ) + .bind(external_job_uid) + .fetch_one(test_db.store().pool()) + .await?; + assert_eq!(rolled_back, ("starting".to_string(), None, 0)); + sqlx::query("UPDATE moa.execution_run SET wake_epoch = $2 WHERE run_uid = $1") + .bind(run.run_uid) + .bind(i64::try_from(run.wake_epoch)?) + .execute(test_db.store().pool()) + .await?; + assert_eq!(repository.list_due_external_jobs(scope, 10).await?.len(), 1); + assert!( + repository + .list_due_external_jobs( + ExecutionScope::Tenant { + tenant_id: other_tenant, + }, + 10, + ) + .await? + .is_empty() + ); + assert!( + repository + .load_external_job(scope, external_job_uid) + .await? + .is_some() + ); + assert!( + repository + .load_external_job( + ExecutionScope::Tenant { + tenant_id: other_tenant, + }, + external_job_uid, + ) + .await? + .is_none() + ); + let ExecutionExternalJobCancellationOutcome::Applied(cancel_requested) = repository + .settle_external_job_cancellation( + scope, + &execution_config, + ExecutionExternalJobCancellation { + external_job_uid, + job_generation: 1, + provider: "batch-provider".to_string(), + provider_job_id: "provider-job-1".to_string(), + state: ExecutionExternalJobState::CancelRequested, + next_reconcile_at: Some(pg_deadline(Duration::minutes(5))), + error: None, + }, + ) + .await? + else { + panic!("exact cancellation request must apply"); + }; + assert_eq!( + cancel_requested.state, + ExecutionExternalJobState::CancelRequested + ); + assert_eq!( + repository + .settle_external_job_cancellation( + scope, + &execution_config, + ExecutionExternalJobCancellation { + external_job_uid, + job_generation: 2, + provider: "batch-provider".to_string(), + provider_job_id: "provider-job-1".to_string(), + state: ExecutionExternalJobState::Cancelled, + next_reconcile_at: None, + error: None, + }, + ) + .await?, + ExecutionExternalJobCancellationOutcome::StaleGeneration + ); + + let progress = callback( + external_job_uid, + 1, + "progress-1", + ExecutionExternalJobCallbackUpdate::Progress { + state: ExecutionExternalJobState::Running, + progress_phase: Some("map".to_string()), + next_reconcile_at: Some(pg_deadline(Duration::minutes(10))), + }, + ); + let progress_write = repository + .apply_external_job_callback_and_activate( + ExecutionScope::ControlPlane, + &execution_config, + progress.clone(), + ) + .await?; + let ExecutionExternalJobCallbackOutcome::Applied(progressed) = progress_write.outcome else { + panic!("exact progress callback must apply"); + }; + assert_eq!(progressed.state, ExecutionExternalJobState::Running); + assert_eq!(progressed.progress_phase.as_deref(), Some("map")); + assert_eq!(progress_write.activation, None); + let duplicate = repository + .apply_external_job_callback_and_activate( + ExecutionScope::ControlPlane, + &execution_config, + progress, + ) + .await?; + assert_eq!( + duplicate.outcome, + ExecutionExternalJobCallbackOutcome::Duplicate + ); + assert_eq!(duplicate.activation, None); + assert_eq!( + repository + .load_external_job(scope, external_job_uid) + .await?, + Some((*progressed).clone()) + ); + assert_eq!( + repository + .apply_external_job_callback_and_activate( + ExecutionScope::ControlPlane, + &execution_config, + callback( + external_job_uid, + 2, + "stale-generation", + ExecutionExternalJobCallbackUpdate::Progress { + state: ExecutionExternalJobState::Running, + progress_phase: Some("reduce".to_string()), + next_reconcile_at: None, + }, + ), + ) + .await? + .outcome, + ExecutionExternalJobCallbackOutcome::StaleGeneration + ); + assert!(matches!( + repository + .apply_external_job_callback_and_activate( + ExecutionScope::ControlPlane, + &execution_config, + callback( + external_job_uid, + 1, + "progress-2", + ExecutionExternalJobCallbackUpdate::Progress { + state: ExecutionExternalJobState::WaitingReconcile, + progress_phase: Some("reduce".to_string()), + next_reconcile_at: Some(pg_deadline(Duration::minutes(10))), + }, + ), + ) + .await?, + moa_execution::repository::external_job::ExecutionExternalJobCallbackWrite { + outcome: ExecutionExternalJobCallbackOutcome::Applied(_), + activation: None, + } + )); + assert_eq!( + repository + .apply_external_job_callback_and_activate( + ExecutionScope::ControlPlane, + &execution_config, + callback( + external_job_uid, + 1, + "progress-1", + ExecutionExternalJobCallbackUpdate::Progress { + state: ExecutionExternalJobState::Running, + progress_phase: Some("map".to_string()), + next_reconcile_at: None, + }, + ), + ) + .await? + .outcome, + ExecutionExternalJobCallbackOutcome::Duplicate + ); + let progress_activation_count: i64 = sqlx::query_scalar( + "SELECT count(*) FROM moa.execution_dispatch_outbox \ + WHERE run_uid=$1 AND dispatch_kind='run_activation' \ + AND payload->>'source'='external_job_callback'", + ) + .bind(run.run_uid) + .fetch_one(test_db.store().pool()) + .await?; + assert_eq!(progress_activation_count, 0); + + repository + .pause_run(scope, &execution_config, run.run_uid, 1) + .await?; + let paused_wake_epoch: i64 = sqlx::query_scalar( + "SELECT wake_epoch FROM moa.execution_run WHERE run_uid=$1 AND status='paused'", + ) + .bind(run.run_uid) + .fetch_one(test_db.store().pool()) + .await?; + + let terminal = callback( + external_job_uid, + 1, + "terminal-1", + ExecutionExternalJobCallbackUpdate::Terminal { + state: ExecutionExternalJobState::Completed, + progress_phase: Some("complete".to_string()), + output: Some(json!({"artifact": "result-1"})), + error: None, + }, + ); + let terminal_write = repository + .apply_external_job_callback_and_activate( + ExecutionScope::ControlPlane, + &execution_config, + terminal.clone(), + ) + .await?; + let ExecutionExternalJobCallbackOutcome::Applied(completed) = terminal_write.outcome else { + panic!("exact terminal callback must apply"); + }; + assert_eq!(terminal_write.activation, None); + assert_eq!(completed.state, ExecutionExternalJobState::Completed); + assert!(completed.completed_at.is_some()); + let paused_after_callback: (String, i64, String) = sqlx::query_as( + "SELECT run.status, run.wake_epoch, task.status \ + FROM moa.execution_run AS run JOIN moa.execution_task AS task USING (run_uid) \ + WHERE run.run_uid=$1 AND task.task_id=$2", + ) + .bind(run.run_uid) + .bind(task.task_id.as_uuid()) + .fetch_one(test_db.store().pool()) + .await?; + assert_eq!( + paused_after_callback, + ( + "paused".to_string(), + paused_wake_epoch, + "completed".to_string() + ) + ); + assert_eq!( + repository + .apply_external_job_callback_and_activate( + ExecutionScope::ControlPlane, + &execution_config, + terminal, + ) + .await?, + moa_execution::repository::external_job::ExecutionExternalJobCallbackWrite { + outcome: ExecutionExternalJobCallbackOutcome::Duplicate, + activation: None, + } + ); + let capacity_after_terminal: Vec<(String, i64)> = sqlx::query_as( + "SELECT scope_kind, reserved_quantity FROM moa.execution_capacity_bucket \ + WHERE resource_dimension = 'external_jobs' ORDER BY scope_kind", + ) + .fetch_all(test_db.store().pool()) + .await?; + assert_eq!( + capacity_after_terminal, + vec![("fleet".to_string(), 0), ("tenant".to_string(), 0)] + ); + repository + .reserve_external_job_intent(scope, &execution_config, second_intent.clone()) + .await?; + repository + .bind_external_job( + scope, + &execution_config, + ExecutionExternalJobBinding { + external_job_uid: second_external_job_uid, + tenant_id, + run_uid: second_run.run_uid, + owner: second_intent.owner, + job_generation: 1, + idempotency_key: second_intent.idempotency_key, + provider: second_intent.provider, + provider_job_id: "provider-job-2".to_string(), + callback_auth_reference: "vault://callback/provider-2".to_string(), + state: ExecutionExternalJobState::Starting, + progress_phase: None, + cancel_supported: true, + next_reconcile_at: Some(pg_deadline(Duration::seconds(-1))), + provider_contract_violation: None, + }, + ) + .await?; + assert_eq!( + repository + .apply_external_job_callback_and_activate( + ExecutionScope::ControlPlane, + &execution_config, + callback( + external_job_uid, + 1, + "late-progress", + ExecutionExternalJobCallbackUpdate::Progress { + state: ExecutionExternalJobState::Running, + progress_phase: Some("late".to_string()), + next_reconcile_at: None, + }, + ), + ) + .await? + .outcome, + ExecutionExternalJobCallbackOutcome::AlreadyTerminal + ); + Ok(()) +} + +#[tokio::test] +async fn terminal_callback_before_task_release_commits_then_settles_once_db() -> TestResult { + // Pins: a terminal callback for an exactly bound active task commits its provider receipt + // without mutating the task; the subsequent attempt release consumes that terminal job once. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let config = execution_capacity_config(); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let mut candidate = new_run( + tenant_id, + None, + "terminal-callback-before-task-release", + ExecutionRunStatus::Queued, + budget(20), + ); + candidate.plan.definition.nodes = vec![watchdog_output_node()]; + let run = create_run(&repository, scope, candidate).await?; + assert!(matches!( + repository + .materialize_ready_page( + scope, + &config, + ReadyMaterializationRequest { + run_uid: run.run_uid, + plan_revision: 1, + node_id: "watchdog-work".to_string(), + expected_cursor: 0, + reduce_cursor: None, + source_exhausted: true, + terminal_output: None, + tasks: vec![logical_task( + run.run_uid, + "watchdog-work", + "callback-race", + estimate(1), + )], + }, + ) + .await?, + ReadyMaterializationOutcome::Applied { .. } + )); + let admission = repository + .admit_ready_attempts(&config, 1, Utc::now()) + .await? + .admitted + .into_iter() + .next() + .expect("one callback-race task must be admitted"); + let fence = TaskAttemptFence { + tenant_id: admission.tenant_id, + run_uid: admission.run_uid, + task_id: admission.task_id, + controller_generation: admission.controller_generation, + attempt_generation: admission.attempt_generation, + dispatch_uid: admission.dispatch_uid, + capacity_reservation_uid: admission.capacity_reservation_uid, + watchdog_trigger_uid: admission.watchdog_trigger_uid, + attempt_deadline_at: admission.attempt_deadline_at, + }; + let TaskAttemptStartOutcome::Started(started) = repository.start_task_attempt(fence).await? + else { + panic!("callback-race fixture must start its admitted attempt"); + }; + let external_job_uid = Uuid::now_v7(); + let intent = NewExecutionExternalJobIntent { + external_job_uid, + tenant_id, + run_uid: run.run_uid, + owner: ExecutionExternalJobOwner::Task { + task_id: fence.task_id.as_uuid(), + attempt_generation: fence.attempt_generation, + }, + job_generation: 1, + provider: "batch-provider".to_string(), + idempotency_key: format!("terminal-before-release-{external_job_uid}"), + expires_at: pg_deadline(Duration::minutes(15)), + }; + repository + .reserve_external_job_intent(scope, &config, intent.clone()) + .await?; + repository + .bind_external_job( + scope, + &config, + ExecutionExternalJobBinding { + external_job_uid, + tenant_id, + run_uid: run.run_uid, + owner: intent.owner, + job_generation: 1, + idempotency_key: intent.idempotency_key, + provider: intent.provider, + provider_job_id: "provider-job-1".to_string(), + callback_auth_reference: "vault://callback/race".to_string(), + state: ExecutionExternalJobState::Running, + progress_phase: Some("running".to_string()), + cancel_supported: true, + next_reconcile_at: None, + provider_contract_violation: None, + }, + ) + .await?; + + let callback_write = repository + .apply_external_job_callback_and_activate( + ExecutionScope::ControlPlane, + &config, + callback( + external_job_uid, + 1, + "terminal-before-release", + ExecutionExternalJobCallbackUpdate::Terminal { + state: ExecutionExternalJobState::Completed, + progress_phase: Some("completed".to_string()), + output: Some(json!({"artifact": "callback-race"})), + error: None, + }, + ), + ) + .await?; + assert!(matches!( + callback_write.outcome, + ExecutionExternalJobCallbackOutcome::Applied(_) + )); + assert_eq!(callback_write.activation, None); + let before_release: (String, String, Option, Uuid, i64) = sqlx::query_as( + "SELECT task.status, task.attempt_state, task.external_job_uid, \ + task.active_dispatch_uid, \ + (SELECT COUNT(*) FROM moa.execution_external_job_callback_receipt receipt \ + WHERE receipt.external_job_uid=$3 \ + AND receipt.provider_event_id='terminal-before-release') \ + FROM moa.execution_task AS task WHERE task.run_uid=$1 AND task.task_id=$2", + ) + .bind(run.run_uid) + .bind(fence.task_id.as_uuid()) + .bind(external_job_uid) + .fetch_one(&pool) + .await?; + assert_eq!( + before_release, + ( + "running".to_string(), + "running".to_string(), + None, + fence.dispatch_uid, + 1, + ), + "the callback must commit without stealing the active attempt's release boundary" + ); + + assert!(matches!( + repository + .begin_task_attempt_release(fence, started.task.generation, "external_job", Utc::now(),) + .await?, + TaskAttemptReleaseClaimOutcome::Applied(_) + )); + let TaskAttemptExternalOutcome::Applied { task, .. } = repository + .yield_task_attempt_to_external_job(fence, external_job_uid, None, None, Utc::now()) + .await? + else { + panic!("the exact task release must consume the committed terminal callback"); + }; + assert_eq!(task.status, ExecutionTaskStatus::Completed); + assert_eq!(task.output, Some(json!({"artifact": "callback-race"}))); + assert!(matches!( + repository + .yield_task_attempt_to_external_job(fence, external_job_uid, None, None, Utc::now(),) + .await?, + TaskAttemptExternalOutcome::Replayed { .. } + )); + Ok(()) +} + +#[tokio::test] +async fn retry_settlement_preserves_cancelling_until_ready_transition_db() -> TestResult { + // Pins: recording a retryable watchdog outcome must not transiently reopen the attempt from + // Cancelling to Running before the exact settlement advances it to the next Ready generation. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let config = execution_capacity_config(); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let mut candidate = new_run( + tenant_id, + None, + "watchdog-retry-settlement", + ExecutionRunStatus::Queued, + budget(20), + ); + candidate.plan.definition.nodes = vec![watchdog_output_node()]; + let run = create_run(&repository, scope, candidate).await?; + repository + .initialize_scheduler_state(scope, run.run_uid) + .await?; + assert!(matches!( + repository + .materialize_ready_page( + scope, + &config, + ReadyMaterializationRequest { + run_uid: run.run_uid, + plan_revision: 1, + node_id: "watchdog-work".to_string(), + expected_cursor: 0, + reduce_cursor: None, + source_exhausted: true, + terminal_output: None, + tasks: vec![logical_task( + run.run_uid, + "watchdog-work", + "retry", + estimate(1), + )], + }, + ) + .await?, + ReadyMaterializationOutcome::Applied { .. } + )); + let admission = repository + .admit_ready_attempts(&config, 1, Utc::now()) + .await? + .admitted + .into_iter() + .next() + .expect("one retry task must be admitted"); + let fence = TaskAttemptFence { + tenant_id: admission.tenant_id, + run_uid: admission.run_uid, + task_id: admission.task_id, + controller_generation: admission.controller_generation, + attempt_generation: admission.attempt_generation, + dispatch_uid: admission.dispatch_uid, + capacity_reservation_uid: admission.capacity_reservation_uid, + watchdog_trigger_uid: admission.watchdog_trigger_uid, + attempt_deadline_at: admission.attempt_deadline_at, + }; + let TaskAttemptStartOutcome::Started(started) = repository.start_task_attempt(fence).await? + else { + panic!("retry fixture must start its admitted attempt"); + }; + let settled_at = Utc::now(); + let TaskAttemptReleaseClaimOutcome::Applied(releasing) = repository + .begin_task_attempt_release(fence, started.task.generation, "watchdog", settled_at) + .await? + else { + panic!("the exact watchdog attempt must enter its release boundary"); + }; + assert_eq!(releasing.task.status, ExecutionTaskStatus::Running); + assert_eq!( + releasing.task.attempt_state, + ExecutionAttemptState::Cancelling + ); + assert_eq!(releasing.task.attempt_generation, fence.attempt_generation); + let retry_at = settled_at + Duration::milliseconds(50); + let TaskAttemptSettlementOutcome::Applied { task, .. } = repository + .settle_released_task_attempt( + &config, + fence, + ExecutionTaskOutcome { + schema_version: 1, + usage: started.task.actual, + result: ExecutionTaskResult::Failed { + class: ExecutionFailureClass::Retryable, + message: "watchdog expired".to_string(), + }, + }, + Some(retry_at), + settled_at, + None, + ) + .await? + else { + panic!("the claimed watchdog retry must settle into the next ready generation"); + }; + assert_eq!(task.status, ExecutionTaskStatus::Ready); + assert_eq!(task.attempt_state, ExecutionAttemptState::Idle); + assert_eq!(task.attempt, 2); + assert_eq!(task.generation, 2); + assert_eq!(task.attempt_generation, 2); + assert_eq!(task.ready_at, Some(retry_at)); + let boundary: (String, String, i64) = sqlx::query_as( + "SELECT trigger.state, capacity.state, \ + (SELECT COUNT(*) FROM moa.execution_dispatch_outbox \ + WHERE run_uid=$1 AND dispatch_kind='run_activation' \ + AND payload->>'source'='task_attempt_settlement') \ + FROM moa.execution_trigger AS trigger \ + JOIN moa.execution_capacity_reservation AS capacity USING (trigger_uid) \ + WHERE trigger.trigger_uid=$2", + ) + .bind(run.run_uid) + .bind(fence.watchdog_trigger_uid) + .fetch_one(&pool) + .await?; + assert_eq!( + boundary, + ("superseded".to_string(), "released".to_string(), 1) + ); + Ok(()) +} + +#[tokio::test] +async fn durable_task_release_receipt_splits_capacity_from_outcome_settlement_db() -> TestResult { + // Pins: entering Cancelling alone and a missing/forged durable hand receipt leave the exact + // ActiveTasks reservation held. The exact persisted receipt releases it once and supersedes + // its watchdog without reconciling the four committed bucket limits; replay is harmless. A + // final outcome can then settle while the fleet bucket is locked, proving the crash-gap + // recovery path does not reacquire either capacity bucket. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let config = execution_capacity_config(); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let mut candidate = new_run( + tenant_id, + None, + "proof-aware-task-release", + ExecutionRunStatus::Queued, + budget(20), + ); + candidate.plan.definition.nodes = vec![watchdog_output_node()]; + let run = create_run(&repository, scope, candidate).await?; + repository + .initialize_scheduler_state(scope, run.run_uid) + .await?; + assert!(matches!( + repository + .materialize_ready_page( + scope, + &config, + ReadyMaterializationRequest { + run_uid: run.run_uid, + plan_revision: 1, + node_id: "watchdog-work".to_string(), + expected_cursor: 0, + reduce_cursor: None, + source_exhausted: true, + terminal_output: None, + tasks: vec![logical_task( + run.run_uid, + "watchdog-work", + "proof-aware", + estimate(1), + )], + }, + ) + .await?, + ReadyMaterializationOutcome::Applied { .. } + )); + let admission = repository + .admit_ready_attempts(&config, 1, Utc::now()) + .await? + .admitted + .into_iter() + .next() + .expect("one proof-aware task must be admitted"); + let fence = TaskAttemptFence { + tenant_id: admission.tenant_id, + run_uid: admission.run_uid, + task_id: admission.task_id, + controller_generation: admission.controller_generation, + attempt_generation: admission.attempt_generation, + dispatch_uid: admission.dispatch_uid, + capacity_reservation_uid: admission.capacity_reservation_uid, + watchdog_trigger_uid: admission.watchdog_trigger_uid, + attempt_deadline_at: admission.attempt_deadline_at, + }; + let TaskAttemptStartOutcome::Started(started) = repository.start_task_attempt(fence).await? + else { + panic!("proof-aware task must start"); + }; + let now = Utc::now(); + let TaskAttemptReleaseClaimOutcome::Applied(releasing) = repository + .begin_task_attempt_release(fence, started.task.generation, "task_outcome", now) + .await? + else { + panic!("proof-aware task must enter Cancelling"); + }; + let reserved_quantity = || async { + sqlx::query_scalar::<_, i64>( + "SELECT reserved_quantity FROM moa.execution_capacity_bucket \ + WHERE scope_kind='fleet' AND resource_dimension='active_tasks'", + ) + .fetch_one(&pool) + .await + }; + assert_eq!( + reserved_quantity().await?, + 1, + "begin-release must retain capacity" + ); + + let receipt = task_release_receipt(&releasing.task, fence, now); + assert_eq!( + repository + .release_released_task_attempt_capacity( + fence, + releasing.task.generation, + receipt.clone(), + ) + .await?, + ReleasedTaskAttemptCapacityOutcome::Stale + ); + assert_eq!( + reserved_quantity().await?, + 1, + "an unpersisted receipt is not proof" + ); + persist_task_release_receipt(&pool, &receipt).await?; + let mut forged = receipt.clone(); + forged.receipt_id = Uuid::now_v7(); + assert_eq!( + repository + .release_released_task_attempt_capacity(fence, releasing.task.generation, forged) + .await?, + ReleasedTaskAttemptCapacityOutcome::Stale + ); + assert_eq!( + reserved_quantity().await?, + 1, + "a forged receipt must roll back release" + ); + let release_limits = [ + ("fleet", None, "active_tasks", 17_i64), + ("tenant", Some(tenant_id.0), "active_tasks", 13_i64), + ("fleet", None, "scheduled_triggers", 19_i64), + ("tenant", Some(tenant_id.0), "scheduled_triggers", 11_i64), + ]; + for (scope_kind, owner, dimension, limit) in release_limits { + sqlx::query( + "UPDATE moa.execution_capacity_bucket SET limit_value=$4 \ + WHERE scope_kind=$1 AND tenant_id IS NOT DISTINCT FROM $2 \ + AND resource_dimension=$3", + ) + .bind(scope_kind) + .bind(owner) + .bind(dimension) + .bind(limit) + .execute(&pool) + .await?; + } + let buckets_before_release = sqlx::query_as::<_, (String, Option, String, i64, i64)>( + "SELECT scope_kind,tenant_id,resource_dimension,limit_value,version \ + FROM moa.execution_capacity_bucket \ + WHERE resource_dimension IN ('active_tasks','scheduled_triggers') \ + AND (scope_kind='fleet' OR tenant_id=$1) \ + ORDER BY resource_dimension,scope_kind", + ) + .bind(tenant_id.0) + .fetch_all(&pool) + .await?; + assert_eq!(buckets_before_release.len(), 4); + assert_eq!( + repository + .release_released_task_attempt_capacity( + fence, + releasing.task.generation, + receipt.clone(), + ) + .await?, + ReleasedTaskAttemptCapacityOutcome::Applied + ); + let buckets_after_release = sqlx::query_as::<_, (String, Option, String, i64, i64)>( + "SELECT scope_kind,tenant_id,resource_dimension,limit_value,version \ + FROM moa.execution_capacity_bucket \ + WHERE resource_dimension IN ('active_tasks','scheduled_triggers') \ + AND (scope_kind='fleet' OR tenant_id=$1) \ + ORDER BY resource_dimension,scope_kind", + ) + .bind(tenant_id.0) + .fetch_all(&pool) + .await?; + assert_eq!(buckets_after_release.len(), 4); + for (before, after) in buckets_before_release.iter().zip(&buckets_after_release) { + assert_eq!(&after.0, &before.0); + assert_eq!(after.1, before.1); + assert_eq!(&after.2, &before.2); + assert_eq!( + after.3, before.3, + "release must not rewrite capacity limits" + ); + assert_eq!( + after.4, + before.4 + 1, + "each exact receipt release must advance its bucket once" + ); + } + assert_eq!(reserved_quantity().await?, 0); + assert_eq!( + repository + .release_released_task_attempt_capacity( + fence, + releasing.task.generation, + receipt.clone(), + ) + .await?, + ReleasedTaskAttemptCapacityOutcome::Replayed + ); + assert_eq!( + reserved_quantity().await?, + 0, + "release replay must not decrement twice" + ); + + let mut capacity_lock = pool.begin().await?; + sqlx::query( + "SELECT capacity_bucket_uid FROM moa.execution_capacity_bucket \ + WHERE scope_kind='fleet' AND resource_dimension='active_tasks' FOR UPDATE", + ) + .fetch_one(&mut *capacity_lock) + .await?; + let settlement = tokio::time::timeout( + StdDuration::from_secs(2), + repository.settle_released_task_attempt( + &config, + fence, + completed(1), + None, + now + Duration::milliseconds(1), + Some(receipt.clone()), + ), + ) + .await + .expect("pre-released settlement must not wait for the fleet capacity lock")?; + capacity_lock.rollback().await?; + assert!(matches!( + settlement, + TaskAttemptSettlementOutcome::Applied { .. } + )); + assert!(matches!( + repository + .settle_released_task_attempt( + &config, + fence, + completed(1), + None, + now + Duration::milliseconds(1), + Some(receipt), + ) + .await?, + TaskAttemptSettlementOutcome::Replayed { .. } + )); + Ok(()) +} + +fn task_release_receipt( + task: &ExecutionTaskRecord, + fence: TaskAttemptFence, + released_at: DateTime, +) -> ExecutionHandReleaseReceipt { + ExecutionHandReleaseReceipt { + receipt_id: Uuid::now_v7(), + tenant_id: fence.tenant_id, + run_id: ExecutionRunScopeId(fence.run_uid), + owner: ExecutionHandReleaseOwner::Task { + task_id: ExecutionTaskScopeId(fence.task_id.as_uuid()), + logical_generation: task.generation, + }, + attempt_generation: fence.attempt_generation, + workspace_id: None, + writer_epoch: None, + instance_generation: None, + hand_provisioning_operation_id: None, + hand_lease_generation: None, + checkpoint_id: None, + checkpoint_generation: None, + checkpoint_manifest_digest: None, + checkpoint_logical_bytes: None, + requested_at: released_at, + released_at, + } +} + +async fn persist_task_release_receipt( + pool: &sqlx::PgPool, + receipt: &ExecutionHandReleaseReceipt, +) -> Result<(), sqlx::Error> { + let ExecutionHandReleaseOwner::Task { + task_id, + logical_generation, + } = receipt.owner + else { + unreachable!("task receipt fixture must have a task owner"); + }; + sqlx::query( + "INSERT INTO moa.sandbox_execution_hand_release_receipts \ + (receipt_id,tenant_id,run_uid,owner_kind,task_id,compensation_id, \ + logical_generation,attempt_generation,workspace_id,writer_epoch,instance_generation, \ + hand_provisioning_operation_id,hand_lease_generation,checkpoint_id, \ + checkpoint_generation,checkpoint_manifest_digest,checkpoint_logical_bytes, \ + receipt_state,destroy_outcome,claim_token,claim_expires_at,requested_at,deadline_at, \ + released_at) VALUES ($1,$2,$3,'task',$4,NULL,$5,$6,NULL,NULL,NULL,NULL,NULL,NULL,NULL, \ + NULL,NULL,'released','verified_absent',NULL,NULL,$7,$7,$7)", + ) + .bind(receipt.receipt_id) + .bind(receipt.tenant_id.0) + .bind(receipt.run_id.0) + .bind(task_id.0) + .bind(i64::try_from(logical_generation).expect("fixture generation fits i64")) + .bind(i64::try_from(receipt.attempt_generation).expect("fixture attempt fits i64")) + .bind(receipt.released_at) + .execute(pool) + .await?; + Ok(()) +} + +fn execution_capacity_config() -> ExecutionConfig { + ExecutionConfig { + planner_repair_attempts: 1, + repeated_failure_limit: 3, + max_in_flight_tasks: 64, + maximum_horizon_seconds: 30 * 24 * 60 * 60, + maximum_activation_steps: 128, + dispatch_batch_size: 32, + active_attempt_timeout_seconds: 10 * 60, + max_tenant_active_runs: 100, + max_fleet_active_runs: 1_000, + max_tenant_active_tasks: 256, + max_fleet_active_tasks: 4_096, + max_tenant_parked_runs: 10_000, + max_fleet_parked_runs: 100_000, + max_tenant_scheduled_triggers: 50_000, + max_fleet_scheduled_triggers: 500_000, + max_tenant_external_jobs: 1_000, + max_fleet_external_jobs: 10_000, + trigger_reconciliation_cadence_seconds: 60, + terminal_detail_retention_days: 30, + max_tasks: 10_000, + max_tokens: 10_000_000, + max_tool_calls: 100_000, + max_retrieved_bytes: 10_000_000_000, + max_cost_microusd: 100_000_000, + unattended_max_cost_microusd: 5_000_000, + agent_turn_cost_microusd: 100_000, + agent_turn_tokens: 8_000, + agent_turn_tool_calls: 8, + agent_turn_retrieved_bytes: 10_000_000, + verifier_turn_cost_microusd: 200_000, + verifier_turn_tokens: 16_000, + verifier_turn_tool_calls: 4, + verifier_turn_retrieved_bytes: 1_000_000, + } +} + +#[tokio::test] +async fn maintenance_checkpoint_is_control_plane_bounded_and_generation_fenced_db() -> TestResult { + // Pins: reconciler health means a completed repair+dispatch pass, not merely a Cron fire; + // stale invocations cannot overwrite a newer generation and errors fit the schema byte bound. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let repository = ExecutionRepository::new(test_db.store().pool().clone()); + let kind = ExecutionMaintenanceJobKind::DispatchReconciliation; + assert!( + repository + .begin_execution_maintenance( + ExecutionScope::Tenant { + tenant_id: TenantId::new(), + }, + kind, + ) + .await + .is_err() + ); + assert!( + repository + .load_execution_maintenance_checkpoint(ExecutionScope::ControlPlane, kind) + .await? + .is_none() + ); + + let first = repository + .begin_execution_maintenance(ExecutionScope::ControlPlane, kind) + .await?; + assert_eq!(first.generation, 1); + assert!(first.last_started_at.is_some()); + let oversized_error = "💥".repeat(2_000); + let ExecutionMaintenanceSettlementOutcome::Applied(failed) = repository + .fail_execution_maintenance( + ExecutionScope::ControlPlane, + kind, + first.generation, + &oversized_error, + ) + .await? + else { + panic!("current maintenance generation must record failure"); + }; + assert!(failed.last_failure_at.is_some()); + assert!( + failed + .last_error + .as_ref() + .is_some_and(|error| error.len() <= 4_096) + ); + + let second = repository + .begin_execution_maintenance(ExecutionScope::ControlPlane, kind) + .await?; + assert_eq!(second.generation, 2); + assert_eq!( + repository + .complete_execution_maintenance(ExecutionScope::ControlPlane, kind, first.generation,) + .await?, + ExecutionMaintenanceSettlementOutcome::StaleOrMissing + ); + let ExecutionMaintenanceSettlementOutcome::Applied(succeeded) = repository + .complete_execution_maintenance(ExecutionScope::ControlPlane, kind, second.generation) + .await? + else { + panic!("current maintenance generation must record success"); + }; + assert!(succeeded.last_succeeded_at.is_some()); + let loaded = repository + .load_execution_maintenance_checkpoint(ExecutionScope::ControlPlane, kind) + .await? + .expect("maintenance health receipt must persist"); + assert_eq!(loaded, succeeded); + Ok(()) +} + +#[tokio::test] +async fn paused_task_review_decision_persists_until_single_resume_activation_db() -> TestResult { + // Pins: a decision for the exact pre-pause review owner is storage-only while paused, is + // replayable, and becomes runnable through the single activation created by resume. + assert_paused_task_review_resolution(ExecutionActionReviewResolution::Completed { + tool_output: json!({"approved": true}), + }) + .await +} + +#[tokio::test] +async fn paused_task_review_timeout_persists_until_single_resume_activation_db() -> TestResult { + // Pins: an exact review timeout cannot become stale merely because pause advanced the run + // controller generation; it remains storage-only and resumes through one activation. + assert_paused_task_review_resolution(ExecutionActionReviewResolution::TimedOut { + reason: "review expired".to_string(), + }) + .await +} + +async fn assert_paused_task_review_resolution( + resolution: ExecutionActionReviewResolution, +) -> TestResult { + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let config = execution_capacity_config(); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let mut candidate = new_run( + tenant_id, + None, + "paused-task-review-resolution", + ExecutionRunStatus::Queued, + budget(20), + ); + candidate.plan.definition.nodes = vec![watchdog_output_node()]; + let run = create_run(&repository, scope, candidate).await?; + assert!( + repository + .initialize_scheduler_state(scope, run.run_uid) + .await? + ); + assert!(matches!( + repository + .materialize_ready_page( + scope, + &config, + ReadyMaterializationRequest { + run_uid: run.run_uid, + plan_revision: 1, + node_id: "watchdog-work".to_string(), + expected_cursor: 0, + reduce_cursor: None, + source_exhausted: true, + terminal_output: None, + tasks: vec![logical_task( + run.run_uid, + "watchdog-work", + "reviewed", + estimate(1), + )], + }, + ) + .await?, + ReadyMaterializationOutcome::Applied { .. } + )); + let admission = repository + .admit_ready_attempts(&config, 1, Utc::now()) + .await? + .admitted + .into_iter() + .next() + .expect("one reviewed task must be admitted"); + let fence = TaskAttemptFence { + tenant_id: admission.tenant_id, + run_uid: admission.run_uid, + task_id: admission.task_id, + controller_generation: admission.controller_generation, + attempt_generation: admission.attempt_generation, + dispatch_uid: admission.dispatch_uid, + capacity_reservation_uid: admission.capacity_reservation_uid, + watchdog_trigger_uid: admission.watchdog_trigger_uid, + attempt_deadline_at: admission.attempt_deadline_at, + }; + let TaskAttemptStartOutcome::Started(started) = repository.start_task_attempt(fence).await? + else { + panic!("review fixture must start its exact admitted attempt"); + }; + let review_uid = Uuid::now_v7(); + let now = Utc::now(); + assert!(matches!( + repository + .begin_task_attempt_release(fence, started.task.generation, "action_review", now) + .await?, + TaskAttemptReleaseClaimOutcome::Applied(_) + )); + let invocation = ToolInvocation { + id: Some("paused-review-call".to_string()), + name: "fixture_reviewed_tool".to_string(), + input: json!({"value": 1}), + }; + assert!(matches!( + repository + .park_task_attempt_on_review( + NewTaskAttemptCheckpoint { + fence, + task_generation: started.task.generation, + kind: TaskAttemptCheckpointKind::CapabilityReview, + schema_version: 1, + payload: json!({ + "state": { + "kind": "capability_review", + "pending_review": { + "review_uid": review_uid, + "expires_at": now + Duration::minutes(5), + "invocation": invocation, + "effect_idempotency": IdempotencyClass::NonIdempotent, + }, + "usage": {}, + }, + "review_resolution": null, + "external_job_resolution": null, + "workspace_release_receipt_id": null, + }), + workspace_release_receipt: None, + created_at: now, + }, + review_uid, + ) + .await?, + TaskAttemptReviewParkOutcome::Applied { .. } + )); + let review_park_activation_count: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM moa.execution_dispatch_outbox \ + WHERE run_uid=$1 AND dispatch_kind='run_activation' \ + AND payload->>'source'='task_attempt_review_park' \ + AND payload->>'task_id'=$2 AND payload->>'dispatch_uid'=$3 \ + AND (payload->>'attempt_generation')::BIGINT=$4", + ) + .bind(run.run_uid) + .bind(fence.task_id.as_uuid().to_string()) + .bind(fence.dispatch_uid.to_string()) + .bind(i64::try_from(fence.attempt_generation)?) + .fetch_one(&pool) + .await?; + assert_eq!( + review_park_activation_count, 1, + "review parking must wake the controller to release ActiveRuns ownership" + ); + let TransitionOutcome::RunApplied(paused) = repository + .pause_run(scope, &config, run.run_uid, run.controller_generation) + .await? + else { + panic!("a run with only a storage-owned review must pause exactly"); + }; + assert_eq!(paused.status, ExecutionRunStatus::Paused); + assert_eq!( + paused.controller_generation, + fence.controller_generation + 1 + ); + + let request = |resolved_at| ResolveTaskAttemptReviewRequest { + scope: ExecutionScope::ControlPlane, + run_uid: paused.run_uid, + task_id: fence.task_id, + expected_task_generation: started.task.generation, + review_uid, + resolution: resolution.clone(), + resolved_at, + }; + let TaskAttemptReviewResolutionOutcome::Applied { task, checkpoint } = repository + .resolve_task_attempt_review(&config, request(now + Duration::seconds(1))) + .await? + else { + panic!("the exact pre-pause task review owner must remain resolvable"); + }; + assert_eq!(task.status, ExecutionTaskStatus::Ready); + assert_eq!( + checkpoint.controller_generation, + paused.controller_generation + ); + assert_eq!( + run_activation_count_for_generation(&pool, paused.run_uid, paused.controller_generation,) + .await?, + 0, + "review resolution must not wake a paused run" + ); + assert!(matches!( + repository + .resolve_task_attempt_review(&config, request(now + Duration::seconds(2))) + .await?, + TaskAttemptReviewResolutionOutcome::Replayed { .. } + )); + + let TransitionOutcome::RunApplied(resumed) = repository + .resume_run(scope, &config, paused.run_uid, paused.controller_generation) + .await? + else { + panic!("paused task review must resume after its decision is persisted"); + }; + assert_eq!( + run_activation_count_for_generation(&pool, resumed.run_uid, resumed.controller_generation) + .await?, + 1, + "resume must enqueue exactly one controller activation" + ); + Ok(()) +} + +async fn run_activation_count_for_generation( + pool: &sqlx::PgPool, + run_uid: Uuid, + controller_generation: u64, +) -> Result { + sqlx::query_scalar( + "SELECT COUNT(*) FROM moa.execution_dispatch_outbox WHERE run_uid=$1 \ + AND controller_generation=$2 AND dispatch_kind='run_activation' \ + AND delivery_state <> 'superseded'", + ) + .bind(run_uid) + .bind(i64::try_from(controller_generation).expect("fixture generation fits i64")) + .fetch_one(pool) + .await +} + +fn run_deadline( + trigger_uid: Uuid, + tenant_id: TenantId, + run_uid: Uuid, + controller_generation: u64, + due_at: chrono::DateTime, +) -> NewExecutionTrigger { + NewExecutionTrigger { + trigger_uid, + tenant_id, + run_uid: Some(run_uid), + task_id: None, + compensation_id: None, + schedule_uid: None, + schedule_incarnation: None, + kind: ExecutionTriggerKind::RunDeadline, + controller_generation: Some(controller_generation), + attempt_generation: None, + compensation_generation: None, + compensation_attempt_generation: None, + occurrence_sequence: None, + due_at, + payload: json!({}), + } +} + +fn run_activation( + tenant_id: TenantId, + run_uid: Uuid, + wake_epoch: u64, + not_before_at: chrono::DateTime, +) -> NewExecutionDispatch { + NewExecutionDispatch { + dispatch_uid: Uuid::now_v7(), + tenant_id, + run_uid: Some(run_uid), + task_id: None, + compensation_id: None, + trigger_uid: None, + external_job_uid: None, + kind: ExecutionDispatchKind::RunActivation, + controller_generation: Some(1), + wake_epoch: Some(wake_epoch), + attempt_generation: None, + compensation_generation: None, + compensation_attempt_generation: None, + not_before_at, + payload: json!({"wake_epoch": wake_epoch}), + } +} + +fn callback( + external_job_uid: Uuid, + job_generation: u64, + provider_event_id: &str, + update: ExecutionExternalJobCallbackUpdate, +) -> ExecutionExternalJobCallback { + ExecutionExternalJobCallback { + external_job_uid, + job_generation, + provider: "batch-provider".to_string(), + provider_job_id: "provider-job-1".to_string(), + provider_event_id: provider_event_id.to_string(), + update, + } +} diff --git a/crates/moa-execution/tests/interpreter.rs b/crates/moa-execution/tests/interpreter.rs index 89a3c0351..f22b82e60 100644 --- a/crates/moa-execution/tests/interpreter.rs +++ b/crates/moa-execution/tests/interpreter.rs @@ -1,11 +1,15 @@ -use std::collections::{BTreeMap, BTreeSet}; +use std::{ + collections::{BTreeMap, BTreeSet}, + str::FromStr, +}; use chrono::{TimeZone, Utc}; use moa_artifacts::execution_plan::{ CapabilityReference, CompletionCheck, CompletionCheckKind, ExecutionBudgetLimit, ExecutionCancelPolicy, ExecutionCondition, ExecutionGoalContract, ExecutionNode, ExecutionOperation, ExecutionPlanDefinition, ExecutionReference, ExecutionRequirement, - ExecutionTaskOutcome, ExecutionTaskResult, ExecutionUsage, MapTask, RetryPolicy, + ExecutionTaskOutcome, ExecutionTaskResult, ExecutionTemporalTarget, ExecutionUsage, + ExecutionWaitExpiryAction, ExecutionWaitPolicy, MapTask, RetryPolicy, }; use moa_config::ExecutionConfig; use moa_core::types::{ @@ -20,11 +24,15 @@ use moa_execution::{ }, compiler::{CanonicalExecutionPlan, ExecutionValidationReport}, completion::{CompletionEvaluationRequest, CompletionStatus, evaluate_completion}, - interpreter::{ScheduleRequest, schedule as schedule_outcome}, + interpreter::{ + ReduceMaterializationPageInput, ScheduleRequest, materialize_node_page, + resolve_temporal_target, schedule as schedule_outcome, + }, state::{ - ExecutionNodeStatus, ExecutionProjection, ExecutionTaskId, ExecutionTaskProjection, - ExecutionTaskStatus, LogicalTaskKind, ScheduleDecision, TerminalProjection, - input_resume_counters, retry_dispatch_counters, supersede_waiting_replan, + ExecutionNodeStatus, ExecutionProjection, ExecutionRunStatus, ExecutionTaskId, + ExecutionTaskProjection, ExecutionTaskStatus, LogicalTaskKind, ScheduleDecision, + TerminalProjection, WaitSettlement, WaitingReason, input_resume_counters, + retry_dispatch_counters, run_status_after_task_outcome, supersede_waiting_replan, task_status_from_outcome, validate_outcome_generation, }, }; @@ -159,6 +167,196 @@ fn scheduler_materializes_every_ready_map_item_with_stable_typed_keys() { } } +#[test] +fn controller_materializes_large_map_in_non_overlapping_cursor_pages() { + // Pins: controller-facing materialization selects one named node and returns at most + // 1,000 deterministic tasks without constructing the complete large-map task vector. + let run_uid = Uuid::from_u128(12); + let items = (0_u64..2_500).map(|item| json!(item)).collect::>(); + let plan = canonical(vec![node( + "inspect", + &[], + ExecutionOperation::Map { + items: Value::Array(items), + item_key: "".to_string(), + max_items: 2_500, + item_output_schema: json!({ "type": "object" }), + task: MapTask::Capability { + reference: capability(), + }, + }, + )]); + let request = request(run_uid, plan, BTreeMap::new(), Vec::new()); + let first = materialize_node_page(&request, "inspect", &BTreeMap::new(), 0, 1_000, None) + .expect("first map page"); + let second = materialize_node_page( + &request, + "inspect", + &BTreeMap::new(), + first.next_cursor, + 1_000, + None, + ) + .expect("second map page"); + let third = materialize_node_page( + &request, + "inspect", + &BTreeMap::new(), + second.next_cursor, + 1_000, + None, + ) + .expect("third map page"); + assert_eq!( + (first.tasks.len(), second.tasks.len(), third.tasks.len()), + (1_000, 1_000, 500) + ); + assert_eq!( + (first.next_cursor, second.next_cursor, third.next_cursor), + (1_000, 2_000, 2_500) + ); + assert!(!first.source_exhausted); + assert!(!second.source_exhausted); + assert!(third.source_exhausted); + let ids = first + .tasks + .iter() + .chain(&second.tasks) + .chain(&third.tasks) + .map(|task| task.task_id) + .collect::>(); + assert_eq!(ids.len(), 2_500); +} + +#[test] +fn controller_pages_more_than_twenty_five_hundred_reduce_batches_across_rounds() { + // Pins: a persisted round/batch cursor materializes every reducer batch exactly once across + // 1,000-task activation bounds, then starts the next round from its own zero-based cursor. + let run_uid = Uuid::from_u128(120); + let items = (0_u64..5_001).map(|item| json!(item)).collect::>(); + let mut reduce = node( + "reduce", + &[], + ExecutionOperation::Reduce { + items: Value::Array(items), + max_items: 5_001, + reducer: moa_artifacts::execution_plan::ExecutionReducer::Capability { + reference: capability(), + }, + batch_size: 2, + }, + ); + reduce.output_schema = json!({}); + let request = request( + run_uid, + canonical(vec![reduce]), + BTreeMap::new(), + Vec::new(), + ); + + let first = materialize_node_page( + &request, + "reduce", + &BTreeMap::new(), + 0, + 1_000, + Some(&ReduceMaterializationPageInput { + round: 1, + batch_cursor: 0, + round_input_count: None, + page_inputs: Vec::new(), + }), + ) + .expect("first reduce page"); + let second = materialize_node_page( + &request, + "reduce", + &BTreeMap::new(), + 1_000, + 1_000, + Some(&ReduceMaterializationPageInput { + round: 1, + batch_cursor: 1_000, + round_input_count: Some(5_001), + page_inputs: Vec::new(), + }), + ) + .expect("second reduce page"); + let third = materialize_node_page( + &request, + "reduce", + &BTreeMap::new(), + 2_000, + 1_000, + Some(&ReduceMaterializationPageInput { + round: 1, + batch_cursor: 2_000, + round_input_count: Some(5_001), + page_inputs: Vec::new(), + }), + ) + .expect("final reduce page"); + assert_eq!( + (first.tasks.len(), second.tasks.len(), third.tasks.len()), + (1_000, 1_000, 501) + ); + assert_eq!( + (first.next_cursor, second.next_cursor, third.next_cursor), + (1_000, 2_000, 2_501) + ); + assert_eq!(first.tasks[0].item_key, "r1:b0"); + assert_eq!(third.tasks[500].item_key, "r1:b2500"); + assert!(!first.source_exhausted); + assert!(!second.source_exhausted); + assert!(third.source_exhausted); + assert_eq!( + first.reduce_cursor.expect("concrete first-round fence"), + moa_execution::ReduceMaterializationCursor { + round: 1, + batch_cursor: 0, + round_input_count: 5_001, + } + ); + + let prior_round_outputs = (0_u64..2_501).map(|value| json!(value)).collect::>(); + let round_two_first = materialize_node_page( + &request, + "reduce", + &BTreeMap::new(), + 2_501, + 1_000, + Some(&ReduceMaterializationPageInput { + round: 2, + batch_cursor: 0, + round_input_count: Some(2_501), + page_inputs: prior_round_outputs[..2_000].to_vec(), + }), + ) + .expect("first second-round page"); + let round_two_second = materialize_node_page( + &request, + "reduce", + &BTreeMap::new(), + 3_501, + 1_000, + Some(&ReduceMaterializationPageInput { + round: 2, + batch_cursor: 1_000, + round_input_count: Some(2_501), + page_inputs: prior_round_outputs[2_000..].to_vec(), + }), + ) + .expect("final second-round page"); + assert_eq!( + (round_two_first.tasks.len(), round_two_second.tasks.len()), + (1_000, 251) + ); + assert_eq!(round_two_first.tasks[0].item_key, "r2:b0"); + assert_eq!(round_two_second.tasks[250].item_key, "r2:b1250"); + assert!(!round_two_first.source_exhausted); + assert!(round_two_second.source_exhausted); +} + #[test] fn scheduler_rejects_duplicate_dynamic_map_keys() { // Pins: duplicate item identities fail materialization before any task can be returned. @@ -653,6 +851,189 @@ fn scheduler_returns_no_progress_for_unbacked_nonterminal_state() { ); } +#[test] +fn scheduler_parks_wait_until_then_settles_exactly_at_the_absolute_target() { + // Pins: WaitUntil consumes no executable task slot while early and settles at `at`, not after it. + let run_uid = Uuid::from_u128(122); + let wake = ExecutionTemporalTarget::At { + at: Utc + .with_ymd_and_hms(2026, 7, 13, 1, 0, 0) + .single() + .expect("wake time"), + }; + let plan = canonical(vec![node( + "timer", + &[], + ExecutionOperation::WaitUntil { + wake: wake.clone(), + result: json!({ "ready": true }), + }, + )]); + let statuses = BTreeMap::from([("timer".to_string(), ExecutionNodeStatus::Waiting)]); + let task = ExecutionTaskProjection { + task_id: ExecutionTaskId::derive(run_uid, "timer", "").expect("timer task id"), + node_id: "timer".to_string(), + item_key: String::new(), + status: ExecutionTaskStatus::WaitingTimer, + attempt: 1, + generation: 1, + input: json!({}), + outcome: None, + }; + + let early = schedule(request( + run_uid, + plan.clone(), + statuses.clone(), + vec![task.clone()], + )) + .expect("timer should park before its target"); + assert_eq!( + early, + ScheduleDecision::Waiting(vec![WaitingReason::Timer { + task_id: task.task_id, + wake: wake.clone(), + }]) + ); + + let mut due_request = request(run_uid, plan, statuses, vec![task.clone()]); + due_request.now = Utc + .with_ymd_and_hms(2026, 7, 13, 1, 0, 0) + .single() + .expect("due time"); + let due = schedule(due_request).expect("timer should settle at its target"); + assert_eq!( + due, + ScheduleDecision::SettleWait(WaitSettlement::TimerElapsed { + task_id: task.task_id, + output: json!({ "ready": true }), + }) + ); +} + +#[test] +fn scheduler_selects_input_wait_expiry_and_resolves_relative_targets_at_wait_entry() { + // Pins: storage-only waits deterministically settle, while relative timers anchor on entry. + let run_uid = Uuid::from_u128(123); + let plan = canonical(vec![node( + "lookup", + &[], + ExecutionOperation::Capability { + reference: capability(), + }, + )]); + let statuses = BTreeMap::from([("lookup".to_string(), ExecutionNodeStatus::Waiting)]); + let task = ExecutionTaskProjection { + task_id: ExecutionTaskId::derive(run_uid, "lookup", "").expect("input task id"), + node_id: "lookup".to_string(), + item_key: String::new(), + status: ExecutionTaskStatus::WaitingInput, + attempt: 1, + generation: 1, + input: json!({}), + outcome: Some(ExecutionTaskOutcome { + schema_version: 1, + usage: ExecutionUsage { + cost_microusd: 0, + tokens: 0, + tool_calls: 0, + retrieved_bytes: 0, + }, + result: ExecutionTaskResult::NeedsInput { + question: "Which order?".to_string(), + audience: moa_artifacts::execution_plan::InputAudience::User, + }, + }), + }; + let mut expiry_request = request(run_uid, plan, statuses, vec![task.clone()]); + expiry_request.now = Utc + .with_ymd_and_hms(2026, 7, 13, 12, 0, 0) + .single() + .expect("expiry time"); + assert_eq!( + schedule(expiry_request).expect("input expiry should settle"), + ScheduleDecision::SettleWait(WaitSettlement::WaitExpired { + task_id: task.task_id, + action: ExecutionWaitExpiryAction::FailTask, + }) + ); + + let entered_at = Utc + .with_ymd_and_hms(2026, 7, 13, 2, 0, 0) + .single() + .expect("wait entry"); + let deadline_at = Utc + .with_ymd_and_hms(2026, 7, 13, 4, 0, 0) + .single() + .expect("run deadline"); + assert_eq!( + resolve_temporal_target( + &ExecutionTemporalTarget::After { + delay_seconds: 3_600 + }, + entered_at, + deadline_at, + ) + .expect("relative target should fit"), + Utc.with_ymd_and_hms(2026, 7, 13, 3, 0, 0) + .single() + .expect("resolved due time") + ); + assert!( + resolve_temporal_target( + &ExecutionTemporalTarget::After { + delay_seconds: 7_200 + }, + entered_at, + deadline_at, + ) + .is_err(), + "a relative target at the deadline must fail closed" + ); +} + +#[test] +fn long_horizon_statuses_use_canonical_snake_case_labels() { + // Pins: persistence and wire projections use one closed label for each new lifecycle state. + for (status, label) in [ + (ExecutionRunStatus::WaitingSignal, "waiting_signal"), + (ExecutionRunStatus::WaitingTimer, "waiting_timer"), + (ExecutionRunStatus::WaitingExternal, "waiting_external"), + (ExecutionRunStatus::PauseRequested, "pause_requested"), + (ExecutionRunStatus::Pausing, "pausing"), + (ExecutionRunStatus::Paused, "paused"), + ] { + assert_eq!(status.as_str(), label); + assert_eq!( + ExecutionRunStatus::from_str(label).expect("run label"), + status + ); + assert_eq!( + serde_json::to_value(status).expect("serialize run status"), + label + ); + } + for (status, label) in [ + (ExecutionTaskStatus::Ready, "ready"), + (ExecutionTaskStatus::Dispatching, "dispatching"), + (ExecutionTaskStatus::WaitingReview, "waiting_review"), + (ExecutionTaskStatus::WaitingSignal, "waiting_signal"), + (ExecutionTaskStatus::WaitingTimer, "waiting_timer"), + (ExecutionTaskStatus::WaitingExternal, "waiting_external"), + (ExecutionTaskStatus::UnknownOutcome, "unknown_outcome"), + ] { + assert_eq!(status.as_str(), label); + assert_eq!( + ExecutionTaskStatus::from_str(label).expect("task label"), + status + ); + assert_eq!( + serde_json::to_value(status).expect("serialize task status"), + label + ); + } +} + #[test] fn task_transition_helpers_pin_retry_resume_generation_and_replan_supersession() { // Pins: durable redispatch counters and WaitingReplan cancellation follow the exact state machine. @@ -737,6 +1118,31 @@ fn task_transition_helpers_pin_retry_resume_generation_and_replan_supersession() ), ExecutionTaskStatus::Cancelled ); + assert_eq!( + task_status_from_outcome( + &outcome(ExecutionTaskResult::UnknownOutcome { + message: "provider outcome is ambiguous".to_string(), + }), + false, + ), + ExecutionTaskStatus::UnknownOutcome + ); + let completed = outcome(ExecutionTaskResult::Completed { + output: json!({}), + citations: vec![], + }); + for waiting in [ + ExecutionRunStatus::WaitingInput, + ExecutionRunStatus::WaitingReview, + ExecutionRunStatus::WaitingSignal, + ExecutionRunStatus::WaitingTimer, + ExecutionRunStatus::WaitingExternal, + ] { + assert_eq!( + run_status_after_task_outcome(waiting, &completed), + ExecutionRunStatus::Running + ); + } let run_uid = Uuid::from_u128(19); let waiting = ExecutionTaskProjection { @@ -895,6 +1301,15 @@ fn canonical(nodes: Vec) -> CanonicalExecutionPlan { CanonicalExecutionPlan { definition: ExecutionPlanDefinition { cancel_policy: ExecutionCancelPolicy::RetainEffects, + input_wait_policy: ExecutionWaitPolicy { + expiry: ExecutionTemporalTarget::At { + at: Utc + .with_ymd_and_hms(2026, 7, 13, 12, 0, 0) + .single() + .expect("input wait expiry"), + }, + on_expiry: ExecutionWaitExpiryAction::FailTask, + }, input_schema: json!({ "type": "object" }), output_schema: json!({ "type": "object" }), nodes, @@ -995,7 +1410,9 @@ fn catalog() -> ExecutionCapabilityCatalog { risk_level: RiskLevel::Low, default_effect: ActionPolicyEffect::Allow, idempotency_class: IdempotencyClass::Idempotent, + async_mode: moa_core::types::tools::ToolAsyncMode::SynchronousOnly, execution_class: ExecutionClass::Data, + requires_sandbox: false, policy_context: CapabilityPolicyContext::registered(source.clone()), source, estimate: ExecutionEstimate { diff --git a/crates/moa-hands/src/adapters/daytona/tests.rs b/crates/moa-hands/src/adapters/daytona/tests.rs index 9e2994c2d..ef66c1497 100644 --- a/crates/moa-hands/src/adapters/daytona/tests.rs +++ b/crates/moa-hands/src/adapters/daytona/tests.rs @@ -295,6 +295,7 @@ async fn daytona_commit_rejects_a_parent_at_generation_zero_before_provider_io() operation, hand, parent_revision: Some(parent), + release_compute: false, }) .await .expect_err("generation-zero parent must fail before provider I/O"); diff --git a/crates/moa-hands/src/adapters/daytona/workspace.rs b/crates/moa-hands/src/adapters/daytona/workspace.rs index 5ec1f9734..d699d3017 100644 --- a/crates/moa-hands/src/adapters/daytona/workspace.rs +++ b/crates/moa-hands/src/adapters/daytona/workspace.rs @@ -277,13 +277,13 @@ impl DaytonaHandProvider { operation: &moa_core::types::sandbox_workspace::WorkspaceStorageOperation, hand: &HandHandle, parent_revision: Option<&WorkspaceRevisionRef>, + release_compute: bool, ) -> Result { - use crate::core::sandbox_workspace::{ - capacity::{CapacityQuantity, CapacityReservationRequest}, - checkpoint::{archive::build_checkpoint_archive, store::CheckpointStoreContext}, + use crate::core::sandbox_workspace::checkpoint::{ + archive::build_checkpoint_archive, store::CheckpointStoreContext, }; use moa_core::types::sandbox_workspace::{ - WorkspaceCapacityDimension, WorkspaceConfirmedDisposition, WorkspaceOperationOutcome, + WorkspaceConfirmedDisposition, WorkspaceOperationOutcome, }; verify_request_resources(operation, Some(hand), None)?; @@ -342,39 +342,10 @@ impl DaytonaHandProvider { .await?; let archive = build_checkpoint_archive(staging.path(), limits).await?; let logical_bytes = archive.manifest.logical_bytes; - let mut quantities = vec![CapacityQuantity { - dimension: WorkspaceCapacityDimension::Checkpoints, - quantity: 1, - }]; - if logical_bytes > 0 { - quantities.push(CapacityQuantity { - dimension: WorkspaceCapacityDimension::LogicalBytes, - quantity: logical_bytes, - }); - } - let capacity_request = CapacityReservationRequest { - tenant_id: operation.binding.tenant_id, - workspace_id: operation.binding.workspace_id, - operation_id: operation.operation_id, - provider_account_id: operation.binding.provider_account_id, - provider_account_generation: i64::try_from( - operation.binding.provider_account_generation, - ) - .map_err(|_| { - MoaError::ValidationError("Daytona account generation overflows bigint".to_string()) - })?, - expected_writer_epoch: i64::try_from(operation.binding.writer_epoch).map_err(|_| { - MoaError::ValidationError("Daytona writer epoch overflows bigint".to_string()) - })?, - expected_instance_generation: i64::try_from(operation.binding.instance_generation) - .map_err(|_| { - MoaError::ValidationError( - "Daytona instance generation overflows bigint".to_string(), - ) - })?, - quantities, - }; - let reservations = dependencies.capacity.reserve(&capacity_request).await?; + dependencies + .capacity + .reserve_checkpoint_publication(operation, logical_bytes) + .await?; let published = dependencies .checkpoint_store .publish( @@ -388,26 +359,15 @@ impl DaytonaHandProvider { archive, ) .await?; - let committed = dependencies - .capacity - .commit_operation_reservations(&capacity_request) - .await?; - if committed - != u64::try_from(reservations.len()).map_err(|_| { - MoaError::StorageError("Daytona reservation count overflows u64".to_string()) - })? - { - return Err(MoaError::StorageError( - "Daytona checkpoint publication lost its exact capacity reservation fence" - .to_string(), - )); - } let checkpoint_publication = WorkspaceCheckpointPublication { revision, storage: published.storage.clone(), manifest_digest: published.manifest_sha256, logical_bytes: published.logical_bytes, }; + if release_compute { + ::destroy(self, hand).await?; + } Ok(WorkspaceStorageOperationResult { outcome: WorkspaceOperationOutcome::Confirmed, confirmed_disposition: Some(WorkspaceConfirmedDisposition::ResourcePresent), @@ -415,7 +375,11 @@ impl DaytonaHandProvider { checkpoint_publication: Some(checkpoint_publication), post_commit_state: (operation.kind == moa_core::types::sandbox_workspace::WorkspaceOperationKind::Commit) - .then_some(WorkspacePostCommitState::AttachmentRetained), + .then_some(if release_compute { + WorkspacePostCommitState::ComputeDestroyed + } else { + WorkspacePostCommitState::AttachmentRetained + }), }) } } @@ -790,6 +754,7 @@ impl SandboxStorageProvider for DaytonaHandProvider { &request.operation, &request.hand, request.parent_revision.as_ref(), + request.release_compute, ) .await } diff --git a/crates/moa-hands/src/adapters/e2b/mod.rs b/crates/moa-hands/src/adapters/e2b/mod.rs index 08d6580a6..08d8ee80f 100644 --- a/crates/moa-hands/src/adapters/e2b/mod.rs +++ b/crates/moa-hands/src/adapters/e2b/mod.rs @@ -67,6 +67,7 @@ use crate::tools::{bash, file_outline, file_read, grep}; use crate::core::provider_credentials::{ ProviderCredentialSource, ProviderEndpoint, ProviderHttpAttempt, ProviderSandboxAttempt, }; +use crate::core::sandbox_workspace::capacity::PostgresWorkspaceCapacityRepository; use crate::core::sandbox_workspace::checkpoint::revision::{ next_workspace_revision, required_current_revision, }; @@ -115,6 +116,7 @@ pub struct E2BHandProvider { credentials: Arc, sandbox_base_url_override: Option, checkpoint_store: Option>, + checkpoint_capacity: Option>, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -144,6 +146,7 @@ impl E2BHandProvider { credentials, sandbox_base_url_override: None, checkpoint_store: None, + checkpoint_capacity: None, } } @@ -154,6 +157,16 @@ impl E2BHandProvider { self } + /// Installs provider-neutral pre-publication checkpoint admission. + #[must_use] + pub fn with_checkpoint_capacity( + mut self, + capacity: Arc, + ) -> Self { + self.checkpoint_capacity = Some(capacity); + self + } + async fn api_attempt( &self, provider_account_id: moa_core::types::identifiers::ProviderAccountId, diff --git a/crates/moa-hands/src/adapters/e2b/tests.rs b/crates/moa-hands/src/adapters/e2b/tests.rs index 05b32dd1d..0019d69d6 100644 --- a/crates/moa-hands/src/adapters/e2b/tests.rs +++ b/crates/moa-hands/src/adapters/e2b/tests.rs @@ -447,6 +447,7 @@ async fn e2b_commit_rejects_a_parent_at_generation_zero_before_provider_io() { operation, hand, parent_revision: Some(parent), + release_compute: false, }) .await .expect_err("generation-zero parent must fail before provider I/O"); diff --git a/crates/moa-hands/src/adapters/e2b/workspace.rs b/crates/moa-hands/src/adapters/e2b/workspace.rs index 3b4485cfe..9ab61b8a6 100644 --- a/crates/moa-hands/src/adapters/e2b/workspace.rs +++ b/crates/moa-hands/src/adapters/e2b/workspace.rs @@ -96,6 +96,7 @@ impl E2BHandProvider { operation: &WorkspaceStorageOperation, hand: &HandHandle, parent_revision: Option<&WorkspaceRevisionRef>, + release_compute: bool, ) -> Result { if chrono::Utc::now() >= operation.deadline { return Err(MoaError::ProviderTimeout( @@ -124,6 +125,11 @@ impl E2BHandProvider { store.archive_limits(), ) .await?; + if let Some(capacity) = self.checkpoint_capacity.as_ref() { + capacity + .reserve_checkpoint_publication(operation, archive.manifest.logical_bytes) + .await?; + } let published = store .publish(Self::checkpoint_context(operation, checkpoint_id), archive) .await?; @@ -133,15 +139,22 @@ impl E2BHandProvider { manifest_digest: published.manifest_sha256, logical_bytes: published.logical_bytes, }; - self.kill_exact_workspace_hand(hand, &operation.binding) - .await?; + if release_compute { + self.kill_exact_workspace_hand(hand, &operation.binding) + .await?; + } Ok(WorkspaceStorageOperationResult { outcome: WorkspaceOperationOutcome::Confirmed, confirmed_disposition: Some(WorkspaceConfirmedDisposition::ResourcePresent), storage: Some(published.storage), checkpoint_publication: Some(checkpoint_publication), - post_commit_state: (operation.kind == WorkspaceOperationKind::Commit) - .then_some(WorkspacePostCommitState::ComputeDestroyed), + post_commit_state: (operation.kind == WorkspaceOperationKind::Commit).then_some( + if release_compute { + WorkspacePostCommitState::ComputeDestroyed + } else { + WorkspacePostCommitState::AttachmentRetained + }, + ), }) } } @@ -375,6 +388,7 @@ impl SandboxStorageProvider for E2BHandProvider { &request.operation, &request.hand, request.parent_revision.as_ref(), + request.release_compute, ) .await } diff --git a/crates/moa-hands/src/adapters/local/mod.rs b/crates/moa-hands/src/adapters/local/mod.rs index 3ee3b7ce5..d75cd3077 100644 --- a/crates/moa-hands/src/adapters/local/mod.rs +++ b/crates/moa-hands/src/adapters/local/mod.rs @@ -65,6 +65,7 @@ use crate::adapters::trusted_command::{ normalized_trusted_skill_path, resolve_trusted_skill_command, rewrite_bash_input, }; use crate::core::leases::LeaseHandle; +use crate::core::sandbox_workspace::capacity::PostgresWorkspaceCapacityRepository; use crate::core::sandbox_workspace::checkpoint::archive::build_checkpoint_archive; use crate::core::sandbox_workspace::checkpoint::revision::{ next_workspace_revision, required_current_revision, @@ -314,6 +315,7 @@ pub struct LocalHandProvider { local_sandboxes: Arc>>, docker_sandboxes: Arc>>, checkpoint_store: Option>, + checkpoint_capacity: Option>, } impl LocalHandProvider { @@ -342,6 +344,7 @@ impl LocalHandProvider { local_sandboxes: Arc::new(RwLock::new(HashMap::new())), docker_sandboxes: Arc::new(RwLock::new(HashMap::new())), checkpoint_store: None, + checkpoint_capacity: None, }) } @@ -364,6 +367,16 @@ impl LocalHandProvider { self } + /// Installs provider-neutral pre-publication checkpoint admission. + #[must_use] + pub fn with_checkpoint_capacity( + mut self, + capacity: Arc, + ) -> Self { + self.checkpoint_capacity = Some(capacity); + self + } + fn sandbox_dir(&self, operation_id: HandProvisioningOperationId) -> PathBuf { self.work_dir .join(format!("{HAND_SANDBOX_PREFIX}{operation_id}")) diff --git a/crates/moa-hands/src/adapters/local/tests.rs b/crates/moa-hands/src/adapters/local/tests.rs index 394624a51..a69a59c1b 100644 --- a/crates/moa-hands/src/adapters/local/tests.rs +++ b/crates/moa-hands/src/adapters/local/tests.rs @@ -228,6 +228,7 @@ async fn local_commit_rejects_a_parent_at_generation_zero_before_storage_work() operation, hand: HandHandle::local(dir.path().join("missing-compute")), parent_revision: Some(parent), + release_compute: false, }) .await .expect_err("generation-zero parent must fail before compute/storage access"); diff --git a/crates/moa-hands/src/adapters/local/workspace.rs b/crates/moa-hands/src/adapters/local/workspace.rs index 770d76258..25b4112ff 100644 --- a/crates/moa-hands/src/adapters/local/workspace.rs +++ b/crates/moa-hands/src/adapters/local/workspace.rs @@ -53,6 +53,7 @@ impl LocalHandProvider { operation: &moa_core::types::sandbox_workspace::WorkspaceStorageOperation, hand: &HandHandle, parent_revision: Option<&WorkspaceRevisionRef>, + release_compute: bool, ) -> Result { if Utc::now() >= operation.deadline { return Err(MoaError::ProviderTimeout( @@ -63,11 +64,12 @@ impl LocalHandProvider { let revision = next_workspace_revision(operation, parent_revision, checkpoint_id, None)?; let root = self.workspace_data_root(hand).await?; let store = self.checkpoint_store()?; - let archive = build_checkpoint_archive( - &root, - crate::core::sandbox_workspace::checkpoint::archive::ArchiveLimits::default(), - ) - .await?; + let archive = build_checkpoint_archive(&root, store.archive_limits()).await?; + if let Some(capacity) = self.checkpoint_capacity.as_ref() { + capacity + .reserve_checkpoint_publication(operation, archive.manifest.logical_bytes) + .await?; + } let published = store .publish(Self::checkpoint_context(operation, checkpoint_id), archive) .await?; @@ -77,13 +79,21 @@ impl LocalHandProvider { manifest_digest: published.manifest_sha256, logical_bytes: published.logical_bytes, }; + if release_compute { + ::destroy(self, hand).await?; + } Ok(WorkspaceStorageOperationResult { outcome: WorkspaceOperationOutcome::Confirmed, confirmed_disposition: Some(WorkspaceConfirmedDisposition::ResourcePresent), storage: Some(published.storage), checkpoint_publication: Some(checkpoint_publication), - post_commit_state: (operation.kind == WorkspaceOperationKind::Commit) - .then_some(WorkspacePostCommitState::AttachmentRetained), + post_commit_state: (operation.kind == WorkspaceOperationKind::Commit).then_some( + if release_compute { + WorkspacePostCommitState::ComputeDestroyed + } else { + WorkspacePostCommitState::AttachmentRetained + }, + ), }) } } @@ -206,6 +216,7 @@ impl SandboxStorageProvider for LocalHandProvider { &request.operation, &request.hand, request.parent_revision.as_ref(), + request.release_compute, ) .await } diff --git a/crates/moa-hands/src/adapters/mcp/mod.rs b/crates/moa-hands/src/adapters/mcp/mod.rs index 2024a263b..e39c9c8ab 100644 --- a/crates/moa-hands/src/adapters/mcp/mod.rs +++ b/crates/moa-hands/src/adapters/mcp/mod.rs @@ -404,6 +404,9 @@ impl RemoteClient { MoaError::ProviderError(format!("failed to call MCP server: {error}")) })?; let status = response.status(); + if !status.is_success() { + return Err(super::http_util::http_error(response).await); + } let content_type = response .headers() .get(reqwest::header::CONTENT_TYPE) @@ -412,16 +415,7 @@ impl RemoteClient { .to_string(); if content_type.contains("text/event-stream") { let body = read_sse_response(response, message_id).await?; - if status.is_success() { - return Ok(body); - } - return parse_jsonrpc_result(body, message_id).and_then(|_| { - Err(MoaError::HttpStatus { - status: status.as_u16(), - retry_after: None, - message: "MCP server request failed".to_string(), - }) - }); + return Ok(body); } if !content_type.contains("application/json") { return Err(MoaError::StreamError(format!( @@ -431,15 +425,6 @@ impl RemoteClient { let body = response.json::().await.map_err(|error| { MoaError::StreamError(format!("invalid MCP JSON response: {error}")) })?; - if !status.is_success() { - return parse_jsonrpc_result(body, message_id).and_then(|_| { - Err(MoaError::HttpStatus { - status: status.as_u16(), - retry_after: None, - message: "MCP server request failed".to_string(), - }) - }); - } Ok(body) } } diff --git a/crates/moa-hands/src/adapters/mcp/tests.rs b/crates/moa-hands/src/adapters/mcp/tests.rs index da27abe8c..7e14eb603 100644 --- a/crates/moa-hands/src/adapters/mcp/tests.rs +++ b/crates/moa-hands/src/adapters/mcp/tests.rs @@ -1,4 +1,5 @@ use std::collections::HashMap; +use std::time::Duration; use moa_config::McpServerConfig; use serde_json::{Value, json}; @@ -238,6 +239,59 @@ async fn sse_client_skips_request_notifications_before_matching_final_response() server.await.expect("fake MCP server should finish"); } +#[tokio::test] +async fn plain_text_rate_limit_preserves_http_status_and_retry_after() { + // Pins: HTTP status and Retry-After are transport metadata, so a non-JSON error body cannot + // turn a retryable MCP rate limit into a fatal unsupported-content-type failure. + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("bind fake MCP server"); + let addr = listener.local_addr().expect("fake MCP server address"); + let server = tokio::spawn(async move { + let (mut discover_socket, _) = listener.accept().await.expect("accept discover"); + let discover = read_request(&mut discover_socket).await; + assert_modern_request(&discover, "server/discover"); + write_json_response( + &mut discover_socket, + r#"{"jsonrpc":"2.0","id":1,"result":{"resultType":"complete","supportedVersions":["2026-07-28"],"capabilities":{"tools":{}},"ttlMs":60000,"cacheScope":"private"}}"#, + ) + .await; + + let (mut call_socket, _) = listener.accept().await.expect("accept tools/call"); + let call = read_request(&mut call_socket).await; + assert_modern_request(&call, "tools/call"); + let body = "fixture rate limit"; + let response = format!( + "HTTP/1.1 429 Too Many Requests\r\ncontent-type: text/plain\r\nretry-after: 7\r\nconnection: close\r\ncontent-length: {}\r\n\r\n{body}", + body.len() + ); + call_socket + .write_all(response.as_bytes()) + .await + .expect("write MCP rate-limit response"); + }); + + let client = MCPClient::connect(&config(addr), HashMap::new()) + .await + .expect("connect to modern MCP server"); + let error = client + .call_tool("rate_limited", json!({}), None, None) + .await + .expect_err("plain-text 429 must remain a typed HTTP error"); + match error { + moa_core::error::MoaError::HttpStatus { + status: 429, + retry_after: Some(delay), + message, + } => assert_eq!( + (delay, message), + (Duration::from_secs(7), "fixture rate limit".to_string()) + ), + other => panic!("plain-text 429 returned the wrong error: {other}"), + } + server.await.expect("fake MCP server should finish"); +} + #[tokio::test] async fn legacy_server_is_rejected_without_initialize_fallback() { // Pins: a server that lacks the exact modern revision fails after one diff --git a/crates/moa-hands/src/core/construction.rs b/crates/moa-hands/src/core/construction.rs index 1aa36398b..d0178d2b2 100644 --- a/crates/moa-hands/src/core/construction.rs +++ b/crates/moa-hands/src/core/construction.rs @@ -152,6 +152,13 @@ impl ToolRouter { } else { Vec::new() }; + let checkpoint_capacity = workspace_pool.as_ref().map(|pool| { + Arc::new( + super::sandbox_workspace::capacity::PostgresWorkspaceCapacityRepository::new( + pool.clone(), + ), + ) + }); let mut providers = HashMap::new(); let mut storage_providers: HashMap> = @@ -169,6 +176,9 @@ impl ToolRouter { if let Some(store) = checkpoint_store.as_ref() { provider = provider.with_checkpoint_store(Arc::clone(store)); } + if let Some(capacity) = checkpoint_capacity.as_ref() { + provider = provider.with_checkpoint_capacity(Arc::clone(capacity)); + } let provider = Arc::new(provider); let provider_trait: Arc = provider.clone(); let storage_trait: Arc = provider.clone(); @@ -228,11 +238,12 @@ impl ToolRouter { pool.clone(), ), ), - capacity: Arc::new( - super::sandbox_workspace::capacity::PostgresWorkspaceCapacityRepository::new( - pool.clone(), - ), - ), + capacity: Arc::clone(checkpoint_capacity.as_ref().ok_or_else(|| { + MoaError::ConfigError( + "Daytona persistent workspaces require checkpoint capacity admission" + .to_string(), + ) + })?), kms: Arc::clone(kms), }, )?); @@ -250,9 +261,16 @@ impl ToolRouter { "E2B persistent workspaces require a checkpoint object store".to_string(), ) })?; + let checkpoint_capacity = checkpoint_capacity.as_ref().ok_or_else(|| { + MoaError::ConfigError( + "E2B persistent workspaces require checkpoint capacity admission" + .to_string(), + ) + })?; let provider = Arc::new( E2BHandProvider::new(Arc::clone(source)) - .with_checkpoint_store(Arc::clone(checkpoint_store)), + .with_checkpoint_store(Arc::clone(checkpoint_store)) + .with_checkpoint_capacity(Arc::clone(checkpoint_capacity)), ); let hand: Arc = provider.clone(); let storage: Arc = provider; @@ -447,9 +465,12 @@ impl ToolRouter { ), Arc::new( super::sandbox_workspace::operations::PostgresWorkspaceOperationRepository::new( - pool, + pool.clone(), ), ), + Arc::new( + super::sandbox_workspace::capacity::PostgresWorkspaceCapacityRepository::new(pool), + ), ); self } diff --git a/crates/moa-hands/src/core/dispatch.rs b/crates/moa-hands/src/core/dispatch.rs index 8ecceb769..735f4fd27 100644 --- a/crates/moa-hands/src/core/dispatch.rs +++ b/crates/moa-hands/src/core/dispatch.rs @@ -13,9 +13,9 @@ use moa_core::{ types::completion::ToolInvocation, types::hands::HandHandle, types::hands::HandStatus, - types::identifiers::ToolCallId, + types::identifiers::{ExecutionRunScopeId, ToolCallId}, types::resource::DeadlineGuard, - types::sandbox_workspace::{SandboxWorkspaceScope, WorkspaceEffect}, + types::sandbox_workspace::{ExecutionHandReleaseOwner, SandboxWorkspaceScope, WorkspaceEffect}, types::security::ToolCapabilityId, types::session::SessionMeta, types::tools::SecuredToolOutput, @@ -112,6 +112,21 @@ pub struct JournaledWorkspaceCommit<'a> { pub scope: ToolCallScope<'a>, } +/// One idempotent request to release an execution attempt's exact sandbox hand. +#[derive(Clone, Copy)] +pub struct ExecutionHandReleaseRequest<'a> { + /// Session whose tenant owns the execution workspace and hand lease. + pub session: &'a SessionMeta, + /// Verified durable execution run. + pub run_id: ExecutionRunScopeId, + /// Verified durable execution owner and logical generation. + pub owner: ExecutionHandReleaseOwner, + /// Exact bounded attempt generation yielding its resources. + pub attempt_generation: u64, + /// Fresh bounded budget for checkpoint publication and verified destroy. + pub scope: ToolCallScope<'a>, +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(super) enum WorkspaceCommitMode { Inline, @@ -845,12 +860,15 @@ impl ToolRouter { && workspace_commit_mode == WorkspaceCommitMode::Inline && let Err(error) = self .commit_workspace_after_tool( - request.session, - workspace_scope, - request.tool_call_id, - provider, - hand, - request.scope, + super::sandbox_workspace::lifecycle::WorkspaceCommitExecution { + session: request.session, + workspace_scope, + tool_call_id: request.tool_call_id, + provider_name: provider, + hand, + call_scope: request.scope, + release_compute: false, + }, ) .await { @@ -1398,6 +1416,7 @@ mod egress_dispatch_tests { diff_strategy: ToolDiffStrategy::None, }, idempotency_class: IdempotencyClass::NonIdempotent, + async_mode: moa_core::types::tools::ToolAsyncMode::SynchronousOnly, rollback: None, max_output_tokens: 4096, } diff --git a/crates/moa-hands/src/core/leases.rs b/crates/moa-hands/src/core/leases.rs index 5e0dc31cf..2aa358fe8 100644 --- a/crates/moa-hands/src/core/leases.rs +++ b/crates/moa-hands/src/core/leases.rs @@ -23,6 +23,8 @@ use sqlx::{PgPool, Row, types::Json}; use tokio::sync::Mutex; use uuid::Uuid; +use super::sandbox_workspace::capacity::release_active_hand_for_reaper_in_transaction; + /// Maximum wall-clock time the platform allows one provider create dispatch. pub(super) const PROVISIONING_TIMEOUT: Duration = Duration::from_secs(5 * 60); /// Provider visibility grace after create dispatch can no longer complete. @@ -363,6 +365,28 @@ pub struct HandLeaseActivateRequest<'a> { pub attachment: HandLeaseWorkspaceAttachment, } +/// Stable keyset cursor for one session's live hand leases. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HandLeaseSessionCursor { + /// Opaque typed-owner key of the last lease returned by the prior page. + pub worker_id: String, + /// Provider name of the last lease returned by the prior page. + pub provider: String, +} + +/// One bounded page of live hand leases for aggregate session teardown. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HandLeaseSessionPage { + /// Live leases ordered by `(worker_id, provider)`. + pub leases: Vec, + /// Cursor to resume after this page, or `None` when the scan is complete. + pub next_cursor: Option, +} + +/// Maximum live hand leases returned by one aggregate session page. +pub const HAND_LEASE_SESSION_PAGE_SIZE: usize = 64; +const HAND_LEASE_SESSION_QUERY_LIMIT: i64 = HAND_LEASE_SESSION_PAGE_SIZE as i64 + 1; + /// Store contract for durable hand lease coordination. /// /// Every foreground method requires the verified tenant in addition to the @@ -392,13 +416,50 @@ pub trait HandLeaseStore: Send + Sync { provider: &str, ) -> Result>; - /// Lists all durable leases for a session, across every typed owner scope. - async fn list_session( + /// Loads one exact owner lease generation by its provider-create identity. + async fn get_exact_generation( &self, tenant_id: TenantId, session_id: SessionId, + worker_id: &str, + provisioning_operation_id: HandProvisioningOperationId, + generation: i64, + ) -> Result>; + + /// Loads at most two live provider rows for one exact typed owner scope. + /// + /// Implementations must use the `(tenant_id, session_id, worker_id)` owner + /// index prefix and a database-side `LIMIT 2`; the second row detects an + /// invalid concurrent replacement without materializing owner history. + async fn list_live_owner_candidates( + &self, + tenant_id: TenantId, + session_id: SessionId, + worker_id: &str, ) -> Result>; + /// Reports whether any non-destroyed provider row exists for an exact owner. + async fn has_live_owner( + &self, + tenant_id: TenantId, + session_id: SessionId, + worker_id: &str, + ) -> Result; + + /// Lists one bounded keyset page of live leases for a session. + /// + /// Rows are ordered by `(worker_id, provider)` and the cursor is exclusive. + /// Implementations must use the tenant/session index prefix, fetch at most + /// one lookahead row beyond [`HAND_LEASE_SESSION_PAGE_SIZE`], and omit + /// already-destroyed history so terminal cleanup stays bounded as a session + /// ages. + async fn list_live_session_page( + &self, + tenant_id: TenantId, + session_id: SessionId, + cursor: Option<&HandLeaseSessionCursor>, + ) -> Result; + /// Marks a claimed generation active with its durable handle payload. /// /// Activation carries no policy or hard deadline: both were fixed by the @@ -563,6 +624,21 @@ fn attachment_columns( }) } +fn finish_live_session_page(leases: &mut Vec) -> HandLeaseSessionPage { + let has_more = leases.len() > HAND_LEASE_SESSION_PAGE_SIZE; + leases.truncate(HAND_LEASE_SESSION_PAGE_SIZE); + let next_cursor = has_more.then(|| { + leases.last().map(|lease| HandLeaseSessionCursor { + worker_id: lease.worker_id.clone(), + provider: lease.provider.clone(), + }) + }); + HandLeaseSessionPage { + leases: std::mem::take(leases), + next_cursor: next_cursor.flatten(), + } +} + #[async_trait] impl HandLeaseStore for PostgresHandLeaseStore { async fn claim_for_provisioning( @@ -623,6 +699,7 @@ impl HandLeaseStore for PostgresHandLeaseStore { reap_attempts = 0, reap_not_before = EXCLUDED.reap_not_before WHERE moa.hand_leases.tenant_id = EXCLUDED.tenant_id + AND moa.hand_leases.handle IS NULL AND ( moa.hand_leases.status IN ('stale', 'destroyed') OR ( @@ -695,22 +772,56 @@ impl HandLeaseStore for PostgresHandLeaseStore { Ok(lease) } - async fn list_session( + async fn get_exact_generation( &self, tenant_id: TenantId, session_id: SessionId, + worker_id: &str, + provisioning_operation_id: HandProvisioningOperationId, + generation: i64, + ) -> Result> { + let mut conn = self.begin(tenant_id).await?; + let row = sqlx::query(&format!( + r#" + SELECT {LEASE_COLUMNS} + FROM moa.hand_leases + WHERE tenant_id = $1 AND session_id = $2 AND worker_id = $3 + AND provisioning_operation_id = $4 AND generation = $5 + "# + )) + .bind(tenant_id) + .bind(session_id) + .bind(worker_id) + .bind(provisioning_operation_id) + .bind(generation) + .fetch_optional(conn.as_mut()) + .await + .map_err(map_sqlx_error)?; + let lease = row.map(|row| hand_lease_from_row(&row)).transpose()?; + conn.commit().await?; + Ok(lease) + } + + async fn list_live_owner_candidates( + &self, + tenant_id: TenantId, + session_id: SessionId, + worker_id: &str, ) -> Result> { let mut conn = self.begin(tenant_id).await?; let rows = sqlx::query(&format!( r#" SELECT {LEASE_COLUMNS} FROM moa.hand_leases - WHERE session_id = $1 AND tenant_id = $2 - ORDER BY worker_id, provider + WHERE tenant_id = $1 AND session_id = $2 AND worker_id = $3 + AND status <> 'destroyed' + ORDER BY provider + LIMIT 2 "# )) - .bind(session_id) .bind(tenant_id) + .bind(session_id) + .bind(worker_id) .fetch_all(conn.as_mut()) .await .map_err(map_sqlx_error)?; @@ -723,6 +834,84 @@ impl HandLeaseStore for PostgresHandLeaseStore { Ok(leases) } + async fn has_live_owner( + &self, + tenant_id: TenantId, + session_id: SessionId, + worker_id: &str, + ) -> Result { + let mut conn = self.begin(tenant_id).await?; + let exists = sqlx::query_scalar::<_, bool>( + "SELECT EXISTS (SELECT 1 FROM moa.hand_leases \ + WHERE tenant_id = $1 AND session_id = $2 AND worker_id = $3 \ + AND status <> 'destroyed')", + ) + .bind(tenant_id) + .bind(session_id) + .bind(worker_id) + .fetch_one(conn.as_mut()) + .await + .map_err(map_sqlx_error)?; + conn.commit().await?; + Ok(exists) + } + + async fn list_live_session_page( + &self, + tenant_id: TenantId, + session_id: SessionId, + cursor: Option<&HandLeaseSessionCursor>, + ) -> Result { + let mut conn = self.begin(tenant_id).await?; + let rows = match cursor { + Some(cursor) => { + sqlx::query(&format!( + r#" + SELECT {LEASE_COLUMNS} + FROM moa.hand_leases + WHERE tenant_id = $1 AND session_id = $2 + AND status <> 'destroyed' + AND (worker_id, provider) > ($3, $4) + ORDER BY worker_id, provider + LIMIT $5 + "# + )) + .bind(tenant_id) + .bind(session_id) + .bind(&cursor.worker_id) + .bind(&cursor.provider) + .bind(HAND_LEASE_SESSION_QUERY_LIMIT) + .fetch_all(conn.as_mut()) + .await + } + None => { + sqlx::query(&format!( + r#" + SELECT {LEASE_COLUMNS} + FROM moa.hand_leases + WHERE tenant_id = $1 AND session_id = $2 + AND status <> 'destroyed' + ORDER BY worker_id, provider + LIMIT $3 + "# + )) + .bind(tenant_id) + .bind(session_id) + .bind(HAND_LEASE_SESSION_QUERY_LIMIT) + .fetch_all(conn.as_mut()) + .await + } + } + .map_err(map_sqlx_error)?; + + let mut leases = rows + .iter() + .map(hand_lease_from_row) + .collect::>>()?; + conn.commit().await?; + Ok(finish_live_session_page(&mut leases)) + } + async fn activate(&self, request: HandLeaseActivateRequest<'_>) -> Result { let HandLeaseActivateRequest { tenant_id, @@ -1010,6 +1199,19 @@ impl HandLeaseStore for PostgresHandLeaseStore { let (workspace_id, writer_epoch, instance_generation, checkpoint_id) = attachment_columns(expected.attachment.as_ref()); let mut conn = self.begin(tenant_id).await?; + if expected.attachment.is_some() + && !release_active_hand_for_reaper_in_transaction( + conn.as_mut(), + tenant_id, + expected.provisioning_operation_id, + expected.generation, + claim_token, + ) + .await? + { + conn.rollback().await?; + return Ok(false); + } let affected = sqlx::query( r#" UPDATE moa.hand_leases @@ -1436,23 +1638,95 @@ impl HandLeaseStore for MemoryHandLeaseStore { .cloned()) } - async fn list_session( + async fn get_exact_generation( &self, tenant_id: TenantId, session_id: SessionId, + worker_id: &str, + provisioning_operation_id: HandProvisioningOperationId, + generation: i64, + ) -> Result> { + Ok(self + .leases + .lock() + .await + .values() + .find(|lease| { + lease.tenant_id == tenant_id + && lease.session_id == session_id + && lease.worker_id == worker_id + && lease.provisioning_operation_id == provisioning_operation_id + && lease.generation == generation + }) + .cloned()) + } + + async fn list_live_owner_candidates( + &self, + tenant_id: TenantId, + session_id: SessionId, + worker_id: &str, ) -> Result> { let mut leases = self .leases .lock() .await .values() - .filter(|lease| lease.tenant_id == tenant_id && lease.session_id == session_id) + .filter(|lease| { + lease.tenant_id == tenant_id + && lease.session_id == session_id + && lease.worker_id == worker_id + && lease.status != HandLeaseStatus::Destroyed + }) .cloned() .collect::>(); leases.sort_by(|left, right| left.provider.cmp(&right.provider)); + leases.truncate(2); Ok(leases) } + async fn has_live_owner( + &self, + tenant_id: TenantId, + session_id: SessionId, + worker_id: &str, + ) -> Result { + Ok(self.leases.lock().await.values().any(|lease| { + lease.tenant_id == tenant_id + && lease.session_id == session_id + && lease.worker_id == worker_id + && lease.status != HandLeaseStatus::Destroyed + })) + } + + async fn list_live_session_page( + &self, + tenant_id: TenantId, + session_id: SessionId, + cursor: Option<&HandLeaseSessionCursor>, + ) -> Result { + let mut leases = self + .leases + .lock() + .await + .values() + .filter(|lease| { + lease.tenant_id == tenant_id + && lease.session_id == session_id + && lease.status != HandLeaseStatus::Destroyed + && cursor.is_none_or(|cursor| { + (&lease.worker_id, &lease.provider) > (&cursor.worker_id, &cursor.provider) + }) + }) + .cloned() + .collect::>(); + leases.sort_by(|left, right| { + (&left.worker_id, &left.provider).cmp(&(&right.worker_id, &right.provider)) + }); + leases.truncate(HAND_LEASE_SESSION_PAGE_SIZE + 1); + Ok(finish_live_session_page(&mut leases)) + } + async fn activate(&self, request: HandLeaseActivateRequest<'_>) -> Result { let HandLeaseActivateRequest { tenant_id, @@ -1729,12 +2003,22 @@ mod tests { worker_id: &'a str, tenant_id: TenantId, policy: &'a HandLeasePolicy, + ) -> HandLeaseProvisionRequest<'a> { + provision_request_for_provider(session_id, worker_id, tenant_id, policy, "local") + } + + fn provision_request_for_provider<'a>( + session_id: SessionId, + worker_id: &'a str, + tenant_id: TenantId, + policy: &'a HandLeasePolicy, + provider: &'a str, ) -> HandLeaseProvisionRequest<'a> { HandLeaseProvisionRequest { session_id, worker_id, tenant_id, - provider: "local", + provider, tier: SandboxTier::Local, attachment: HandLeaseWorkspaceAttachment::new(SandboxWorkspaceId::new(), 1, 1, None) .expect("test attachment should validate"), @@ -1799,12 +2083,145 @@ mod tests { .expect("load worker lease") .is_some() ); - // list_session reclaims every scope under the session at once. let listed = store - .list_session(tenant_id, session_id) + .list_live_session_page(tenant_id, session_id, None) .await .expect("list leases"); - assert_eq!(listed.len(), 2, "both scopes belong to the session"); + assert_eq!( + listed.leases.len(), + 2, + "both live scopes belong to the session" + ); + assert_eq!(listed.next_cursor, None); + } + + #[tokio::test] + async fn memory_store_live_session_pages_are_bounded_replayable_and_complete() { + // Pins: terminal cleanup of a session with more than one page of live + // hands resumes by keyset cursor without materializing destroyed history, + // skipping a live lease, or changing a replayed page. + let store = MemoryHandLeaseStore::shared(); + let session_id = SessionId::new(); + let tenant_id = TenantId::new(); + let policy = lease_policy(Some(300), Some(3600), "cap-session-page"); + + for index in 0..128 { + let worker_id = format!("destroyed-owner-{index:03}"); + let claim = store + .claim_for_provisioning(provision_request( + session_id, &worker_id, tenant_id, &policy, + )) + .await + .expect("seed destroyed lease") + .expect("destroyed-history owner should claim its own row"); + assert!( + store + .transition_status(tenant_id, &claim, HandLeaseStatus::Destroyed) + .await + .expect("mark historical lease destroyed"), + "seeded historical lease should retain its fence" + ); + } + + let live_count = HAND_LEASE_SESSION_PAGE_SIZE + 7; + for index in 0..live_count { + let worker_id = format!("live-owner-{index:03}"); + store + .claim_for_provisioning(provision_request( + session_id, &worker_id, tenant_id, &policy, + )) + .await + .expect("seed live lease") + .expect("live owner should claim its own row"); + } + + let first = store + .list_live_session_page(tenant_id, session_id, None) + .await + .expect("load first live session page"); + let replay = store + .list_live_session_page(tenant_id, session_id, None) + .await + .expect("replay first live session page"); + assert_eq!(first, replay, "the same cursor must replay the same page"); + assert_eq!(first.leases.len(), HAND_LEASE_SESSION_PAGE_SIZE); + let cursor = first + .next_cursor + .as_ref() + .expect("a saturated first page must expose continuation"); + assert_eq!( + first + .leases + .last() + .map(|lease| (&lease.worker_id, &lease.provider)), + Some((&cursor.worker_id, &cursor.provider)), + "continuation must start after the last returned lease" + ); + + let second = store + .list_live_session_page(tenant_id, session_id, Some(cursor)) + .await + .expect("load final live session page"); + assert_eq!(second.leases.len(), 7); + assert_eq!(second.next_cursor, None); + + let keys = first + .leases + .iter() + .chain(&second.leases) + .map(|lease| (lease.worker_id.as_str(), lease.provider.as_str())) + .collect::>(); + assert_eq!(keys.len(), live_count); + assert!( + keys.windows(2).all(|pair| pair[0] < pair[1]), + "keyset pages must be strictly ordered without duplicates" + ); + assert!( + keys.iter() + .all(|(worker_id, _)| worker_id.starts_with("live-owner-")), + "destroyed session history must not consume a live cleanup page" + ); + } + + #[tokio::test] + async fn memory_store_exact_owner_lookup_excludes_unrelated_session_history() { + // Pins: owner-scoped teardown observes only the exact compensation scope, + // even when the same session carries a large unrelated lease history. + let store = MemoryHandLeaseStore::shared(); + let session_id = SessionId::new(); + let tenant_id = TenantId::new(); + let policy = lease_policy(Some(300), Some(3600), "cap-owner-lookup"); + for index in 0..128 { + let worker_id = format!("unrelated-owner-{index}"); + store + .claim_for_provisioning(provision_request( + session_id, &worker_id, tenant_id, &policy, + )) + .await + .expect("seed unrelated lease") + .expect("unrelated owner should claim its own row"); + } + let target = "execution_compensation:run-1:compensation-1"; + for provider in ["local", "daytona", "e2b"] { + store + .claim_for_provisioning(provision_request_for_provider( + session_id, target, tenant_id, &policy, provider, + )) + .await + .expect("seed target lease") + .expect("target owner should claim its own row"); + } + + let leases = store + .list_live_owner_candidates(tenant_id, session_id, target) + .await + .expect("load exact target owner"); + assert_eq!( + leases.len(), + 2, + "the release probe must not materialize every live replacement" + ); + assert!(leases.iter().all(|lease| lease.worker_id == target)); } #[tokio::test] diff --git a/crates/moa-hands/src/core/lifecycle.rs b/crates/moa-hands/src/core/lifecycle.rs index 274c87b33..f69f4277a 100644 --- a/crates/moa-hands/src/core/lifecycle.rs +++ b/crates/moa-hands/src/core/lifecycle.rs @@ -32,12 +32,14 @@ use super::leases::{ HandLeaseRenewRequest, HandLeaseStatus, LeaseHandle, provisioning_deadline, }; use super::reaper::{ProvisioningAbsenceProof, destroy_provisioning_operations}; +use super::sandbox_workspace::capacity::ActiveHandCapacityRequest; use super::sandbox_workspace::lifecycle::lease_attachment; #[cfg(test)] use super::sandbox_workspace::lifecycle::validate_managed_restore_target; use super::{ - ActiveHand, DEFAULT_PROVIDER_NAME, DEFAULT_TOOL_TIMEOUT, HandRoute, HandScopeKey, - InstalledManifestMarker, ToolCallScope, ToolRouter, TrustedSandboxManifest, + ActiveHand, DEFAULT_PROVIDER_NAME, DEFAULT_TOOL_TIMEOUT, HandProviderCacheKey, HandRoute, + HandScopeKey, InstalledManifestMarker, SessionCacheDrainCursor, ToolCallScope, ToolRouter, + TrustedSandboxManifest, }; /// Builds a sandbox-provisioning span parented to the active turn root when present. @@ -97,17 +99,58 @@ const HAND_LEASE_PROVISION_WAIT_MS: u64 = 25; const HAND_DESTROY_CLAIM_TTL: StdDuration = StdDuration::from_secs(5 * 60); const HAND_DESTROY_RETRY_DELAY: StdDuration = StdDuration::from_secs(15); +/// Result of one bounded session-wide hand cleanup page. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SessionHandReleasePageOutcome { + /// No live durable lease or process-local session cache entry remains. + Complete, + /// This page made forward progress and another bounded page is required. + Progressed, + /// Live ownership remains, but this page could not make forward progress. + Waiting, +} + struct DurableHandProvisionContext<'a> { route: &'a HandRoute, session: &'a SessionMeta, worker_id: Option<&'a str>, workspace_binding: &'a WorkspaceBinding, - key: String, + key: HandProviderCacheKey, effective: &'a EffectiveSandboxProfile, policy: &'a HandLeasePolicy, call_scope: ToolCallScope<'a>, } +fn active_hand_capacity_request( + workspace_binding: &WorkspaceBinding, + lease: &HandLease, +) -> Result { + Ok(ActiveHandCapacityRequest { + tenant_id: workspace_binding.tenant_id, + workspace_id: workspace_binding.workspace_id, + provider_account_id: workspace_binding.provider_account_id, + provider_account_generation: i64::try_from(workspace_binding.provider_account_generation) + .map_err(|_| { + MoaError::ValidationError( + "hand capacity provider-account generation overflows Postgres bigint".to_string(), + ) + })?, + provisioning_operation_id: lease.provisioning_operation_id, + hand_lease_generation: lease.generation, + expected_writer_epoch: i64::try_from(workspace_binding.writer_epoch).map_err(|_| { + MoaError::ValidationError( + "hand capacity writer epoch overflows Postgres bigint".to_string(), + ) + })?, + expected_instance_generation: i64::try_from(workspace_binding.instance_generation) + .map_err(|_| { + MoaError::ValidationError( + "hand capacity instance generation overflows Postgres bigint".to_string(), + ) + })?, + }) +} + struct CreateHandContext<'a> { route: &'a HandRoute, workspace_binding: &'a WorkspaceBinding, @@ -122,7 +165,7 @@ struct ProvisionedHandRecoveryContext<'a> { worker_id: &'a str, provider: &'a str, workspace_binding: &'a WorkspaceBinding, - cache_key: &'a str, + cache_key: &'a HandProviderCacheKey, lease: &'a HandLease, policy: &'a HandLeasePolicy, call_scope: ToolCallScope<'a>, @@ -134,7 +177,7 @@ struct ProvisionedHandActivationContext<'a> { worker_id: &'a str, provider: &'a str, workspace_binding: &'a WorkspaceBinding, - cache_key: &'a str, + cache_key: &'a HandProviderCacheKey, lease: &'a HandLease, handle: HandHandle, call_scope: ToolCallScope<'a>, @@ -199,9 +242,7 @@ impl ToolRouter { return; } - // Installed markers are nested under the exact structured scope, so a - // manifest change never scans or invalidates another session. - self.hands.installed_files.write().await.remove(&scope); + self.hands.clear_installed_files_for_scope(&scope).await; } pub(super) async fn install_trusted_files_for_hand( @@ -333,32 +374,63 @@ impl ToolRouter { session_id: &moa_core::types::identifiers::SessionId, scope: Option<&str>, ) -> bool { + matches!( + self.reclaim_hands_page(tenant_id, session_id, scope).await, + SessionHandReleasePageOutcome::Complete + ) + } + + /// Reclaims one bounded page of all hand ownership under a terminal session. + pub async fn reclaim_session_hands_page( + &self, + tenant_id: TenantId, + session_id: &moa_core::types::identifiers::SessionId, + ) -> SessionHandReleasePageOutcome { + self.reclaim_hands_page(tenant_id, session_id, None).await + } + + async fn reclaim_hands_page( + &self, + tenant_id: TenantId, + session_id: &moa_core::types::identifiers::SessionId, + scope: Option<&str>, + ) -> SessionHandReleasePageOutcome { let mut complete = true; - let session_prefix = format!("{session_id}:"); - let match_prefix = match scope { - Some(worker_id) => format!("{session_prefix}{worker_id}:"), - None => session_prefix.clone(), - }; - match scope { - Some(worker_id) => { - let scope_key = format!("{session_prefix}{worker_id}"); - self.hands - .preferred_hand_routes - .write() - .await - .remove(&scope_key); - } - None => { - self.hands - .preferred_hand_routes - .write() - .await - .retain(|key, _| !key.starts_with(&session_prefix)); - } + let mut made_progress = false; + if let Some(worker_id) = scope { + let scope_key = HandScopeKey::new(tenant_id, *session_id, worker_id); + made_progress |= self + .hands + .preferred_hand_routes + .write() + .await + .remove(&scope_key) + .is_some(); } if let Some(lease_store) = &self.hands.hand_leases { - match lease_store.list_session(tenant_id, *session_id).await { + let (leases, batch_saturated) = match scope { + Some(worker_id) => { + let leases = lease_store + .list_live_owner_candidates(tenant_id, *session_id, worker_id) + .await; + let saturated = leases.as_ref().is_ok_and(|leases| leases.len() == 2); + (leases, saturated) + } + None => { + let page = lease_store + .list_live_session_page(tenant_id, *session_id, None) + .await; + let saturated = page.as_ref().is_ok_and(|page| page.next_cursor.is_some()); + (page.map(|page| page.leases), saturated) + } + }; + if batch_saturated { + // Process a bounded owner batch, but require another pass before + // declaring an owner with multiple live providers fully released. + complete = false; + } + match leases { Ok(leases) => { for lease in leases { if let Some(worker_id) = scope @@ -370,6 +442,10 @@ impl ToolRouter { continue; } if self.hands.workspace_repository.is_some() && lease.attachment.is_some() { + // Durable workspace teardown is owned by the reaper after this handoff. + // Keep the session continuation live until a later page observes that + // the exact lease has reached Destroyed and no longer appears here. + complete = false; let terminal_status = match lease.status { HandLeaseStatus::Provisioning | HandLeaseStatus::Failed => { HandLeaseStatus::Failed @@ -388,7 +464,7 @@ impl ToolRouter { .transition_status(tenant_id, &lease, terminal_status) .await { - Ok(true) => {} + Ok(true) => made_progress = true, Ok(false) => { complete = false; continue; @@ -409,6 +485,7 @@ impl ToolRouter { } if let Some(lease_handle) = lease.handle.as_ref() { let key = session_provider_key_from_parts( + tenant_id, lease.session_id, &lease.worker_id, &lease.provider, @@ -421,7 +498,11 @@ impl ToolRouter { .await; } self.remove_installed_marker( - manifest_scope_key_from_parts(lease.session_id, &lease.worker_id), + manifest_scope_key_from_parts( + tenant_id, + lease.session_id, + &lease.worker_id, + ), &lease.provider, ) .await; @@ -452,6 +533,7 @@ impl ToolRouter { } }; let key = session_provider_key_from_parts( + tenant_id, lease.session_id, &lease.worker_id, &lease.provider, @@ -465,7 +547,11 @@ impl ToolRouter { .await; } self.remove_installed_marker( - manifest_scope_key_from_parts(lease.session_id, &lease.worker_id), + manifest_scope_key_from_parts( + tenant_id, + lease.session_id, + &lease.worker_id, + ), &lease.provider, ) .await; @@ -513,7 +599,7 @@ impl ToolRouter { .finalize_destroy(tenant_id, &lease, claim_token) .await { - Ok(true) => {} + Ok(true) => made_progress = true, Ok(false) => complete = false, Err(error) => { complete = false; @@ -568,40 +654,78 @@ impl ToolRouter { ); } } - } else { - let hands = { - let mut active_hands = self.hands.active_hands.write().await; - let keys = active_hands - .keys() - .filter(|key| key.starts_with(&match_prefix)) - .cloned() - .collect::>(); - keys.into_iter() - .filter_map(|key| active_hands.remove(&key).map(|hand| (key, hand))) - .collect::>() - }; - for (key, hand) in hands { - let remainder = key.strip_prefix(&session_prefix).unwrap_or_default(); - let (_, provider_name) = remainder.rsplit_once(':').unwrap_or(("", remainder)); - let Some(provider) = self.hands.providers.get(provider_name) else { - complete = false; - continue; - }; - if let Err(error) = provider.destroy(&hand.handle).await { - complete = false; - tracing::warn!( - session_id = %session_id, - provider = %provider_name, - hand_id = %hand_id(&hand.handle), - error = %error, - "failed to destroy cached hand" - ); + } + if scope.is_none() { + let mut cursor = SessionCacheDrainCursor::start(); + loop { + match self + .hands + .drain_session_cache_page(tenant_id, *session_id, cursor) + .await + { + Ok(page) => { + made_progress |= page.removed_entries > page.active_hands.len(); + for cached in page.active_hands { + let Some(provider) = self.hands.providers.get(&cached.provider) else { + complete = false; + self.hands + .active_hands + .write() + .await + .insert(cached.key, cached.hand); + continue; + }; + if let Err(error) = provider.destroy(&cached.hand.handle).await { + complete = false; + let hand_id = hand_id(&cached.hand.handle); + tracing::warn!( + session_id = %session_id, + provider = %cached.provider, + hand_id = %hand_id, + error = %error, + "failed to destroy cached hand" + ); + self.hands + .active_hands + .write() + .await + .insert(cached.key, cached.hand); + } else { + made_progress = true; + } + } + let page_was_empty = page.removed_entries == 0; + cursor = page.next_cursor; + if matches!(cursor, SessionCacheDrainCursor::Complete) { + break; + } + if !page_was_empty { + complete = false; + break; + } + } + Err(error) => { + complete = false; + tracing::warn!( + session_id = %session_id, + error = %error, + "failed to drain session hand caches" + ); + break; + } } } + } else { + self.clear_manifest_scopes(tenant_id, *session_id, scope) + .await; + } + if complete { + SessionHandReleasePageOutcome::Complete + } else if made_progress { + SessionHandReleasePageOutcome::Progressed + } else { + SessionHandReleasePageOutcome::Waiting } - - self.clear_manifest_scopes(*session_id, scope).await; - complete } /// Provisions or reuses a hand on behalf of a run with `budget` left. @@ -813,7 +937,7 @@ impl ToolRouter { pub(in crate::core) async fn remove_cached_binding_if_matches( &self, - key: &str, + key: &HandProviderCacheKey, expected_handle: &HandHandle, expected_generation: Option, ) { @@ -862,63 +986,16 @@ impl ToolRouter { }) .await? { - let mut claim = claim; + let claim = claim; if let Err(error) = call_scope.admit() { let _ = lease_store .transition_status(session.tenant_id, &claim, HandLeaseStatus::Failed) .await?; return Err(error); } - if let Some(previous_handle) = claim.handle.as_ref() { - let Some(provider_impl) = self.hands.providers.get(provider) else { - let _ = lease_store - .transition_status(session.tenant_id, &claim, HandLeaseStatus::Failed) - .await?; - return Err(MoaError::ProviderError(format!( - "unknown hand provider: {provider}" - ))); - }; - if let Err(error) = call_scope.admit() { - let _ = lease_store - .transition_status(session.tenant_id, &claim, HandLeaseStatus::Failed) - .await?; - return Err(error); - } - // Once destroy is dispatched it is deliberately outside a - // DeadlineGuard. Its exact provisioning fence is finalized - // even if cancellation arrives while the provider runs. - if let Err(error) = destroy_provisioning_operations( - provider_impl.as_ref(), - previous_handle - .handle - .provider_account() - .map_or(ProviderAccountId(Uuid::nil()), |context| context.0), - previous_handle - .handle - .provider_account() - .map_or(0, |context| context.1), - claim.provisioning_operation_id, - Some(previous_handle), - ProvisioningAbsenceProof::Immediate, - ) - .await - { - let _ = lease_store - .transition_status(session.tenant_id, &claim, HandLeaseStatus::Failed) - .await?; - return Err(error); - } - if !lease_store - .clear_handle_for_provisioning(session.tenant_id, &claim) - .await? - { - return Err(MoaError::StorageError(format!( - "hand lease replacement lost generation fence for session {} provider {provider}", - session.id - ))); - } - claim.handle = None; - if let Err(error) = call_scope.admit() { + if let Some(capacity) = self.hands.workspace_capacity.as_ref() { + let request = active_hand_capacity_request(workspace_binding, &claim)?; + if let Err(error) = capacity.reserve_active_hand(&request).await { let _ = lease_store .transition_status(session.tenant_id, &claim, HandLeaseStatus::Failed) .await?; @@ -1324,6 +1401,19 @@ impl ToolRouter { }; match activated { Ok(true) => { + if let Some(capacity) = self.hands.workspace_capacity.as_ref() + && !capacity + .commit_active_hand(&active_hand_capacity_request( + workspace_binding, + lease, + )?) + .await? + { + return Err(MoaError::StorageError(format!( + "hand activation lost its exact capacity fence for session {} provider {provider}", + session.id + ))); + } let active = ActiveHand { handle: handle.clone(), generation: Some(lease.generation), @@ -1332,7 +1422,7 @@ impl ToolRouter { .active_hands .write() .await - .insert(cache_key.to_string(), active.clone()); + .insert(cache_key.clone(), active.clone()); self.remember_preactivation_manifest_install( session, worker_id, @@ -1356,6 +1446,10 @@ impl ToolRouter { && current.attachment == lease.attachment }); if already_active { + if let Some(capacity) = self.hands.workspace_capacity.as_ref() { + let request = active_hand_capacity_request(workspace_binding, lease)?; + let _ = capacity.commit_active_hand(&request).await?; + } let active = ActiveHand { handle: handle.clone(), generation: Some(lease.generation), @@ -1364,7 +1458,7 @@ impl ToolRouter { .active_hands .write() .await - .insert(cache_key.to_string(), active.clone()); + .insert(cache_key.clone(), active.clone()); self.remember_preactivation_manifest_install( session, worker_id, @@ -1559,7 +1653,7 @@ impl ToolRouter { &self, provider: &str, lease: &HandLease, - key: &str, + key: &HandProviderCacheKey, call_scope: ToolCallScope<'_>, ) -> Result { let lease_handle = lease.handle.as_ref().ok_or_else(|| { @@ -1590,7 +1684,7 @@ impl ToolRouter { } } self.hands.active_hands.write().await.insert( - key.to_string(), + key.clone(), ActiveHand { handle: handle.clone(), generation: Some(lease.generation), @@ -1658,31 +1752,18 @@ impl ToolRouter { async fn clear_manifest_scopes( &self, + tenant_id: TenantId, session_id: moa_core::types::identifiers::SessionId, worker_id: Option<&str>, ) { - match worker_id { - Some(worker_id) => { - let scope = manifest_scope_key_from_parts(session_id, worker_id); - self.hands - .trusted_sandbox_files - .write() - .await - .remove(&scope); - self.hands.installed_files.write().await.remove(&scope); - } - None => { - self.hands - .trusted_sandbox_files - .write() - .await - .retain(|scope, _| scope.session_id != session_id); - self.hands - .installed_files - .write() - .await - .retain(|scope, _| scope.session_id != session_id); - } + if let Some(worker_id) = worker_id { + let scope = manifest_scope_key_from_parts(tenant_id, session_id, worker_id); + self.hands + .trusted_sandbox_files + .write() + .await + .remove(&scope); + self.hands.clear_installed_files_for_scope(&scope).await; } } @@ -1800,51 +1881,52 @@ pub(super) fn workspace_lease_scope(scope: &SandboxWorkspaceScope) -> String { /// Returns the scope key that namespaces a session's hands by typed owner. /// -/// A populated lease key yields `"{session_id}:{owner_key}"`. `None` yields the -/// non-owning `"{session_id}:"` aggregate key used only by session-wide -/// bookkeeping. Sandbox admission never accepts that aggregate key as a -/// workspace owner. All keys share the `"{session_id}:"` prefix so session -/// teardown can match every typed owner scope at once. -pub(super) fn scope_key(session: &SessionMeta, worker_id: Option<&str>) -> String { - format!("{}:{}", session.id, worker_id.unwrap_or_default()) +/// Tenant and session identity remain typed fields, so aggregate cleanup can +/// use a bounded ordered range without parsing or prefix-scanning string keys. +/// `None` uses an empty non-owning worker key for sandbox-free bookkeeping; +/// sandbox admission never accepts it as a workspace owner. +pub(super) fn scope_key(session: &SessionMeta, worker_id: Option<&str>) -> HandScopeKey { + HandScopeKey::new(session.tenant_id, session.id, worker_id.unwrap_or_default()) } pub(in crate::core) fn manifest_scope_key( session: &SessionMeta, worker_id: Option<&str>, ) -> HandScopeKey { - manifest_scope_key_from_parts(session.id, worker_id.unwrap_or_default()) + manifest_scope_key_from_parts(session.tenant_id, session.id, worker_id.unwrap_or_default()) } fn manifest_scope_key_from_parts( + tenant_id: TenantId, session_id: moa_core::types::identifiers::SessionId, worker_id: &str, ) -> HandScopeKey { - HandScopeKey { - session_id, - worker_id: worker_id.to_string(), - } + HandScopeKey::new(tenant_id, session_id, worker_id) } /// Returns the cache/lease key for one hand within a typed owner scope. /// -/// The format is `"{session_id}:{owner_key}:{provider}"`. A missing owner key -/// produces the reserved non-owning aggregate form -/// `"{session_id}::{provider}"`; sandbox dispatch never provisions against it. +/// Tenant, session, owner, and provider remain separate typed fields. A missing +/// owner uses the reserved empty non-owning worker key; sandbox dispatch never +/// provisions against it. pub(super) fn session_provider_key( session: &SessionMeta, worker_id: Option<&str>, provider: &str, -) -> String { - format!("{}:{provider}", scope_key(session, worker_id)) +) -> HandProviderCacheKey { + HandProviderCacheKey::new(scope_key(session, worker_id), provider) } fn session_provider_key_from_parts( + tenant_id: TenantId, session_id: moa_core::types::identifiers::SessionId, worker_id: &str, provider: &str, -) -> String { - format!("{session_id}:{worker_id}:{provider}") +) -> HandProviderCacheKey { + HandProviderCacheKey::new( + HandScopeKey::new(tenant_id, session_id, worker_id), + provider, + ) } /// Returns the idle deadline a renewal should ask for under `policy`. diff --git a/crates/moa-hands/src/core/lifecycle/tests.rs b/crates/moa-hands/src/core/lifecycle/tests.rs index c29038d7a..4ec58bf8e 100644 --- a/crates/moa-hands/src/core/lifecycle/tests.rs +++ b/crates/moa-hands/src/core/lifecycle/tests.rs @@ -181,12 +181,56 @@ impl HandLeaseStore for BarrierHandLeaseStore { .await } - async fn list_session( + async fn get_exact_generation( &self, tenant_id: TenantId, session_id: moa_core::types::identifiers::SessionId, + worker_id: &str, + provisioning_operation_id: moa_core::types::identifiers::HandProvisioningOperationId, + generation: i64, + ) -> Result> { + self.inner + .get_exact_generation( + tenant_id, + session_id, + worker_id, + provisioning_operation_id, + generation, + ) + .await + } + + async fn list_live_owner_candidates( + &self, + tenant_id: TenantId, + session_id: moa_core::types::identifiers::SessionId, + worker_id: &str, ) -> Result> { - self.inner.list_session(tenant_id, session_id).await + self.inner + .list_live_owner_candidates(tenant_id, session_id, worker_id) + .await + } + + async fn has_live_owner( + &self, + tenant_id: TenantId, + session_id: moa_core::types::identifiers::SessionId, + worker_id: &str, + ) -> Result { + self.inner + .has_live_owner(tenant_id, session_id, worker_id) + .await + } + + async fn list_live_session_page( + &self, + tenant_id: TenantId, + session_id: moa_core::types::identifiers::SessionId, + cursor: Option<&crate::core::leases::HandLeaseSessionCursor>, + ) -> Result { + self.inner + .list_live_session_page(tenant_id, session_id, cursor) + .await } async fn activate(&self, request: HandLeaseActivateRequest<'_>) -> Result { @@ -1108,13 +1152,13 @@ async fn stale_manifest_install_completion_cannot_replace_new_hand_marker() { .expect("the first install task should join") .expect_err("the stale hand A install must lose its active-binding fence"); - let scope = manifest_scope_key(&session, Some(TEST_WORKER_ID)); + let marker_scope = manifest_scope_key(&session, Some(TEST_WORKER_ID)); let marker = router .hands .installed_files .read() .await - .get(&scope) + .get(&marker_scope) .and_then(|providers| providers.get("manifest-race")) .cloned() .expect("replacement hand B has an installed marker"); @@ -1386,6 +1430,54 @@ async fn lifecycle_destroy_session_reads_durable_leases_not_only_cache() { assert_eq!(provider.destroy_calls(), 1); } +#[tokio::test] +async fn lifecycle_session_cleanup_reports_incomplete_until_every_cache_page_is_destroyed() { + // Pins: terminal session cleanup processes at most one non-empty 64-entry cache page per + // activation, reports incomplete while another page remains, and reaches exact completion + // without leaking the final short page. + let lease_store = MemoryHandLeaseStore::shared(); + let provider = Arc::new(CountingProvider::new("paged-session-cleanup")); + let session = session(); + let router = router(provider.clone(), lease_store); + let count = crate::core::HAND_LEASE_SESSION_PAGE_SIZE + 7; + + for index in 0..count { + let scope = HandScopeKey::new( + session.tenant_id, + session.id, + format!("session-owner-{index:03}"), + ); + router.hands.active_hands.write().await.insert( + crate::core::HandProviderCacheKey::new(scope, provider.provider_name()), + ActiveHand { + handle: HandHandle::local(std::path::PathBuf::from(format!( + "/tmp/session-cleanup-{index}" + ))), + generation: None, + }, + ); + } + + assert!( + !router + .reclaim_hands(session.tenant_id, &session.id, None) + .await, + "the first bounded page must request a durable continuation" + ); + assert_eq!( + provider.destroy_calls(), + crate::core::HAND_LEASE_SESSION_PAGE_SIZE + ); + assert!( + router + .reclaim_hands(session.tenant_id, &session.id, None) + .await, + "the final short page must prove complete cleanup" + ); + assert_eq!(provider.destroy_calls(), count); + assert!(router.hands.active_hands.read().await.is_empty()); +} + #[tokio::test] async fn lifecycle_cached_active_hand_is_renewed_and_stale_cache_not_reused() { // Pins: cached durable hands are revalidated and renewed before reuse. diff --git a/crates/moa-hands/src/core/maintenance_provider_inventory.rs b/crates/moa-hands/src/core/maintenance_provider_inventory.rs new file mode 100644 index 000000000..3107046da --- /dev/null +++ b/crates/moa-hands/src/core/maintenance_provider_inventory.rs @@ -0,0 +1,177 @@ +//! Maintenance-only construction of sandbox compute and storage providers. + +use std::sync::Arc; + +use moa_config::{CloudHandProviderKind, MoaConfig}; +use moa_core::{ + error::{MoaError, Result}, + traits::{HandProvider, SandboxStorageProvider}, +}; +use moa_crypto::KeyManagementProvider; +use sqlx::PgPool; + +use super::{ + DEFAULT_TOOL_TIMEOUT, + normalization::expand_local_path, + provider_credentials::{FileProviderCredentialSource, ProviderCredentialSource}, + sandbox_workspace::{ + capacity::PostgresWorkspaceCapacityRepository, checkpoint::store::CheckpointObjectStore, + operations::PostgresWorkspaceOperationRepository, repository::PostgresWorkspaceRepository, + storage_resources::PostgresWorkspaceStorageResourceRepository, + }, +}; +use crate::adapters::{ + daytona::{DaytonaHandProvider, storage::DaytonaStorageDependencies}, + e2b::E2BHandProvider, + local::LocalHandProvider, +}; + +/// Exact provider adapters needed by durable sandbox maintenance. +/// +/// This inventory intentionally contains no tool catalog, action-policy owner, +/// session store, connector state, or background task. The orchestrator owns +/// maintenance loops separately and uses these adapters only for durable +/// provider inventory, reconciliation, and destruction. +pub struct SandboxProviderInventory { + hand_providers: Vec>, + storage_providers: Vec>, +} + +impl SandboxProviderInventory { + /// Constructs every provider kind with a configured durable account mapping. + /// + /// Maintenance selection follows provider-account inventory rather than the + /// active tool route. An account generation may still own resources after it + /// stops being the default or fallback admission route, so cleanup must keep + /// its provider adapter available until the durable state is drained. + pub async fn for_maintenance( + config: &MoaConfig, + workspace_pool: &PgPool, + checkpoint_store: Arc, + workspace_kms: Arc, + ) -> Result { + if !config.sandbox_workspaces.mode.maintenance_enabled() { + return Err(MoaError::ConfigError( + "sandbox provider maintenance inventory requires maintenance or admit mode" + .to_string(), + )); + } + if !workspace_kms.is_durable() { + return Err(MoaError::ConfigError( + "sandbox provider maintenance inventory requires durable KMS authority".to_string(), + )); + } + + let capacity = Arc::new(PostgresWorkspaceCapacityRepository::new( + workspace_pool.clone(), + )); + let mut hand_providers: Vec> = Vec::new(); + let mut storage_providers: Vec> = Vec::new(); + + if config.local.provider_account.is_some() { + let sandbox_root = expand_local_path(&config.local.sandbox_dir)?; + let provider = Arc::new( + LocalHandProvider::new_with_docker_detection( + sandbox_root, + config.local.docker_enabled, + ) + .await? + .with_command_timeout(DEFAULT_TOOL_TIMEOUT) + .with_checkpoint_store(Arc::clone(&checkpoint_store)) + .with_checkpoint_capacity(Arc::clone(&capacity)), + ); + hand_providers.push(provider.clone()); + storage_providers.push(provider); + } + + if let Some(hands) = &config.cloud.hands { + let has_daytona = hands + .provider_accounts + .iter() + .any(|account| account.provider == CloudHandProviderKind::Daytona); + let has_e2b = hands + .provider_accounts + .iter() + .any(|account| account.provider == CloudHandProviderKind::E2b); + let credentials = if has_daytona || has_e2b { + let source = Arc::new(FileProviderCredentialSource::from_config(hands)?); + source.validate_all().await?; + Some(source as Arc) + } else { + None + }; + + if has_daytona { + let credentials = credentials.as_ref().ok_or_else(|| { + MoaError::ConfigError( + "Daytona maintenance credential source is unavailable".to_string(), + ) + })?; + let provider = Arc::new(DaytonaHandProvider::new_with_storage( + Arc::clone(credentials), + DaytonaStorageDependencies { + config: config.cloud.daytona_storage.clone(), + checkpoint_store: Arc::clone(&checkpoint_store), + workspaces: Arc::new(PostgresWorkspaceRepository::new( + workspace_pool.clone(), + )), + storage_resources: Arc::new( + PostgresWorkspaceStorageResourceRepository::new(workspace_pool.clone()), + ), + operations: Arc::new(PostgresWorkspaceOperationRepository::new( + workspace_pool.clone(), + )), + capacity: Arc::clone(&capacity), + kms: Arc::clone(&workspace_kms), + }, + )?); + hand_providers.push(provider.clone()); + storage_providers.push(provider); + } + + if has_e2b { + let credentials = credentials.as_ref().ok_or_else(|| { + MoaError::ConfigError( + "E2B maintenance credential source is unavailable".to_string(), + ) + })?; + let provider = Arc::new( + E2BHandProvider::new(Arc::clone(credentials)) + .with_checkpoint_store(Arc::clone(&checkpoint_store)) + .with_checkpoint_capacity(Arc::clone(&capacity)), + ); + hand_providers.push(provider.clone()); + storage_providers.push(provider); + } + } + + if hand_providers.is_empty() || storage_providers.is_empty() { + return Err(MoaError::ConfigError( + "sandbox provider maintenance inventory requires a configured provider account" + .to_string(), + )); + } + hand_providers.sort_by(|left, right| left.provider_name().cmp(right.provider_name())); + storage_providers.sort_by(|left, right| { + left.storage_provider_name() + .cmp(right.storage_provider_name()) + }); + + Ok(Self { + hand_providers, + storage_providers, + }) + } + + /// Returns hand providers in stable provider-name order. + #[must_use] + pub fn hand_providers(&self) -> Vec> { + self.hand_providers.clone() + } + + /// Returns workspace-storage providers in stable provider-name order. + #[must_use] + pub fn storage_providers(&self) -> Vec> { + self.storage_providers.clone() + } +} diff --git a/crates/moa-hands/src/core/mod.rs b/crates/moa-hands/src/core/mod.rs index ebd151a6f..0c3135ce8 100644 --- a/crates/moa-hands/src/core/mod.rs +++ b/crates/moa-hands/src/core/mod.rs @@ -4,6 +4,7 @@ mod construction; mod dispatch; pub mod leases; mod lifecycle; +mod maintenance_provider_inventory; pub mod mcp_catalog; mod normalization; mod output_budget; @@ -16,7 +17,11 @@ mod registration; pub mod sandbox_workspace; pub mod telemetry; +pub use lifecycle::SessionHandReleasePageOutcome; + +use std::cmp::Ordering as CmpOrdering; use std::collections::{BTreeMap, HashMap}; +use std::ops::Bound; use std::path::PathBuf; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; @@ -43,10 +48,11 @@ use tokio_util::sync::CancellationToken; use crate::adapters::local::LocalHandProvider; pub use dispatch::{ - AuthorizedToolCall, DeferredWorkspaceToolOutput, JournaledWorkspaceCommit, - PendingConnectorToolOutput, + AuthorizedToolCall, DeferredWorkspaceToolOutput, ExecutionHandReleaseRequest, + JournaledWorkspaceCommit, PendingConnectorToolOutput, }; -use leases::HandLeaseStore; +use leases::{HAND_LEASE_SESSION_PAGE_SIZE, HandLeaseStore}; +pub use maintenance_provider_inventory::SandboxProviderInventory; pub use mcp_catalog::{ CandidateConnector, CatalogDefect, McpCatalogActivation, McpCatalogRefresh, McpConnectorHealth, PinnedToolContract, PinnedToolOwner, ToolCatalogDrift, ToolCatalogPin, @@ -67,6 +73,7 @@ pub use registration::{ HandRoute, MCP_TOOL_REFERENCE_PREFIX, ToolExecution, ToolRegistry, governed_tool_contract_revision, installed_connector_tool_name, mcp_tool_reference, }; +use sandbox_workspace::capacity::PostgresWorkspaceCapacityRepository; use sandbox_workspace::operations::PostgresWorkspaceOperationRepository; use sandbox_workspace::repository::PostgresWorkspaceRepository; pub use telemetry::truncate_tool_span_text; @@ -448,17 +455,18 @@ struct HandLifecycleOwner { providers: HashMap>, storage_providers: HashMap>, local_provider: Option>, - active_hands: RwLock>, - preferred_hand_routes: RwLock>, + active_hands: RwLock>, + preferred_hand_routes: RwLock>, hand_leases: Option>, workspace_repository: Option>, workspace_operations: Option>, + workspace_capacity: Option>, checkpoint_store: Option>, deployment_sandbox_policy: SandboxPolicySnapshot, tenant_sandbox_policy: Option>, hand_lease_reaper_installed: bool, - trusted_sandbox_files: RwLock>>, - installed_files: RwLock>>, + trusted_sandbox_files: RwLock>>, + installed_files: RwLock>>, workspace_roots: RwLock>, sandbox_root: Option, } @@ -473,10 +481,168 @@ struct ActiveHand { /// Exact conversational scope used by trusted and installed manifest caches. #[derive(Debug, Clone, PartialEq, Eq, Hash)] struct HandScopeKey { + tenant_id: TenantId, session_id: SessionId, worker_id: String, } +impl HandScopeKey { + fn new(tenant_id: TenantId, session_id: SessionId, worker_id: impl Into) -> Self { + Self { + tenant_id, + session_id, + worker_id: worker_id.into(), + } + } +} + +impl PartialOrd for HandScopeKey { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for HandScopeKey { + fn cmp(&self, other: &Self) -> CmpOrdering { + (self.tenant_id.0, self.session_id.0, &self.worker_id).cmp(&( + other.tenant_id.0, + other.session_id.0, + &other.worker_id, + )) + } +} + +/// Exact process-local cache key for one provider binding under a typed owner. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct HandProviderCacheKey { + scope: HandScopeKey, + provider: String, +} + +impl HandProviderCacheKey { + fn new(scope: HandScopeKey, provider: impl Into) -> Self { + Self { + scope, + provider: provider.into(), + } + } +} + +impl PartialOrd for HandProviderCacheKey { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for HandProviderCacheKey { + fn cmp(&self, other: &Self) -> CmpOrdering { + (&self.scope, &self.provider).cmp(&(&other.scope, &other.provider)) + } +} + +/// Cursor through the bounded process-local cache cleanup phases for a session. +#[derive(Debug, Clone, PartialEq, Eq)] +enum SessionCacheDrainCursor { + PreferredRoutes(Option), + TrustedSandboxFiles(Option), + InstalledFiles(Option), + ActiveHands(Option), + Complete, +} + +impl SessionCacheDrainCursor { + fn start() -> Self { + Self::PreferredRoutes(None) + } +} + +/// One active process-local hand removed for provider teardown. +#[derive(Debug)] +struct CachedActiveHandRelease { + key: HandProviderCacheKey, + provider: String, + hand: ActiveHand, +} + +/// One bounded process-local session-cache cleanup page. +#[derive(Debug)] +struct SessionCacheDrainPage { + removed_entries: usize, + active_hands: Vec, + next_cursor: SessionCacheDrainCursor, +} + +fn validate_scope_cache_cursor( + tenant_id: TenantId, + session_id: SessionId, + cursor: Option<&HandScopeKey>, +) -> Result<()> { + if cursor.is_some_and(|cursor| cursor.tenant_id != tenant_id || cursor.session_id != session_id) + { + return Err(MoaError::ValidationError( + "session cache cursor does not belong to the requested tenant and session".to_string(), + )); + } + Ok(()) +} + +fn validate_provider_cache_cursor( + tenant_id: TenantId, + session_id: SessionId, + cursor: Option<&HandProviderCacheKey>, +) -> Result<()> { + validate_scope_cache_cursor(tenant_id, session_id, cursor.map(|cursor| &cursor.scope)) +} + +fn session_scope_cache_keys( + entries: &BTreeMap, + tenant_id: TenantId, + session_id: SessionId, + after: Option<&HandScopeKey>, +) -> Vec { + let lower = after + .cloned() + .unwrap_or_else(|| HandScopeKey::new(tenant_id, session_id, String::new())); + let lower_bound = if after.is_some() { + Bound::Excluded(lower) + } else { + Bound::Included(lower) + }; + entries + .range((lower_bound, Bound::Unbounded)) + .take_while(|(key, _)| key.tenant_id == tenant_id && key.session_id == session_id) + .take(HAND_LEASE_SESSION_PAGE_SIZE + 1) + .map(|(key, _)| key.clone()) + .collect() +} + +fn session_provider_cache_keys( + entries: &BTreeMap, + tenant_id: TenantId, + session_id: SessionId, + after: Option<&HandProviderCacheKey>, +) -> Vec { + let lower = after.cloned().unwrap_or_else(|| { + HandProviderCacheKey::new( + HandScopeKey::new(tenant_id, session_id, String::new()), + String::new(), + ) + }); + let lower_bound = if after.is_some() { + Bound::Excluded(lower) + } else { + Bound::Included(lower) + }; + entries + .range((lower_bound, Bound::Unbounded)) + .take_while(|(key, _)| { + key.scope.tenant_id == tenant_id && key.scope.session_id == session_id + }) + .take(HAND_LEASE_SESSION_PAGE_SIZE + 1) + .map(|(key, _)| key.clone()) + .collect() +} + /// One immutable trusted-file publication shared cheaply across dispatches. #[derive(Debug)] struct TrustedSandboxManifest { @@ -502,17 +668,18 @@ impl HandLifecycleOwner { providers, storage_providers: HashMap::new(), local_provider: None, - active_hands: RwLock::new(HashMap::new()), - preferred_hand_routes: RwLock::new(HashMap::new()), + active_hands: RwLock::new(BTreeMap::new()), + preferred_hand_routes: RwLock::new(BTreeMap::new()), hand_leases: None, workspace_repository: None, workspace_operations: None, + workspace_capacity: None, checkpoint_store: None, deployment_sandbox_policy, tenant_sandbox_policy: None, hand_lease_reaper_installed: false, - trusted_sandbox_files: RwLock::new(HashMap::new()), - installed_files: RwLock::new(HashMap::new()), + trusted_sandbox_files: RwLock::new(BTreeMap::new()), + installed_files: RwLock::new(BTreeMap::new()), workspace_roots: RwLock::new(HashMap::new()), sandbox_root: None, } @@ -528,6 +695,123 @@ impl HandLifecycleOwner { .collect() } + /// Removes one bounded, session-indexed page of process-local lifecycle state. + /// + /// Each call advances exactly one cache family. Active bindings are returned + /// to the caller for provider teardown; the other cache families carry no + /// external resources. Supplying a cursor for another tenant or session is + /// rejected before any entry is removed. + async fn drain_session_cache_page( + &self, + tenant_id: TenantId, + session_id: SessionId, + cursor: SessionCacheDrainCursor, + ) -> Result { + match cursor { + SessionCacheDrainCursor::PreferredRoutes(after) => { + validate_scope_cache_cursor(tenant_id, session_id, after.as_ref())?; + let mut routes = self.preferred_hand_routes.write().await; + let mut keys = + session_scope_cache_keys(&routes, tenant_id, session_id, after.as_ref()); + let has_more = keys.len() > HAND_LEASE_SESSION_PAGE_SIZE; + keys.truncate(HAND_LEASE_SESSION_PAGE_SIZE); + for key in &keys { + routes.remove(key); + } + let next_cursor = if has_more { + SessionCacheDrainCursor::PreferredRoutes(keys.last().cloned()) + } else { + SessionCacheDrainCursor::TrustedSandboxFiles(None) + }; + Ok(SessionCacheDrainPage { + removed_entries: keys.len(), + active_hands: Vec::new(), + next_cursor, + }) + } + SessionCacheDrainCursor::TrustedSandboxFiles(after) => { + validate_scope_cache_cursor(tenant_id, session_id, after.as_ref())?; + let mut manifests = self.trusted_sandbox_files.write().await; + let mut keys = + session_scope_cache_keys(&manifests, tenant_id, session_id, after.as_ref()); + let has_more = keys.len() > HAND_LEASE_SESSION_PAGE_SIZE; + keys.truncate(HAND_LEASE_SESSION_PAGE_SIZE); + for key in &keys { + manifests.remove(key); + } + let next_cursor = if has_more { + SessionCacheDrainCursor::TrustedSandboxFiles(keys.last().cloned()) + } else { + SessionCacheDrainCursor::InstalledFiles(None) + }; + Ok(SessionCacheDrainPage { + removed_entries: keys.len(), + active_hands: Vec::new(), + next_cursor, + }) + } + SessionCacheDrainCursor::InstalledFiles(after) => { + validate_scope_cache_cursor(tenant_id, session_id, after.as_ref())?; + let mut installed = self.installed_files.write().await; + let mut keys = + session_scope_cache_keys(&installed, tenant_id, session_id, after.as_ref()); + let has_more = keys.len() > HAND_LEASE_SESSION_PAGE_SIZE; + keys.truncate(HAND_LEASE_SESSION_PAGE_SIZE); + for key in &keys { + installed.remove(key); + } + let next_cursor = if has_more { + SessionCacheDrainCursor::InstalledFiles(keys.last().cloned()) + } else { + SessionCacheDrainCursor::ActiveHands(None) + }; + Ok(SessionCacheDrainPage { + removed_entries: keys.len(), + active_hands: Vec::new(), + next_cursor, + }) + } + SessionCacheDrainCursor::ActiveHands(after) => { + validate_provider_cache_cursor(tenant_id, session_id, after.as_ref())?; + let mut hands = self.active_hands.write().await; + let mut keys = + session_provider_cache_keys(&hands, tenant_id, session_id, after.as_ref()); + let has_more = keys.len() > HAND_LEASE_SESSION_PAGE_SIZE; + keys.truncate(HAND_LEASE_SESSION_PAGE_SIZE); + let mut active_hands = Vec::with_capacity(keys.len()); + for key in &keys { + if let Some(hand) = hands.remove(key) { + active_hands.push(CachedActiveHandRelease { + key: key.clone(), + provider: key.provider.clone(), + hand, + }); + } + } + let next_cursor = if has_more { + SessionCacheDrainCursor::ActiveHands(keys.last().cloned()) + } else { + SessionCacheDrainCursor::Complete + }; + Ok(SessionCacheDrainPage { + removed_entries: active_hands.len(), + active_hands, + next_cursor, + }) + } + SessionCacheDrainCursor::Complete => Ok(SessionCacheDrainPage { + removed_entries: 0, + active_hands: Vec::new(), + next_cursor: SessionCacheDrainCursor::Complete, + }), + } + } + + /// Clears every installed-manifest marker for one exact typed owner. + async fn clear_installed_files_for_scope(&self, scope: &HandScopeKey) { + self.installed_files.write().await.remove(scope); + } + /// Installs the provider-neutral persistent-workspace adapters. fn set_storage_providers( &mut self, @@ -561,9 +845,11 @@ impl HandLifecycleOwner { &mut self, workspaces: Arc, operations: Arc, + capacity: Arc, ) { self.workspace_repository = Some(workspaces); self.workspace_operations = Some(operations); + self.workspace_capacity = Some(capacity); } /// Records that the deployment starts its durable lease reaper. @@ -797,4 +1083,171 @@ mod tests { .await .expect("a failed or immediately stale refresh must be retryable"); } + + #[tokio::test] + async fn session_cache_drain_is_bounded_and_tenant_session_indexed_offline() { + // Pins: terminal cleanup with more than one cache page removes only the + // exact tenant/session through ordered ranges and returns every active + // binding for teardown without a global retain or unbounded collect. + let owner = HandLifecycleOwner::new( + HashMap::new(), + crate::core::profile::local_development_sandbox_policy(), + ); + let tenant_id = TenantId::new(); + let unrelated_tenant_id = TenantId::new(); + let session_id = SessionId::new(); + let count = HAND_LEASE_SESSION_PAGE_SIZE + 7; + + for index in 0..count { + let scope = HandScopeKey::new(tenant_id, session_id, format!("owner-{index:03}")); + let provider_key = HandProviderCacheKey::new(scope.clone(), "local"); + owner + .preferred_hand_routes + .write() + .await + .insert(scope.clone(), "local".to_string()); + owner.trusted_sandbox_files.write().await.insert( + scope, + Arc::new(TrustedSandboxManifest { + identity: uuid::Uuid::new_v4(), + files: Vec::::new().into(), + }), + ); + owner + .installed_files + .write() + .await + .entry(provider_key.scope.clone()) + .or_default() + .insert( + "local".to_string(), + InstalledManifestMarker { + manifest_identity: uuid::Uuid::new_v4(), + handle: HandHandle::local(PathBuf::from(format!("/tmp/installed-{index}"))), + generation: Some(1), + }, + ); + owner.active_hands.write().await.insert( + provider_key, + ActiveHand { + handle: HandHandle::local(PathBuf::from(format!("/tmp/active-{index}"))), + generation: Some(1), + }, + ); + } + + let unrelated_scope = HandScopeKey::new(unrelated_tenant_id, session_id, "unrelated-owner"); + let unrelated_provider_key = HandProviderCacheKey::new(unrelated_scope.clone(), "local"); + owner + .preferred_hand_routes + .write() + .await + .insert(unrelated_scope.clone(), "local".to_string()); + owner.trusted_sandbox_files.write().await.insert( + unrelated_scope, + Arc::new(TrustedSandboxManifest { + identity: uuid::Uuid::new_v4(), + files: Vec::::new().into(), + }), + ); + owner + .installed_files + .write() + .await + .entry(unrelated_provider_key.scope.clone()) + .or_default() + .insert( + "local".to_string(), + InstalledManifestMarker { + manifest_identity: uuid::Uuid::new_v4(), + handle: HandHandle::local(PathBuf::from("/tmp/unrelated-installed")), + generation: Some(1), + }, + ); + owner.active_hands.write().await.insert( + unrelated_provider_key, + ActiveHand { + handle: HandHandle::local(PathBuf::from("/tmp/unrelated-active")), + generation: Some(1), + }, + ); + + let first = owner + .drain_session_cache_page(tenant_id, session_id, SessionCacheDrainCursor::start()) + .await + .expect("drain first preferred-route page"); + assert_eq!(first.removed_entries, HAND_LEASE_SESSION_PAGE_SIZE); + assert!(matches!( + first.next_cursor, + SessionCacheDrainCursor::PreferredRoutes(Some(_)) + )); + + let mut cursor = first.next_cursor; + let mut removed = first.removed_entries; + let mut released = first.active_hands.len(); + let mut calls = 1; + while cursor != SessionCacheDrainCursor::Complete { + let page = owner + .drain_session_cache_page(tenant_id, session_id, cursor) + .await + .expect("drain next exact-session cache page"); + assert!(page.removed_entries <= HAND_LEASE_SESSION_PAGE_SIZE); + removed += page.removed_entries; + released += page.active_hands.len(); + calls += 1; + assert!(calls <= 12, "the four two-page phases must terminate"); + cursor = page.next_cursor; + } + + assert_eq!(removed, count * 4); + assert_eq!(released, count); + assert_eq!(owner.preferred_hand_routes.read().await.len(), 1); + assert_eq!(owner.trusted_sandbox_files.read().await.len(), 1); + assert_eq!(owner.installed_files.read().await.len(), 1); + assert_eq!(owner.active_hands.read().await.len(), 1); + } + + #[tokio::test] + async fn installed_manifest_scope_removal_is_one_logical_owner_operation_offline() { + // Pins: an exact owner with more providers than one cleanup page is + // invalidated by removing its nested scope entry, without a provider-key + // loop, while another tenant's identically shaped scope remains intact. + let owner = HandLifecycleOwner::new( + HashMap::new(), + crate::core::profile::local_development_sandbox_policy(), + ); + let session_id = SessionId::new(); + let scope = HandScopeKey::new(TenantId::new(), session_id, "owner"); + let unrelated_scope = HandScopeKey::new(TenantId::new(), session_id, "owner"); + let mut providers = BTreeMap::new(); + for index in 0..=HAND_LEASE_SESSION_PAGE_SIZE { + providers.insert( + format!("provider-{index:03}"), + InstalledManifestMarker { + manifest_identity: uuid::Uuid::new_v4(), + handle: HandHandle::local(PathBuf::from(format!("/tmp/provider-{index:03}"))), + generation: Some(1), + }, + ); + } + let unrelated = providers + .first_key_value() + .map(|(_, marker)| marker.clone()) + .expect("provider fixture is not empty"); + owner + .installed_files + .write() + .await + .insert(scope.clone(), providers); + owner.installed_files.write().await.insert( + unrelated_scope.clone(), + BTreeMap::from([("local".to_string(), unrelated)]), + ); + + owner.clear_installed_files_for_scope(&scope).await; + + let installed = owner.installed_files.read().await; + assert!(!installed.contains_key(&scope)); + assert_eq!(installed.get(&unrelated_scope).map(BTreeMap::len), Some(1)); + } } diff --git a/crates/moa-hands/src/core/reaper.rs b/crates/moa-hands/src/core/reaper.rs index c906e889e..0f08969d8 100644 --- a/crates/moa-hands/src/core/reaper.rs +++ b/crates/moa-hands/src/core/reaper.rs @@ -14,8 +14,8 @@ //! is never returned to `Active`: a sandbox the reaper decided to destroy is //! not a sandbox anyone should get back. -use std::sync::Arc; -use std::time::Duration; +use std::sync::{Arc, RwLock}; +use std::time::{Duration, Instant}; use chrono::{DateTime, Utc}; use futures_util::{StreamExt, stream}; @@ -30,12 +30,14 @@ use moa_core::{ use moa_db::ScopedConn; use sqlx::{PgConnection, PgPool, Row}; use tokio::task::JoinHandle; +use tokio_util::sync::CancellationToken; use uuid::Uuid; use super::leases::{ HandLeaseWorkspaceAttachment, LeaseHandle, PROVISIONING_EMPTY_CONFIRMATION, PROVISIONING_VISIBILITY_GRACE, map_sqlx_error, }; +use super::sandbox_workspace::capacity::release_active_hand_for_reaper_in_transaction; /// One generation the reaper owns and must destroy. #[derive(Debug, Clone, PartialEq, Eq)] @@ -293,6 +295,19 @@ impl ExpiredHandLeaseClaims for PostgresExpiredHandLeaseClaims { conn.rollback().await?; return Ok(false); } + if claimed.attachment.is_some() + && !release_active_hand_for_reaper_in_transaction( + conn.as_mut(), + claimed.tenant_id, + claimed.provisioning_operation_id, + claimed.generation, + claimed.claim_token, + ) + .await? + { + conn.rollback().await?; + return Ok(false); + } let affected = sqlx::query( r#" UPDATE moa.hand_leases @@ -637,6 +652,8 @@ pub struct HandLeaseReaperConfig { pub base_retry_delay: Duration, /// Ceiling the exponential backoff never exceeds. pub max_retry_delay: Duration, + /// Maximum acceptable age of the last complete successful sweep. + pub heartbeat_maximum_age: Duration, } impl Default for HandLeaseReaperConfig { @@ -648,6 +665,7 @@ impl Default for HandLeaseReaperConfig { max_destroy_concurrency: 4, base_retry_delay: Duration::from_secs(15), max_retry_delay: Duration::from_secs(15 * 60), + heartbeat_maximum_age: Duration::from_secs(90), } } } @@ -670,6 +688,126 @@ pub struct HandLeaseReaper { config: HandLeaseReaperConfig, } +/// Supervised process handle for durable hand-lease cleanup. +pub struct HandLeaseReaperHandle { + state: Arc, + shutdown: CancellationToken, + task: JoinHandle>, + heartbeat_maximum_age: Duration, +} + +/// Cloneable readiness projection for the supervised hand-lease reaper. +#[derive(Clone)] +pub struct HandLeaseReaperReadiness { + state: Arc, + heartbeat_maximum_age: Duration, +} + +#[derive(Debug)] +struct HandLeaseReaperHealth { + started_at: Instant, + last_heartbeat: RwLock>, + unready_reason: RwLock>, + exited: std::sync::atomic::AtomicBool, +} + +impl HandLeaseReaperHandle { + /// Returns the age of the most recent complete successful sweep. + #[must_use] + pub fn heartbeat_age(&self) -> Duration { + self.readiness().heartbeat_age() + } + + /// Returns a cloneable health projection for process readiness. + #[must_use] + pub fn readiness(&self) -> HandLeaseReaperReadiness { + HandLeaseReaperReadiness { + state: Arc::clone(&self.state), + heartbeat_maximum_age: self.heartbeat_maximum_age, + } + } + + /// Awaits the task result so unexpected exit can be process-fatal. + pub async fn task_result(&mut self) -> Result<()> { + match (&mut self.task).await { + Ok(result) => result, + Err(error) => Err(MoaError::StorageError(format!( + "hand lease reaper task join failed: {error}" + ))), + } + } + + /// Cancels and joins the supervised task during graceful shutdown. + pub async fn shutdown(mut self) -> Result<()> { + self.shutdown.cancel(); + match tokio::time::timeout(Duration::from_secs(10), &mut self.task).await { + Ok(Ok(result)) => result, + Ok(Err(error)) => Err(MoaError::StorageError(format!( + "hand lease reaper task join failed: {error}" + ))), + Err(_) => { + self.task.abort(); + let _ = (&mut self.task).await; + Err(MoaError::StorageError( + "hand lease reaper exceeded its shutdown deadline".to_string(), + )) + } + } + } +} + +impl Drop for HandLeaseReaperHandle { + fn drop(&mut self) { + self.state + .exited + .store(true, std::sync::atomic::Ordering::Release); + self.shutdown.cancel(); + self.task.abort(); + } +} + +impl HandLeaseReaperReadiness { + /// Returns the age of the most recent complete successful sweep. + #[must_use] + pub fn heartbeat_age(&self) -> Duration { + let heartbeat = self + .state + .last_heartbeat + .read() + .ok() + .and_then(|guard| *guard); + heartbeat.map_or_else( + || self.state.started_at.elapsed(), + |heartbeat| heartbeat.elapsed(), + ) + } + + /// Returns a bounded reason readiness must refuse sandbox traffic. + #[must_use] + pub fn unready_reason(&self) -> Option { + if self.state.exited.load(std::sync::atomic::Ordering::Acquire) { + return Some("hand lease reaper exited unexpectedly".to_string()); + } + if self.heartbeat_age() > self.heartbeat_maximum_age { + return Some("hand lease reaper heartbeat is stale".to_string()); + } + match self.state.unready_reason.read() { + Ok(reason) => reason.clone(), + Err(_) => Some("hand lease reaper health lock is poisoned".to_string()), + } + } +} + +fn set_reaper_heartbeat(state: &HandLeaseReaperHealth) -> Result<()> { + *state.last_heartbeat.write().map_err(|_| { + MoaError::StorageError("hand lease reaper heartbeat lock is poisoned".to_string()) + })? = Some(Instant::now()); + *state.unready_reason.write().map_err(|_| { + MoaError::StorageError("hand lease reaper health lock is poisoned".to_string()) + })? = None; + Ok(()) +} + impl HandLeaseReaper { /// Creates a reaper over one claim surface and the providers that can /// destroy the sandboxes it will claim. @@ -814,21 +952,58 @@ impl HandLeaseReaper { } } - /// Spawns the sweep loop and returns its join handle. - #[must_use] - pub fn spawn(self) -> JoinHandle<()> { - tokio::spawn(async move { - let mut ticker = tokio::time::interval(self.config.interval); - ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); - loop { - ticker.tick().await; - if let Err(error) = self.sweep().await { - tracing::warn!( - error = %error, - "durable hand-lease reaper sweep failed; retrying next interval" - ); + /// Spawns the supervised sweep loop. + pub fn spawn(self) -> Result { + if self.config.interval.is_zero() + || self.config.batch_size <= 0 + || self.config.max_destroy_concurrency == 0 + || self.config.heartbeat_maximum_age.is_zero() + || self.config.interval >= self.config.heartbeat_maximum_age + { + return Err(MoaError::ConfigError( + "hand lease reaper requires positive sweep bounds and an interval shorter than heartbeat freshness" + .to_string(), + )); + } + let heartbeat_maximum_age = self.config.heartbeat_maximum_age; + let state = Arc::new(HandLeaseReaperHealth { + started_at: Instant::now(), + last_heartbeat: RwLock::new(None), + unready_reason: RwLock::new(Some( + "hand lease reaper has not completed its first pass".to_string(), + )), + exited: std::sync::atomic::AtomicBool::new(false), + }); + let shutdown = CancellationToken::new(); + let task_state = Arc::clone(&state); + let task_shutdown = shutdown.clone(); + let task = tokio::spawn(async move { + let result = async { + loop { + self.sweep().await?; + set_reaper_heartbeat(&task_state)?; + tokio::select! { + () = task_shutdown.cancelled() => return Ok(()), + () = tokio::time::sleep(self.config.interval) => {} + } } } + .await; + task_state + .exited + .store(true, std::sync::atomic::Ordering::Release); + if result.is_err() + && let Ok(mut reason) = task_state.unready_reason.write() + { + *reason = Some("hand lease reaper pass failed".to_string()); + } + result + }); + Ok(HandLeaseReaperHandle { + state, + shutdown, + task, + heartbeat_maximum_age, }) } } @@ -1200,6 +1375,119 @@ mod tests { ); } + #[tokio::test] + async fn supervised_reaper_opens_readiness_after_a_complete_pass_and_shuts_down() { + // Pins: sandbox traffic stays unready until the destruction owner has + // completed a full database pass, after which a graceful shutdown joins + // the owner instead of abandoning it. + let config = HandLeaseReaperConfig { + interval: Duration::from_secs(60), + heartbeat_maximum_age: Duration::from_secs(120), + ..HandLeaseReaperConfig::default() + }; + let handle = HandLeaseReaper::new(Arc::new(RecordingClaims::default()), Vec::new(), config) + .spawn() + .expect("valid supervised reaper config should start"); + let readiness = handle.readiness(); + + tokio::time::timeout(Duration::from_secs(1), async { + while readiness.unready_reason().is_some() { + tokio::task::yield_now().await; + } + }) + .await + .expect("first empty sweep should open readiness"); + assert!( + handle.heartbeat_age() < Duration::from_secs(1), + "the first complete sweep should record a fresh heartbeat" + ); + handle + .shutdown() + .await + .expect("graceful shutdown should join the reaper"); + } + + #[tokio::test] + async fn supervised_reaper_failure_closes_readiness_and_surfaces_task_result() { + // Pins: a failed cleanup pass cannot leave the process reporting ready; + // the task result reaches the process supervisor as a fatal error. + struct FailingClaims; + + #[async_trait::async_trait] + impl ExpiredHandLeaseClaims for FailingClaims { + async fn claim_expired( + &self, + _limit: i64, + _claim_ttl: Duration, + ) -> Result> { + Err(MoaError::StorageError("forced reaper failure".to_string())) + } + + async fn finalize_destroyed(&self, _claimed: &ClaimedHandLease) -> Result { + panic!("a failed claim pass cannot finalize work") + } + + async fn release_for_retry( + &self, + _claimed: &ClaimedHandLease, + _retry_after: Duration, + ) -> Result { + panic!("a failed claim pass cannot release work") + } + + async fn renew_claim( + &self, + _claimed: &ClaimedHandLease, + _claim_ttl: Duration, + ) -> Result { + panic!("a failed claim pass cannot renew work") + } + } + + let config = HandLeaseReaperConfig { + interval: Duration::from_millis(10), + heartbeat_maximum_age: Duration::from_secs(1), + ..HandLeaseReaperConfig::default() + }; + let mut handle = HandLeaseReaper::new(Arc::new(FailingClaims), Vec::new(), config) + .spawn() + .expect("valid supervised reaper config should start"); + let readiness = handle.readiness(); + + let error = handle + .task_result() + .await + .expect_err("failed sweep must terminate the supervised owner"); + assert!( + matches!(error, MoaError::StorageError(message) if message == "forced reaper failure") + ); + assert_eq!( + readiness.unready_reason().as_deref(), + Some("hand lease reaper exited unexpectedly") + ); + } + + #[tokio::test] + async fn dropping_supervised_reaper_immediately_closes_readiness() { + // Pins: losing the cleanup-owner handle cannot leave a cloned process + // readiness projection healthy until its heartbeat eventually ages out. + let config = HandLeaseReaperConfig { + interval: Duration::from_secs(60), + heartbeat_maximum_age: Duration::from_secs(120), + ..HandLeaseReaperConfig::default() + }; + let handle = HandLeaseReaper::new(Arc::new(RecordingClaims::default()), Vec::new(), config) + .spawn() + .expect("valid supervised reaper config should start"); + let readiness = handle.readiness(); + drop(handle); + + assert_eq!( + readiness.unready_reason().as_deref(), + Some("hand lease reaper exited unexpectedly") + ); + } + #[test] fn retry_backoff_grows_and_is_capped() { // Pins: repeated destroy failures back off exponentially and stop at the diff --git a/crates/moa-hands/src/core/recovery.rs b/crates/moa-hands/src/core/recovery.rs index 3d747a35e..fc16d5e99 100644 --- a/crates/moa-hands/src/core/recovery.rs +++ b/crates/moa-hands/src/core/recovery.rs @@ -14,7 +14,7 @@ use tracing::Instrument; use super::dispatch::{ AuthorizedToolCall, DeferredWorkspaceToolOutput, McpDispatch, WorkspaceCommitMode, }; -use super::lifecycle::{hand_id, scope_key, workspace_lease_scope}; +use super::lifecycle::{hand_id, manifest_scope_key, workspace_lease_scope}; use super::registration::{McpClientRoute, McpRouteGeneration}; use super::{HandRoute, ToolCallScope, ToolExecution, ToolRouter}; @@ -457,7 +457,7 @@ impl ToolRouter { )); } let mut ordered = routes.to_vec(); - let scope = scope_key(session, worker_id); + let scope = manifest_scope_key(session, worker_id); let preferred = self .hands .preferred_hand_routes @@ -484,7 +484,7 @@ impl ToolRouter { .preferred_hand_routes .write() .await - .insert(scope_key(session, worker_id), provider.to_string()); + .insert(manifest_scope_key(session, worker_id), provider.to_string()); } async fn try_fallback_hand_route( @@ -519,8 +519,8 @@ impl ToolRouter { .workspace_scope .map(workspace_lease_scope) .map_or_else( - || scope_key(request.session, None), - |lease| scope_key(request.session, Some(lease.as_str())), + || manifest_scope_key(request.session, None), + |lease| manifest_scope_key(request.session, Some(lease.as_str())), ); let mut preferred = self.hands.preferred_hand_routes.write().await; if preferred diff --git a/crates/moa-hands/src/core/registration.rs b/crates/moa-hands/src/core/registration.rs index dd946c4d4..d992c3bda 100644 --- a/crates/moa-hands/src/core/registration.rs +++ b/crates/moa-hands/src/core/registration.rs @@ -413,6 +413,7 @@ impl RegisteredTool { schema, policy, idempotency_class, + async_mode: moa_core::types::tools::ToolAsyncMode::SynchronousOnly, rollback: None, max_output_tokens: default_budget_for_tool(name), }, @@ -475,6 +476,7 @@ impl RegisteredTool { diff_strategy: ToolDiffStrategy::None, }, idempotency_class, + async_mode: moa_core::types::tools::ToolAsyncMode::SynchronousOnly, rollback: None, max_output_tokens: 8_000, }, @@ -672,6 +674,7 @@ impl ToolRegistry { diff_strategy: ToolDiffStrategy::None, }, idempotency_class: operation_policy.idempotency, + async_mode: moa_core::types::tools::ToolAsyncMode::SynchronousOnly, rollback: None, max_output_tokens: default_budget_for_tool(&name), }; @@ -885,6 +888,25 @@ impl ToolRegistry { } impl ToolRouter { + /// Adds one deployment-owned built-in before the router is shared. + /// + /// The returned router publishes the tool through the same immutable catalog + /// snapshot used by prompt compilation and effect admission. Duplicate names + /// are rejected so an integration cannot silently replace a production tool. + pub fn with_additional_builtin(self, tool: Arc) -> Result { + let name = tool.name(); + let mut registry = (*self.registry()).clone(); + if registry.tools.contains_key(name) { + return Err(MoaError::ValidationError(format!( + "additional built-in tool `{name}` is already registered" + ))); + } + registry.register_builtin(tool); + registry.apply_budgets(self.bindings.tool_budgets()); + self.publish_registry(registry); + Ok(self) + } + /// Returns live registered definitions with their executable owners in stable name order. pub fn capability_registrations(&self) -> Vec<(ToolDefinition, ToolExecution)> { self.registry().capability_registrations() @@ -903,6 +925,9 @@ fn default_budget_for_tool(tool_name: &str) -> u32 { #[cfg(test)] mod tests { + use std::{collections::HashMap, sync::Arc}; + + use moa_core::types::hands::{BuiltinPolicyRevision, SandboxPolicySnapshot}; use moa_core::types::sandbox_workspace::WorkspaceEffect; use serde_json::json; @@ -911,7 +936,27 @@ mod tests { default_sandbox_tool_descriptors, sandbox_tool_descriptors, }; - use super::{ToolRegistry, mcp_tool_reference}; + use super::{ToolRegistry, ToolRouter, mcp_tool_reference}; + + #[test] + fn additional_builtin_publishes_once_without_replacing_an_owner_offline() { + // Pins: integration composition publishes through the immutable router catalog and a + // duplicate fixture name cannot silently replace a production executable owner. + let router = ToolRouter::new( + ToolRegistry::new(), + HashMap::new(), + SandboxPolicySnapshot::builtin(BuiltinPolicyRevision::RouteUnset), + ) + .with_additional_builtin(Arc::new(crate::tools::memory::MemoryRememberTool)) + .expect("a unique deployment built-in should publish"); + assert!(router.tool_definition("memory_remember").is_some()); + + let error = router + .with_additional_builtin(Arc::new(crate::tools::memory::MemoryRememberTool)) + .err() + .expect("a duplicate built-in must be rejected"); + assert!(error.to_string().contains("already registered")); + } fn discovered_tool(name: &str, description: &str) -> McpDiscoveredTool { McpDiscoveredTool { diff --git a/crates/moa-hands/src/core/sandbox_workspace/capacity.rs b/crates/moa-hands/src/core/sandbox_workspace/capacity.rs index 81bc2b2d8..eb0698e23 100644 --- a/crates/moa-hands/src/core/sandbox_workspace/capacity.rs +++ b/crates/moa-hands/src/core/sandbox_workspace/capacity.rs @@ -2,11 +2,17 @@ use std::collections::{BTreeMap, HashMap, HashSet}; +use chrono::{DateTime, Utc}; use moa_core::{ error::{MoaError, Result}, types::{ - identifiers::{ProviderAccountId, SandboxWorkspaceId, TenantId, WorkspaceOperationId}, - sandbox_workspace::WorkspaceCapacityDimension, + identifiers::{ + HandProvisioningOperationId, ProviderAccountId, SandboxWorkspaceId, TenantId, + WorkspaceOperationId, + }, + sandbox_workspace::{ + WorkspaceCapacityDimension, WorkspaceOperationKind, WorkspaceStorageOperation, + }, }, }; use moa_db::ScopedConn; @@ -57,6 +63,27 @@ pub struct CapacityReservation { pub quantity: u64, } +/// Exact active-compute reservation owned by one hand lease generation. +#[derive(Debug, Clone, Copy)] +pub struct ActiveHandCapacityRequest { + /// Verified tenant owner. + pub tenant_id: TenantId, + /// Workspace whose writer owns the compute. + pub workspace_id: SandboxWorkspaceId, + /// Provider account and isolation cell. + pub provider_account_id: ProviderAccountId, + /// Exact provider-account generation. + pub provider_account_generation: i64, + /// Provider-visible idempotent hand creation identity. + pub provisioning_operation_id: HandProvisioningOperationId, + /// Exact durable hand lease generation. + pub hand_lease_generation: i64, + /// Exact workspace writer fence. + pub expected_writer_epoch: i64, + /// Exact workspace compute-instance fence. + pub expected_instance_generation: i64, +} + /// Postgres-backed atomic capacity admission repository. #[derive(Clone)] pub struct PostgresWorkspaceCapacityRepository { @@ -90,6 +117,14 @@ impl PostgresWorkspaceCapacityRepository { pub async fn reserve( &self, request: &CapacityReservationRequest, + ) -> Result> { + self.reserve_with_expiry(request, None).await + } + + async fn reserve_with_expiry( + &self, + request: &CapacityReservationRequest, + expires_at: Option>, ) -> Result> { let quantities = validated_quantities(request)?; let mut conn = self.begin().await?; @@ -150,83 +185,54 @@ impl PostgresWorkspaceCapacityRepository { )); } - let provider_limits = sqlx::query( - r#" - SELECT configured_limits - FROM moa.sandbox_provider_accounts - WHERE provider_account_id = $1 AND generation = $2 - "#, - ) - .bind(request.provider_account_id) - .bind(request.provider_account_generation) - .fetch_optional(conn.as_mut()) - .await - .map_err(map_sqlx_error)? - .ok_or_else(|| MoaError::StorageError("provider-account generation not found".to_string()))? - .try_get::, _>("configured_limits") - .map_err(map_sqlx_error)? - .0; + let existing = load_operation_reservations(conn.as_mut(), request).await?; + if !existing.is_empty() { + if existing.len() != quantities.len() + || existing.iter().any(|reservation| { + quantities.get(&reservation.dimension).copied() + != i64::try_from(reservation.quantity).ok() + }) + { + return Err(MoaError::StorageError( + "capacity reservation replay changed its exact dimensions or quantities" + .to_string(), + )); + } + conn.commit().await?; + return Ok(existing); + } - let tenant_limits = sqlx::query( - r#" - SELECT configured_limits - FROM moa.sandbox_tenant_capacity_limits - WHERE tenant_id = $1 - FOR UPDATE - "#, + enforce_capacity( + conn.as_mut(), + request.tenant_id, + request.provider_account_id, + request.provider_account_generation, + &quantities, ) - .bind(request.tenant_id) - .fetch_optional(conn.as_mut()) - .await - .map_err(map_sqlx_error)? - .map(|row| { - row.try_get::, _>("configured_limits") - .map(|value| value.0) - .map_err(map_sqlx_error) - }) - .transpose()? - .unwrap_or_else(|| Value::Object(serde_json::Map::new())); - - let tenant_limits = parse_limits(&tenant_limits, "tenant")?; - let provider_limits = parse_limits(&provider_limits, "provider account")?; - - for (dimension, quantity) in &quantities { - let tenant_used = - reserved_total(conn.as_mut(), Some(request.tenant_id), None, *dimension).await?; - let provider_used = reserved_total( - conn.as_mut(), - None, - Some(request.provider_account_id), - *dimension, - ) - .await?; - enforce_limit( - "tenant", - *dimension, - tenant_used, - *quantity, - tenant_limits.get(dimension).copied(), - )?; - enforce_limit( - "provider account", - *dimension, - provider_used, - *quantity, - provider_limits.get(dimension).copied(), - )?; - } + .await?; let mut reservations = Vec::with_capacity(quantities.len()); for (dimension, quantity) in quantities { let reservation_id = Uuid::now_v7(); - sqlx::query( + let row = sqlx::query( r#" INSERT INTO moa.sandbox_capacity_reservations ( reservation_id, tenant_id, provider_account_id, provider_account_generation, workspace_id, operation_id, expected_writer_epoch, expected_instance_generation, - resource_dimension, quantity - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + resource_dimension, quantity, expires_at + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) + ON CONFLICT (tenant_id, operation_id, resource_dimension) + WHERE operation_id IS NOT NULL + DO UPDATE SET updated_at = moa.sandbox_capacity_reservations.updated_at + WHERE moa.sandbox_capacity_reservations.workspace_id = EXCLUDED.workspace_id + AND moa.sandbox_capacity_reservations.provider_account_id = EXCLUDED.provider_account_id + AND moa.sandbox_capacity_reservations.provider_account_generation = EXCLUDED.provider_account_generation + AND moa.sandbox_capacity_reservations.expected_writer_epoch = EXCLUDED.expected_writer_epoch + AND moa.sandbox_capacity_reservations.expected_instance_generation = EXCLUDED.expected_instance_generation + AND moa.sandbox_capacity_reservations.quantity = EXCLUDED.quantity + AND moa.sandbox_capacity_reservations.reservation_state IN ('pending', 'committed', 'reconciling') + RETURNING reservation_id "#, ) .bind(reservation_id) @@ -239,11 +245,18 @@ impl PostgresWorkspaceCapacityRepository { .bind(request.expected_instance_generation) .bind(dimension.as_str()) .bind(quantity) - .execute(conn.as_mut()) + .bind(expires_at) + .fetch_optional(conn.as_mut()) .await - .map_err(map_sqlx_error)?; + .map_err(map_sqlx_error)? + .ok_or_else(|| { + MoaError::StorageError( + "conflicting replay attempted to reuse an operation capacity identity" + .to_string(), + ) + })?; reservations.push(CapacityReservation { - reservation_id, + reservation_id: row.try_get("reservation_id").map_err(map_sqlx_error)?, dimension, quantity: u64::try_from(quantity).map_err(|_| { MoaError::StorageError("negative persisted capacity quantity".to_string()) @@ -254,6 +267,258 @@ impl PostgresWorkspaceCapacityRepository { Ok(reservations) } + /// Reserves provider-independent checkpoint count and logical bytes before upload. + /// + /// The caller supplies the logical byte count from the already-built, + /// bounded portable archive. A zero-byte archive reserves only its one + /// immutable checkpoint row because reservation quantities are positive. + pub async fn reserve_checkpoint_publication( + &self, + operation: &WorkspaceStorageOperation, + logical_bytes: u64, + ) -> Result> { + if !matches!( + operation.kind, + WorkspaceOperationKind::Commit | WorkspaceOperationKind::Checkpoint + ) { + return Err(MoaError::ValidationError( + "checkpoint capacity requires a commit or checkpoint operation".to_string(), + )); + } + let mut request = CapacityReservationRequest { + tenant_id: operation.binding.tenant_id, + workspace_id: operation.binding.workspace_id, + operation_id: operation.operation_id, + provider_account_id: operation.binding.provider_account_id, + provider_account_generation: i64::try_from( + operation.binding.provider_account_generation, + ) + .map_err(|_| { + MoaError::ValidationError( + "checkpoint provider-account generation overflows Postgres bigint".to_string(), + ) + })?, + expected_writer_epoch: i64::try_from(operation.binding.writer_epoch).map_err(|_| { + MoaError::ValidationError( + "checkpoint writer epoch overflows Postgres bigint".to_string(), + ) + })?, + expected_instance_generation: i64::try_from(operation.binding.instance_generation) + .map_err(|_| { + MoaError::ValidationError( + "checkpoint instance generation overflows Postgres bigint".to_string(), + ) + })?, + quantities: Vec::with_capacity(2), + }; + request.quantities.push(CapacityQuantity { + dimension: WorkspaceCapacityDimension::Checkpoints, + quantity: 1, + }); + if logical_bytes > 0 { + request.quantities.push(CapacityQuantity { + dimension: WorkspaceCapacityDimension::LogicalBytes, + quantity: logical_bytes, + }); + } + self.reserve_with_expiry(&request, Some(operation.deadline)) + .await + } + + /// Reserves one active hand before its exact provider creation operation starts. + pub async fn reserve_active_hand( + &self, + request: &ActiveHandCapacityRequest, + ) -> Result { + validate_active_hand_request(request)?; + let mut conn = self.begin().await?; + lock_capacity_scope_values( + conn.as_mut(), + request.tenant_id, + request.provider_account_id, + ) + .await?; + let lease_exists = sqlx::query_scalar::<_, bool>( + r#" + SELECT TRUE + FROM moa.hand_leases AS lease + JOIN moa.sandbox_workspaces AS workspace + ON workspace.tenant_id = lease.tenant_id + AND workspace.workspace_id = lease.workspace_id + WHERE lease.tenant_id = $1 + AND lease.provisioning_operation_id = $2 + AND lease.generation = $3 + AND lease.status = 'provisioning' + AND lease.handle IS NULL + AND lease.workspace_id = $4 + AND lease.workspace_writer_epoch = $5 + AND lease.workspace_instance_generation = $6 + AND workspace.provider = lease.provider + AND workspace.provider_account_id = $7 + AND workspace.provider_account_generation = $8 + AND workspace.writer_epoch = $5 + AND workspace.instance_generation = $6 + AND workspace.lifecycle_state = 'restoring' + AND workspace.access_fenced_at IS NULL + FOR UPDATE OF lease, workspace + "#, + ) + .bind(request.tenant_id) + .bind(request.provisioning_operation_id) + .bind(request.hand_lease_generation) + .bind(request.workspace_id) + .bind(request.expected_writer_epoch) + .bind(request.expected_instance_generation) + .bind(request.provider_account_id) + .bind(request.provider_account_generation) + .fetch_optional(conn.as_mut()) + .await + .map_err(map_sqlx_error)? + .unwrap_or(false); + if !lease_exists { + return Err(MoaError::StorageError( + "active-hand reservation lost its exact lease or workspace generation fence" + .to_string(), + )); + } + + let existing = load_active_hand_reservation(conn.as_mut(), request).await?; + if let Some(existing) = existing { + conn.commit().await?; + return Ok(existing); + } + + let quantities = BTreeMap::from([(WorkspaceCapacityDimension::ActiveHands, 1_i64)]); + enforce_capacity( + conn.as_mut(), + request.tenant_id, + request.provider_account_id, + request.provider_account_generation, + &quantities, + ) + .await?; + let reservation_id = Uuid::now_v7(); + let row = sqlx::query( + r#" + INSERT INTO moa.sandbox_capacity_reservations ( + reservation_id, tenant_id, provider_account_id, + provider_account_generation, workspace_id, operation_id, + expected_writer_epoch, expected_instance_generation, + resource_dimension, quantity, hand_provisioning_operation_id, + hand_lease_generation + ) VALUES ($1, $2, $3, $4, $5, NULL, $6, $7, + 'active_hands', 1, $8, $9) + ON CONFLICT (tenant_id, hand_provisioning_operation_id, resource_dimension) + WHERE resource_dimension = 'active_hands' + DO UPDATE SET updated_at = moa.sandbox_capacity_reservations.updated_at + WHERE moa.sandbox_capacity_reservations.workspace_id = EXCLUDED.workspace_id + AND moa.sandbox_capacity_reservations.provider_account_id = EXCLUDED.provider_account_id + AND moa.sandbox_capacity_reservations.provider_account_generation = EXCLUDED.provider_account_generation + AND moa.sandbox_capacity_reservations.expected_writer_epoch = EXCLUDED.expected_writer_epoch + AND moa.sandbox_capacity_reservations.expected_instance_generation = EXCLUDED.expected_instance_generation + AND moa.sandbox_capacity_reservations.hand_lease_generation = EXCLUDED.hand_lease_generation + AND moa.sandbox_capacity_reservations.reservation_state IN ('pending', 'committed', 'reconciling') + RETURNING reservation_id, reservation_state + "#, + ) + .bind(reservation_id) + .bind(request.tenant_id) + .bind(request.provider_account_id) + .bind(request.provider_account_generation) + .bind(request.workspace_id) + .bind(request.expected_writer_epoch) + .bind(request.expected_instance_generation) + .bind(request.provisioning_operation_id) + .bind(request.hand_lease_generation) + .fetch_optional(conn.as_mut()) + .await + .map_err(map_sqlx_error)? + .ok_or_else(|| { + MoaError::StorageError( + "conflicting replay attempted to reuse an active-hand capacity identity" + .to_string(), + ) + })?; + let reservation_id = row.try_get("reservation_id").map_err(map_sqlx_error)?; + conn.commit().await?; + Ok(CapacityReservation { + reservation_id, + dimension: WorkspaceCapacityDimension::ActiveHands, + quantity: 1, + }) + } + + /// Commits active-hand capacity only after the exact lease is active. + pub async fn commit_active_hand(&self, request: &ActiveHandCapacityRequest) -> Result { + validate_active_hand_request(request)?; + let mut conn = self.begin().await?; + let changed = commit_active_hand_in_transaction(conn.as_mut(), request).await?; + conn.commit().await?; + Ok(changed) + } + + /// Releases active-hand capacity after exact durable reaper ownership is established. + pub async fn release_active_hand_to_reaper( + &self, + request: &ActiveHandCapacityRequest, + claim_token: Uuid, + ) -> Result { + validate_active_hand_request(request)?; + let mut conn = self.begin().await?; + let owns_reaping = sqlx::query_scalar::<_, bool>( + r#" + SELECT TRUE + FROM moa.hand_leases + WHERE tenant_id = $1 + AND provisioning_operation_id = $2 + AND generation = $3 + AND status = 'reaping' + AND reap_claim_token = $4 + AND reap_claim_expires_at > now() + FOR UPDATE + "#, + ) + .bind(request.tenant_id) + .bind(request.provisioning_operation_id) + .bind(request.hand_lease_generation) + .bind(claim_token) + .fetch_optional(conn.as_mut()) + .await + .map_err(map_sqlx_error)? + .unwrap_or(false); + if !owns_reaping { + conn.rollback().await?; + return Ok(false); + } + let changed = release_active_hand_row(conn.as_mut(), request).await?; + conn.commit().await?; + Ok(changed) + } + + /// Releases one logical workspace charge after exact deletion finalization. + pub async fn release_workspace( + &self, + tenant_id: TenantId, + workspace_id: SandboxWorkspaceId, + delete_generation: i64, + ) -> Result { + if delete_generation <= 0 { + return Err(MoaError::ValidationError( + "workspace capacity release requires a positive delete generation".to_string(), + )); + } + let mut conn = self.begin().await?; + let changed = release_workspace_in_transaction( + conn.as_mut(), + tenant_id, + workspace_id, + delete_generation, + ) + .await?; + conn.commit().await?; + Ok(changed) + } + /// Atomically reserves one lifetime Daytona volume without double-counting inventory. /// /// Live durable rows, provider-observed IDs, and pending/unknown operation @@ -397,7 +662,9 @@ impl PostgresWorkspaceCapacityRepository { storage_resource_id, expected_writer_epoch, expected_instance_generation, resource_dimension, quantity ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, 'volumes', 1) - ON CONFLICT (tenant_id, operation_id, resource_dimension) DO UPDATE + ON CONFLICT (tenant_id, operation_id, resource_dimension) + WHERE operation_id IS NOT NULL + DO UPDATE SET updated_at = moa.sandbox_capacity_reservations.updated_at WHERE moa.sandbox_capacity_reservations.storage_resource_id = EXCLUDED.storage_resource_id AND moa.sandbox_capacity_reservations.provider_account_id = EXCLUDED.provider_account_id @@ -508,59 +775,156 @@ impl PostgresWorkspaceCapacityRepository { conn.commit().await?; Ok(changed) } +} - /// Commits all checkpoint-count/logical-byte reservations for an exact operation. - pub async fn commit_operation_reservations( - &self, - request: &CapacityReservationRequest, - ) -> Result { - let mut conn = self.begin().await?; - let changed = sqlx::query( - r#" - UPDATE moa.sandbox_capacity_reservations - SET reservation_state = 'committed', expires_at = NULL, updated_at = now() - WHERE tenant_id = $1 AND operation_id = $2 - AND provider_account_id = $3 AND provider_account_generation = $4 - AND expected_writer_epoch = $5 AND expected_instance_generation = $6 - AND resource_dimension IN ('checkpoints', 'logical_bytes') - AND reservation_state = 'pending' - "#, - ) - .bind(request.tenant_id) - .bind(request.operation_id) - .bind(request.provider_account_id) - .bind(request.provider_account_generation) - .bind(request.expected_writer_epoch) - .bind(request.expected_instance_generation) - .execute(conn.as_mut()) - .await - .map_err(map_sqlx_error)? - .rows_affected(); - conn.commit().await?; - Ok(changed) - } +/// Releases a lifetime workspace owner after exact deletion finalization. +pub(crate) async fn release_workspace_in_transaction( + conn: &mut sqlx::PgConnection, + tenant_id: TenantId, + workspace_id: SandboxWorkspaceId, + delete_generation: i64, +) -> Result { + Ok(sqlx::query( + r#" + UPDATE moa.sandbox_capacity_reservations AS reservation + SET reservation_state = 'released', updated_at = now() + FROM moa.sandbox_workspaces AS workspace + WHERE reservation.tenant_id = $1 + AND reservation.workspace_id = $2 + AND reservation.resource_dimension = 'workspaces' + AND reservation.reservation_state = 'committed' + AND reservation.expected_delete_generation + 1 = $3 + AND workspace.tenant_id = reservation.tenant_id + AND workspace.workspace_id = reservation.workspace_id + AND workspace.provider_account_id = reservation.provider_account_id + AND workspace.provider_account_generation = reservation.provider_account_generation + AND workspace.lifecycle_state = 'deleted' + AND workspace.delete_generation = $3 + "#, + ) + .bind(tenant_id) + .bind(workspace_id) + .bind(delete_generation) + .execute(conn) + .await + .map_err(map_sqlx_error)? + .rows_affected() + == 1) } async fn lock_capacity_scopes( conn: &mut sqlx::PgConnection, request: &CapacityReservationRequest, +) -> Result<()> { + lock_capacity_scope_values(conn, request.tenant_id, request.provider_account_id).await +} + +async fn lock_capacity_scope_values( + conn: &mut sqlx::PgConnection, + tenant_id: TenantId, + provider_account_id: ProviderAccountId, ) -> Result<()> { sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1, 0))") - .bind(format!("sandbox-capacity:tenant:{}", request.tenant_id)) + .bind(format!("sandbox-capacity:tenant:{tenant_id}")) .execute(&mut *conn) .await .map_err(map_sqlx_error)?; sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1, 0))") - .bind(format!( - "sandbox-capacity:provider:{}", - request.provider_account_id - )) + .bind(format!("sandbox-capacity:provider:{provider_account_id}")) .execute(conn) .await .map_err(map_sqlx_error)?; Ok(()) } +async fn load_operation_reservations( + conn: &mut sqlx::PgConnection, + request: &CapacityReservationRequest, +) -> Result> { + let rows = sqlx::query( + r#" + SELECT reservation_id, resource_dimension, quantity + FROM moa.sandbox_capacity_reservations + WHERE tenant_id = $1 AND workspace_id = $2 AND operation_id = $3 + AND provider_account_id = $4 AND provider_account_generation = $5 + AND expected_writer_epoch = $6 AND expected_instance_generation = $7 + AND reservation_state IN ('pending', 'committed', 'reconciling') + ORDER BY resource_dimension + "#, + ) + .bind(request.tenant_id) + .bind(request.workspace_id) + .bind(request.operation_id) + .bind(request.provider_account_id) + .bind(request.provider_account_generation) + .bind(request.expected_writer_epoch) + .bind(request.expected_instance_generation) + .fetch_all(conn) + .await + .map_err(map_sqlx_error)?; + rows.iter() + .map(|row| { + let quantity: i64 = row.try_get("quantity").map_err(map_sqlx_error)?; + Ok(CapacityReservation { + reservation_id: row.try_get("reservation_id").map_err(map_sqlx_error)?, + dimension: WorkspaceCapacityDimension::from_label( + &row.try_get::("resource_dimension") + .map_err(map_sqlx_error)?, + )?, + quantity: u64::try_from(quantity).map_err(|_| { + MoaError::StorageError( + "persisted capacity reservation quantity is not positive".to_string(), + ) + })?, + }) + }) + .collect() +} + +async fn load_active_hand_reservation( + conn: &mut sqlx::PgConnection, + request: &ActiveHandCapacityRequest, +) -> Result> { + let row = sqlx::query( + r#" + SELECT reservation_id, quantity + FROM moa.sandbox_capacity_reservations + WHERE tenant_id = $1 AND workspace_id = $2 + AND provider_account_id = $3 AND provider_account_generation = $4 + AND hand_provisioning_operation_id = $5 + AND hand_lease_generation = $6 + AND expected_writer_epoch = $7 AND expected_instance_generation = $8 + AND resource_dimension = 'active_hands' AND quantity = 1 + AND reservation_state IN ('pending', 'committed', 'reconciling') + FOR UPDATE + "#, + ) + .bind(request.tenant_id) + .bind(request.workspace_id) + .bind(request.provider_account_id) + .bind(request.provider_account_generation) + .bind(request.provisioning_operation_id) + .bind(request.hand_lease_generation) + .bind(request.expected_writer_epoch) + .bind(request.expected_instance_generation) + .fetch_optional(conn) + .await + .map_err(map_sqlx_error)?; + row.map(|row| { + Ok(CapacityReservation { + reservation_id: row.try_get("reservation_id").map_err(map_sqlx_error)?, + dimension: WorkspaceCapacityDimension::ActiveHands, + quantity: u64::try_from(row.try_get::("quantity").map_err(map_sqlx_error)?) + .map_err(|_| { + MoaError::StorageError( + "persisted active-hand capacity quantity is invalid".to_string(), + ) + })?, + }) + }) + .transpose() +} + fn validated_quantities( request: &CapacityReservationRequest, ) -> Result> { @@ -576,9 +940,16 @@ fn validated_quantities( let mut seen = HashSet::new(); let mut quantities = BTreeMap::new(); for item in &request.quantities { - if item.quantity == 0 || !seen.insert(item.dimension) { + if item.quantity == 0 + || matches!( + item.dimension, + WorkspaceCapacityDimension::Workspaces | WorkspaceCapacityDimension::ActiveHands + ) + || !seen.insert(item.dimension) + { return Err(MoaError::ValidationError( - "capacity quantities must be positive and dimensions unique".to_string(), + "capacity quantities must be positive, operation-bound, and dimensions unique" + .to_string(), )); } let quantity = i64::try_from(item.quantity).map_err(|_| { @@ -592,6 +963,210 @@ fn validated_quantities( Ok(quantities) } +fn validate_active_hand_request(request: &ActiveHandCapacityRequest) -> Result<()> { + if request.provider_account_generation <= 0 + || request.hand_lease_generation <= 0 + || request.expected_writer_epoch < 0 + || request.expected_instance_generation < 0 + { + return Err(MoaError::ValidationError( + "active-hand capacity requires positive account/lease generations and valid workspace fences" + .to_string(), + )); + } + Ok(()) +} + +async fn enforce_capacity( + conn: &mut sqlx::PgConnection, + tenant_id: TenantId, + provider_account_id: ProviderAccountId, + provider_account_generation: i64, + quantities: &BTreeMap, +) -> Result<()> { + let provider_limits = sqlx::query( + r#" + SELECT configured_limits + FROM moa.sandbox_provider_accounts + WHERE provider_account_id = $1 AND generation = $2 + "#, + ) + .bind(provider_account_id) + .bind(provider_account_generation) + .fetch_optional(&mut *conn) + .await + .map_err(map_sqlx_error)? + .ok_or_else(|| MoaError::StorageError("provider-account generation not found".to_string()))? + .try_get::, _>("configured_limits") + .map_err(map_sqlx_error)? + .0; + let tenant_limits = sqlx::query( + r#" + SELECT configured_limits + FROM moa.sandbox_tenant_capacity_limits + WHERE tenant_id = $1 + FOR UPDATE + "#, + ) + .bind(tenant_id) + .fetch_optional(&mut *conn) + .await + .map_err(map_sqlx_error)? + .map(|row| { + row.try_get::, _>("configured_limits") + .map(|value| value.0) + .map_err(map_sqlx_error) + }) + .transpose()? + .unwrap_or_else(|| Value::Object(serde_json::Map::new())); + let tenant_limits = parse_limits(&tenant_limits, "tenant")?; + let provider_limits = parse_limits(&provider_limits, "provider account")?; + for (dimension, quantity) in quantities { + let tenant_used = reserved_total(conn, Some(tenant_id), None, *dimension).await?; + let provider_used = + reserved_total(conn, None, Some(provider_account_id), *dimension).await?; + enforce_limit( + "tenant", + *dimension, + tenant_used, + *quantity, + tenant_limits.get(dimension).copied(), + )?; + enforce_limit( + "provider account", + *dimension, + provider_used, + *quantity, + provider_limits.get(dimension).copied(), + )?; + } + Ok(()) +} + +/// Commits one exact active-hand reservation inside an existing transaction. +pub async fn commit_active_hand_in_transaction( + conn: &mut sqlx::PgConnection, + request: &ActiveHandCapacityRequest, +) -> Result { + validate_active_hand_request(request)?; + Ok(sqlx::query( + r#" + UPDATE moa.sandbox_capacity_reservations AS reservation + SET reservation_state = 'committed', expires_at = NULL, updated_at = now() + FROM moa.hand_leases AS lease + WHERE reservation.tenant_id = $1 + AND reservation.workspace_id = $2 + AND reservation.provider_account_id = $3 + AND reservation.provider_account_generation = $4 + AND reservation.hand_provisioning_operation_id = $5 + AND reservation.hand_lease_generation = $6 + AND reservation.expected_writer_epoch = $7 + AND reservation.expected_instance_generation = $8 + AND reservation.resource_dimension = 'active_hands' + AND reservation.reservation_state = 'pending' + AND lease.tenant_id = reservation.tenant_id + AND lease.provisioning_operation_id = reservation.hand_provisioning_operation_id + AND lease.generation = reservation.hand_lease_generation + AND lease.workspace_id = reservation.workspace_id + AND lease.workspace_writer_epoch = reservation.expected_writer_epoch + AND lease.workspace_instance_generation = reservation.expected_instance_generation + AND lease.status = 'active' + AND lease.handle IS NOT NULL + "#, + ) + .bind(request.tenant_id) + .bind(request.workspace_id) + .bind(request.provider_account_id) + .bind(request.provider_account_generation) + .bind(request.provisioning_operation_id) + .bind(request.hand_lease_generation) + .bind(request.expected_writer_epoch) + .bind(request.expected_instance_generation) + .execute(conn) + .await + .map_err(map_sqlx_error)? + .rows_affected() + == 1) +} + +/// Releases the active-compute owner held by one exact live durable reaper claim. +pub(crate) async fn release_active_hand_for_reaper_in_transaction( + conn: &mut sqlx::PgConnection, + tenant_id: TenantId, + provisioning_operation_id: HandProvisioningOperationId, + hand_lease_generation: i64, + claim_token: Uuid, +) -> Result { + Ok(sqlx::query( + r#" + UPDATE moa.sandbox_capacity_reservations AS reservation + SET reservation_state = 'released', updated_at = now() + FROM moa.hand_leases AS lease + JOIN moa.sandbox_workspaces AS workspace + ON workspace.tenant_id = lease.tenant_id + AND workspace.workspace_id = lease.workspace_id + WHERE lease.tenant_id = $1 + AND lease.provisioning_operation_id = $2 + AND lease.generation = $3 + AND lease.status = 'reaping' + AND lease.reap_claim_token = $4 + AND lease.reap_claim_expires_at > now() + AND reservation.tenant_id = lease.tenant_id + AND reservation.workspace_id = lease.workspace_id + AND reservation.provider_account_id = workspace.provider_account_id + AND reservation.provider_account_generation = workspace.provider_account_generation + AND reservation.hand_provisioning_operation_id = lease.provisioning_operation_id + AND reservation.hand_lease_generation = lease.generation + AND reservation.expected_writer_epoch = lease.workspace_writer_epoch + AND reservation.expected_instance_generation = lease.workspace_instance_generation + AND reservation.resource_dimension = 'active_hands' + AND reservation.reservation_state IN ('pending', 'committed', 'reconciling') + "#, + ) + .bind(tenant_id) + .bind(provisioning_operation_id) + .bind(hand_lease_generation) + .bind(claim_token) + .execute(conn) + .await + .map_err(map_sqlx_error)? + .rows_affected() + == 1) +} + +async fn release_active_hand_row( + conn: &mut sqlx::PgConnection, + request: &ActiveHandCapacityRequest, +) -> Result { + Ok(sqlx::query( + r#" + UPDATE moa.sandbox_capacity_reservations + SET reservation_state = 'released', updated_at = now() + WHERE tenant_id = $1 AND workspace_id = $2 + AND provider_account_id = $3 AND provider_account_generation = $4 + AND hand_provisioning_operation_id = $5 + AND hand_lease_generation = $6 + AND expected_writer_epoch = $7 + AND expected_instance_generation = $8 + AND resource_dimension = 'active_hands' + AND reservation_state IN ('pending', 'committed', 'reconciling') + "#, + ) + .bind(request.tenant_id) + .bind(request.workspace_id) + .bind(request.provider_account_id) + .bind(request.provider_account_generation) + .bind(request.provisioning_operation_id) + .bind(request.hand_lease_generation) + .bind(request.expected_writer_epoch) + .bind(request.expected_instance_generation) + .execute(conn) + .await + .map_err(map_sqlx_error)? + .rows_affected() + == 1) +} + fn parse_limits(value: &Value, scope: &str) -> Result> { let object = value.as_object().ok_or_else(|| { MoaError::StorageError(format!("{scope} capacity limits must be a JSON object")) @@ -692,4 +1267,27 @@ mod tests { assert!(validated_quantities(&request(u64::MAX)).is_err()); assert!(validated_quantities(&request(1)).is_ok()); } + + #[test] + fn lifetime_dimensions_require_their_generation_fenced_admission_paths_offline() { + // Pins: generic operation capacity cannot bypass the workspace lifetime + // or active-hand lease-generation admission contracts. + for dimension in [ + WorkspaceCapacityDimension::Workspaces, + WorkspaceCapacityDimension::ActiveHands, + ] { + let mut candidate = request(1); + candidate.quantities[0].dimension = dimension; + assert!( + validated_quantities(&candidate).is_err(), + "{dimension:?} must use its specialized fenced reservation path" + ); + } + let mut checkpoint = request(1); + checkpoint.quantities[0].dimension = WorkspaceCapacityDimension::Checkpoints; + assert!( + validated_quantities(&checkpoint).is_ok(), + "checkpoint capacity remains operation-bound" + ); + } } diff --git a/crates/moa-hands/src/core/sandbox_workspace/lifecycle.rs b/crates/moa-hands/src/core/sandbox_workspace/lifecycle.rs index 11c4cf676..5a6450815 100644 --- a/crates/moa-hands/src/core/sandbox_workspace/lifecycle.rs +++ b/crates/moa-hands/src/core/sandbox_workspace/lifecycle.rs @@ -6,10 +6,12 @@ use moa_core::{ types::{ hands::HandHandle, identifiers::{ - SandboxWorkspaceId, ToolCallId, WorkspaceCheckpointId, WorkspaceOperationId, + ExecutionCompensationScopeId, SandboxWorkspaceId, ToolCallId, WorkspaceCheckpointId, + WorkspaceOperationId, }, sandbox_workspace::{ - ProviderStorageKind, ProviderStorageRef, SandboxWorkspaceScope, SandboxWorkspaceState, + ExecutionHandReleaseOwner, ExecutionHandReleaseReceipt, ProviderStorageKind, + ProviderStorageRef, SandboxWorkspaceScope, SandboxWorkspaceState, WorkspaceAttachRequest, WorkspaceBinding, WorkspaceCheckpointPublishRequest, WorkspaceCheckpointState, WorkspaceConfirmedDisposition, WorkspaceOperationKind, WorkspaceOperationOutcome, WorkspacePostCommitState, WorkspaceReconcileRequest, @@ -27,18 +29,75 @@ use super::{ model::{CreateCheckpointRequest, PublishCheckpointCommitRequest}, }, failpoints, - model::{SandboxWorkspace, WorkspaceTransition, WorkspaceWriterClaim}, + model::{ + AbsentTaskHandReleaseIntent, CompensationHandReleaseIntent, SandboxWorkspace, + TaskHandReleaseIntent, WorkspaceTransition, WorkspaceWriterClaim, + }, operations::WorkspaceOperationIntent, }; use crate::core::{ - ActiveHand, HandRoute, InstalledManifestMarker, JournaledWorkspaceCommit, ToolCallScope, - ToolExecution, ToolRouter, TrustedSandboxManifest, + ActiveHand, ExecutionHandReleaseRequest, HandProviderCacheKey, HandRoute, + InstalledManifestMarker, JournaledWorkspaceCommit, ToolCallScope, ToolExecution, ToolRouter, + TrustedSandboxManifest, leases::{HandLease, HandLeaseStatus, HandLeaseWorkspaceAttachment}, lifecycle::{ manifest_scope_key, session_provider_key, workspace_binding_for_hand, workspace_lease_scope, }, }; +#[derive(Clone, Copy)] +/// Internal identity and policy for one deterministic workspace commit. +pub(in crate::core) struct WorkspaceCommitExecution<'a> { + /// Session owning the workspace. + pub(in crate::core) session: &'a SessionMeta, + /// Typed durable workspace owner. + pub(in crate::core) workspace_scope: &'a SandboxWorkspaceScope, + /// Deterministic tool or yield identity. + pub(in crate::core) tool_call_id: ToolCallId, + /// Pinned hand and storage provider. + pub(in crate::core) provider_name: &'a str, + /// Exact active compute handle. + pub(in crate::core) hand: &'a HandHandle, + /// Bounded execution scope. + pub(in crate::core) call_scope: ToolCallScope<'a>, + /// Whether verified publication must destroy compute. + pub(in crate::core) release_compute: bool, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum ExecutionReleaseStep { + DurableReconciliation, + ProviderIo, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum PersistedLeaseReleaseState { + Missing, + Destroyed, + LiveOrAmbiguous, +} + +const fn compensation_release_identity_is_verified( + persisted_identity_present: bool, + lease_state: PersistedLeaseReleaseState, +) -> bool { + matches!( + (persisted_identity_present, lease_state), + (true, PersistedLeaseReleaseState::Destroyed) + | (false, PersistedLeaseReleaseState::Missing) + ) +} + +fn admit_execution_release_step( + scope: ToolCallScope<'_>, + step: ExecutionReleaseStep, +) -> Result<()> { + match step { + ExecutionReleaseStep::DurableReconciliation => Ok(()), + ExecutionReleaseStep::ProviderIo => scope.admit(), + } +} + impl ToolRouter { /// Materializes the exact authorized worker workspace on its pinned provider. pub async fn attach_managed_workspace( @@ -548,6 +607,7 @@ impl ToolRouter { operation: storage_operation, hand: hand.clone(), parent_revision: binding.current_revision.clone(), + release_compute: false, }) .await }; @@ -598,6 +658,8 @@ impl ToolRouter { }) .await? { + self.delete_abandoned_checkpoint_prefix(&binding, publication.revision.checkpoint_id) + .await?; operations .mark_unknown(binding.tenant_id, operation_id) .await?; @@ -1041,7 +1103,7 @@ impl ToolRouter { session: &SessionMeta, worker_id: &str, provider: &str, - cache_key: &str, + cache_key: &HandProviderCacheKey, active: &ActiveHand, manifest: Option<&std::sync::Arc>, ) { @@ -1219,26 +1281,675 @@ impl ToolRouter { .ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { operation_id: format!("workspace-tool-call:{}", request.tool_call_id), })?; - self.commit_workspace_after_tool( - request.session, + self.commit_workspace_after_tool(WorkspaceCommitExecution { + session: request.session, workspace_scope, - request.tool_call_id, - &workspace.provider, - &hand, - request.scope, - ) + tool_call_id: request.tool_call_id, + provider_name: &workspace.provider, + hand: &hand, + call_scope: request.scope, + release_compute: false, + }) .await } + /// Checkpoints one execution-task workspace and releases its exact compute lease. + /// + /// The returned receipt is the durable proof required before a task may yield to + /// a timer, external callback, pause, or long backoff. Retries return the same + /// receipt. Provider teardown errors remain ambiguous and never produce release + /// proof; a later retry reconciles the checkpoint and repeats exact destruction. + pub async fn checkpoint_and_release_execution_hand( + &self, + request: ExecutionHandReleaseRequest<'_>, + ) -> Result { + if request.attempt_generation == 0 { + return Err(MoaError::ValidationError( + "execution task attempt generation must be positive".to_string(), + )); + } + let (task_id, logical_generation) = match request.owner { + ExecutionHandReleaseOwner::Task { + task_id, + logical_generation, + } if logical_generation > 0 => (task_id, logical_generation), + ExecutionHandReleaseOwner::Task { .. } => { + return Err(MoaError::ValidationError( + "execution task logical generation must be positive".to_string(), + )); + } + ExecutionHandReleaseOwner::Compensation { + compensation_id, + logical_generation, + } => { + return self + .release_execution_compensation_hand( + request, + compensation_id, + logical_generation, + ) + .await; + } + }; + let repository = + self.hands.workspace_repository.as_ref().ok_or_else(|| { + MoaError::StorageError("workspace repository missing".to_string()) + })?; + if let Some(receipt) = repository + .get_task_execution_hand_release_receipt( + request.session.tenant_id, + request.run_id, + task_id, + logical_generation, + request.attempt_generation, + ) + .await? + { + return Ok(receipt); + } + + let absence_receipt_id = Uuid::new_v5( + &request.run_id.0, + format!( + "execution-task-hand-absence-v1:{task_id}:{logical_generation}:{}", + request.attempt_generation + ) + .as_bytes(), + ); + match repository + .record_absent_task_execution_hand_release_receipt(AbsentTaskHandReleaseIntent { + receipt_id: absence_receipt_id, + tenant_id: request.session.tenant_id, + run_id: request.run_id, + task_id, + logical_generation, + attempt_generation: request.attempt_generation, + verified_at: Utc::now(), + }) + .await + { + Ok(receipt) => return Ok(receipt), + Err(MoaError::ExternalEffectUnknownOutcome { .. }) => {} + Err(error) => return Err(error), + } + + let workspace_scope = SandboxWorkspaceScope::ExecutionTask { + run_id: request.run_id, + task_id, + }; + let initial_workspace = repository + .get_by_scope(request.session.tenant_id, &workspace_scope) + .await? + .ok_or_else(|| { + MoaError::PermissionDenied( + "execution-task workspace disappeared before hand release".to_string(), + ) + })?; + let release_key = format!( + "execution-task-yield-v1:{}:{}:{}", + request.run_id, task_id, request.attempt_generation + ); + let tool_call_id = ToolCallId(Uuid::new_v5( + &initial_workspace.workspace_id.0, + release_key.as_bytes(), + )); + let candidate_receipt_id = Uuid::new_v5( + &initial_workspace.workspace_id.0, + format!("release-receipt-v1:{release_key}").as_bytes(), + ); + let lease_scope = workspace_lease_scope(&workspace_scope); + let lease_store = self.hands.hand_leases.as_ref().ok_or_else(|| { + MoaError::StorageError("durable hand lease store missing".to_string()) + })?; + let initial_lease = lease_store + .get( + request.session.tenant_id, + request.session.id, + &lease_scope, + &initial_workspace.provider, + ) + .await? + .ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { + operation_id: release_key.clone(), + })?; + let (receipt_id, release_claim_token, requested_at) = repository + .begin_task_execution_hand_release(TaskHandReleaseIntent { + receipt_id: candidate_receipt_id, + run_id: request.run_id, + task_id, + logical_generation, + attempt_generation: request.attempt_generation, + deadline_at: request + .scope + .budget + .deadline + .unwrap_or_else(|| Utc::now() + ChronoDuration::minutes(5)), + recovery_claim_expires_at: Utc::now() + ChronoDuration::minutes(5), + workspace: &initial_workspace, + lease: &initial_lease, + }) + .await?; + + admit_execution_release_step( + request.scope, + if initial_lease.status == HandLeaseStatus::Active { + ExecutionReleaseStep::ProviderIo + } else { + ExecutionReleaseStep::DurableReconciliation + }, + )?; + + match initial_lease.status { + HandLeaseStatus::Active => { + let hand = initial_lease + .handle + .as_ref() + .map(|handle| handle.handle.clone()) + .ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { + operation_id: release_key.clone(), + })?; + self.commit_workspace_after_tool(WorkspaceCommitExecution { + session: request.session, + workspace_scope: &workspace_scope, + tool_call_id, + provider_name: &initial_workspace.provider, + hand: &hand, + call_scope: request.scope, + release_compute: true, + }) + .await?; + } + HandLeaseStatus::Destroyed => { + if !self + .confirmed_workspace_commit_replay(&initial_workspace, tool_call_id) + .await? + { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: release_key.clone(), + }); + } + } + HandLeaseStatus::Provisioning + | HandLeaseStatus::Stale + | HandLeaseStatus::Failed + | HandLeaseStatus::Reaping => { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: release_key.clone(), + }); + } + } + + let mut final_workspace = repository + .get_by_scope(request.session.tenant_id, &workspace_scope) + .await? + .ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { + operation_id: release_key.clone(), + })?; + let mut final_lease = lease_store + .get( + request.session.tenant_id, + request.session.id, + &lease_scope, + &initial_workspace.provider, + ) + .await? + .ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { + operation_id: release_key.clone(), + })?; + + // An unknown provider outcome may reconcile the already-verified bytes + // while conservatively retaining the attachment. Finish the destroy as a + // separate exact step, then atomically release lease and capacity ownership. + if final_lease.status == HandLeaseStatus::Active { + if !self + .confirmed_workspace_commit_replay(&final_workspace, tool_call_id) + .await? + { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: release_key.clone(), + }); + } + let hand = final_lease + .handle + .as_ref() + .map(|handle| handle.handle.clone()) + .ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { + operation_id: release_key.clone(), + })?; + let provider = self + .hands + .providers + .get(&initial_workspace.provider) + .ok_or_else(|| { + MoaError::ProviderError(format!( + "hand provider {} is not registered", + initial_workspace.provider + )) + })?; + self.run_within_scope(request.scope, provider.destroy(&hand)) + .await + .map_err(|error| { + tracing::warn!( + operation_id = %release_key, + error = %error, + "execution-task hand destroy outcome is ambiguous" + ); + MoaError::ExternalEffectUnknownOutcome { + operation_id: release_key.clone(), + } + })?; + if !repository + .finalize_task_yield_destroy(&final_workspace.binding()?, &final_lease) + .await? + { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: release_key.clone(), + }); + } + let key = session_provider_key( + request.session, + Some(&lease_scope), + &initial_workspace.provider, + ); + self.remove_cached_binding_if_matches(&key, &hand, Some(initial_lease.generation)) + .await; + self.remove_installed_marker( + manifest_scope_key(request.session, Some(&lease_scope)), + &initial_workspace.provider, + ) + .await; + final_workspace = repository + .get_by_scope(request.session.tenant_id, &workspace_scope) + .await? + .ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { + operation_id: release_key.clone(), + })?; + final_lease = lease_store + .get( + request.session.tenant_id, + request.session.id, + &lease_scope, + &initial_workspace.provider, + ) + .await? + .ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { + operation_id: release_key.clone(), + })?; + } + + let operation_id = WorkspaceOperationId(Uuid::new_v5( + &initial_workspace.workspace_id.0, + format!("tool-commit-v1:{tool_call_id}").as_bytes(), + )); + let checkpoint_id = WorkspaceCheckpointId(operation_id.0); + let checkpoint = repository + .get_checkpoint( + request.session.tenant_id, + final_workspace.workspace_id, + checkpoint_id, + ) + .await? + .ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { + operation_id: release_key.clone(), + })?; + if final_workspace.state != SandboxWorkspaceState::Ready + || final_workspace.writer_epoch != initial_workspace.writer_epoch + || final_workspace.instance_generation != initial_workspace.instance_generation + || final_workspace.checkpoint_id != Some(checkpoint_id) + || final_workspace.checkpoint_generation != checkpoint.generation + || final_lease.status != HandLeaseStatus::Destroyed + || final_lease.handle.is_some() + || final_lease.generation != initial_lease.generation + || final_lease.provisioning_operation_id != initial_lease.provisioning_operation_id + || checkpoint.state != WorkspaceCheckpointState::Available + || checkpoint.manifest_digest.is_none() + || checkpoint.logical_bytes.is_none() + { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: release_key, + }); + } + let receipt = ExecutionHandReleaseReceipt { + receipt_id, + tenant_id: request.session.tenant_id, + run_id: request.run_id, + owner: request.owner, + attempt_generation: request.attempt_generation, + workspace_id: Some(final_workspace.workspace_id), + writer_epoch: Some(u64::try_from(final_workspace.writer_epoch).map_err(|_| { + MoaError::StorageError("workspace writer epoch is invalid".to_string()) + })?), + instance_generation: Some(u64::try_from(final_workspace.instance_generation).map_err( + |_| MoaError::StorageError("workspace instance generation is invalid".to_string()), + )?), + hand_provisioning_operation_id: Some(initial_lease.provisioning_operation_id), + hand_lease_generation: Some(u64::try_from(initial_lease.generation).map_err(|_| { + MoaError::StorageError("hand lease generation is invalid".to_string()) + })?), + checkpoint_id: Some(checkpoint_id), + checkpoint_generation: Some(u64::try_from(checkpoint.generation).map_err(|_| { + MoaError::StorageError("checkpoint generation is invalid".to_string()) + })?), + checkpoint_manifest_digest: Some(checkpoint.manifest_digest.ok_or_else(|| { + MoaError::StorageError("verified checkpoint digest is missing".to_string()) + })?), + checkpoint_logical_bytes: Some( + u64::try_from(checkpoint.logical_bytes.ok_or_else(|| { + MoaError::StorageError("verified checkpoint bytes are missing".to_string()) + })?) + .map_err(|_| MoaError::StorageError("checkpoint bytes are negative".to_string()))?, + ), + requested_at, + released_at: Utc::now(), + }; + repository + .record_task_execution_hand_release_receipt(&receipt, release_claim_token) + .await + } + + async fn release_execution_compensation_hand( + &self, + request: ExecutionHandReleaseRequest<'_>, + compensation_id: ExecutionCompensationScopeId, + logical_generation: u64, + ) -> Result { + if logical_generation == 0 { + return Err(MoaError::ValidationError( + "execution compensation logical generation must be positive".to_string(), + )); + } + let repository = + self.hands.workspace_repository.as_ref().ok_or_else(|| { + MoaError::StorageError("workspace repository missing".to_string()) + })?; + if let Some(receipt) = repository + .get_compensation_execution_hand_release_receipt( + request.session.tenant_id, + request.run_id, + compensation_id, + logical_generation, + request.attempt_generation, + ) + .await? + { + return Ok(receipt); + } + + let hand_scope = format!( + "execution_compensation:{}:{}", + request.run_id, compensation_id + ); + let lease_store = self.hands.hand_leases.as_ref().ok_or_else(|| { + MoaError::StorageError("durable hand lease store missing".to_string()) + })?; + if let Some(claim) = repository + .claim_pending_compensation_execution_hand_release( + request.session.tenant_id, + request.run_id, + compensation_id, + logical_generation, + request.attempt_generation, + Utc::now() + ChronoDuration::minutes(5), + ) + .await? + { + let persisted_identity = match ( + claim.hand_provisioning_operation_id, + claim.hand_lease_generation, + ) { + (Some(operation_id), Some(generation)) => Some((operation_id, generation)), + (None, None) => None, + _ => { + return Err(MoaError::StorageError( + "pending compensation release has a partial hand identity".to_string(), + )); + } + }; + let exact_lease = match persisted_identity { + Some((operation_id, generation)) => { + lease_store + .get_exact_generation( + request.session.tenant_id, + request.session.id, + &hand_scope, + operation_id, + generation, + ) + .await? + } + None => None, + }; + let provider_io_required = exact_lease + .as_ref() + .is_some_and(|lease| lease.status != HandLeaseStatus::Destroyed); + admit_execution_release_step( + request.scope, + if provider_io_required { + ExecutionReleaseStep::ProviderIo + } else { + ExecutionReleaseStep::DurableReconciliation + }, + )?; + if provider_io_required + && !self + .reclaim_hands( + request.session.tenant_id, + &request.session.id, + Some(&hand_scope), + ) + .await + { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: format!( + "execution-compensation-hand-release:{}:{compensation_id}:{logical_generation}:{}", + request.run_id, request.attempt_generation + ), + }); + } + let exact_lease = match persisted_identity { + Some((operation_id, generation)) => { + lease_store + .get_exact_generation( + request.session.tenant_id, + request.session.id, + &hand_scope, + operation_id, + generation, + ) + .await? + } + None => None, + }; + let lease_state = match exact_lease.as_ref() { + None => PersistedLeaseReleaseState::Missing, + Some(lease) + if lease.status == HandLeaseStatus::Destroyed && lease.handle.is_none() => + { + PersistedLeaseReleaseState::Destroyed + } + Some(_) => PersistedLeaseReleaseState::LiveOrAmbiguous, + }; + let exact_released = compensation_release_identity_is_verified( + persisted_identity.is_some(), + lease_state, + ); + let replacement = lease_store + .has_live_owner(request.session.tenant_id, request.session.id, &hand_scope) + .await?; + if !exact_released || replacement { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: format!( + "execution-compensation-hand-release:{}:{compensation_id}:{logical_generation}:{}", + request.run_id, request.attempt_generation + ), + }); + } + let hand_lease_generation = claim + .hand_lease_generation + .map(u64::try_from) + .transpose() + .map_err(|_| { + MoaError::StorageError("hand lease generation is invalid".to_string()) + })?; + return repository + .record_compensation_execution_hand_release_receipt( + &ExecutionHandReleaseReceipt { + receipt_id: claim.receipt_id, + tenant_id: request.session.tenant_id, + run_id: request.run_id, + owner: request.owner, + attempt_generation: request.attempt_generation, + workspace_id: None, + writer_epoch: None, + instance_generation: None, + hand_provisioning_operation_id: claim.hand_provisioning_operation_id, + hand_lease_generation, + checkpoint_id: None, + checkpoint_generation: None, + checkpoint_manifest_digest: None, + checkpoint_logical_bytes: None, + requested_at: claim.requested_at, + released_at: Utc::now(), + }, + request.session.id, + &hand_scope, + claim.claim_token, + ) + .await; + } + let leases = lease_store + .list_live_owner_candidates(request.session.tenant_id, request.session.id, &hand_scope) + .await?; + if leases.len() > 1 { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: format!( + "execution-compensation-hand-release:{}:{compensation_id}:{logical_generation}:{}", + request.run_id, request.attempt_generation + ), + }); + } + let initial_lease = leases.into_iter().next(); + + let release_key = format!( + "execution-compensation-release-v1:{}:{compensation_id}:{logical_generation}:{}", + request.run_id, request.attempt_generation + ); + let receipt_id = Uuid::new_v5(&request.run_id.0, release_key.as_bytes()); + let (receipt_id, claim_token, requested_at) = repository + .begin_compensation_execution_hand_release(CompensationHandReleaseIntent { + receipt_id, + tenant_id: request.session.tenant_id, + session_id: request.session.id, + run_id: request.run_id, + compensation_id, + logical_generation, + attempt_generation: request.attempt_generation, + hand_scope: &hand_scope, + lease: initial_lease.as_ref(), + deadline_at: request + .scope + .budget + .deadline + .unwrap_or_else(|| Utc::now() + ChronoDuration::minutes(5)), + recovery_claim_expires_at: Utc::now() + ChronoDuration::minutes(5), + }) + .await?; + admit_execution_release_step( + request.scope, + if initial_lease.is_some() { + ExecutionReleaseStep::ProviderIo + } else { + ExecutionReleaseStep::DurableReconciliation + }, + )?; + if initial_lease.is_some() + && !self + .reclaim_hands( + request.session.tenant_id, + &request.session.id, + Some(&hand_scope), + ) + .await + { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: release_key, + }); + } + if let Some(initial_lease) = initial_lease.as_ref() { + let exact_lease = lease_store + .get( + request.session.tenant_id, + request.session.id, + &hand_scope, + &initial_lease.provider, + ) + .await?; + let exact_destroyed = exact_lease.as_ref().is_some_and(|lease| { + lease.worker_id == hand_scope + && lease.provisioning_operation_id == initial_lease.provisioning_operation_id + && lease.generation == initial_lease.generation + && lease.status == HandLeaseStatus::Destroyed + && lease.handle.is_none() + }); + let replacement = lease_store + .has_live_owner(request.session.tenant_id, request.session.id, &hand_scope) + .await?; + if !exact_destroyed || replacement { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: release_key, + }); + } + } + let hand_lease_generation = initial_lease + .as_ref() + .map(|lease| { + u64::try_from(lease.generation).map_err(|_| { + MoaError::StorageError("hand lease generation is invalid".to_string()) + }) + }) + .transpose()?; + repository + .record_compensation_execution_hand_release_receipt( + &ExecutionHandReleaseReceipt { + receipt_id, + tenant_id: request.session.tenant_id, + run_id: request.run_id, + owner: request.owner, + attempt_generation: request.attempt_generation, + workspace_id: None, + writer_epoch: None, + instance_generation: None, + hand_provisioning_operation_id: initial_lease + .as_ref() + .map(|lease| lease.provisioning_operation_id), + hand_lease_generation, + checkpoint_id: None, + checkpoint_generation: None, + checkpoint_manifest_digest: None, + checkpoint_logical_bytes: None, + requested_at, + released_at: Utc::now(), + }, + request.session.id, + &hand_scope, + claim_token, + ) + .await + } + pub(in crate::core) async fn commit_workspace_after_tool( &self, - session: &SessionMeta, - workspace_scope: &SandboxWorkspaceScope, - tool_call_id: ToolCallId, - provider_name: &str, - hand: &HandHandle, - call_scope: ToolCallScope<'_>, + request: WorkspaceCommitExecution<'_>, ) -> Result<()> { + let WorkspaceCommitExecution { + session, + workspace_scope, + tool_call_id, + provider_name, + hand, + call_scope, + release_compute, + } = request; let Some(repository) = self.hands.workspace_repository.as_ref() else { return Ok(()); }; @@ -1496,6 +2207,7 @@ impl ToolRouter { operation: storage_operation, hand: hand.clone(), parent_revision: binding.current_revision.clone(), + release_compute, }), ) .await @@ -1547,6 +2259,8 @@ impl ToolRouter { }) .await? { + self.delete_abandoned_checkpoint_prefix(&binding, publication.revision.checkpoint_id) + .await?; operations .mark_unknown(binding.tenant_id, operation_id) .await?; @@ -1566,6 +2280,29 @@ impl ToolRouter { } Ok(()) } + + async fn delete_abandoned_checkpoint_prefix( + &self, + binding: &WorkspaceBinding, + checkpoint_id: WorkspaceCheckpointId, + ) -> Result<()> { + let store = self.hands.checkpoint_store.as_ref().ok_or_else(|| { + MoaError::ConfigError( + "checkpoint CAS cleanup requires the durable checkpoint store".to_string(), + ) + })?; + store + .delete( + crate::core::sandbox_workspace::checkpoint::store::CheckpointStoreContext { + tenant_id: binding.tenant_id, + workspace_id: binding.workspace_id, + checkpoint_id, + provider_account_id: binding.provider_account_id, + provider_account_generation: binding.provider_account_generation, + }, + ) + .await + } } fn revision_from_checkpoint_parent( @@ -1634,3 +2371,56 @@ pub(in crate::core) fn lease_attachment( .map(|revision| revision.checkpoint_id), ) } + +#[cfg(test)] +mod tests { + use chrono::{Duration, Utc}; + use moa_core::{error::MoaError, types::resource::ResourceBudget}; + + use super::{ + ExecutionReleaseStep, PersistedLeaseReleaseState, ToolCallScope, + admit_execution_release_step, compensation_release_identity_is_verified, + }; + + #[test] + fn expired_release_budget_allows_reconciliation_but_rejects_new_provider_io() { + // Pins: retrying an exact release after its five-minute I/O window may return a durable + // receipt or finalize verified absence, but it must not start fresh provider operations. + let release_started_at = Utc::now() - Duration::minutes(6); + let scope = ToolCallScope::unbounded().with_budget(ResourceBudget::until( + release_started_at + Duration::minutes(5), + )); + + assert!( + admit_execution_release_step(scope, ExecutionReleaseStep::DurableReconciliation) + .is_ok() + ); + assert!(matches!( + admit_execution_release_step(scope, ExecutionReleaseStep::ProviderIo), + Err(MoaError::BudgetExhausted(_)) + )); + } + + #[test] + fn persisted_compensation_lease_identity_requires_the_exact_destroyed_row() { + // Pins: after provider teardown, a persisted op/generation may finalize only against its + // exact Destroyed row; a missing row is not interchangeable with an attempt that proved + // it never acquired a hand. + assert!(compensation_release_identity_is_verified( + true, + PersistedLeaseReleaseState::Destroyed, + )); + assert!(!compensation_release_identity_is_verified( + true, + PersistedLeaseReleaseState::Missing, + )); + assert!(!compensation_release_identity_is_verified( + true, + PersistedLeaseReleaseState::LiveOrAmbiguous, + )); + assert!(compensation_release_identity_is_verified( + false, + PersistedLeaseReleaseState::Missing, + )); + } +} diff --git a/crates/moa-hands/src/core/sandbox_workspace/maintenance/inventory.rs b/crates/moa-hands/src/core/sandbox_workspace/maintenance/inventory.rs index 5be3621bb..c88f83201 100644 --- a/crates/moa-hands/src/core/sandbox_workspace/maintenance/inventory.rs +++ b/crates/moa-hands/src/core/sandbox_workspace/maintenance/inventory.rs @@ -3,60 +3,202 @@ use super::*; impl WorkspaceMaintenanceCoordinator { - /// Reconciles every persisted provider-account generation and persists drift. - pub async fn reconcile_provider_inventory_once(&self) -> Result { - let accounts = self.provider_accounts().await?; - let mut observed_keys = HashSet::new(); - let mut observed_accounts = HashSet::new(); - let mut pass = WorkspaceInventoryPass::default(); - let mut counts: BTreeMap<(String, InventoryFindingKind), u64> = BTreeMap::new(); + /// Claims and reconciles one bounded shard of provider-account generations. + pub async fn reconcile_claimed_provider_inventory_once( + &self, + batch_size: usize, + ) -> Result { + let accounts = self.claim_provider_accounts(batch_size).await?; + let mut pass = WorkspaceInventoryPass { + accounts: accounts.len() as u64, + ..WorkspaceInventoryPass::default() + }; + let mut first_error = None; for account in accounts { - let provider = self.storage_provider(&account.provider)?; - let inventory = provider - .enumerate_account_storage(account.id, account.generation) - .await?; - validate_inventory_identity(&account, &inventory)?; - observed_accounts.insert((account.id, account.generation)); - let durable = self.durable_inventory(&account).await?; - pass.accounts += 1; - pass.resources += inventory.resources.len() as u64; - let findings = compare_inventory(&account, &inventory, &durable); - for finding in findings { - observed_keys.insert(finding.key()); - *counts - .entry(( - provider_metric_label(&account.provider).to_string(), - finding.kind, - )) - .or_default() += 1; - self.upsert_inventory_finding(&finding).await?; + let result = self.reconcile_claimed_provider_account(&account).await; + match result { + Ok((resources, account_counts)) => { + pass.resources += resources; + pass.unresolved_findings += account_counts.values().sum::(); + if !self.complete_provider_inventory_claim(&account).await? { + first_error.get_or_insert_with(|| MoaError::ExternalEffectUnknownOutcome { + operation_id: format!( + "provider-inventory-claim:{}:{}:{}", + account.id, account.generation, account.claim_generation + ), + }); + } + } + Err(error) => { + let released = self + .fail_provider_inventory_claim(&account, &error.to_string()) + .await?; + if !released { + first_error.get_or_insert_with(|| MoaError::ExternalEffectUnknownOutcome { + operation_id: format!( + "provider-inventory-claim:{}:{}:{}", + account.id, account.generation, account.claim_generation + ), + }); + } else if first_error.is_none() { + first_error = Some(error); + } + } } } - self.resolve_unseen_findings(&observed_keys, &observed_accounts) - .await?; - pass.unresolved_findings = counts.values().sum(); - emit_complete_inventory_metrics(&counts); + let fleet_counts = self.unresolved_inventory_counts().await?; + emit_complete_inventory_metrics(&fleet_counts); + if let Some(error) = first_error { + return Err(error); + } Ok(pass) } - /// Fences access, removes all external sandbox state, and returns exact absence evidence. - /// - /// Relational metadata is deliberately untouched after the access fence. A - /// provider outage therefore leaves every ownership and reconciliation row - /// available for the next Restate replay. - async fn provider_accounts(&self) -> Result> { + async fn unresolved_inventory_counts( + &self, + ) -> Result> { + let mut conn = maintenance_conn(&self.pool).await?; + let rows = sqlx::query( + r#" + SELECT account.provider, finding.finding_kind, count(*)::BIGINT AS count + FROM moa.sandbox_provider_inventory_findings AS finding + JOIN moa.sandbox_provider_accounts AS account + ON account.provider_account_id = finding.provider_account_id + AND account.generation = finding.provider_account_generation + WHERE finding.quarantine_state <> 'resolved' + GROUP BY account.provider, finding.finding_kind + "#, + ) + .fetch_all(conn.as_mut()) + .await + .map_err(map_sqlx)?; + let mut counts = BTreeMap::new(); + for row in rows { + let provider: String = row.try_get("provider").map_err(map_sqlx)?; + let kind = InventoryFindingKind::from_label( + &row.try_get::("finding_kind").map_err(map_sqlx)?, + )?; + let count: i64 = row.try_get("count").map_err(map_sqlx)?; + counts.insert( + (provider_metric_label(&provider).to_string(), kind), + u64::try_from(count).map_err(|_| { + MoaError::StorageError("inventory finding count is negative".to_string()) + })?, + ); + } + conn.commit().await?; + Ok(counts) + } + + async fn reconcile_claimed_provider_account( + &self, + claimed: &ClaimedProviderAccount, + ) -> Result<(u64, BTreeMap<(String, InventoryFindingKind), u64>)> { + let account = claimed.account(); + let provider = self.storage_provider(&account.provider)?; + let inventory = provider + .enumerate_account_storage(account.id, account.generation) + .await?; + validate_inventory_identity(&account, &inventory)?; + let durable = self.durable_inventory(&account).await?; + let findings = compare_inventory(&account, &inventory, &durable); + let mut observed_keys = HashSet::new(); + let mut counts = BTreeMap::new(); + for finding in findings { + observed_keys.insert(finding.key()); + *counts + .entry(( + provider_metric_label(&account.provider).to_string(), + finding.kind, + )) + .or_default() += 1; + self.upsert_inventory_finding(&finding).await?; + } + self.resolve_unseen_findings( + &observed_keys, + &HashSet::from([(account.id, account.generation)]), + ) + .await?; + Ok((inventory.resources.len() as u64, counts)) + } + + async fn claim_provider_accounts( + &self, + batch_size: usize, + ) -> Result> { + if batch_size == 0 { + return Err(MoaError::ValidationError( + "provider inventory claim batch must be positive".to_string(), + )); + } + let batch_size = i64::try_from(batch_size).map_err(|_| { + MoaError::ValidationError("provider inventory batch overflows bigint".to_string()) + })?; + let ttl_seconds = i64::try_from(self.reconciliation_claim_ttl.as_secs()).map_err(|_| { + MoaError::ValidationError("provider inventory claim TTL overflows bigint".to_string()) + })?; + let claim_token = Uuid::now_v7(); let mut conn = maintenance_conn(&self.pool).await?; + sqlx::query( + r#" + INSERT INTO moa.sandbox_provider_inventory_claims AS claim ( + provider_account_id, provider_account_generation, provider + ) + SELECT provider_account_id, generation, provider + FROM moa.sandbox_provider_accounts + WHERE health <> 'disabled' + ON CONFLICT (provider_account_id, provider_account_generation) + DO UPDATE SET provider = EXCLUDED.provider + WHERE claim.provider IS DISTINCT FROM EXCLUDED.provider + "#, + ) + .execute(conn.as_mut()) + .await + .map_err(map_sqlx)?; let rows = sqlx::query( - "SELECT provider_account_id, generation, provider FROM moa.sandbox_provider_accounts WHERE health <> 'disabled' ORDER BY provider, provider_account_id", + r#" + WITH candidates AS ( + SELECT claim.provider_account_id, claim.provider_account_generation + FROM moa.sandbox_provider_inventory_claims AS claim + JOIN moa.sandbox_provider_accounts AS account + ON account.provider_account_id = claim.provider_account_id + AND account.generation = claim.provider_account_generation + WHERE account.health <> 'disabled' + AND (claim.claim_token IS NULL OR claim.claim_expires_at <= now()) + ORDER BY claim.last_succeeded_at NULLS FIRST, + claim.provider, claim.provider_account_id, + claim.provider_account_generation + LIMIT $1 + FOR UPDATE OF claim SKIP LOCKED + ) + UPDATE moa.sandbox_provider_inventory_claims AS claim + SET claim_generation = claim.claim_generation + 1, + claim_owner = $2, claim_token = $3, + claimed_at = now(), + claim_expires_at = now() + make_interval(secs => $4), + updated_at = now() + FROM candidates + WHERE claim.provider_account_id = candidates.provider_account_id + AND claim.provider_account_generation = candidates.provider_account_generation + RETURNING claim.provider_account_id, claim.provider_account_generation, + claim.provider, claim.claim_generation, claim.claim_token + "#, ) + .bind(batch_size) + .bind(self.inventory_claim_owner) + .bind(claim_token) + .bind(ttl_seconds) .fetch_all(conn.as_mut()) .await .map_err(map_sqlx)?; let accounts = rows .iter() .map(|row| { - let generation: i64 = row.try_get("generation").map_err(map_sqlx)?; - Ok(ProviderAccount { + let generation: i64 = row + .try_get("provider_account_generation") + .map_err(map_sqlx)?; + let claim_generation: i64 = row.try_get("claim_generation").map_err(map_sqlx)?; + Ok(ClaimedProviderAccount { id: row.try_get("provider_account_id").map_err(map_sqlx)?, generation: u64::try_from(generation).map_err(|_| { MoaError::StorageError( @@ -64,6 +206,12 @@ impl WorkspaceMaintenanceCoordinator { ) })?, provider: row.try_get("provider").map_err(map_sqlx)?, + claim_generation: u64::try_from(claim_generation).map_err(|_| { + MoaError::StorageError( + "provider inventory claim generation is invalid".to_string(), + ) + })?, + claim_token: row.try_get("claim_token").map_err(map_sqlx)?, }) }) .collect::>>()?; @@ -71,6 +219,86 @@ impl WorkspaceMaintenanceCoordinator { Ok(accounts) } + async fn complete_provider_inventory_claim( + &self, + claim: &ClaimedProviderAccount, + ) -> Result { + self.finish_provider_inventory_claim(claim, None).await + } + + async fn fail_provider_inventory_claim( + &self, + claim: &ClaimedProviderAccount, + error: &str, + ) -> Result { + self.finish_provider_inventory_claim(claim, Some(error)) + .await + } + + async fn finish_provider_inventory_claim( + &self, + claim: &ClaimedProviderAccount, + error: Option<&str>, + ) -> Result { + let mut conn = maintenance_conn(&self.pool).await?; + let affected = if let Some(error) = error { + let error = truncate_inventory_error(error); + sqlx::query( + r#" + UPDATE moa.sandbox_provider_inventory_claims + SET claim_owner = NULL, claim_token = NULL, + claimed_at = NULL, claim_expires_at = NULL, + last_error = $6, last_error_at = now(), updated_at = now() + WHERE provider_account_id = $1 AND provider_account_generation = $2 + AND claim_generation = $3 AND claim_owner = $4 AND claim_token = $5 + AND claim_expires_at > now() + "#, + ) + .bind(claim.id) + .bind(i64::try_from(claim.generation).map_err(|_| { + MoaError::StorageError("provider account generation overflows bigint".to_string()) + })?) + .bind(i64::try_from(claim.claim_generation).map_err(|_| { + MoaError::StorageError("inventory claim generation overflows bigint".to_string()) + })?) + .bind(self.inventory_claim_owner) + .bind(claim.claim_token) + .bind(error) + .execute(conn.as_mut()) + .await + .map_err(map_sqlx)? + .rows_affected() + } else { + sqlx::query( + r#" + UPDATE moa.sandbox_provider_inventory_claims + SET claim_owner = NULL, claim_token = NULL, + claimed_at = NULL, claim_expires_at = NULL, + last_succeeded_at = now(), last_error = NULL, + last_error_at = NULL, updated_at = now() + WHERE provider_account_id = $1 AND provider_account_generation = $2 + AND claim_generation = $3 AND claim_owner = $4 AND claim_token = $5 + AND claim_expires_at > now() + "#, + ) + .bind(claim.id) + .bind(i64::try_from(claim.generation).map_err(|_| { + MoaError::StorageError("provider account generation overflows bigint".to_string()) + })?) + .bind(i64::try_from(claim.claim_generation).map_err(|_| { + MoaError::StorageError("inventory claim generation overflows bigint".to_string()) + })?) + .bind(self.inventory_claim_owner) + .bind(claim.claim_token) + .execute(conn.as_mut()) + .await + .map_err(map_sqlx)? + .rows_affected() + }; + conn.commit().await?; + Ok(affected == 1) + } + /// Lists exact provider-account generations referenced by one tenant. pub(super) async fn tenant_provider_accounts( &self, @@ -142,9 +370,14 @@ impl WorkspaceMaintenanceCoordinator { SELECT workspace_id, tenant_id, writer_epoch, instance_generation, provider_account_id, provider_account_generation FROM moa.sandbox_workspaces - WHERE lifecycle_state <> 'deleted' + WHERE provider_account_id = $1 AND provider_account_generation = $2 + AND lifecycle_state <> 'deleted' "#, ) + .bind(account.id) + .bind(i64::try_from(account.generation).map_err(|_| { + MoaError::StorageError("provider account generation overflows Postgres".to_string()) + })?) .fetch_all(conn.as_mut()) .await .map_err(map_sqlx)?; @@ -340,6 +573,30 @@ pub(super) struct ProviderAccount { pub(super) provider: String, } +#[derive(Debug, Clone)] +struct ClaimedProviderAccount { + id: ProviderAccountId, + generation: u64, + provider: String, + claim_generation: u64, + claim_token: Uuid, +} + +impl ClaimedProviderAccount { + fn account(&self) -> ProviderAccount { + ProviderAccount { + id: self.id, + generation: self.generation, + provider: self.provider.clone(), + } + } +} + +fn truncate_inventory_error(error: &str) -> String { + const LIMIT: usize = 2_048; + error.chars().take(LIMIT).collect() +} + #[derive(Debug, Clone)] /// Durable ownership fences for one workspace provider resource. pub(super) struct DurableWorkspaceOwner { diff --git a/crates/moa-hands/src/core/sandbox_workspace/maintenance/mod.rs b/crates/moa-hands/src/core/sandbox_workspace/maintenance/mod.rs index b514b4a54..7d8ec00f4 100644 --- a/crates/moa-hands/src/core/sandbox_workspace/maintenance/mod.rs +++ b/crates/moa-hands/src/core/sandbox_workspace/maintenance/mod.rs @@ -76,7 +76,7 @@ pub struct WorkspaceRetentionPass { /// One provider-inventory reconciliation pass. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub struct WorkspaceInventoryPass { - /// Provider-account generations observed. + /// Provider-account generations exclusively claimed by this replica. pub accounts: u64, /// Provider resources observed. pub resources: u64, @@ -150,6 +150,7 @@ pub struct WorkspaceMaintenanceCoordinator { hand_providers: Arc>>, retention: CheckpointRetentionConfig, reconciliation_claim_ttl: Duration, + inventory_claim_owner: Uuid, } impl WorkspaceMaintenanceCoordinator { @@ -259,6 +260,7 @@ impl WorkspaceMaintenanceCoordinator { hand_providers: Arc::new(hand_providers), retention, reconciliation_claim_ttl, + inventory_claim_owner: Uuid::now_v7(), }) } diff --git a/crates/moa-hands/src/core/sandbox_workspace/model.rs b/crates/moa-hands/src/core/sandbox_workspace/model.rs index 625c54566..815768a90 100644 --- a/crates/moa-hands/src/core/sandbox_workspace/model.rs +++ b/crates/moa-hands/src/core/sandbox_workspace/model.rs @@ -4,7 +4,10 @@ use chrono::{DateTime, Utc}; use moa_core::{ error::{MoaError, Result}, types::{ - identifiers::{ProviderAccountId, SandboxWorkspaceId, TenantId, WorkspaceCheckpointId}, + identifiers::{ + HandProvisioningOperationId, ProviderAccountId, SandboxWorkspaceId, TenantId, + WorkspaceCheckpointId, + }, sandbox_workspace::{ DurabilityClass, SandboxWorkspaceScope, SandboxWorkspaceState, WorkspaceBinding, }, @@ -15,6 +18,87 @@ use uuid::Uuid; use super::checkpoint::archive::CHECKPOINT_ARCHIVE_FORMAT_VERSION; use crate::core::leases::{HandLease, LeaseHandle}; +/// Exact durable identity claimed before an execution task releases its hand. +pub struct TaskHandReleaseIntent<'a> { + /// Deterministic receipt identity. + pub receipt_id: Uuid, + /// Owning execution run. + pub run_id: moa_core::types::identifiers::ExecutionRunScopeId, + /// Stable task identity within the run. + pub task_id: moa_core::types::identifiers::ExecutionTaskScopeId, + /// Exact logical task generation being suspended. + pub logical_generation: u64, + /// Exact attempt generation being suspended. + pub attempt_generation: u64, + /// Absolute recovery deadline for this release operation. + pub deadline_at: DateTime, + /// Short database claim expiry used for storage-only finalization retries. + pub recovery_claim_expires_at: DateTime, + /// Exact workspace generation being checkpointed. + pub workspace: &'a SandboxWorkspace, + /// Exact hand generation that must be destroyed. + pub lease: &'a HandLease, +} + +/// Exact task attempt whose sandbox absence must be durably proven. +pub struct AbsentTaskHandReleaseIntent { + /// Deterministic receipt identity. + pub receipt_id: Uuid, + /// Tenant owning the execution. + pub tenant_id: TenantId, + /// Owning execution run. + pub run_id: moa_core::types::identifiers::ExecutionRunScopeId, + /// Stable task identity. + pub task_id: moa_core::types::identifiers::ExecutionTaskScopeId, + /// Exact logical task generation. + pub logical_generation: u64, + /// Exact bounded attempt generation. + pub attempt_generation: u64, + /// Time at which the database absence proof was established. + pub verified_at: DateTime, +} + +/// Exact durable identity claimed before a compensation releases its scoped hand. +pub struct CompensationHandReleaseIntent<'a> { + /// Deterministic receipt identity. + pub receipt_id: Uuid, + /// Tenant owning the execution. + pub tenant_id: TenantId, + /// Parent session whose hand scope is inspected. + pub session_id: moa_core::types::identifiers::SessionId, + /// Owning execution run. + pub run_id: moa_core::types::identifiers::ExecutionRunScopeId, + /// Stable compensation identity. + pub compensation_id: moa_core::types::identifiers::ExecutionCompensationScopeId, + /// Exact logical compensation generation. + pub logical_generation: u64, + /// Exact bounded attempt generation. + pub attempt_generation: u64, + /// Opaque deterministic hand scope for this compensation. + pub hand_scope: &'a str, + /// Exact durable lease claimed before destroy, or `None` for verified absence. + pub lease: Option<&'a HandLease>, + /// Absolute recovery deadline for this release operation. + pub deadline_at: DateTime, + /// Short database claim expiry used for storage-only finalization retries. + pub recovery_claim_expires_at: DateTime, +} + +/// Renewed recovery authority for one already-persisted compensation release. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CompensationHandReleaseClaim { + /// Deterministic receipt identity selected before provider teardown. + pub receipt_id: Uuid, + /// Exact short-lived database claim that may finalize this receipt. + pub claim_token: Uuid, + /// Original release request time preserved across recovery. + pub requested_at: DateTime, + /// Persisted provider create identity, absent only for a proven no-hand attempt. + pub hand_provisioning_operation_id: Option, + /// Persisted lease generation paired with the provider create identity. + pub hand_lease_generation: Option, +} + /// One durable logical workspace row. #[derive(Debug, Clone, PartialEq, Eq)] pub struct SandboxWorkspace { diff --git a/crates/moa-hands/src/core/sandbox_workspace/reaper.rs b/crates/moa-hands/src/core/sandbox_workspace/reaper.rs index b0fa61a13..85328c0d6 100644 --- a/crates/moa-hands/src/core/sandbox_workspace/reaper.rs +++ b/crates/moa-hands/src/core/sandbox_workspace/reaper.rs @@ -2,7 +2,7 @@ use std::{ sync::{Arc, RwLock}, - time::{Duration, Instant}, + time::Duration, }; use async_trait::async_trait; @@ -22,7 +22,7 @@ use super::{ maintenance::WorkspaceMaintenanceCoordinator, repository::PostgresWorkspaceRepository, }; use crate::core::{leases::HandLease, telemetry::record_workspace_reaper_health}; -use tokio::task::JoinHandle; +use tokio::{task::JoinHandle, time::Instant}; use tokio_util::sync::CancellationToken; /// One provider inventory observation for an exact fenced operation. @@ -78,6 +78,43 @@ pub struct WorkspaceReaperPass { pub retrying: usize, } +/// Independent maintenance cadences for safety and resource-intensive workspace work. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct WorkspaceReaperCadenceConfig { + /// Prompt cadence for due/backlog reconciliation and readiness heartbeat. + pub safety_interval: Duration, + /// Initial checkpoint-retention cadence after work or failure. + pub retention_initial_interval: Duration, + /// Maximum checkpoint-retention cadence after consecutive idle passes. + pub retention_maximum_interval: Duration, + /// Initial full provider-inventory cadence after work or failure. + pub inventory_initial_interval: Duration, + /// Maximum full provider-inventory cadence after consecutive idle passes. + pub inventory_maximum_interval: Duration, + /// Fixed low-frequency fleet metric refresh cadence. + pub fleet_metrics_interval: Duration, +} + +impl WorkspaceReaperCadenceConfig { + fn validate(self, heartbeat_maximum_age: Duration) -> Result<()> { + if self.safety_interval.is_zero() + || self.retention_initial_interval.is_zero() + || self.retention_maximum_interval < self.retention_initial_interval + || self.inventory_initial_interval.is_zero() + || self.inventory_maximum_interval < self.inventory_initial_interval + || self.fleet_metrics_interval.is_zero() + || heartbeat_maximum_age.is_zero() + || self.safety_interval >= heartbeat_maximum_age + { + return Err(MoaError::ConfigError( + "workspace reaper requires positive ordered cadences and a safety interval shorter than heartbeat freshness" + .to_string(), + )); + } + Ok(()) + } +} + /// Cross-replica workspace cleanup owner. pub struct WorkspaceReaper { operations: Arc, @@ -113,23 +150,52 @@ struct WorkspaceReaperHealth { exited: std::sync::atomic::AtomicBool, } +#[derive(Debug, Clone, Copy)] +struct AdaptiveCadence { + initial: Duration, + maximum: Duration, + current: Duration, + idle_streak: u32, +} + +impl AdaptiveCadence { + fn new(initial: Duration, maximum: Duration) -> Self { + Self { + initial, + maximum, + current: initial, + idle_streak: 0, + } + } + + fn record_success(&mut self, idle: bool) { + if idle { + self.idle_streak = self.idle_streak.saturating_add(1); + self.current = self.current.saturating_mul(2).min(self.maximum); + } else { + self.reset(); + } + } + + fn reset(&mut self) { + self.current = self.initial; + self.idle_streak = 0; + } +} + impl WorkspaceReaperHandle { /// Starts the supervised reaper before listener readiness. pub fn spawn( coordinator: Arc, reaper: WorkspaceReaper, - interval: Duration, + cadences: WorkspaceReaperCadenceConfig, batch_size: i64, heartbeat_maximum_age: Duration, ) -> Result { - if interval.is_zero() - || batch_size <= 0 - || heartbeat_maximum_age.is_zero() - || interval >= heartbeat_maximum_age - { + cadences.validate(heartbeat_maximum_age)?; + if batch_size <= 0 { return Err(MoaError::ConfigError( - "workspace reaper requires positive batch/heartbeat bounds and an interval shorter than heartbeat freshness" - .to_string(), + "workspace reaper requires a positive reconciliation batch".to_string(), )); } let state = Arc::new(WorkspaceReaperHealth { @@ -146,33 +212,14 @@ impl WorkspaceReaperHandle { let task_state = Arc::clone(&state); let task_shutdown = shutdown.clone(); let task = tokio::spawn(async move { - let result = async { - loop { - let backlog = coordinator.backlog().await?; - task_state - .backlog - .store(backlog.count, std::sync::atomic::Ordering::Release); - task_state.oldest_work_seconds.store( - backlog.oldest_age.as_secs(), - std::sync::atomic::Ordering::Release, - ); - reaper.run_once(batch_size).await?; - coordinator.run_retention_once().await?; - coordinator.reconcile_provider_inventory_once().await?; - coordinator.emit_fleet_metrics().await?; - set_reaper_heartbeat(&task_state)?; - record_workspace_reaper_health( - true, - Duration::ZERO, - backlog.count, - backlog.oldest_age, - ); - tokio::select! { - () = task_shutdown.cancelled() => return Ok(()), - () = tokio::time::sleep(interval) => {} - } - } - } + let result = supervise_workspace_maintenance( + coordinator, + reaper, + cadences, + batch_size, + Arc::clone(&task_state), + task_shutdown, + ) .await; task_state .exited @@ -372,6 +419,195 @@ impl WorkspaceReaperReadiness { } } +async fn supervise_workspace_maintenance( + coordinator: Arc, + reaper: WorkspaceReaper, + cadences: WorkspaceReaperCadenceConfig, + batch_size: i64, + state: Arc, + shutdown: CancellationToken, +) -> Result<()> { + let mut lanes = tokio::task::JoinSet::new(); + lanes.spawn(run_safety_lane( + Arc::clone(&coordinator), + reaper, + cadences.safety_interval, + batch_size, + Arc::clone(&state), + shutdown.clone(), + )); + lanes.spawn(run_retention_lane( + Arc::clone(&coordinator), + AdaptiveCadence::new( + cadences.retention_initial_interval, + cadences.retention_maximum_interval, + ), + shutdown.clone(), + )); + lanes.spawn(run_inventory_lane( + Arc::clone(&coordinator), + AdaptiveCadence::new( + cadences.inventory_initial_interval, + cadences.inventory_maximum_interval, + ), + usize::try_from(batch_size).map_err(|_| { + MoaError::ConfigError( + "workspace inventory reconciliation batch exceeds this platform".to_string(), + ) + })?, + shutdown.clone(), + )); + lanes.spawn(run_metrics_lane( + coordinator, + cadences.fleet_metrics_interval, + shutdown.clone(), + )); + + let mut failure = None; + while let Some(joined) = lanes.join_next().await { + match joined { + Ok(Ok(())) if shutdown.is_cancelled() => {} + Ok(Ok(())) => { + failure = Some(MoaError::StorageError( + "workspace maintenance lane exited unexpectedly".to_string(), + )); + shutdown.cancel(); + } + Ok(Err(error)) => { + failure = Some(error); + shutdown.cancel(); + } + Err(error) => { + failure = Some(MoaError::StorageError(format!( + "workspace maintenance lane join failed: {error}" + ))); + shutdown.cancel(); + } + } + } + failure.map_or(Ok(()), Err) +} + +async fn run_safety_lane( + coordinator: Arc, + reaper: WorkspaceReaper, + interval: Duration, + batch_size: i64, + state: Arc, + shutdown: CancellationToken, +) -> Result<()> { + loop { + if shutdown.is_cancelled() { + return Ok(()); + } + let backlog = coordinator.backlog().await?; + state + .backlog + .store(backlog.count, std::sync::atomic::Ordering::Release); + state.oldest_work_seconds.store( + backlog.oldest_age.as_secs(), + std::sync::atomic::Ordering::Release, + ); + reaper.run_once(batch_size).await?; + set_reaper_heartbeat(&state)?; + record_workspace_reaper_health(true, Duration::ZERO, backlog.count, backlog.oldest_age); + if wait_for_lane(&shutdown, interval).await { + return Ok(()); + } + } +} + +async fn run_retention_lane( + coordinator: Arc, + mut cadence: AdaptiveCadence, + shutdown: CancellationToken, +) -> Result<()> { + loop { + if shutdown.is_cancelled() { + return Ok(()); + } + match coordinator.run_retention_once().await { + Ok(pass) => { + let idle = pass.claimed == 0 + && pass.deleted == 0 + && pass.awaiting_absence == 0 + && pass.retrying == 0; + cadence.record_success(idle); + } + Err(error) => { + cadence.reset(); + tracing::warn!( + error_code = safe_error_code(&error), + "workspace checkpoint retention pass failed and will retry" + ); + } + } + if wait_for_lane(&shutdown, cadence.current).await { + return Ok(()); + } + } +} + +async fn run_inventory_lane( + coordinator: Arc, + mut cadence: AdaptiveCadence, + batch_size: usize, + shutdown: CancellationToken, +) -> Result<()> { + loop { + if shutdown.is_cancelled() { + return Ok(()); + } + match coordinator + .reconcile_claimed_provider_inventory_once(batch_size) + .await + { + Ok(pass) => { + let idle = pass.resources == 0 && pass.unresolved_findings == 0; + cadence.record_success(idle); + } + Err(error) => { + cadence.reset(); + tracing::warn!( + error_code = safe_error_code(&error), + "workspace provider inventory pass failed and will retry" + ); + } + } + if wait_for_lane(&shutdown, cadence.current).await { + return Ok(()); + } + } +} + +async fn run_metrics_lane( + coordinator: Arc, + interval: Duration, + shutdown: CancellationToken, +) -> Result<()> { + loop { + if shutdown.is_cancelled() { + return Ok(()); + } + if let Err(error) = coordinator.emit_fleet_metrics().await { + tracing::warn!( + error_code = safe_error_code(&error), + "workspace fleet metric refresh failed and will retry" + ); + } + if wait_for_lane(&shutdown, interval).await { + return Ok(()); + } + } +} + +async fn wait_for_lane(shutdown: &CancellationToken, delay: Duration) -> bool { + tokio::select! { + () = shutdown.cancelled() => true, + () = tokio::time::sleep(delay) => false, + } +} + fn set_reaper_heartbeat(state: &WorkspaceReaperHealth) -> Result<()> { *state.last_heartbeat.write().map_err(|_| { MoaError::StorageError("workspace reaper heartbeat lock is poisoned".to_string()) @@ -569,12 +805,13 @@ fn safe_error_code(error: &MoaError) -> &'static str { mod tests { use std::{ sync::{Arc, RwLock}, - time::{Duration, Instant}, + time::Duration, }; + use tokio::time::Instant; use super::{ - WorkspaceReaperHealth, WorkspaceReaperReadiness, reconciliation_backoff, - set_reaper_heartbeat, + AdaptiveCadence, WorkspaceReaperCadenceConfig, WorkspaceReaperHealth, + WorkspaceReaperReadiness, reconciliation_backoff, set_reaper_heartbeat, }; #[test] @@ -616,4 +853,73 @@ mod tests { Some("workspace reaper exited unexpectedly") ); } + + #[test] + fn adaptive_cadence_backs_off_only_on_idle_and_resets_on_work_or_failure_offline() { + // Pins: empty expensive passes exponentially reduce provider/database load, while + // discovered work and pass failures both restore the prompt initial retry cadence. + let mut cadence = AdaptiveCadence::new(Duration::from_secs(10), Duration::from_secs(80)); + cadence.record_success(true); + assert_eq!( + (cadence.current, cadence.idle_streak), + (Duration::from_secs(20), 1) + ); + cadence.record_success(true); + cadence.record_success(true); + cadence.record_success(true); + assert_eq!( + (cadence.current, cadence.idle_streak), + (Duration::from_secs(80), 4) + ); + cadence.record_success(true); + assert_eq!( + (cadence.current, cadence.idle_streak), + (Duration::from_secs(80), 5) + ); + cadence.record_success(false); + assert_eq!( + (cadence.current, cadence.idle_streak), + (Duration::from_secs(10), 0) + ); + cadence.record_success(true); + cadence.reset(); + assert_eq!( + (cadence.current, cadence.idle_streak), + (Duration::from_secs(10), 0) + ); + } + + #[test] + fn cadence_config_rejects_hot_or_inverted_bounds_offline() { + // Pins: construction cannot accidentally enable a zero-delay hot loop or allow the + // safety heartbeat cadence to be slower than readiness freshness. + let valid = WorkspaceReaperCadenceConfig { + safety_interval: Duration::from_secs(5), + retention_initial_interval: Duration::from_secs(30), + retention_maximum_interval: Duration::from_secs(300), + inventory_initial_interval: Duration::from_secs(60), + inventory_maximum_interval: Duration::from_secs(600), + fleet_metrics_interval: Duration::from_secs(15), + }; + valid + .validate(Duration::from_secs(20)) + .expect("ordered nonzero cadences validate"); + assert!( + WorkspaceReaperCadenceConfig { + retention_maximum_interval: Duration::from_secs(1), + ..valid + } + .validate(Duration::from_secs(20)) + .is_err() + ); + assert!(valid.validate(Duration::from_secs(5)).is_err()); + assert!( + WorkspaceReaperCadenceConfig { + inventory_initial_interval: Duration::ZERO, + ..valid + } + .validate(Duration::from_secs(20)) + .is_err() + ); + } } diff --git a/crates/moa-hands/src/core/sandbox_workspace/repository/base.rs b/crates/moa-hands/src/core/sandbox_workspace/repository/base.rs index a6e4773d3..83cc6694e 100644 --- a/crates/moa-hands/src/core/sandbox_workspace/repository/base.rs +++ b/crates/moa-hands/src/core/sandbox_workspace/repository/base.rs @@ -85,7 +85,17 @@ impl PostgresWorkspaceRepository { .bind(request.provider_account_generation) .bind(request.durability_class.as_str()) .bind(request.retention_deadline_at) - .fetch_one(conn) + .fetch_one(&mut *conn) + .await + .map_err(map_sqlx_error)?; + sqlx::query_scalar::<_, Uuid>( + "SELECT moa.reserve_sandbox_workspace_capacity($1, $2, $3, $4, 0)", + ) + .bind(request.tenant_id) + .bind(request.workspace_id) + .bind(request.provider_account_id) + .bind(request.provider_account_generation) + .fetch_one(&mut *conn) .await .map_err(map_sqlx_error)?; workspace_from_row(&row) diff --git a/crates/moa-hands/src/core/sandbox_workspace/repository/checkpoints.rs b/crates/moa-hands/src/core/sandbox_workspace/repository/checkpoints.rs index 481a4f4e4..4b89249f4 100644 --- a/crates/moa-hands/src/core/sandbox_workspace/repository/checkpoints.rs +++ b/crates/moa-hands/src/core/sandbox_workspace/repository/checkpoints.rs @@ -94,6 +94,18 @@ impl PostgresWorkspaceRepository { )?; let mut conn = self.begin(binding.tenant_id).await?; + if !checkpoint_capacity_matches( + conn.as_mut(), + binding, + operation_id, + operation_kind, + publication_fields.logical_bytes, + ) + .await? + { + conn.rollback().await?; + return Ok(false); + } let checkpoint_affected = sqlx::query( r#" UPDATE moa.sandbox_workspace_checkpoints AS checkpoint @@ -182,6 +194,38 @@ impl PostgresWorkspaceRepository { conn.rollback().await?; return Ok(false); } + if post_commit_state == WorkspacePostCommitState::ComputeDestroyed { + let released = sqlx::query( + r#" + UPDATE moa.sandbox_capacity_reservations + SET reservation_state = 'released', updated_at = now() + WHERE tenant_id = $1 AND workspace_id = $2 + AND provider_account_id = $3 AND provider_account_generation = $4 + AND hand_provisioning_operation_id = $5 + AND hand_lease_generation = $6 + AND expected_writer_epoch = $7 + AND expected_instance_generation = $8 + AND resource_dimension = 'active_hands' + AND reservation_state = 'committed' + "#, + ) + .bind(binding.tenant_id) + .bind(binding.workspace_id) + .bind(binding.provider_account_id) + .bind(fence.provider_account_generation) + .bind(lease.provisioning_operation_id) + .bind(lease.generation) + .bind(fence.writer_epoch) + .bind(fence.instance_generation) + .execute(conn.as_mut()) + .await + .map_err(map_sqlx_error)? + .rows_affected(); + if released != 1 { + conn.rollback().await?; + return Ok(false); + } + } let workspace_state = match post_commit_state { WorkspacePostCommitState::AttachmentRetained => "active", @@ -222,6 +266,18 @@ impl PostgresWorkspaceRepository { .rows_affected(); if workspace_affected != 1 { conn.rollback().await?; + if !self + .abandon_checkpoint_after_cas_loss( + binding, + operation_id, + publication.revision.checkpoint_id, + ) + .await? + { + return Err(MoaError::StorageError( + "checkpoint CAS loss could not release its exact capacity owner".to_string(), + )); + } return Ok(false); } @@ -275,6 +331,7 @@ impl PostgresWorkspaceRepository { AND reservation.provider_account_generation = operation.provider_account_generation AND reservation.expected_writer_epoch = operation.expected_writer_epoch AND reservation.expected_instance_generation = operation.expected_instance_generation + AND reservation.resource_dimension IN ('checkpoints', 'logical_bytes') AND reservation.reservation_state IN ('pending', 'reconciling') "#, ) @@ -288,6 +345,92 @@ impl PostgresWorkspaceRepository { Ok(true) } + async fn abandon_checkpoint_after_cas_loss( + &self, + binding: &WorkspaceBinding, + operation_id: WorkspaceOperationId, + checkpoint_id: WorkspaceCheckpointId, + ) -> Result { + let fence = WorkspaceBindingFence::try_from(binding)?; + let mut conn = self.begin(binding.tenant_id).await?; + let checkpoint_affected = sqlx::query( + r#" + UPDATE moa.sandbox_workspace_checkpoints + SET lifecycle_state = 'failed' + WHERE tenant_id = $1 AND workspace_id = $2 + AND checkpoint_id = $3 AND operation_id = $4 + AND source_writer_epoch = $5 AND source_instance_generation = $6 + AND source_checkpoint_generation = $7 + AND lifecycle_state = 'creating' + AND object_reference IS NULL AND manifest_digest IS NULL + AND logical_bytes IS NULL AND verified_at IS NULL + "#, + ) + .bind(binding.tenant_id) + .bind(binding.workspace_id) + .bind(checkpoint_id) + .bind(operation_id) + .bind(fence.writer_epoch) + .bind(fence.instance_generation) + .bind(fence.checkpoint_generation) + .execute(conn.as_mut()) + .await + .map_err(map_sqlx_error)? + .rows_affected(); + if checkpoint_affected != 1 { + conn.rollback().await?; + return Ok(false); + } + sqlx::query( + r#" + UPDATE moa.sandbox_workspace_operations + SET outcome_class = 'unknown', updated_at = now() + WHERE tenant_id = $1 AND workspace_id = $2 AND operation_id = $3 + AND provider_account_id = $4 AND provider_account_generation = $5 + AND expected_writer_epoch = $6 AND expected_instance_generation = $7 + AND outcome_class IN ('not_sent', 'unknown') + "#, + ) + .bind(binding.tenant_id) + .bind(binding.workspace_id) + .bind(operation_id) + .bind(binding.provider_account_id) + .bind(fence.provider_account_generation) + .bind(fence.writer_epoch) + .bind(fence.instance_generation) + .execute(conn.as_mut()) + .await + .map_err(map_sqlx_error)?; + let released = sqlx::query( + r#" + UPDATE moa.sandbox_capacity_reservations + SET reservation_state = 'released', updated_at = now() + WHERE tenant_id = $1 AND workspace_id = $2 AND operation_id = $3 + AND provider_account_id = $4 AND provider_account_generation = $5 + AND expected_writer_epoch = $6 AND expected_instance_generation = $7 + AND resource_dimension IN ('checkpoints', 'logical_bytes') + AND reservation_state IN ('pending', 'reconciling') + "#, + ) + .bind(binding.tenant_id) + .bind(binding.workspace_id) + .bind(operation_id) + .bind(binding.provider_account_id) + .bind(fence.provider_account_generation) + .bind(fence.writer_epoch) + .bind(fence.instance_generation) + .execute(conn.as_mut()) + .await + .map_err(map_sqlx_error)? + .rows_affected(); + if released == 0 { + conn.rollback().await?; + return Ok(false); + } + conn.commit().await?; + Ok(true) + } + /// Inserts an immutable `creating` checkpoint before any byte or provider I/O. pub async fn create_checkpoint( &self, @@ -450,6 +593,69 @@ impl PostgresWorkspaceRepository { Ok(checkpoint) } } + +async fn checkpoint_capacity_matches( + conn: &mut PgConnection, + binding: &WorkspaceBinding, + operation_id: WorkspaceOperationId, + operation_kind: WorkspaceOperationKind, + logical_bytes: i64, +) -> Result { + sqlx::query_scalar( + r#" + SELECT + count(*) FILTER ( + WHERE reservation.resource_dimension = 'checkpoints' + AND reservation.quantity = 1 + ) = 1 + AND count(*) FILTER ( + WHERE reservation.resource_dimension = 'logical_bytes' + AND reservation.quantity = $8 + ) = CASE WHEN $8 = 0 THEN 0 ELSE 1 END + AND count(*) = CASE WHEN $8 = 0 THEN 1 ELSE 2 END + FROM moa.sandbox_capacity_reservations AS reservation + JOIN moa.sandbox_workspace_operations AS operation + ON operation.tenant_id = reservation.tenant_id + AND operation.workspace_id = reservation.workspace_id + AND operation.operation_id = reservation.operation_id + WHERE reservation.tenant_id = $1 + AND reservation.workspace_id = $2 + AND reservation.operation_id = $3 + AND reservation.provider_account_id = $4 + AND reservation.provider_account_generation = $5 + AND reservation.expected_writer_epoch = $6 + AND reservation.expected_instance_generation = $7 + AND reservation.resource_dimension IN ('checkpoints', 'logical_bytes') + AND reservation.reservation_state IN ('pending', 'committed', 'reconciling') + AND operation.operation_kind = $9 + AND operation.outcome_class IN ('not_sent', 'unknown') + "#, + ) + .bind(binding.tenant_id) + .bind(binding.workspace_id) + .bind(operation_id) + .bind(binding.provider_account_id) + .bind( + i64::try_from(binding.provider_account_generation).map_err(|_| { + MoaError::ValidationError( + "workspace provider-account generation overflows Postgres bigint".to_string(), + ) + })?, + ) + .bind(i64::try_from(binding.writer_epoch).map_err(|_| { + MoaError::ValidationError("workspace writer epoch overflows Postgres bigint".to_string()) + })?) + .bind(i64::try_from(binding.instance_generation).map_err(|_| { + MoaError::ValidationError( + "workspace instance generation overflows Postgres bigint".to_string(), + ) + })?) + .bind(logical_bytes) + .bind(operation_kind.as_str()) + .fetch_one(conn) + .await + .map_err(map_sqlx_error) +} const CHECKPOINT_COLUMNS: &str = "checkpoint_id, tenant_id, workspace_id, generation, \ parent_checkpoint_id, source_writer_epoch, source_instance_generation, operation_id, \ lifecycle_state, object_reference, manifest_digest, logical_bytes, \ diff --git a/crates/moa-hands/src/core/sandbox_workspace/repository/lifecycle.rs b/crates/moa-hands/src/core/sandbox_workspace/repository/lifecycle.rs index 8310c7318..608ee5bcd 100644 --- a/crates/moa-hands/src/core/sandbox_workspace/repository/lifecycle.rs +++ b/crates/moa-hands/src/core/sandbox_workspace/repository/lifecycle.rs @@ -1,8 +1,1073 @@ //! Fenced sandbox-workspace lifecycle transitions and deletion. use super::*; +use crate::core::sandbox_workspace::capacity::release_workspace_in_transaction; impl PostgresWorkspaceRepository { + /// Persists the exact task-attempt release intent before checkpoint or destroy I/O. + /// + /// While this row is pending, the migration guard prevents the task attempt + /// generation from advancing. That closes the validation-to-provider-I/O race + /// without holding a database transaction open across an external call. + pub async fn begin_task_execution_hand_release( + &self, + intent: TaskHandReleaseIntent<'_>, + ) -> Result<(Uuid, Uuid, chrono::DateTime)> { + let TaskHandReleaseIntent { + receipt_id, + run_id, + task_id, + logical_generation, + attempt_generation, + deadline_at, + recovery_claim_expires_at, + workspace, + lease, + } = intent; + let active_identity = workspace.state == SandboxWorkspaceState::Active + && lease.status == HandLeaseStatus::Active + && lease.handle.is_some() + && lease.attachment.as_ref().map(|attachment| { + ( + attachment.workspace_id, + attachment.workspace_writer_epoch, + attachment.workspace_instance_generation, + ) + }) == Some(( + workspace.workspace_id, + workspace.writer_epoch, + workspace.instance_generation, + )); + let released_identity = workspace.state == SandboxWorkspaceState::Ready + && lease.status == HandLeaseStatus::Destroyed + && lease.handle.is_none(); + if workspace.scope != (SandboxWorkspaceScope::ExecutionTask { run_id, task_id }) + || !(active_identity || released_identity) + { + return Err(MoaError::ValidationError( + "task hand release intent does not match the active workspace and lease" + .to_string(), + )); + } + let logical_generation = i64::try_from(logical_generation).map_err(|_| { + MoaError::ValidationError( + "execution task logical generation overflows Postgres bigint".to_string(), + ) + })?; + let attempt_generation = i64::try_from(attempt_generation).map_err(|_| { + MoaError::ValidationError( + "execution task attempt generation overflows Postgres bigint".to_string(), + ) + })?; + let claim_token = Uuid::now_v7(); + let mut conn = self.begin(workspace.tenant_id).await?; + let row = sqlx::query( + r#" + INSERT INTO moa.sandbox_execution_hand_release_receipts ( + receipt_id, tenant_id, run_uid, owner_kind, task_id, + logical_generation, attempt_generation, + workspace_id, writer_epoch, instance_generation, + hand_provisioning_operation_id, hand_lease_generation, + receipt_state, claim_token, claim_expires_at, + requested_at, deadline_at + ) + SELECT $1, $2, $3, 'task', $4, $5, $6, $7, $8, $9, $10, $11, + 'pending', $13, $14, now(), $12 + FROM moa.execution_task AS task + JOIN moa.sandbox_workspaces AS workspace + ON workspace.tenant_id = task.tenant_id AND workspace.workspace_id = $7 + JOIN moa.hand_leases AS lease + ON lease.tenant_id = task.tenant_id + AND lease.provisioning_operation_id = $10 AND lease.generation = $11 + JOIN moa.sandbox_capacity_reservations AS capacity + ON capacity.tenant_id = task.tenant_id + AND capacity.workspace_id = workspace.workspace_id + AND capacity.hand_provisioning_operation_id = lease.provisioning_operation_id + AND capacity.hand_lease_generation = lease.generation + AND capacity.resource_dimension = 'active_hands' + WHERE task.tenant_id = $2 AND task.run_uid = $3 AND task.task_id = $4 + AND task.generation = $5 AND task.attempt_generation = $6 + AND task.attempt_state = 'cancelling' + AND workspace.writer_epoch = $8 AND workspace.instance_generation = $9 + AND workspace.access_fenced_at IS NULL + AND capacity.expected_writer_epoch = $8 + AND capacity.expected_instance_generation = $9 + AND ( + ( + workspace.lifecycle_state = 'active' + AND lease.status = 'active' AND lease.handle IS NOT NULL + AND lease.workspace_id = workspace.workspace_id + AND lease.workspace_writer_epoch = workspace.writer_epoch + AND lease.workspace_instance_generation = workspace.instance_generation + AND capacity.reservation_state = 'committed' + ) + OR + ( + workspace.lifecycle_state = 'ready' + AND lease.status = 'destroyed' AND lease.handle IS NULL + AND capacity.reservation_state = 'released' + AND EXISTS ( + SELECT 1 + FROM moa.sandbox_execution_hand_release_receipts AS pending + WHERE pending.tenant_id = $2 AND pending.run_uid = $3 + AND pending.task_id = $4 AND pending.logical_generation = $5 + AND pending.attempt_generation = $6 + AND pending.receipt_id = $1 AND pending.workspace_id = $7 + AND pending.writer_epoch = $8 AND pending.instance_generation = $9 + AND pending.hand_provisioning_operation_id = $10 + AND pending.hand_lease_generation = $11 + AND pending.receipt_state = 'pending' + ) + ) + ) + ON CONFLICT (tenant_id, run_uid, task_id, logical_generation, attempt_generation) + WHERE owner_kind = 'task' + DO UPDATE SET claim_token = $13, claim_expires_at = $14, + updated_at = now() + WHERE sandbox_execution_hand_release_receipts.receipt_state = 'pending' + AND sandbox_execution_hand_release_receipts.claim_expires_at <= now() + AND sandbox_execution_hand_release_receipts.receipt_id = $1 + AND sandbox_execution_hand_release_receipts.logical_generation = $5 + AND sandbox_execution_hand_release_receipts.attempt_generation = $6 + AND sandbox_execution_hand_release_receipts.workspace_id = $7 + AND sandbox_execution_hand_release_receipts.writer_epoch = $8 + AND sandbox_execution_hand_release_receipts.instance_generation = $9 + AND sandbox_execution_hand_release_receipts.hand_provisioning_operation_id = $10 + AND sandbox_execution_hand_release_receipts.hand_lease_generation = $11 + RETURNING receipt_id, claim_token, requested_at + "#, + ) + .bind(receipt_id) + .bind(workspace.tenant_id) + .bind(run_id) + .bind(task_id) + .bind(logical_generation) + .bind(attempt_generation) + .bind(workspace.workspace_id) + .bind(workspace.writer_epoch) + .bind(workspace.instance_generation) + .bind(lease.provisioning_operation_id) + .bind(lease.generation) + .bind(deadline_at) + .bind(claim_token) + .bind(recovery_claim_expires_at) + .fetch_optional(conn.as_mut()) + .await + .map_err(map_sqlx_error)?; + let row = row.ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { + operation_id: format!( + "execution-task-hand-release:{run_id}:{task_id}:{attempt_generation}" + ), + })?; + let persisted_id = row.try_get("receipt_id").map_err(map_sqlx_error)?; + let persisted_claim = row.try_get("claim_token").map_err(map_sqlx_error)?; + let requested_at = row.try_get("requested_at").map_err(map_sqlx_error)?; + conn.commit().await?; + Ok((persisted_id, persisted_claim, requested_at)) + } + + /// Loads the verified release receipt for one exact execution-task attempt. + pub async fn get_task_execution_hand_release_receipt( + &self, + tenant_id: TenantId, + run_id: ExecutionRunScopeId, + task_id: ExecutionTaskScopeId, + logical_generation: u64, + attempt_generation: u64, + ) -> Result> { + let logical_generation = i64::try_from(logical_generation).map_err(|_| { + MoaError::ValidationError( + "execution task logical generation overflows Postgres bigint".to_string(), + ) + })?; + let attempt_generation = i64::try_from(attempt_generation).map_err(|_| { + MoaError::ValidationError( + "execution task attempt generation overflows Postgres bigint".to_string(), + ) + })?; + let mut conn = self.begin(tenant_id).await?; + let row = sqlx::query( + r#" + SELECT receipt_id, tenant_id, run_uid, owner_kind, task_id, compensation_id, + logical_generation, attempt_generation, + workspace_id, writer_epoch, instance_generation, + hand_provisioning_operation_id, hand_lease_generation, + checkpoint_id, checkpoint_generation, + checkpoint_manifest_digest, checkpoint_logical_bytes, + requested_at, released_at + FROM moa.sandbox_execution_hand_release_receipts + WHERE tenant_id = $1 AND run_uid = $2 AND task_id = $3 + AND owner_kind = 'task' AND logical_generation = $4 + AND attempt_generation = $5 + AND receipt_state = 'released' + AND destroy_outcome = 'verified_absent' + "#, + ) + .bind(tenant_id) + .bind(run_id) + .bind(task_id) + .bind(logical_generation) + .bind(attempt_generation) + .fetch_optional(conn.as_mut()) + .await + .map_err(map_sqlx_error)?; + let receipt = row + .as_ref() + .map(execution_hand_release_receipt_from_row) + .transpose()?; + conn.commit().await?; + Ok(receipt) + } + + /// Persists a verified-absence receipt for a cancelling task with no live hand. + /// + /// A ready workspace left by an earlier attempt is permitted only when its + /// active-hand capacity is already released. Any live lease, active workspace, + /// or unreleased active-hand reservation rejects the absence proof. + pub async fn record_absent_task_execution_hand_release_receipt( + &self, + intent: AbsentTaskHandReleaseIntent, + ) -> Result { + let logical_generation = i64::try_from(intent.logical_generation).map_err(|_| { + MoaError::ValidationError( + "execution task logical generation overflows Postgres bigint".to_string(), + ) + })?; + let attempt_generation = i64::try_from(intent.attempt_generation).map_err(|_| { + MoaError::ValidationError( + "execution task attempt generation overflows Postgres bigint".to_string(), + ) + })?; + let mut conn = self.begin(intent.tenant_id).await?; + sqlx::query( + r#" + WITH locked_task AS MATERIALIZED ( + SELECT task.tenant_id, task.run_uid, task.task_id, + task.generation, task.attempt_generation, run.session_id + FROM moa.execution_task AS task + JOIN moa.execution_run AS run + ON run.tenant_id = task.tenant_id AND run.run_uid = task.run_uid + WHERE task.tenant_id = $2 AND task.run_uid = $3 AND task.task_id = $4 + AND task.generation = $5 AND task.attempt_generation = $6 + AND task.attempt_state = 'cancelling' + FOR UPDATE OF task + ) + INSERT INTO moa.sandbox_execution_hand_release_receipts ( + receipt_id, tenant_id, run_uid, owner_kind, task_id, + logical_generation, attempt_generation, receipt_state, + destroy_outcome, requested_at, deadline_at, released_at + ) + SELECT $1, task.tenant_id, task.run_uid, 'task', task.task_id, + task.generation, task.attempt_generation, 'released', + 'verified_absent', $7, $7, $7 + FROM locked_task AS task + WHERE NOT EXISTS ( + SELECT 1 + FROM moa.hand_leases AS lease + WHERE lease.tenant_id = task.tenant_id + AND lease.session_id = task.session_id + AND lease.worker_id = + 'execution:' || task.run_uid::text || ':' || task.task_id::text + AND lease.status <> 'destroyed' + ) + AND NOT EXISTS ( + SELECT 1 + FROM moa.sandbox_workspaces AS workspace + WHERE workspace.tenant_id = task.tenant_id + AND workspace.scope_kind = 'execution_task' + AND workspace.scope_run_id = task.run_uid + AND workspace.scope_task_id = task.task_id + AND workspace.lifecycle_state <> 'deleted' + AND ( + workspace.lifecycle_state <> 'ready' + OR EXISTS ( + SELECT 1 + FROM moa.sandbox_capacity_reservations AS capacity + WHERE capacity.tenant_id = workspace.tenant_id + AND capacity.workspace_id = workspace.workspace_id + AND capacity.resource_dimension = 'active_hands' + AND capacity.reservation_state <> 'released' + ) + ) + ) + ON CONFLICT (tenant_id, run_uid, task_id, logical_generation, attempt_generation) + WHERE owner_kind = 'task' + DO NOTHING + "#, + ) + .bind(intent.receipt_id) + .bind(intent.tenant_id) + .bind(intent.run_id) + .bind(intent.task_id) + .bind(logical_generation) + .bind(attempt_generation) + .bind(intent.verified_at) + .execute(conn.as_mut()) + .await + .map_err(map_sqlx_error)?; + let row = sqlx::query( + r#" + SELECT receipt_id, tenant_id, run_uid, owner_kind, task_id, compensation_id, + logical_generation, attempt_generation, + workspace_id, writer_epoch, instance_generation, + hand_provisioning_operation_id, hand_lease_generation, + checkpoint_id, checkpoint_generation, + checkpoint_manifest_digest, checkpoint_logical_bytes, + requested_at, released_at + FROM moa.sandbox_execution_hand_release_receipts + WHERE receipt_id = $1 AND tenant_id = $2 AND run_uid = $3 + AND owner_kind = 'task' AND task_id = $4 + AND logical_generation = $5 AND attempt_generation = $6 + AND receipt_state = 'released' AND destroy_outcome = 'verified_absent' + AND workspace_id IS NULL AND hand_provisioning_operation_id IS NULL + "#, + ) + .bind(intent.receipt_id) + .bind(intent.tenant_id) + .bind(intent.run_id) + .bind(intent.task_id) + .bind(logical_generation) + .bind(attempt_generation) + .fetch_optional(conn.as_mut()) + .await + .map_err(map_sqlx_error)? + .ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { + operation_id: format!( + "execution-task-hand-absence:{}:{}:{}", + intent.run_id, intent.task_id, intent.attempt_generation + ), + })?; + let receipt = execution_hand_release_receipt_from_row(&row)?; + conn.commit().await?; + Ok(receipt) + } + + /// Persists a compensation release intent after proving the exact attempt is cancelling. + pub async fn begin_compensation_execution_hand_release( + &self, + intent: CompensationHandReleaseIntent<'_>, + ) -> Result<(Uuid, Uuid, chrono::DateTime)> { + let CompensationHandReleaseIntent { + receipt_id, + tenant_id, + session_id, + run_id, + compensation_id, + logical_generation, + attempt_generation, + hand_scope, + lease, + deadline_at, + recovery_claim_expires_at, + } = intent; + if lease.is_some_and(|lease| { + lease.tenant_id != tenant_id + || lease.session_id != session_id + || lease.worker_id != hand_scope + || lease.status == HandLeaseStatus::Destroyed + }) { + return Err(MoaError::ValidationError( + "compensation hand release lease does not match its exact scope".to_string(), + )); + } + let logical_generation = i64::try_from(logical_generation).map_err(|_| { + MoaError::ValidationError( + "compensation logical generation overflows Postgres bigint".to_string(), + ) + })?; + let attempt_generation = i64::try_from(attempt_generation).map_err(|_| { + MoaError::ValidationError( + "compensation attempt generation overflows Postgres bigint".to_string(), + ) + })?; + let claim_token = Uuid::now_v7(); + let mut conn = self.begin(tenant_id).await?; + let row = sqlx::query( + r#" + INSERT INTO moa.sandbox_execution_hand_release_receipts ( + receipt_id, tenant_id, run_uid, owner_kind, compensation_id, + logical_generation, attempt_generation, + hand_provisioning_operation_id, hand_lease_generation, receipt_state, + claim_token, claim_expires_at, requested_at, deadline_at + ) + SELECT $1, $2, $3, 'compensation', $4, $5, $6, $11, $12, 'pending', + $10, $13, now(), $9 + FROM moa.execution_compensation AS compensation + WHERE compensation.tenant_id = $2 AND compensation.run_uid = $3 + AND compensation.compensation_id = $4 AND compensation.generation = $5 + AND compensation.attempt_generation = $6 + AND compensation.attempt_state = 'cancelling' + AND (($11::uuid IS NULL AND $12::bigint IS NULL AND NOT EXISTS ( + SELECT 1 FROM moa.hand_leases AS lease + WHERE lease.tenant_id = $2 AND lease.session_id = $7 + AND lease.worker_id = $8 AND lease.status <> 'destroyed' + )) OR ($11::uuid IS NOT NULL AND $12::bigint IS NOT NULL AND EXISTS ( + SELECT 1 FROM moa.hand_leases AS lease + WHERE lease.tenant_id = $2 AND lease.session_id = $7 + AND lease.worker_id = $8 + AND lease.provisioning_operation_id = $11 + AND lease.generation = $12 AND lease.status <> 'destroyed' + ))) + ON CONFLICT ( + tenant_id, run_uid, compensation_id, logical_generation, attempt_generation + ) WHERE owner_kind = 'compensation' + DO UPDATE SET claim_token = $10, claim_expires_at = $13, + updated_at = now() + WHERE sandbox_execution_hand_release_receipts.receipt_state = 'pending' + AND sandbox_execution_hand_release_receipts.claim_expires_at <= now() + AND sandbox_execution_hand_release_receipts.receipt_id = $1 + AND sandbox_execution_hand_release_receipts.hand_provisioning_operation_id + IS NOT DISTINCT FROM $11 + AND sandbox_execution_hand_release_receipts.hand_lease_generation + IS NOT DISTINCT FROM $12 + RETURNING receipt_id, claim_token, requested_at + "#, + ) + .bind(receipt_id) + .bind(tenant_id) + .bind(run_id) + .bind(compensation_id) + .bind(logical_generation) + .bind(attempt_generation) + .bind(session_id) + .bind(hand_scope) + .bind(deadline_at) + .bind(claim_token) + .bind(lease.map(|lease| lease.provisioning_operation_id.0)) + .bind(lease.map(|lease| lease.generation)) + .bind(recovery_claim_expires_at) + .fetch_optional(conn.as_mut()) + .await + .map_err(map_sqlx_error)? + .ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { + operation_id: format!( + "execution-compensation-hand-release:{run_id}:{compensation_id}:{logical_generation}:{attempt_generation}" + ), + })?; + let persisted_id = row.try_get("receipt_id").map_err(map_sqlx_error)?; + let persisted_claim = row.try_get("claim_token").map_err(map_sqlx_error)?; + let requested_at = row.try_get("requested_at").map_err(map_sqlx_error)?; + conn.commit().await?; + Ok((persisted_id, persisted_claim, requested_at)) + } + + /// Loads verified absence proof for one exact compensation attempt. + pub async fn get_compensation_execution_hand_release_receipt( + &self, + tenant_id: TenantId, + run_id: ExecutionRunScopeId, + compensation_id: ExecutionCompensationScopeId, + logical_generation: u64, + attempt_generation: u64, + ) -> Result> { + let logical_generation = i64::try_from(logical_generation).map_err(|_| { + MoaError::ValidationError( + "compensation logical generation overflows Postgres bigint".to_string(), + ) + })?; + let attempt_generation = i64::try_from(attempt_generation).map_err(|_| { + MoaError::ValidationError( + "compensation attempt generation overflows Postgres bigint".to_string(), + ) + })?; + let mut conn = self.begin(tenant_id).await?; + let row = sqlx::query( + r#" + SELECT receipt_id, tenant_id, run_uid, owner_kind, task_id, compensation_id, + logical_generation, attempt_generation, workspace_id, writer_epoch, + instance_generation, hand_provisioning_operation_id, + hand_lease_generation, checkpoint_id, checkpoint_generation, + checkpoint_manifest_digest, checkpoint_logical_bytes, + requested_at, released_at + FROM moa.sandbox_execution_hand_release_receipts + WHERE tenant_id = $1 AND run_uid = $2 AND owner_kind = 'compensation' + AND compensation_id = $3 AND logical_generation = $4 + AND attempt_generation = $5 AND receipt_state = 'released' + AND destroy_outcome = 'verified_absent' + "#, + ) + .bind(tenant_id) + .bind(run_id) + .bind(compensation_id) + .bind(logical_generation) + .bind(attempt_generation) + .fetch_optional(conn.as_mut()) + .await + .map_err(map_sqlx_error)?; + let receipt = row + .as_ref() + .map(execution_hand_release_receipt_from_row) + .transpose()?; + conn.commit().await?; + Ok(receipt) + } + + /// Reclaims one expired pending compensation release without rediscovering its lease. + /// + /// The immutable provider deadline continues to prohibit new provider I/O. + /// This short database claim exists only to verify already-absent compute and + /// finalize the persisted receipt using its original provisioning identity. + pub async fn claim_pending_compensation_execution_hand_release( + &self, + tenant_id: TenantId, + run_id: ExecutionRunScopeId, + compensation_id: ExecutionCompensationScopeId, + logical_generation: u64, + attempt_generation: u64, + recovery_claim_expires_at: chrono::DateTime, + ) -> Result> { + let logical_generation = i64::try_from(logical_generation).map_err(|_| { + MoaError::ValidationError( + "compensation logical generation overflows Postgres bigint".to_string(), + ) + })?; + let attempt_generation = i64::try_from(attempt_generation).map_err(|_| { + MoaError::ValidationError( + "compensation attempt generation overflows Postgres bigint".to_string(), + ) + })?; + let claim_token = Uuid::now_v7(); + let mut conn = self.begin(tenant_id).await?; + let row = sqlx::query( + r#" + UPDATE moa.sandbox_execution_hand_release_receipts AS receipt + SET claim_token = $6, claim_expires_at = $7, updated_at = now() + FROM moa.execution_compensation AS compensation + WHERE receipt.tenant_id = $1 AND receipt.run_uid = $2 + AND receipt.owner_kind = 'compensation' + AND receipt.compensation_id = $3 + AND receipt.logical_generation = $4 + AND receipt.attempt_generation = $5 + AND receipt.receipt_state = 'pending' + AND receipt.claim_expires_at <= now() AND $7 > now() + AND compensation.tenant_id = $1 AND compensation.run_uid = $2 + AND compensation.compensation_id = $3 AND compensation.generation = $4 + AND compensation.attempt_generation = $5 + AND compensation.attempt_state = 'cancelling' + RETURNING receipt.receipt_id, receipt.claim_token, receipt.requested_at, + receipt.hand_provisioning_operation_id, + receipt.hand_lease_generation + "#, + ) + .bind(tenant_id) + .bind(run_id) + .bind(compensation_id) + .bind(logical_generation) + .bind(attempt_generation) + .bind(claim_token) + .bind(recovery_claim_expires_at) + .fetch_optional(conn.as_mut()) + .await + .map_err(map_sqlx_error)?; + let claim = row + .as_ref() + .map(|row| { + let operation_id: Option = row + .try_get("hand_provisioning_operation_id") + .map_err(map_sqlx_error)?; + let generation: Option = row + .try_get("hand_lease_generation") + .map_err(map_sqlx_error)?; + if operation_id.is_some() != generation.is_some() { + return Err(MoaError::StorageError( + "pending compensation release has a partial hand identity".to_string(), + )); + } + Ok(CompensationHandReleaseClaim { + receipt_id: row.try_get("receipt_id").map_err(map_sqlx_error)?, + claim_token: row.try_get("claim_token").map_err(map_sqlx_error)?, + requested_at: row.try_get("requested_at").map_err(map_sqlx_error)?, + hand_provisioning_operation_id: operation_id.map(HandProvisioningOperationId), + hand_lease_generation: generation, + }) + }) + .transpose()?; + conn.commit().await?; + Ok(claim) + } + + /// Finalizes compensation release only while exact cancelling ownership persists. + pub async fn record_compensation_execution_hand_release_receipt( + &self, + receipt: &ExecutionHandReleaseReceipt, + session_id: SessionId, + hand_scope: &str, + claim_token: Uuid, + ) -> Result { + let (compensation_id, logical_generation) = match receipt.owner { + ExecutionHandReleaseOwner::Compensation { + compensation_id, + logical_generation, + } => (compensation_id, logical_generation), + ExecutionHandReleaseOwner::Task { .. } => { + return Err(MoaError::ValidationError( + "compensation hand release receipt has a task owner".to_string(), + )); + } + }; + let logical_generation = i64::try_from(logical_generation).map_err(|_| { + MoaError::ValidationError( + "compensation logical generation overflows Postgres bigint".to_string(), + ) + })?; + let attempt_generation = i64::try_from(receipt.attempt_generation).map_err(|_| { + MoaError::ValidationError( + "compensation attempt generation overflows Postgres bigint".to_string(), + ) + })?; + let hand_operation_id = receipt + .hand_provisioning_operation_id + .map(|operation_id| operation_id.0); + let hand_generation = receipt + .hand_lease_generation + .map(i64::try_from) + .transpose() + .map_err(|_| { + MoaError::ValidationError( + "compensation hand generation overflows Postgres bigint".to_string(), + ) + })?; + if hand_operation_id.is_some() != hand_generation.is_some() { + return Err(MoaError::ValidationError( + "compensation hand release identity must be wholly present or absent".to_string(), + )); + } + let mut conn = self.begin(receipt.tenant_id).await?; + let row = sqlx::query( + r#" + UPDATE moa.sandbox_execution_hand_release_receipts AS receipt + SET receipt_state = 'released', destroy_outcome = 'verified_absent', + claim_token = NULL, claim_expires_at = NULL, + released_at = $9, updated_at = now() + FROM moa.execution_compensation AS compensation + WHERE receipt.receipt_id = $1 AND receipt.tenant_id = $2 + AND receipt.run_uid = $3 AND receipt.owner_kind = 'compensation' + AND receipt.compensation_id = $4 AND receipt.logical_generation = $5 + AND receipt.attempt_generation = $6 AND receipt.receipt_state = 'pending' + AND receipt.claim_token = $10 AND receipt.claim_expires_at > now() + AND receipt.requested_at = $8 + AND compensation.tenant_id = $2 AND compensation.run_uid = $3 + AND compensation.compensation_id = $4 AND compensation.generation = $5 + AND compensation.attempt_generation = $6 + AND compensation.attempt_state = 'cancelling' + AND NOT EXISTS ( + SELECT 1 FROM moa.hand_leases AS lease + WHERE lease.tenant_id = $2 AND lease.session_id = $7 + AND lease.worker_id = $11 AND lease.status <> 'destroyed' + ) + AND (($12::uuid IS NULL AND $13::bigint IS NULL + AND receipt.hand_provisioning_operation_id IS NULL + AND receipt.hand_lease_generation IS NULL) + OR ($12::uuid IS NOT NULL AND $13::bigint IS NOT NULL + AND receipt.hand_provisioning_operation_id = $12 + AND receipt.hand_lease_generation = $13 + AND EXISTS ( + SELECT 1 FROM moa.hand_leases AS released_lease + WHERE released_lease.tenant_id = $2 + AND released_lease.session_id = $7 + AND released_lease.worker_id = $11 + AND released_lease.provisioning_operation_id = $12 + AND released_lease.generation = $13 + AND released_lease.status = 'destroyed' + AND released_lease.handle IS NULL + ))) + RETURNING receipt.receipt_id, receipt.tenant_id, receipt.run_uid, + receipt.owner_kind, receipt.task_id, receipt.compensation_id, + receipt.logical_generation, receipt.attempt_generation, + receipt.workspace_id, receipt.writer_epoch, receipt.instance_generation, + receipt.hand_provisioning_operation_id, receipt.hand_lease_generation, + receipt.checkpoint_id, receipt.checkpoint_generation, + receipt.checkpoint_manifest_digest, receipt.checkpoint_logical_bytes, + receipt.requested_at, receipt.released_at + "#, + ) + .bind(receipt.receipt_id) + .bind(receipt.tenant_id) + .bind(receipt.run_id) + .bind(compensation_id) + .bind(logical_generation) + .bind(attempt_generation) + .bind(session_id) + .bind(receipt.requested_at) + .bind(receipt.released_at) + .bind(claim_token) + .bind(hand_scope) + .bind(hand_operation_id) + .bind(hand_generation) + .fetch_optional(conn.as_mut()) + .await + .map_err(map_sqlx_error)? + .ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { + operation_id: format!( + "execution-compensation-hand-release:{}:{compensation_id}:{logical_generation}:{attempt_generation}", + receipt.run_id + ), + })?; + let persisted = execution_hand_release_receipt_from_row(&row)?; + conn.commit().await?; + Ok(persisted) + } + + /// Persists a release receipt only after every exact durable fence proves release. + /// + /// A racing retry receives the original receipt. A conflicting identity for the + /// same task attempt is rejected rather than silently replacing recovery truth. + pub async fn record_task_execution_hand_release_receipt( + &self, + receipt: &ExecutionHandReleaseReceipt, + claim_token: Uuid, + ) -> Result { + let (task_id, logical_generation) = match receipt.owner { + ExecutionHandReleaseOwner::Task { + task_id, + logical_generation, + } => (task_id, logical_generation), + ExecutionHandReleaseOwner::Compensation { .. } => { + return Err(MoaError::ValidationError( + "task hand release receipt has a compensation owner".to_string(), + )); + } + }; + let workspace_id = receipt.workspace_id.ok_or_else(|| { + MoaError::ValidationError("task hand release receipt has no workspace".to_string()) + })?; + let writer_epoch = receipt.writer_epoch.ok_or_else(|| { + MoaError::ValidationError("task hand release receipt has no writer epoch".to_string()) + })?; + let instance_generation = receipt.instance_generation.ok_or_else(|| { + MoaError::ValidationError( + "task hand release receipt has no instance generation".to_string(), + ) + })?; + let hand_provisioning_operation_id = + receipt.hand_provisioning_operation_id.ok_or_else(|| { + MoaError::ValidationError( + "task hand release receipt has no provisioning identity".to_string(), + ) + })?; + let hand_lease_generation = receipt.hand_lease_generation.ok_or_else(|| { + MoaError::ValidationError( + "task hand release receipt has no hand lease generation".to_string(), + ) + })?; + let checkpoint_id = receipt.checkpoint_id.ok_or_else(|| { + MoaError::ValidationError("task hand release receipt has no checkpoint".to_string()) + })?; + let checkpoint_generation = receipt.checkpoint_generation.ok_or_else(|| { + MoaError::ValidationError( + "task hand release receipt has no checkpoint generation".to_string(), + ) + })?; + let checkpoint_manifest_digest = + receipt + .checkpoint_manifest_digest + .as_deref() + .ok_or_else(|| { + MoaError::ValidationError( + "task hand release receipt has no checkpoint digest".to_string(), + ) + })?; + let checkpoint_logical_bytes = receipt.checkpoint_logical_bytes.ok_or_else(|| { + MoaError::ValidationError( + "task hand release receipt has no checkpoint byte count".to_string(), + ) + })?; + let logical_generation = i64::try_from(logical_generation).map_err(|_| { + MoaError::ValidationError( + "execution task logical generation overflows Postgres bigint".to_string(), + ) + })?; + let attempt_generation = i64::try_from(receipt.attempt_generation).map_err(|_| { + MoaError::ValidationError( + "execution task attempt generation overflows Postgres bigint".to_string(), + ) + })?; + let writer_epoch = i64::try_from(writer_epoch).map_err(|_| { + MoaError::ValidationError( + "workspace writer epoch overflows Postgres bigint".to_string(), + ) + })?; + let instance_generation = i64::try_from(instance_generation).map_err(|_| { + MoaError::ValidationError( + "workspace instance generation overflows Postgres bigint".to_string(), + ) + })?; + let hand_lease_generation = i64::try_from(hand_lease_generation).map_err(|_| { + MoaError::ValidationError("hand lease generation overflows Postgres bigint".to_string()) + })?; + let checkpoint_generation = i64::try_from(checkpoint_generation).map_err(|_| { + MoaError::ValidationError("checkpoint generation overflows Postgres bigint".to_string()) + })?; + let checkpoint_logical_bytes = i64::try_from(checkpoint_logical_bytes).map_err(|_| { + MoaError::ValidationError( + "checkpoint logical bytes overflow Postgres bigint".to_string(), + ) + })?; + let mut conn = self.begin(receipt.tenant_id).await?; + let row = sqlx::query( + r#" + UPDATE moa.sandbox_execution_hand_release_receipts AS receipt + SET checkpoint_id = $12, checkpoint_generation = $13, + checkpoint_manifest_digest = $14, checkpoint_logical_bytes = $15, + receipt_state = 'released', destroy_outcome = 'verified_absent', + claim_token = NULL, claim_expires_at = NULL, + released_at = $17, updated_at = now() + FROM moa.execution_task AS task + JOIN moa.sandbox_workspaces AS workspace + ON workspace.tenant_id = task.tenant_id + AND workspace.workspace_id = $7 + JOIN moa.hand_leases AS lease + ON lease.tenant_id = task.tenant_id + AND lease.provisioning_operation_id = $10 + AND lease.generation = $11 + JOIN moa.sandbox_workspace_checkpoints AS checkpoint + ON checkpoint.tenant_id = task.tenant_id + AND checkpoint.workspace_id = workspace.workspace_id + AND checkpoint.checkpoint_id = $12 + JOIN moa.sandbox_capacity_reservations AS capacity + ON capacity.tenant_id = task.tenant_id + AND capacity.workspace_id = workspace.workspace_id + AND capacity.hand_provisioning_operation_id = lease.provisioning_operation_id + AND capacity.hand_lease_generation = lease.generation + AND capacity.expected_writer_epoch = $8 + AND capacity.expected_instance_generation = $9 + AND capacity.resource_dimension = 'active_hands' + WHERE receipt.receipt_id = $1 AND receipt.tenant_id = $2 + AND receipt.run_uid = $3 AND receipt.owner_kind = 'task' + AND receipt.task_id = $4 AND receipt.logical_generation = $5 + AND receipt.attempt_generation = $6 + AND receipt.workspace_id = $7 + AND receipt.writer_epoch = $8 AND receipt.instance_generation = $9 + AND receipt.hand_provisioning_operation_id = $10 + AND receipt.hand_lease_generation = $11 + AND receipt.receipt_state = 'pending' + AND receipt.claim_token = $18 AND receipt.claim_expires_at > now() + AND receipt.requested_at = $16 + AND task.tenant_id = $2 AND task.run_uid = $3 AND task.task_id = $4 + AND task.generation = $5 AND task.attempt_generation = $6 + AND task.attempt_state = 'cancelling' + AND workspace.writer_epoch = $8 AND workspace.instance_generation = $9 + AND workspace.lifecycle_state = 'ready' + AND workspace.current_checkpoint_id = $12 + AND workspace.current_checkpoint_generation = $13 + AND lease.status = 'destroyed' AND lease.handle IS NULL + AND lease.workspace_id IS NULL + AND checkpoint.lifecycle_state = 'available' + AND checkpoint.generation = $13 + AND checkpoint.manifest_digest = $14 + AND checkpoint.logical_bytes = $15 + AND capacity.reservation_state = 'released' + RETURNING receipt.receipt_id, receipt.tenant_id, receipt.run_uid, + receipt.owner_kind, receipt.task_id, receipt.compensation_id, + receipt.logical_generation, receipt.attempt_generation, receipt.workspace_id, + receipt.writer_epoch, receipt.instance_generation, + receipt.hand_provisioning_operation_id, receipt.hand_lease_generation, + receipt.checkpoint_id, receipt.checkpoint_generation, + receipt.checkpoint_manifest_digest, receipt.checkpoint_logical_bytes, + receipt.requested_at, receipt.released_at + "#, + ) + .bind(receipt.receipt_id) + .bind(receipt.tenant_id) + .bind(receipt.run_id) + .bind(task_id) + .bind(logical_generation) + .bind(attempt_generation) + .bind(workspace_id) + .bind(writer_epoch) + .bind(instance_generation) + .bind(hand_provisioning_operation_id) + .bind(hand_lease_generation) + .bind(checkpoint_id) + .bind(checkpoint_generation) + .bind(checkpoint_manifest_digest) + .bind(checkpoint_logical_bytes) + .bind(receipt.requested_at) + .bind(receipt.released_at) + .bind(claim_token) + .fetch_optional(conn.as_mut()) + .await + .map_err(map_sqlx_error)?; + let row = if let Some(row) = row { + row + } else { + sqlx::query( + r#" + SELECT receipt_id, tenant_id, run_uid, owner_kind, task_id, compensation_id, + logical_generation, attempt_generation, + workspace_id, writer_epoch, instance_generation, + hand_provisioning_operation_id, hand_lease_generation, + checkpoint_id, checkpoint_generation, + checkpoint_manifest_digest, checkpoint_logical_bytes, + requested_at, released_at + FROM moa.sandbox_execution_hand_release_receipts + WHERE receipt_id = $1 AND tenant_id = $2 AND run_uid = $3 + AND owner_kind = 'task' AND task_id = $4 AND logical_generation = $5 + AND attempt_generation = $6 AND workspace_id = $7 + AND writer_epoch = $8 AND instance_generation = $9 + AND hand_provisioning_operation_id = $10 AND hand_lease_generation = $11 + AND checkpoint_id = $12 AND checkpoint_generation = $13 + AND checkpoint_manifest_digest = $14 AND checkpoint_logical_bytes = $15 + AND receipt_state = 'released' AND destroy_outcome = 'verified_absent' + AND requested_at = $16 + "#, + ) + .bind(receipt.receipt_id) + .bind(receipt.tenant_id) + .bind(receipt.run_id) + .bind(task_id) + .bind(logical_generation) + .bind(attempt_generation) + .bind(workspace_id) + .bind(writer_epoch) + .bind(instance_generation) + .bind(hand_provisioning_operation_id) + .bind(hand_lease_generation) + .bind(checkpoint_id) + .bind(checkpoint_generation) + .bind(checkpoint_manifest_digest) + .bind(checkpoint_logical_bytes) + .bind(receipt.requested_at) + .fetch_optional(conn.as_mut()) + .await + .map_err(map_sqlx_error)? + .ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { + operation_id: format!( + "execution-task-hand-release:{}:{}:{}", + receipt.run_id, task_id, receipt.attempt_generation + ), + })? + }; + let persisted = execution_hand_release_receipt_from_row(&row)?; + conn.commit().await?; + Ok(persisted) + } + + /// Finalizes verified compute destruction after a checkpoint was already committed. + /// + /// This is the recovery seam for an ambiguous checkpoint attempt that was later + /// reconciled with its attachment retained. Provider destruction happens before + /// this call; the lease, capacity charge, and workspace state then advance under + /// the exact hand and workspace generations in one transaction. + pub async fn finalize_task_yield_destroy( + &self, + binding: &WorkspaceBinding, + lease: &HandLease, + ) -> Result { + let fence = WorkspaceBindingFence::try_from(binding)?; + let checkpoint_id = fence.checkpoint_id.ok_or_else(|| { + MoaError::ValidationError( + "task hand release requires a verified checkpoint head".to_string(), + ) + })?; + if lease.status != HandLeaseStatus::Active + || lease.attachment + != Some(crate::core::leases::HandLeaseWorkspaceAttachment::new( + binding.workspace_id, + fence.writer_epoch, + fence.instance_generation, + Some(checkpoint_id), + )?) + || lease.handle.is_none() + { + return Err(MoaError::ValidationError( + "task hand release lease does not match the committed workspace head".to_string(), + )); + } + let mut conn = self.begin(binding.tenant_id).await?; + let lease_affected = sqlx::query( + r#" + UPDATE moa.hand_leases + SET status = 'destroyed', handle = NULL, workspace_id = NULL, + workspace_writer_epoch = NULL, workspace_instance_generation = NULL, + restored_checkpoint_id = NULL, reap_not_before = NULL, + reap_claim_token = NULL, reap_claim_expires_at = NULL, updated_at = now() + WHERE tenant_id = $1 AND session_id = $2 AND worker_id = $3 + AND provider = $4 AND generation = $5 + AND provisioning_operation_id = $6 AND status = 'active' + AND workspace_id = $7 AND workspace_writer_epoch = $8 + AND workspace_instance_generation = $9 + AND restored_checkpoint_id = $10 AND handle IS NOT NULL + "#, + ) + .bind(binding.tenant_id) + .bind(lease.session_id) + .bind(&lease.worker_id) + .bind(&lease.provider) + .bind(lease.generation) + .bind(lease.provisioning_operation_id) + .bind(binding.workspace_id) + .bind(fence.writer_epoch) + .bind(fence.instance_generation) + .bind(checkpoint_id) + .execute(conn.as_mut()) + .await + .map_err(map_sqlx_error)? + .rows_affected(); + if lease_affected != 1 { + conn.rollback().await?; + return Ok(false); + } + let capacity_affected = sqlx::query( + r#" + UPDATE moa.sandbox_capacity_reservations + SET reservation_state = 'released', updated_at = now() + WHERE tenant_id = $1 AND workspace_id = $2 + AND provider_account_id = $3 AND provider_account_generation = $4 + AND hand_provisioning_operation_id = $5 + AND hand_lease_generation = $6 + AND expected_writer_epoch = $7 AND expected_instance_generation = $8 + AND resource_dimension = 'active_hands' + AND reservation_state = 'committed' + "#, + ) + .bind(binding.tenant_id) + .bind(binding.workspace_id) + .bind(binding.provider_account_id) + .bind(fence.provider_account_generation) + .bind(lease.provisioning_operation_id) + .bind(lease.generation) + .bind(fence.writer_epoch) + .bind(fence.instance_generation) + .execute(conn.as_mut()) + .await + .map_err(map_sqlx_error)? + .rows_affected(); + if capacity_affected != 1 { + conn.rollback().await?; + return Ok(false); + } + let workspace_affected = sqlx::query( + r#" + UPDATE moa.sandbox_workspaces + SET lifecycle_state = 'ready', updated_at = now() + WHERE tenant_id = $1 AND workspace_id = $2 + AND provider_account_id = $3 AND provider_account_generation = $4 + AND writer_epoch = $5 AND instance_generation = $6 + AND current_checkpoint_generation = $7 AND current_checkpoint_id = $8 + AND lifecycle_state = 'active' AND access_fenced_at IS NULL + "#, + ) + .bind(binding.tenant_id) + .bind(binding.workspace_id) + .bind(binding.provider_account_id) + .bind(fence.provider_account_generation) + .bind(fence.writer_epoch) + .bind(fence.instance_generation) + .bind(fence.checkpoint_generation) + .bind(checkpoint_id) + .execute(conn.as_mut()) + .await + .map_err(map_sqlx_error)? + .rows_affected(); + if workspace_affected != 1 { + conn.rollback().await?; + return Ok(false); + } + conn.commit().await?; + Ok(true) + } + /// Applies one documented lifecycle transition under writer and instance fences. pub async fn transition(&self, transition: WorkspaceTransition) -> Result { if !allowed_transition(transition.from, transition.to) { @@ -279,11 +1344,117 @@ impl PostgresWorkspaceRepository { .await .map_err(map_sqlx_error)? .rows_affected(); + if affected == 1 + && !release_workspace_in_transaction( + conn.as_mut(), + tenant_id, + workspace_id, + delete_generation, + ) + .await? + { + conn.rollback().await?; + return Ok(false); + } conn.commit().await?; Ok(affected == 1) } } +fn execution_hand_release_receipt_from_row( + row: &sqlx::postgres::PgRow, +) -> Result { + let owner_kind: String = row.try_get("owner_kind").map_err(map_sqlx_error)?; + let logical_generation: i64 = row.try_get("logical_generation").map_err(map_sqlx_error)?; + let logical_generation = u64::try_from(logical_generation).map_err(|_| { + MoaError::StorageError("release receipt logical generation is not positive".to_string()) + })?; + let owner = match owner_kind.as_str() { + "task" => ExecutionHandReleaseOwner::Task { + task_id: row.try_get("task_id").map_err(map_sqlx_error)?, + logical_generation, + }, + "compensation" => ExecutionHandReleaseOwner::Compensation { + compensation_id: row.try_get("compensation_id").map_err(map_sqlx_error)?, + logical_generation, + }, + other => { + return Err(MoaError::StorageError(format!( + "unknown execution hand release owner kind {other}" + ))); + } + }; + let attempt_generation: i64 = row.try_get("attempt_generation").map_err(map_sqlx_error)?; + let writer_epoch: Option = row.try_get("writer_epoch").map_err(map_sqlx_error)?; + let instance_generation: Option = + row.try_get("instance_generation").map_err(map_sqlx_error)?; + let hand_lease_generation: Option = row + .try_get("hand_lease_generation") + .map_err(map_sqlx_error)?; + let checkpoint_generation: Option = row + .try_get("checkpoint_generation") + .map_err(map_sqlx_error)?; + let checkpoint_logical_bytes: Option = row + .try_get("checkpoint_logical_bytes") + .map_err(map_sqlx_error)?; + let hand_provisioning_operation_id: Option = row + .try_get("hand_provisioning_operation_id") + .map_err(map_sqlx_error)?; + Ok(ExecutionHandReleaseReceipt { + receipt_id: row.try_get("receipt_id").map_err(map_sqlx_error)?, + tenant_id: row.try_get("tenant_id").map_err(map_sqlx_error)?, + run_id: row.try_get("run_uid").map_err(map_sqlx_error)?, + owner, + attempt_generation: u64::try_from(attempt_generation).map_err(|_| { + MoaError::StorageError("release receipt attempt generation is not positive".to_string()) + })?, + workspace_id: row.try_get("workspace_id").map_err(map_sqlx_error)?, + writer_epoch: writer_epoch.map(u64::try_from).transpose().map_err(|_| { + MoaError::StorageError("release receipt writer epoch is negative".to_string()) + })?, + instance_generation: instance_generation + .map(u64::try_from) + .transpose() + .map_err(|_| { + MoaError::StorageError( + "release receipt instance generation is negative".to_string(), + ) + })?, + hand_provisioning_operation_id: hand_provisioning_operation_id + .map(HandProvisioningOperationId), + hand_lease_generation: hand_lease_generation + .map(u64::try_from) + .transpose() + .map_err(|_| { + MoaError::StorageError( + "release receipt hand lease generation is not positive".to_string(), + ) + })?, + checkpoint_id: row.try_get("checkpoint_id").map_err(map_sqlx_error)?, + checkpoint_generation: checkpoint_generation + .map(u64::try_from) + .transpose() + .map_err(|_| { + MoaError::StorageError( + "release receipt checkpoint generation is not positive".to_string(), + ) + })?, + checkpoint_manifest_digest: row + .try_get("checkpoint_manifest_digest") + .map_err(map_sqlx_error)?, + checkpoint_logical_bytes: checkpoint_logical_bytes + .map(u64::try_from) + .transpose() + .map_err(|_| { + MoaError::StorageError( + "release receipt checkpoint logical bytes are negative".to_string(), + ) + })?, + requested_at: row.try_get("requested_at").map_err(map_sqlx_error)?, + released_at: row.try_get("released_at").map_err(map_sqlx_error)?, + }) +} + fn allowed_transition(from: SandboxWorkspaceState, to: SandboxWorkspaceState) -> bool { use SandboxWorkspaceState::{ Active, Committing, Creating, Deleting, Failed, Quiescing, Ready, Reconciling, Restoring, diff --git a/crates/moa-hands/src/core/sandbox_workspace/repository/mod.rs b/crates/moa-hands/src/core/sandbox_workspace/repository/mod.rs index cc53f965b..f9d486b03 100644 --- a/crates/moa-hands/src/core/sandbox_workspace/repository/mod.rs +++ b/crates/moa-hands/src/core/sandbox_workspace/repository/mod.rs @@ -9,14 +9,16 @@ use moa_core::{ error::{MoaError, Result}, types::{ identifiers::{ - ExecutionRunScopeId, ExecutionTaskScopeId, ProviderAccountId, SandboxWorkspaceId, - SessionId, TenantId, WorkspaceCheckpointId, WorkspaceOperationId, + ExecutionCompensationScopeId, ExecutionRunScopeId, ExecutionTaskScopeId, + HandProvisioningOperationId, ProviderAccountId, SandboxWorkspaceId, SessionId, + TenantId, WorkspaceCheckpointId, WorkspaceOperationId, }, memory::RlsContext, sandbox_workspace::{ - DurabilityClass, ProviderStorageKind, SandboxWorkspaceScope, SandboxWorkspaceState, - WorkspaceBinding, WorkspaceCheckpointPublication, WorkspaceCheckpointState, - WorkspaceOperationKind, WorkspacePostCommitState, + DurabilityClass, ExecutionHandReleaseOwner, ExecutionHandReleaseReceipt, + ProviderStorageKind, SandboxWorkspaceScope, SandboxWorkspaceState, WorkspaceBinding, + WorkspaceCheckpointPublication, WorkspaceCheckpointState, WorkspaceOperationKind, + WorkspacePostCommitState, }, }, }; @@ -31,9 +33,11 @@ use super::{ }, failpoints, model::{ - ActivateHydratedWorkspaceRequest, CreateWorkspaceRequest, SandboxWorkspace, WorkspaceGrant, - WorkspaceGrantRelation, WorkspaceGrantSubjectType, WorkspaceProviderAccount, - WorkspaceTransition, WorkspaceWriterClaim, + AbsentTaskHandReleaseIntent, ActivateHydratedWorkspaceRequest, + CompensationHandReleaseClaim, CompensationHandReleaseIntent, CreateWorkspaceRequest, + SandboxWorkspace, TaskHandReleaseIntent, WorkspaceGrant, WorkspaceGrantRelation, + WorkspaceGrantSubjectType, WorkspaceProviderAccount, WorkspaceTransition, + WorkspaceWriterClaim, }, operations::ClaimedWorkspaceOperation, }; diff --git a/crates/moa-hands/src/lib.rs b/crates/moa-hands/src/lib.rs index c5cfdb1a7..909571dc5 100644 --- a/crates/moa-hands/src/lib.rs +++ b/crates/moa-hands/src/lib.rs @@ -10,12 +10,13 @@ pub use adapters::local::{LOCAL_HAND_CAPABILITIES, LocalHandProvider}; pub use adapters::mcp::{MCPClient, McpDiscoveredTool}; pub use core::{ ActionOrigin, AuthorizedToolCall, CandidateConnector, CatalogDefect, - DeferredWorkspaceToolOutput, FileProviderCredentialSource, HandLeaseReaper, - HandLeaseReaperConfig, HandRoute, JournaledWorkspaceCommit, MCP_TOOL_REFERENCE_PREFIX, - McpCatalogActivation, McpCatalogRefresh, McpConnectorHealth, PendingConnectorToolOutput, - PinnedToolContract, PinnedToolOwner, PostgresExpiredHandLeaseClaims, - PostgresTenantSandboxPolicyStore, PreparedActionInvocation, ProviderCredentialSource, - ProviderEndpoint, ProviderHttpAttempt, ProviderSandboxAttempt, TenantSandboxPolicyStore, + DeferredWorkspaceToolOutput, ExecutionHandReleaseRequest, FileProviderCredentialSource, + HandLeaseReaper, HandLeaseReaperConfig, HandRoute, JournaledWorkspaceCommit, + MCP_TOOL_REFERENCE_PREFIX, McpCatalogActivation, McpCatalogRefresh, McpConnectorHealth, + PendingConnectorToolOutput, PinnedToolContract, PinnedToolOwner, + PostgresExpiredHandLeaseClaims, PostgresTenantSandboxPolicyStore, PreparedActionInvocation, + ProviderCredentialSource, ProviderEndpoint, ProviderHttpAttempt, ProviderSandboxAttempt, + SandboxProviderInventory, SessionHandReleasePageOutcome, TenantSandboxPolicyStore, ToolCallScope, ToolCatalogDrift, ToolCatalogPin, ToolCatalogSnapshot, ToolExecution, ToolRegistry, ToolRouter, deployment_sandbox_policy, governed_tool_contract_revision, local_development_sandbox_policy, mcp_tool_reference, route_sandbox_policy, diff --git a/crates/moa-hands/src/tools/sandbox_descriptor.rs b/crates/moa-hands/src/tools/sandbox_descriptor.rs index bd5720c2a..3e9c29631 100644 --- a/crates/moa-hands/src/tools/sandbox_descriptor.rs +++ b/crates/moa-hands/src/tools/sandbox_descriptor.rs @@ -129,6 +129,7 @@ impl SandboxToolDescriptor { schema: (self.schema)(), policy: self.policy.clone(), idempotency_class: self.idempotency_class, + async_mode: moa_core::types::tools::ToolAsyncMode::SynchronousOnly, rollback: None, max_output_tokens, } diff --git a/crates/moa-hands/tests/daytona_live.rs b/crates/moa-hands/tests/daytona_live.rs index 314534626..00c0116fb 100644 --- a/crates/moa-hands/tests/daytona_live.rs +++ b/crates/moa-hands/tests/daytona_live.rs @@ -989,6 +989,7 @@ async fn daytona_volume_workspace_survives_compute_replacement_live() { .await, hand: source.clone(), parent_revision: None, + release_compute: false, }, ) .await?; @@ -1132,6 +1133,7 @@ async fn daytona_workspace_restores_after_tenant_volume_replacement_live() { .await, hand: source.clone(), parent_revision: None, + release_compute: false, }, ) .await? diff --git a/crates/moa-hands/tests/e2b_live.rs b/crates/moa-hands/tests/e2b_live.rs index 9ddcf170e..b23bdd38e 100644 --- a/crates/moa-hands/tests/e2b_live.rs +++ b/crates/moa-hands/tests/e2b_live.rs @@ -595,6 +595,7 @@ async fn e2b_workspace_restores_into_fresh_compute_live() { ), hand: source.clone(), parent_revision: source_binding.current_revision.clone(), + release_compute: false, }, ) .await diff --git a/crates/moa-hands/tests/hands_db/hand_lease_reaper_db.rs b/crates/moa-hands/tests/hands_db/hand_lease_reaper_db.rs index 54dfcf7fc..8d4a612aa 100644 --- a/crates/moa-hands/tests/hands_db/hand_lease_reaper_db.rs +++ b/crates/moa-hands/tests/hands_db/hand_lease_reaper_db.rs @@ -31,14 +31,15 @@ use moa_core::types::identifiers::{ }; use moa_hands::PostgresTenantSandboxPolicyStore; use moa_hands::core::leases::{ - HandLeaseActivateRequest, HandLeasePolicy, HandLeaseProvisionRequest, HandLeaseRenewRequest, - HandLeaseStatus, HandLeaseStore, HandLeaseWorkspaceAttachment, LeaseHandle, - PostgresHandLeaseStore, + HAND_LEASE_SESSION_PAGE_SIZE, HandLeaseActivateRequest, HandLeasePolicy, + HandLeaseProvisionRequest, HandLeaseRenewRequest, HandLeaseStatus, HandLeaseStore, + HandLeaseWorkspaceAttachment, LeaseHandle, PostgresHandLeaseStore, }; use moa_hands::core::reaper::{ExpiredHandLeaseClaims, PostgresExpiredHandLeaseClaims}; use moa_hands::{TenantSandboxPolicyStore, deployment_sandbox_policy}; use sqlx::PgPool; use sqlx::postgres::PgPoolOptions; +use uuid::Uuid; use super::{database_url, seed_session}; @@ -51,6 +52,251 @@ async fn pool() -> PgPool { .expect("test Postgres should be reachable") } +#[tokio::test] +async fn exact_owner_lookup_uses_index_under_large_session_history_db() { + // Pins: compensation teardown must use the exact owner-key index rather + // than fetching and filtering every lease accumulated by the session. + let pool = pool().await; + let tenant_id = TenantId::new(); + let session_id = SessionId::new(); + seed_session(&pool, session_id, tenant_id).await; + sqlx::query( + "INSERT INTO moa.hand_leases (\ + session_id, worker_id, tenant_id, provider, tier, handle, status, generation,\ + provisioning_operation_id, provisioning_deadline_at, created_at, updated_at\ + ) SELECT $1, 'unrelated-owner-' || series::TEXT, $2, 'local', 'local', NULL,\ + 'destroyed', 1, gen_random_uuid(), now(), now(), now()\ + FROM generate_series(1, 2000) AS series", + ) + .bind(session_id) + .bind(tenant_id) + .execute(&pool) + .await + .expect("seed unrelated lease history"); + let target = format!( + "execution_compensation:{}:{}", + Uuid::now_v7(), + Uuid::now_v7() + ); + sqlx::query( + "INSERT INTO moa.hand_leases (\ + session_id, worker_id, tenant_id, provider, tier, handle, status, generation,\ + provisioning_operation_id, provisioning_deadline_at, created_at, updated_at\ + ) VALUES ($1, $2, $3, 'local', 'local', NULL, 'stale', 7,\ + gen_random_uuid(), now(), now(), now())", + ) + .bind(session_id) + .bind(&target) + .bind(tenant_id) + .execute(&pool) + .await + .expect("seed exact compensation lease"); + sqlx::query("ANALYZE moa.hand_leases") + .execute(&pool) + .await + .expect("refresh hand lease planner statistics"); + + let leases = PostgresHandLeaseStore::new(pool.clone()) + .list_live_owner_candidates(tenant_id, session_id, &target) + .await + .expect("load exact compensation owner"); + assert_eq!(leases.len(), 1); + assert_eq!(leases[0].worker_id, target); + assert_eq!(leases[0].generation, 7); + + sqlx::query( + "INSERT INTO moa.hand_leases (\ + session_id, worker_id, tenant_id, provider, tier, handle, status, generation,\ + provisioning_operation_id, provisioning_deadline_at, created_at, updated_at\ + ) SELECT $1, $2, $3, provider, 'local', NULL, 'stale', 8,\ + gen_random_uuid(), now(), now(), now()\ + FROM (VALUES ('daytona'), ('e2b')) AS replacements(provider)", + ) + .bind(session_id) + .bind(&target) + .bind(tenant_id) + .execute(&pool) + .await + .expect("seed multiple invalid live replacements"); + let store = PostgresHandLeaseStore::new(pool.clone()); + let bounded = store + .list_live_owner_candidates(tenant_id, session_id, &target) + .await + .expect("load bounded live candidates"); + assert_eq!( + bounded.len(), + 2, + "the release probe must never materialize more than two candidates" + ); + assert!( + store + .has_live_owner(tenant_id, session_id, &target) + .await + .expect("check exact live replacement"), + "the exact owner replacement probe must observe live ownership" + ); + + let candidate_plan: serde_json::Value = sqlx::query_scalar( + "EXPLAIN (ANALYZE, FORMAT JSON)\ + SELECT session_id, worker_id, provider FROM moa.hand_leases\ + WHERE tenant_id = $1 AND session_id = $2 AND worker_id = $3\ + AND status <> 'destroyed'\ + ORDER BY provider LIMIT 2", + ) + .bind(tenant_id) + .bind(session_id) + .bind(&target) + .fetch_one(&pool) + .await + .expect("explain exact owner lookup"); + let candidate_plan = candidate_plan.to_string(); + assert!( + candidate_plan.contains("hand_leases_tenant_live_owner_idx") + || candidate_plan.contains("idx_hand_leases_tenant_owner") + || candidate_plan.contains("hand_leases_pkey"), + "exact owner lookup must use an owner-key index: {candidate_plan}" + ); + assert!( + !candidate_plan.contains("Seq Scan"), + "exact owner lookup must not scan unrelated session history: {candidate_plan}" + ); + let exists_plan: serde_json::Value = sqlx::query_scalar( + "EXPLAIN (ANALYZE, FORMAT JSON)\ + SELECT EXISTS (SELECT 1 FROM moa.hand_leases\ + WHERE tenant_id = $1 AND session_id = $2 AND worker_id = $3\ + AND status <> 'destroyed')", + ) + .bind(tenant_id) + .bind(session_id) + .bind(&target) + .fetch_one(&pool) + .await + .expect("explain exact owner existence probe"); + let exists_plan = exists_plan.to_string(); + assert!( + exists_plan.contains("hand_leases_tenant_live_owner_idx") + || exists_plan.contains("idx_hand_leases_tenant_owner") + || exists_plan.contains("hand_leases_pkey"), + "exact replacement probe must use an owner-key index: {exists_plan}" + ); + assert!( + !exists_plan.contains("Seq Scan"), + "exact replacement probe must not scan unrelated session history: {exists_plan}" + ); + + sqlx::query("DELETE FROM moa.hand_leases WHERE session_id = $1") + .bind(session_id) + .execute(&pool) + .await + .expect("delete lease history fixture"); + sqlx::query("DELETE FROM public.session_agent_context WHERE session_id = $1") + .bind(session_id) + .execute(&pool) + .await + .expect("delete session agent fixture"); + sqlx::query("DELETE FROM public.sessions WHERE id = $1") + .bind(session_id) + .execute(&pool) + .await + .expect("delete session fixture"); +} + +#[tokio::test] +async fn live_session_paging_is_indexed_bounded_and_replayable_db() { + // Pins: terminal teardown pages every live hand through the partial owner + // index while destroyed history remains outside both the result and scan. + let pool = pool().await; + let tenant_id = TenantId::new(); + let session_id = SessionId::new(); + seed_session(&pool, session_id, tenant_id).await; + sqlx::query( + "INSERT INTO moa.hand_leases (\ + session_id, worker_id, tenant_id, provider, tier, handle, status, generation,\ + provisioning_operation_id, provisioning_deadline_at, created_at, updated_at\ + ) SELECT $1, 'destroyed-owner-' || lpad(series::TEXT, 4, '0'), $2,\ + 'local', 'local', NULL, 'destroyed', 1, gen_random_uuid(), now(), now(), now()\ + FROM generate_series(1, 2000) AS series", + ) + .bind(session_id) + .bind(tenant_id) + .execute(&pool) + .await + .expect("seed destroyed lease history"); + sqlx::query( + "INSERT INTO moa.hand_leases (\ + session_id, worker_id, tenant_id, provider, tier, handle, status, generation,\ + provisioning_operation_id, provisioning_deadline_at, created_at, updated_at\ + ) SELECT $1, 'live-owner-' || lpad(series::TEXT, 4, '0'), $2,\ + 'local', 'local', NULL, 'stale', 1, gen_random_uuid(), now(), now(), now()\ + FROM generate_series(1, $3) AS series", + ) + .bind(session_id) + .bind(tenant_id) + .bind(i64::try_from(HAND_LEASE_SESSION_PAGE_SIZE + 7).expect("page fixture fits i64")) + .execute(&pool) + .await + .expect("seed live lease pages"); + sqlx::query("ANALYZE moa.hand_leases") + .execute(&pool) + .await + .expect("refresh hand lease planner statistics"); + + let store = PostgresHandLeaseStore::new(pool.clone()); + let first = store + .list_live_session_page(tenant_id, session_id, None) + .await + .expect("load first live session page"); + let replay = store + .list_live_session_page(tenant_id, session_id, None) + .await + .expect("replay first live session page"); + assert_eq!(first, replay); + assert_eq!(first.leases.len(), HAND_LEASE_SESSION_PAGE_SIZE); + let second = store + .list_live_session_page(tenant_id, session_id, first.next_cursor.as_ref()) + .await + .expect("load final live session page"); + assert_eq!(second.leases.len(), 7); + assert_eq!(second.next_cursor, None); + + let plan: serde_json::Value = sqlx::query_scalar( + "EXPLAIN (ANALYZE, FORMAT JSON)\ + SELECT session_id, worker_id, provider FROM moa.hand_leases\ + WHERE tenant_id = $1 AND session_id = $2 AND status <> 'destroyed'\ + ORDER BY worker_id, provider LIMIT 65", + ) + .bind(tenant_id) + .bind(session_id) + .fetch_one(&pool) + .await + .expect("explain bounded live-session lookup"); + let plan = plan.to_string(); + assert!( + plan.contains("hand_leases_tenant_live_owner_idx"), + "live session page must use the partial owner index: {plan}" + ); + assert!( + !plan.contains("Seq Scan"), + "live session page must not scan destroyed history: {plan}" + ); + + sqlx::query("DELETE FROM moa.hand_leases WHERE session_id = $1") + .bind(session_id) + .execute(&pool) + .await + .expect("delete lease paging fixture"); + sqlx::query("DELETE FROM public.session_agent_context WHERE session_id = $1") + .bind(session_id) + .execute(&pool) + .await + .expect("delete session agent fixture"); + sqlx::query("DELETE FROM public.sessions WHERE id = $1") + .bind(session_id) + .execute(&pool) + .await + .expect("delete session fixture"); +} + async fn seed_workspace( pool: &PgPool, tenant_id: TenantId, diff --git a/crates/moa-hands/tests/hands_db/sandbox_workspace/capacity_db.rs b/crates/moa-hands/tests/hands_db/sandbox_workspace/capacity_db.rs index f64c04498..64b232600 100644 --- a/crates/moa-hands/tests/hands_db/sandbox_workspace/capacity_db.rs +++ b/crates/moa-hands/tests/hands_db/sandbox_workspace/capacity_db.rs @@ -9,14 +9,14 @@ use moa_core::{ TenantId, WorkspaceOperationId, }, sandbox_workspace::{ - DurabilityClass, SandboxWorkspaceScope, WorkspaceCapacityDimension, - WorkspaceOperationKind, + DurabilityClass, SandboxWorkspaceScope, SandboxWorkspaceState, + WorkspaceCapacityDimension, WorkspaceOperationKind, }, }, }; use moa_hands::core::sandbox_workspace::{ capacity::{CapacityQuantity, CapacityReservationRequest, PostgresWorkspaceCapacityRepository}, - model::CreateWorkspaceRequest, + model::{CreateWorkspaceRequest, WorkspaceTransition}, operations::{PostgresWorkspaceOperationRepository, WorkspaceOperationIntent}, repository::PostgresWorkspaceRepository, storage_resources::{PostgresWorkspaceStorageResourceRepository, StorageResourceCreateIntent}, @@ -142,10 +142,10 @@ async fn cleanup_volume_account(pool: &sqlx::PgPool, account_id: ProviderAccount } #[tokio::test] -#[ignore = "requires a fresh V58 compose Postgres via MOA_DATABASE_URL"] +#[ignore = "requires a fresh V60 compose Postgres via MOA_DATABASE_URL"] async fn exact_capacity_succeeds_and_exact_limit_plus_one_is_deterministic_db() { - // Pins: one atomic reservation consumes the exact tenant/provider limit and - // the next replica is rejected without a partial reservation. + // Pins: workspace creation atomically consumes the exact tenant/provider + // limit, and only finalized deletion at the exact generation releases it. let pool = PgPoolOptions::new() .max_connections(6) .connect(&database_url()) @@ -176,81 +176,144 @@ async fn exact_capacity_succeeds_and_exact_limit_plus_one_is_deterministic_db() .expect("seed tenant capacity"); let workspaces = PostgresWorkspaceRepository::new(pool.clone()); - let operations = PostgresWorkspaceOperationRepository::new(pool.clone()); - let capacity = PostgresWorkspaceCapacityRepository::new(pool.clone()); - let now = Utc::now(); - let mut requests = Vec::new(); - for ordinal in 0..2 { - let workspace_id = SandboxWorkspaceId::new(); - let operation_id = WorkspaceOperationId::new(); + let create = |workspace_id| CreateWorkspaceRequest { + workspace_id, + tenant_id, + scope: SandboxWorkspaceScope::ExecutionTask { + run_id: ExecutionRunScopeId::new(), + task_id: ExecutionTaskScopeId::new(), + }, + provider: "local".to_string(), + provider_account_id: account_id, + provider_account_generation: 1, + durability_class: DurabilityClass::PortableFilesystem, + retention_deadline_at: None, + }; + let first_workspace_id = SandboxWorkspaceId::new(); + workspaces + .create(&create(first_workspace_id)) + .await + .expect("workspace creation atomically reserves the exact capacity"); + let second_workspace_id = SandboxWorkspaceId::new(); + let error = workspaces + .create(&create(second_workspace_id)) + .await + .expect_err("limit plus one must roll back workspace creation"); + assert!( + matches!(error, MoaError::StorageError(ref detail) if detail.contains("tenant workspaces capacity exceeded")), + "workspace admission must report the exact exhausted dimension: {error}" + ); + let reservation_count = sqlx::query_scalar::<_, i64>( + "SELECT count(*) FROM moa.sandbox_capacity_reservations WHERE tenant_id = $1 AND resource_dimension = 'workspaces' AND reservation_state = 'committed'", + ) + .bind(tenant_id) + .fetch_one(&pool) + .await + .expect("count atomic reservations"); + assert_eq!( + reservation_count, 1, + "a rejected batch leaves no partial row" + ); + let workspace_count = sqlx::query_scalar::<_, i64>( + "SELECT count(*) FROM moa.sandbox_workspaces WHERE tenant_id = $1", + ) + .bind(tenant_id) + .fetch_one(&pool) + .await + .expect("count atomically admitted workspaces"); + assert_eq!( + workspace_count, 1, + "capacity failure rolls back workspace metadata" + ); + assert_ne!(first_workspace_id, second_workspace_id); + + assert!( workspaces - .create(&CreateWorkspaceRequest { - workspace_id, + .transition(WorkspaceTransition { tenant_id, - scope: SandboxWorkspaceScope::ExecutionTask { - run_id: ExecutionRunScopeId::new(), - task_id: ExecutionTaskScopeId::new(), - }, - provider: "local".to_string(), - provider_account_id: account_id, - provider_account_generation: 1, - durability_class: DurabilityClass::PortableFilesystem, - retention_deadline_at: None, + workspace_id: first_workspace_id, + from: SandboxWorkspaceState::Creating, + to: SandboxWorkspaceState::Ready, + writer_epoch: 0, + instance_generation: 0, }) .await - .expect("create logical workspace"); - operations - .persist_intent(&WorkspaceOperationIntent { - operation_id, - tenant_id, - workspace_id, - provider_account_id: account_id, - provider_account_generation: 1, - kind: WorkspaceOperationKind::Create, - request_hash: format!("sha256:capacity-{ordinal}"), - expected_writer_epoch: 0, - expected_instance_generation: 0, - expected_checkpoint_generation: 0, - deadline_at: now + ChronoDuration::seconds(10), - reconcile_not_before: now + ChronoDuration::seconds(20), - }) + .expect("make admitted workspace ready for deletion") + ); + assert!( + workspaces + .fence_for_deletion(tenant_id, first_workspace_id, 0, 0) .await - .expect("persist create intent"); - requests.push(CapacityReservationRequest { + .expect("fence exact workspace generation for deletion") + ); + let capacity = PostgresWorkspaceCapacityRepository::new(pool.clone()); + assert!( + !capacity + .release_workspace(tenant_id, first_workspace_id, 1) + .await + .expect("deleting workspace cannot release capacity before finalized absence") + ); + + let delete_operation_id = WorkspaceOperationId::new(); + let now = Utc::now(); + PostgresWorkspaceOperationRepository::new(pool.clone()) + .persist_intent(&WorkspaceOperationIntent { + operation_id: delete_operation_id, tenant_id, - workspace_id, - operation_id, + workspace_id: first_workspace_id, provider_account_id: account_id, provider_account_generation: 1, + kind: WorkspaceOperationKind::Delete, + request_hash: format!("sha256:workspace-delete-{delete_operation_id}"), expected_writer_epoch: 0, expected_instance_generation: 0, - quantities: vec![CapacityQuantity { - dimension: WorkspaceCapacityDimension::Workspaces, - quantity: 1, - }], - }); - } - - let first = capacity - .reserve(&requests[0]) + expected_checkpoint_generation: 0, + deadline_at: now + ChronoDuration::seconds(10), + reconcile_not_before: now + ChronoDuration::seconds(20), + }) .await - .expect("exact capacity is admitted"); - assert_eq!(first.len(), 1); + .expect("persist exact delete operation"); + sqlx::query( + r#" + UPDATE moa.sandbox_workspace_operations + SET outcome_class = 'confirmed', confirmed_disposition = 'resource_absent', + absence_observation_count = 2, + absence_first_observed_at = now() - interval '2 seconds', + absence_last_observed_at = now(), + absence_inventory_digest = 'sha256:capacity-delete-absence' + WHERE operation_id = $1 + "#, + ) + .bind(delete_operation_id) + .execute(&pool) + .await + .expect("record verified provider absence for deletion"); assert!( - capacity.reserve(&requests[1]).await.is_err(), - "exact limit plus one must be rejected" + workspaces + .finalize_deleted(tenant_id, first_workspace_id, 1, delete_operation_id) + .await + .expect("finalize exact deleted workspace") ); - let reservation_count = sqlx::query_scalar::<_, i64>( - "SELECT count(*) FROM moa.sandbox_capacity_reservations WHERE tenant_id = $1", + assert!( + !capacity + .release_workspace(tenant_id, first_workspace_id, 2) + .await + .expect("future delete generation is a fenced miss") + ); + assert!( + !capacity + .release_workspace(tenant_id, first_workspace_id, 1) + .await + .expect("finalize atomically released the exact workspace owner") + ); + let released_state = sqlx::query_scalar::<_, String>( + "SELECT reservation_state FROM moa.sandbox_capacity_reservations WHERE workspace_id = $1 AND resource_dimension = 'workspaces'", ) - .bind(tenant_id) + .bind(first_workspace_id) .fetch_one(&pool) .await - .expect("count atomic reservations"); - assert_eq!( - reservation_count, 1, - "a rejected batch leaves no partial row" - ); + .expect("load released lifetime workspace reservation"); + assert_eq!(released_state, "released"); sqlx::query("DELETE FROM moa.sandbox_capacity_reservations WHERE tenant_id = $1") .bind(tenant_id) @@ -261,7 +324,7 @@ async fn exact_capacity_succeeds_and_exact_limit_plus_one_is_deterministic_db() .bind(tenant_id) .execute(&pool) .await - .expect("clean operations"); + .expect("clean workspace operations"); sqlx::query("DELETE FROM moa.sandbox_workspaces WHERE tenant_id = $1") .bind(tenant_id) .execute(&pool) @@ -281,7 +344,7 @@ async fn exact_capacity_succeeds_and_exact_limit_plus_one_is_deterministic_db() } #[tokio::test] -#[ignore = "requires a fresh V58 compose Postgres via MOA_DATABASE_URL"] +#[ignore = "requires a fresh V60 compose Postgres via MOA_DATABASE_URL"] async fn volume_inventory_overlap_headroom_and_exact_limit_are_atomic_db() { // Pins: one volume seen in both durable state and Daytona inventory counts // once, while provider-only inventory and reserved headroom still make the diff --git a/crates/moa-hands/tests/hands_db/sandbox_workspace/dispatch_db.rs b/crates/moa-hands/tests/hands_db/sandbox_workspace/dispatch_db.rs index 8f53524b2..bcc5efe0e 100644 --- a/crates/moa-hands/tests/hands_db/sandbox_workspace/dispatch_db.rs +++ b/crates/moa-hands/tests/hands_db/sandbox_workspace/dispatch_db.rs @@ -51,6 +51,7 @@ use moa_hands::{ HandLeaseWorkspaceAttachment, LeaseHandle, PostgresHandLeaseStore, }, sandbox_workspace::{ + capacity::PostgresWorkspaceCapacityRepository, checkpoint::model::{CreateCheckpointRequest, PublishCheckpointCommitRequest}, model::{ ActivateHydratedWorkspaceRequest, CreateWorkspaceRequest, SandboxWorkspace, @@ -469,6 +470,19 @@ async fn hydration_and_checkpoint_commits_are_atomic_replay_safe_and_generation_ Some(creating.clone()) ); let publication = publication(&commit_binding, operation_id, 17 + commit_index); + PostgresWorkspaceCapacityRepository::new(pool.clone()) + .reserve_checkpoint_publication( + &WorkspaceStorageOperation { + operation_id, + kind: WorkspaceOperationKind::Commit, + binding: commit_binding.clone(), + deadline: intent.deadline_at, + request_hash: intent.request_hash.clone(), + }, + publication.logical_bytes, + ) + .await + .expect("reserve checkpoint capacity before publication"); if commit_index == 1 { let mut stale_lease = active_lease.clone(); @@ -574,6 +588,119 @@ async fn hydration_and_checkpoint_commits_are_atomic_replay_safe_and_generation_ .and_then(|attachment| attachment.restored_checkpoint_id), Some(checkpoint_id) ); + + // Pins: if another writer wins after immutable bytes are uploaded, + // the losing checkpoint is failed and its exact count/bytes charge is + // released so the upload can be garbage-collected without a quota leak. + for (from, to) in [ + ( + SandboxWorkspaceState::Active, + SandboxWorkspaceState::Quiescing, + ), + ( + SandboxWorkspaceState::Quiescing, + SandboxWorkspaceState::Committing, + ), + ] { + assert!( + workspaces + .transition(WorkspaceTransition { + tenant_id, + workspace_id, + from, + to, + writer_epoch: active_workspace.writer_epoch, + instance_generation: active_workspace.instance_generation, + }) + .await + .expect("enter abandoned checkpoint barrier") + ); + } + let abandoned_workspace = workspaces + .get(tenant_id, workspace_id) + .await + .expect("load abandoned workspace") + .expect("abandoned workspace exists"); + let abandoned_binding = binding(&abandoned_workspace); + let abandoned_operation_id = WorkspaceOperationId::new(); + let abandoned_intent = operation_intent( + tenant_id, + &abandoned_workspace, + abandoned_operation_id, + WorkspaceOperationKind::Commit, + "sha256:abandoned-cas-loss", + ); + operations + .persist_intent(&abandoned_intent) + .await + .expect("persist abandoned operation"); + let abandoned_checkpoint_id = WorkspaceCheckpointId(abandoned_operation_id.0); + workspaces + .create_checkpoint(CreateCheckpointRequest { + checkpoint_id: abandoned_checkpoint_id, + tenant_id, + workspace_id, + parent_checkpoint_id: Some(checkpoint_id), + operation_id: abandoned_operation_id, + expected_writer_epoch: abandoned_workspace.writer_epoch, + expected_instance_generation: abandoned_workspace.instance_generation, + expected_checkpoint_generation: abandoned_workspace.checkpoint_generation, + }) + .await + .expect("create abandoned checkpoint") + .expect("abandoned checkpoint fence matches"); + let abandoned_publication = + self::publication(&abandoned_binding, abandoned_operation_id, 23); + PostgresWorkspaceCapacityRepository::new(pool.clone()) + .reserve_checkpoint_publication( + &WorkspaceStorageOperation { + operation_id: abandoned_operation_id, + kind: WorkspaceOperationKind::Commit, + binding: abandoned_binding.clone(), + deadline: abandoned_intent.deadline_at, + request_hash: abandoned_intent.request_hash.clone(), + }, + abandoned_publication.logical_bytes, + ) + .await + .expect("reserve abandoned checkpoint before upload"); + sqlx::query( + "UPDATE moa.sandbox_workspaces SET lifecycle_state = 'active' WHERE workspace_id = $1", + ) + .bind(workspace_id) + .execute(&pool) + .await + .expect("simulate another writer winning the lifecycle CAS"); + assert!( + !workspaces + .publish_checkpoint_commit(PublishCheckpointCommitRequest { + binding: &abandoned_binding, + operation_id: abandoned_operation_id, + publication: &abandoned_publication, + post_commit_state: WorkspacePostCommitState::AttachmentRetained, + lease: &active_lease, + }) + .await + .expect("CAS loss must be safely abandoned") + ); + let abandoned_state = sqlx::query_as::<_, (String, String)>( + r#" + SELECT checkpoint.lifecycle_state, reservation.reservation_state + FROM moa.sandbox_workspace_checkpoints AS checkpoint + JOIN moa.sandbox_capacity_reservations AS reservation + ON reservation.operation_id = checkpoint.operation_id + AND reservation.resource_dimension = 'checkpoints' + WHERE checkpoint.checkpoint_id = $1 + "#, + ) + .bind(abandoned_checkpoint_id) + .fetch_one(&pool) + .await + .expect("load abandoned checkpoint and capacity"); + assert_eq!( + abandoned_state, + ("failed".to_string(), "released".to_string()) + ); } else { assert_eq!(active_workspace.state, SandboxWorkspaceState::Ready); assert_eq!(active_lease.status, HandLeaseStatus::Destroyed); diff --git a/crates/moa-hands/tests/hands_db/sandbox_workspace/lifecycle_db.rs b/crates/moa-hands/tests/hands_db/sandbox_workspace/lifecycle_db.rs index aee09cdec..0f1a47283 100644 --- a/crates/moa-hands/tests/hands_db/sandbox_workspace/lifecycle_db.rs +++ b/crates/moa-hands/tests/hands_db/sandbox_workspace/lifecycle_db.rs @@ -11,13 +11,15 @@ use moa_core::types::{ MemoryLimit, SandboxPolicySnapshot, SandboxProfile, SandboxTier, }, identifiers::{ - ExecutionRunScopeId, ExecutionTaskScopeId, ProviderAccountId, SandboxWorkspaceId, - SessionId, TenantId, WorkspaceCheckpointId, WorkspaceOperationId, + ExecutionCompensationScopeId, ExecutionRunScopeId, ExecutionTaskScopeId, + HandProvisioningOperationId, ProviderAccountId, SandboxWorkspaceId, SessionId, TenantId, + WorkspaceCheckpointId, WorkspaceOperationId, }, sandbox_workspace::{ - DurabilityClass, ProviderStorageKind, ProviderStorageRef, SandboxWorkspaceScope, - SandboxWorkspaceState, WorkspaceBinding, WorkspaceCheckpointPublication, - WorkspaceOperationKind, WorkspacePostCommitState, WorkspaceRevisionRef, + DurabilityClass, ExecutionHandReleaseOwner, ExecutionHandReleaseReceipt, + ProviderStorageKind, ProviderStorageRef, SandboxWorkspaceScope, SandboxWorkspaceState, + WorkspaceBinding, WorkspaceCheckpointPublication, WorkspaceOperationKind, + WorkspacePostCommitState, WorkspaceRevisionRef, WorkspaceStorageOperation, }, }; use moa_hands::core::{ @@ -26,10 +28,12 @@ use moa_hands::core::{ HandLeaseWorkspaceAttachment, LeaseHandle, PostgresHandLeaseStore, }, sandbox_workspace::{ + capacity::{ActiveHandCapacityRequest, PostgresWorkspaceCapacityRepository}, checkpoint::model::{CreateCheckpointRequest, PublishCheckpointCommitRequest}, model::{ - ActivateHydratedWorkspaceRequest, CreateWorkspaceRequest, SandboxWorkspace, - WorkspaceTransition, WorkspaceWriterClaim, + AbsentTaskHandReleaseIntent, ActivateHydratedWorkspaceRequest, + CompensationHandReleaseIntent, CreateWorkspaceRequest, SandboxWorkspace, + TaskHandReleaseIntent, WorkspaceTransition, WorkspaceWriterClaim, }, operations::{ AbsenceObservation, PostgresWorkspaceOperationRepository, WorkspaceOperationIntent, @@ -58,6 +62,161 @@ async fn seed_account(pool: &sqlx::PgPool, account_id: ProviderAccountId) { .expect("seed provider account"); } +async fn seed_cancelling_compensation( + pool: &sqlx::PgPool, + tenant_id: TenantId, + session_id: SessionId, +) -> ( + ExecutionRunScopeId, + ExecutionTaskScopeId, + ExecutionCompensationScopeId, +) { + let planning_context_uid = uuid::Uuid::now_v7(); + let run_id = ExecutionRunScopeId::new(); + let task_id = ExecutionTaskScopeId::new(); + let compensation_id = ExecutionCompensationScopeId::new(); + let plan_hash = "1".repeat(64); + let context_hash = "2".repeat(64); + let plan = serde_json::json!({ + "definition": { + "cancel_policy": "retain_effects", + "input_schema": {}, + "output_schema": {}, + "input_wait_policy": { + "expiry": {"kind": "after", "delay_seconds": 1}, + "on_expiry": {"kind": "fail_run"} + }, + "nodes": [{ + "id": "output", "requirement_ids": [], "depends_on": [], "when": null, + "input": {}, "output_schema": {}, + "operation": {"kind": "output", "value": {}}, + "compensation": null, + "retry": {"max_attempts": 1, "initial_backoff_ms": 1, "max_backoff_ms": 1}, + "budget": null + }] + }, + "plan_hash": plan_hash, + "catalog_hash": "0".repeat(64), + "estimate": {"cost_microusd": 0, "tokens": 0, "tool_calls": 0, + "retrieved_bytes": 0, "tasks": 1}, + "report": {"issues": []} + }); + sqlx::query( + "INSERT INTO moa.execution_planning_context (\ + planning_context_uid, tenant_id, session_id, originating_user_sequence_num,\ + originating_user_event_hash, owner_user_id, planning_context_hash, snapshot\ + ) VALUES ($1, $2, $3, 0, $4, 'hands-release-test', $4, '{}'::JSONB)", + ) + .bind(planning_context_uid) + .bind(tenant_id) + .bind(session_id) + .bind(&context_hash) + .execute(pool) + .await + .expect("seed execution planning context"); + sqlx::query( + "INSERT INTO moa.execution_run (\ + run_uid, tenant_id, session_id, originating_user_sequence_num,\ + planning_context_uid, planning_context_hash, owner_user_id, goal_contract,\ + initial_plan, active_plan, initial_plan_hash, active_plan_hash,\ + capability_catalog, authorization_envelope, source_provenance, source_kind,\ + input, admitted_identity, status\ + ) VALUES ($1, $2, $3, 0, $4, $5, 'hands-release-test', $6, $7, $7, $8, $8,\ + $9, $10, $11, 'generated_plan', '{}'::JSONB, $12, 'queued')", + ) + .bind(run_id) + .bind(tenant_id) + .bind(session_id) + .bind(planning_context_uid) + .bind(&context_hash) + .bind(serde_json::json!({ + "objective": "release", "requirements": [], "deliverables": [], + "coverage": [], "constraints": [], "completion_checks": [] + })) + .bind(&plan) + .bind(&plan_hash) + .bind(serde_json::json!({"capabilities": [], "catalog_hash": "0".repeat(64)})) + .bind(serde_json::json!({"capability_refs": [], "skill_refs": []})) + .bind(serde_json::json!({ + "kind": "generated_plan", + "planner": {"model": "hands-release-test", "prompt_version": "test", + "candidate_hash": "3".repeat(64), "compiler_report_hash": "4".repeat(64), + "final_plan_hash": plan_hash, "repair_attempts": 0} + })) + .bind(serde_json::json!({ + "identity_type": "operator", "id": run_id, "tenant_id": tenant_id, + "api_key_id": null, "acting_on_behalf_of": null + })) + .execute(pool) + .await + .expect("seed execution run"); + sqlx::query( + "INSERT INTO moa.execution_task (\ + task_id, run_uid, tenant_id, node_id, item_key, plan_revision, status, input,\ + task_kind, retry_policy, estimate_cost_microusd, estimate_tokens, estimate_tasks,\ + estimate_tool_calls, estimate_retrieved_bytes\ + ) VALUES ($1, $2, $3, 'forward', 'forward', 1, 'completed', '{}',\ + '{\"kind\":\"output\",\"value\":null}',\ + '{\"max_attempts\":2,\"initial_backoff_ms\":1,\"max_backoff_ms\":1}',\ + 0, 0, 1, 0, 0)", + ) + .bind(task_id) + .bind(run_id) + .bind(tenant_id) + .execute(pool) + .await + .expect("seed forward task"); + sqlx::query( + "INSERT INTO moa.execution_compensation (\ + compensation_id, run_uid, forward_task_id, tenant_id, registered_sequence,\ + forward_generation, compensator, mapped_input, status, started_at,\ + attempt_state, attempt_started_at, attempt_deadline_at, release_intent\ + ) VALUES ($1, $2, $3, $4, 1, 1, $5, '{}', 'running', now(),\ + 'cancelling', now(), now() + interval '10 minutes', 'pause')", + ) + .bind(compensation_id) + .bind(run_id) + .bind(task_id) + .bind(tenant_id) + .bind(serde_json::json!({ + "compensator": {"name": "test.undo", "version": "contract"}, + "input_mapping": {"bindings": []} + })) + .execute(pool) + .await + .expect("seed cancelling compensation"); + (run_id, task_id, compensation_id) +} + +async fn seed_cancelling_task( + pool: &sqlx::PgPool, + tenant_id: TenantId, + run_id: ExecutionRunScopeId, + node_id: &str, +) -> ExecutionTaskScopeId { + let task_id = ExecutionTaskScopeId::new(); + sqlx::query( + "INSERT INTO moa.execution_task (\ + task_id, run_uid, tenant_id, node_id, item_key, plan_revision, status, input,\ + task_kind, retry_policy, estimate_cost_microusd, estimate_tokens, estimate_tasks,\ + estimate_tool_calls, estimate_retrieved_bytes, reserved_tasks, reserved_at,\ + started_at, attempt_state, attempt_started_at, attempt_deadline_at\ + ) VALUES ($1, $2, $3, $4, $4, 1, 'running', '{}',\ + '{\"kind\":\"output\",\"value\":null}',\ + '{\"max_attempts\":2,\"initial_backoff_ms\":1,\"max_backoff_ms\":1}',\ + 0, 0, 1, 0, 0, 1, now(), now(), 'cancelling', now(),\ + now() + interval '10 minutes')", + ) + .bind(task_id) + .bind(run_id) + .bind(tenant_id) + .bind(node_id) + .execute(pool) + .await + .expect("seed cancelling execution task"); + task_id +} + fn lease_policy() -> HandLeasePolicy { let profile = SandboxProfile::new( CpuLimit::Unbounded, @@ -107,7 +266,7 @@ async fn activate_hydrated_workspace( session_id: SessionId, worker_id: &str, workspace: &SandboxWorkspace, -) { +) -> ActiveHandCapacityRequest { let leases = PostgresHandLeaseStore::new(pool.clone()); let attachment = HandLeaseWorkspaceAttachment::new( workspace.workspace_id, @@ -139,6 +298,21 @@ async fn activate_hydrated_workspace( ))), ); let workspaces = PostgresWorkspaceRepository::new(pool.clone()); + let capacity = PostgresWorkspaceCapacityRepository::new(pool.clone()); + let active_capacity = ActiveHandCapacityRequest { + tenant_id: workspace.tenant_id, + workspace_id: workspace.workspace_id, + provider_account_id: workspace.provider_account_id, + provider_account_generation: workspace.provider_account_generation, + provisioning_operation_id: provisioning.provisioning_operation_id, + hand_lease_generation: provisioning.generation, + expected_writer_epoch: workspace.writer_epoch, + expected_instance_generation: workspace.instance_generation, + }; + capacity + .reserve_active_hand(&active_capacity) + .await + .expect("reserve exact active hand before provider creation"); assert!( workspaces .activate_hydrated(ActivateHydratedWorkspaceRequest { @@ -149,6 +323,12 @@ async fn activate_hydrated_workspace( .await .expect("atomically activate exact hydrated lease and workspace") ); + assert!( + capacity + .commit_active_hand(&active_capacity) + .await + .expect("commit exact active hand after activation") + ); let active_lease = leases .get(workspace.tenant_id, session_id, worker_id, "local") .await @@ -156,6 +336,7 @@ async fn activate_hydrated_workspace( .expect("activated hand lease exists"); assert_eq!(active_lease.status, HandLeaseStatus::Active); assert_eq!(active_lease.attachment, provisioning.attachment); + active_capacity } fn create_request( @@ -179,7 +360,509 @@ fn create_request( } #[tokio::test] -#[ignore = "requires a fresh V58 compose Postgres via MOA_DATABASE_URL"] +#[ignore = "requires a fresh V60 compose Postgres via MOA_DATABASE_URL"] +async fn cancelling_task_without_owned_compute_gets_exact_absence_receipt_db() { + // Pins: a sandbox-capable task denied before provisioning still obtains a + // durable exact-attempt absence receipt, while a live lease cannot be hidden + // behind that no-workspace path. + let pool = PgPoolOptions::new() + .max_connections(4) + .connect(&database_url()) + .await + .expect("test Postgres should be reachable"); + let tenant_id = TenantId::new(); + let session_id = SessionId::new(); + seed_session(&pool, session_id, tenant_id).await; + let (run_id, _, _) = seed_cancelling_compensation(&pool, tenant_id, session_id).await; + let task_id = seed_cancelling_task(&pool, tenant_id, run_id, "never-provisioned").await; + let repository = PostgresWorkspaceRepository::new(pool.clone()); + let intent = AbsentTaskHandReleaseIntent { + receipt_id: uuid::Uuid::now_v7(), + tenant_id, + run_id, + task_id, + logical_generation: 1, + attempt_generation: 1, + verified_at: Utc::now(), + }; + let receipt = repository + .record_absent_task_execution_hand_release_receipt(intent) + .await + .expect("record exact no-owned-compute receipt"); + assert_eq!( + receipt.owner, + ExecutionHandReleaseOwner::Task { + task_id, + logical_generation: 1, + } + ); + assert_eq!(receipt.attempt_generation, 1); + assert_eq!( + ( + receipt.workspace_id, + receipt.hand_provisioning_operation_id, + receipt.checkpoint_id, + ), + (None, None, None), + "database-verified absence must not fabricate provider ownership" + ); + assert_eq!( + repository + .get_task_execution_hand_release_receipt(tenant_id, run_id, task_id, 1, 1) + .await + .expect("replay exact absence receipt"), + Some(receipt) + ); + + let live_task_id = seed_cancelling_task(&pool, tenant_id, run_id, "live-owner").await; + let live_scope = format!("execution:{run_id}:{live_task_id}"); + sqlx::query( + "INSERT INTO moa.hand_leases (\ + session_id, worker_id, tenant_id, provider, tier, handle, status, generation,\ + provisioning_operation_id, provisioning_deadline_at, created_at, updated_at\ + ) VALUES ($1, $2, $3, 'local', 'local', NULL, 'stale', 1,\ + gen_random_uuid(), now(), now(), now())", + ) + .bind(session_id) + .bind(&live_scope) + .bind(tenant_id) + .execute(&pool) + .await + .expect("seed live exact task owner"); + assert!( + repository + .record_absent_task_execution_hand_release_receipt(AbsentTaskHandReleaseIntent { + receipt_id: uuid::Uuid::now_v7(), + tenant_id, + run_id, + task_id: live_task_id, + logical_generation: 1, + attempt_generation: 1, + verified_at: Utc::now(), + }) + .await + .is_err(), + "a live exact owner must block the task absence proof" + ); + pool.close().await; +} + +#[tokio::test] +#[ignore = "requires Postgres for an isolated current-schema test database"] +async fn checkpointed_task_destroy_records_exact_release_receipt_db() { + // Pins: after the portable checkpoint CAS destroys an execution-task hand, + // the final receipt CAS recognizes the available checkpoint row and makes + // the exact task-attempt release replayable. + let test_db = moa_test_support::postgres::bootstrap_test_db() + .await + .expect("bootstrap isolated current-schema Postgres"); + let pool = test_db.store().pool().clone(); + let tenant_id = TenantId::new(); + let session_id = SessionId::new(); + let account_id = ProviderAccountId::new(); + let workspace_id = SandboxWorkspaceId::new(); + seed_session(&pool, session_id, tenant_id).await; + seed_account(&pool, account_id).await; + let (run_id, _, _) = seed_cancelling_compensation(&pool, tenant_id, session_id).await; + let task_id = seed_cancelling_task(&pool, tenant_id, run_id, "checkpointed-release").await; + let worker_id = format!("execution:{run_id}:{task_id}"); + let workspace_scope = SandboxWorkspaceScope::ExecutionTask { run_id, task_id }; + let workspaces = PostgresWorkspaceRepository::new(pool.clone()); + let operations = PostgresWorkspaceOperationRepository::new(pool.clone()); + workspaces + .create(&CreateWorkspaceRequest { + workspace_id, + tenant_id, + scope: workspace_scope, + provider: "local".to_string(), + provider_account_id: account_id, + provider_account_generation: 1, + durability_class: DurabilityClass::PortableFilesystem, + retention_deadline_at: None, + }) + .await + .expect("create task workspace"); + assert!( + workspaces + .transition(WorkspaceTransition { + tenant_id, + workspace_id, + from: SandboxWorkspaceState::Creating, + to: SandboxWorkspaceState::Ready, + writer_epoch: 0, + instance_generation: 0, + }) + .await + .expect("make task workspace ready") + ); + let restoring = workspaces + .claim_writer(WorkspaceWriterClaim { + tenant_id, + workspace_id, + expected_state: SandboxWorkspaceState::Ready, + expected_writer_epoch: 0, + expected_instance_generation: 0, + }) + .await + .expect("claim task workspace writer") + .expect("task workspace writer claim succeeds"); + activate_hydrated_workspace(&pool, session_id, &worker_id, &restoring).await; + let active = workspaces + .get(tenant_id, workspace_id) + .await + .expect("load active task workspace") + .expect("active task workspace exists"); + let leases = PostgresHandLeaseStore::new(pool.clone()); + let active_lease = leases + .get(tenant_id, session_id, &worker_id, "local") + .await + .expect("load active task lease") + .expect("active task lease exists"); + let receipt_id = uuid::Uuid::now_v7(); + let (persisted_receipt_id, claim_token, requested_at) = workspaces + .begin_task_execution_hand_release(TaskHandReleaseIntent { + receipt_id, + run_id, + task_id, + logical_generation: 1, + attempt_generation: 1, + deadline_at: Utc::now() + ChronoDuration::minutes(5), + recovery_claim_expires_at: Utc::now() + ChronoDuration::minutes(5), + workspace: &active, + lease: &active_lease, + }) + .await + .expect("persist task release intent before provider work"); + assert_eq!(persisted_receipt_id, receipt_id); + + for (from, to) in [ + ( + SandboxWorkspaceState::Active, + SandboxWorkspaceState::Quiescing, + ), + ( + SandboxWorkspaceState::Quiescing, + SandboxWorkspaceState::Committing, + ), + ] { + assert!( + workspaces + .transition(WorkspaceTransition { + tenant_id, + workspace_id, + from, + to, + writer_epoch: active.writer_epoch, + instance_generation: active.instance_generation, + }) + .await + .expect("enter task release checkpoint barrier") + ); + } + let operation_id = WorkspaceOperationId::new(); + let checkpoint_id = WorkspaceCheckpointId(operation_id.0); + let now = Utc::now(); + operations + .persist_intent(&WorkspaceOperationIntent { + operation_id, + tenant_id, + workspace_id, + provider_account_id: account_id, + provider_account_generation: 1, + kind: WorkspaceOperationKind::Commit, + request_hash: "sha256:task-release".to_string(), + expected_writer_epoch: active.writer_epoch, + expected_instance_generation: active.instance_generation, + expected_checkpoint_generation: active.checkpoint_generation, + deadline_at: now + ChronoDuration::minutes(5), + reconcile_not_before: now + ChronoDuration::minutes(6), + }) + .await + .expect("persist task release checkpoint intent"); + workspaces + .create_checkpoint(CreateCheckpointRequest { + checkpoint_id, + tenant_id, + workspace_id, + parent_checkpoint_id: None, + operation_id, + expected_writer_epoch: active.writer_epoch, + expected_instance_generation: active.instance_generation, + expected_checkpoint_generation: active.checkpoint_generation, + }) + .await + .expect("create task release checkpoint row") + .expect("task release checkpoint fences match"); + let active_binding = binding(&active); + let storage_operation = WorkspaceStorageOperation { + operation_id, + kind: WorkspaceOperationKind::Commit, + binding: active_binding.clone(), + deadline: now + ChronoDuration::minutes(5), + request_hash: "sha256:task-release".to_string(), + }; + PostgresWorkspaceCapacityRepository::new(pool.clone()) + .reserve_checkpoint_publication(&storage_operation, 0) + .await + .expect("reserve task release checkpoint before publication"); + let publication = WorkspaceCheckpointPublication { + revision: WorkspaceRevisionRef { + checkpoint_id, + generation: 1, + format_version: 1, + }, + storage: ProviderStorageRef { + provider_account_id: account_id, + provider_account_generation: 1, + kind: ProviderStorageKind::PortableCheckpoint, + resource_id: format!("object://{checkpoint_id}"), + workspace_locator: None, + }, + manifest_digest: "sha256:task-release-manifest".to_string(), + logical_bytes: 0, + }; + assert!( + workspaces + .publish_checkpoint_commit(PublishCheckpointCommitRequest { + binding: &active_binding, + operation_id, + publication: &publication, + post_commit_state: WorkspacePostCommitState::ComputeDestroyed, + lease: &active_lease, + }) + .await + .expect("atomically publish task release checkpoint and destroy lease") + ); + let receipt = ExecutionHandReleaseReceipt { + receipt_id, + tenant_id, + run_id, + owner: ExecutionHandReleaseOwner::Task { + task_id, + logical_generation: 1, + }, + attempt_generation: 1, + workspace_id: Some(workspace_id), + writer_epoch: Some(u64::try_from(active.writer_epoch).expect("valid writer epoch")), + instance_generation: Some( + u64::try_from(active.instance_generation).expect("valid instance generation"), + ), + hand_provisioning_operation_id: Some(active_lease.provisioning_operation_id), + hand_lease_generation: Some( + u64::try_from(active_lease.generation).expect("valid hand lease generation"), + ), + checkpoint_id: Some(checkpoint_id), + checkpoint_generation: Some(1), + checkpoint_manifest_digest: Some(publication.manifest_digest.clone()), + checkpoint_logical_bytes: Some(0), + requested_at, + released_at: Utc::now(), + }; + let finalized = workspaces + .record_task_execution_hand_release_receipt(&receipt, claim_token) + .await + .expect("available checkpoint must finalize the task release receipt"); + assert_eq!(finalized, receipt); + assert_eq!( + workspaces + .get_task_execution_hand_release_receipt(tenant_id, run_id, task_id, 1, 1) + .await + .expect("replay finalized task release receipt"), + Some(receipt) + ); +} + +#[tokio::test] +#[ignore = "requires a fresh V60 compose Postgres via MOA_DATABASE_URL"] +async fn compensation_release_recovers_persisted_destroyed_identity_after_deadline_db() { + // Pins: a crash after provider teardown but before receipt finalization reuses + // the pending receipt's exact lease identity after the provider-I/O deadline; + // deleting that exact destroyed row remains fail-closed. + let pool = PgPoolOptions::new() + .max_connections(4) + .connect(&database_url()) + .await + .expect("test Postgres should be reachable"); + let tenant_id = TenantId::new(); + let session_id = SessionId::new(); + seed_session(&pool, session_id, tenant_id).await; + let (run_id, _task_id, compensation_id) = + seed_cancelling_compensation(&pool, tenant_id, session_id).await; + let hand_scope = format!("execution_compensation:{run_id}:{compensation_id}"); + let provisioning_operation_id = HandProvisioningOperationId::new(); + let generation = 7_i64; + let lease_handle = LeaseHandle::new( + provisioning_operation_id, + HandHandle::local(PathBuf::from(format!("/tmp/{provisioning_operation_id}"))), + ); + sqlx::query( + "INSERT INTO moa.hand_leases (\ + session_id, worker_id, tenant_id, provider, tier, handle, status, generation,\ + provisioning_operation_id, provisioning_deadline_at, created_at, updated_at\ + ) VALUES ($1, $2, $3, 'local', 'local', $4, 'stale', $5, $6, now(), now(), now())", + ) + .bind(session_id) + .bind(&hand_scope) + .bind(tenant_id) + .bind(sqlx::types::Json(&lease_handle)) + .bind(generation) + .bind(provisioning_operation_id) + .execute(&pool) + .await + .expect("seed stale compensation lease"); + let leases = PostgresHandLeaseStore::new(pool.clone()); + let initial_lease = leases + .get(tenant_id, session_id, &hand_scope, "local") + .await + .expect("load exact compensation lease") + .expect("seeded compensation lease exists"); + let repository = PostgresWorkspaceRepository::new(pool.clone()); + let receipt_id = uuid::Uuid::now_v7(); + let deadline_at = Utc::now() + ChronoDuration::minutes(1); + repository + .begin_compensation_execution_hand_release(CompensationHandReleaseIntent { + receipt_id, + tenant_id, + session_id, + run_id, + compensation_id, + logical_generation: 1, + attempt_generation: 1, + hand_scope: &hand_scope, + lease: Some(&initial_lease), + deadline_at, + recovery_claim_expires_at: deadline_at, + }) + .await + .expect("persist exact compensation release intent"); + let destroy_claim = leases + .claim_for_destroy(tenant_id, &initial_lease, Duration::from_secs(30)) + .await + .expect("claim exact lease destroy") + .expect("stale lease is destroyable"); + assert!( + leases + .finalize_destroy(tenant_id, &initial_lease, destroy_claim) + .await + .expect("finalize exact lease destroy") + ); + sqlx::query( + "UPDATE moa.sandbox_execution_hand_release_receipts\ + SET requested_at = now() - interval '10 minutes',\ + deadline_at = now() - interval '5 minutes',\ + claim_expires_at = now() - interval '6 minutes'\ + WHERE receipt_id = $1", + ) + .bind(receipt_id) + .execute(&pool) + .await + .expect("advance pending release beyond its provider deadline"); + let claim = repository + .claim_pending_compensation_execution_hand_release( + tenant_id, + run_id, + compensation_id, + 1, + 1, + Utc::now() + ChronoDuration::minutes(5), + ) + .await + .expect("renew storage-only recovery claim") + .expect("expired pending release is reclaimable"); + assert_eq!(claim.receipt_id, receipt_id); + assert_eq!( + claim.hand_provisioning_operation_id, + Some(provisioning_operation_id) + ); + assert_eq!(claim.hand_lease_generation, Some(generation)); + let destroyed = leases + .get_exact_generation( + tenant_id, + session_id, + &hand_scope, + provisioning_operation_id, + generation, + ) + .await + .expect("load exact destroyed generation") + .expect("destroyed generation remains durable"); + assert_eq!(destroyed.status, HandLeaseStatus::Destroyed); + assert!(destroyed.handle.is_none()); + + sqlx::query( + "DELETE FROM moa.hand_leases WHERE tenant_id = $1 AND session_id = $2\ + AND worker_id = $3 AND provisioning_operation_id = $4 AND generation = $5", + ) + .bind(tenant_id) + .bind(session_id) + .bind(&hand_scope) + .bind(provisioning_operation_id) + .bind(generation) + .execute(&pool) + .await + .expect("simulate corrupt missing exact lease row"); + let release_receipt = ExecutionHandReleaseReceipt { + receipt_id, + tenant_id, + run_id, + owner: ExecutionHandReleaseOwner::Compensation { + compensation_id, + logical_generation: 1, + }, + attempt_generation: 1, + workspace_id: None, + writer_epoch: None, + instance_generation: None, + hand_provisioning_operation_id: Some(provisioning_operation_id), + hand_lease_generation: Some(u64::try_from(generation).expect("positive generation")), + checkpoint_id: None, + checkpoint_generation: None, + checkpoint_manifest_digest: None, + checkpoint_logical_bytes: None, + requested_at: claim.requested_at, + released_at: Utc::now(), + }; + assert!( + repository + .record_compensation_execution_hand_release_receipt( + &release_receipt, + session_id, + &hand_scope, + claim.claim_token, + ) + .await + .is_err(), + "missing exact destroyed lease evidence must not finalize absence" + ); + sqlx::query( + "INSERT INTO moa.hand_leases (\ + session_id, worker_id, tenant_id, provider, tier, handle, status, generation,\ + provisioning_operation_id, provisioning_deadline_at, created_at, updated_at\ + ) VALUES ($1, $2, $3, 'local', 'local', NULL, 'destroyed', $4, $5, now(), now(), now())", + ) + .bind(session_id) + .bind(&hand_scope) + .bind(tenant_id) + .bind(generation) + .bind(provisioning_operation_id) + .execute(&pool) + .await + .expect("restore exact destroyed evidence"); + let finalized = repository + .record_compensation_execution_hand_release_receipt( + &release_receipt, + session_id, + &hand_scope, + claim.claim_token, + ) + .await + .expect("finalize recovered exact receipt"); + assert_eq!(finalized, release_receipt); + pool.close().await; +} + +#[tokio::test] +#[ignore = "requires a fresh V60 compose Postgres via MOA_DATABASE_URL"] async fn workspace_writer_and_reconciliation_callbacks_are_generation_fenced_db() { // Pins: competing writers have one CAS winner, and changed inventory cannot // satisfy the two-separated-empty absence proof. @@ -242,7 +925,8 @@ async fn workspace_writer_and_reconciliation_callbacks_are_generation_fenced_db( (restoring.writer_epoch, restoring.instance_generation), (1, 1) ); - activate_hydrated_workspace(&pool, session_id, &worker_id, &restoring).await; + let active_capacity = + activate_hydrated_workspace(&pool, session_id, &worker_id, &restoring).await; let active = workspaces .get(tenant_id, workspace_id) .await @@ -251,6 +935,50 @@ async fn workspace_writer_and_reconciliation_callbacks_are_generation_fenced_db( assert_eq!(active.state, SandboxWorkspaceState::Active); assert_eq!((active.writer_epoch, active.instance_generation), (1, 1)); + let reaper_claim = uuid::Uuid::now_v7(); + sqlx::query( + r#" + UPDATE moa.hand_leases + SET status = 'reaping', reap_claim_token = $2, + reap_claim_expires_at = now() + interval '30 seconds' + WHERE session_id = $1 + "#, + ) + .bind(session_id) + .bind(reaper_claim) + .execute(&pool) + .await + .expect("establish exact durable reaper ownership"); + let capacity = PostgresWorkspaceCapacityRepository::new(pool.clone()); + let mut stale_capacity = active_capacity; + stale_capacity.hand_lease_generation += 1; + assert!( + !capacity + .release_active_hand_to_reaper(&stale_capacity, reaper_claim) + .await + .expect("stale generation is a fenced miss") + ); + assert!( + !capacity + .release_active_hand_to_reaper(&active_capacity, uuid::Uuid::now_v7()) + .await + .expect("wrong reaper token is a fenced miss") + ); + assert!( + capacity + .release_active_hand_to_reaper(&active_capacity, reaper_claim) + .await + .expect("exact live reaper claim releases active compute capacity") + ); + let active_capacity_state = sqlx::query_scalar::<_, String>( + "SELECT reservation_state FROM moa.sandbox_capacity_reservations WHERE hand_provisioning_operation_id = $1", + ) + .bind(active_capacity.provisioning_operation_id) + .fetch_one(&pool) + .await + .expect("load released active capacity row"); + assert_eq!(active_capacity_state, "released"); + assert!( !workspaces .transition(WorkspaceTransition { @@ -331,6 +1059,11 @@ async fn workspace_writer_and_reconciliation_callbacks_are_generation_fenced_db( AbsenceObservation::First ); + sqlx::query("DELETE FROM moa.sandbox_capacity_reservations WHERE workspace_id = $1") + .bind(workspace_id) + .execute(&pool) + .await + .expect("clean workspace capacity reservations"); sqlx::query("DELETE FROM moa.sandbox_workspace_operations WHERE operation_id = $1") .bind(operation_id) .execute(&pool) @@ -360,7 +1093,7 @@ async fn workspace_writer_and_reconciliation_callbacks_are_generation_fenced_db( } #[tokio::test] -#[ignore = "requires a fresh V58 compose Postgres via MOA_DATABASE_URL"] +#[ignore = "requires a fresh V60 compose Postgres via MOA_DATABASE_URL"] async fn checkpoint_metadata_is_created_before_bytes_and_remains_immutable_db() { // Pins: checkpoint bytes cannot exist as a published revision without an // earlier exact operation row, and the atomic production publication path @@ -406,7 +1139,8 @@ async fn checkpoint_metadata_is_created_before_bytes_and_remains_immutable_db() .expect("claim writer") .expect("writer claim succeeds"); assert_eq!(restoring.state, SandboxWorkspaceState::Restoring); - activate_hydrated_workspace(&pool, session_id, &worker_id, &restoring).await; + let _active_capacity = + activate_hydrated_workspace(&pool, session_id, &worker_id, &restoring).await; let active = workspaces .get(tenant_id, workspace_id) .await @@ -538,6 +1272,35 @@ async fn checkpoint_metadata_is_created_before_bytes_and_remains_immutable_db() manifest_digest: "sha256:manifest-one".to_string(), logical_bytes: 17, }; + let storage_operation = WorkspaceStorageOperation { + operation_id, + kind: WorkspaceOperationKind::Checkpoint, + binding: active_binding.clone(), + deadline: now + ChronoDuration::seconds(30), + request_hash: "sha256:checkpoint-one".to_string(), + }; + PostgresWorkspaceCapacityRepository::new(pool.clone()) + .reserve_checkpoint_publication(&storage_operation, publication.logical_bytes) + .await + .expect("reserve provider-independent checkpoint count and bytes before upload"); + // Pins: an upload/verification failure leaves only an indexed, bounded + // pending reservation that maintenance can reclaim at the operation deadline. + let pending_expiries = sqlx::query_scalar::<_, chrono::DateTime>( + r#" + SELECT expires_at + FROM moa.sandbox_capacity_reservations + WHERE tenant_id = $1 AND operation_id = $2 + AND resource_dimension IN ('checkpoints', 'logical_bytes') + AND reservation_state = 'pending' + ORDER BY resource_dimension + "#, + ) + .bind(tenant_id) + .bind(operation_id) + .fetch_all(&pool) + .await + .expect("load reclaimable pre-upload reservations"); + assert_eq!(pending_expiries, vec![storage_operation.deadline; 2]); assert!( workspaces .publish_workspace_checkpoint(PublishCheckpointCommitRequest { @@ -569,6 +1332,11 @@ async fn checkpoint_metadata_is_created_before_bytes_and_remains_immutable_db() .await .expect("detach checkpoint head for fixture cleanup"); + sqlx::query("DELETE FROM moa.sandbox_capacity_reservations WHERE workspace_id = $1") + .bind(workspace_id) + .execute(&pool) + .await + .expect("clean workspace, hand, and checkpoint capacity reservations"); sqlx::query("DELETE FROM moa.hand_leases WHERE session_id = $1") .bind(session_id) .execute(&pool) diff --git a/crates/moa-hands/tests/hands_db/sandbox_workspace/reconciliation_db.rs b/crates/moa-hands/tests/hands_db/sandbox_workspace/reconciliation_db.rs index ac906a06d..e8b3463d9 100644 --- a/crates/moa-hands/tests/hands_db/sandbox_workspace/reconciliation_db.rs +++ b/crates/moa-hands/tests/hands_db/sandbox_workspace/reconciliation_db.rs @@ -3,24 +3,175 @@ use moa_config::CheckpointRetentionConfig; use moa_core::types::{ identifiers::{ProviderAccountId, TenantId}, - sandbox_workspace::{ProviderInventoryResource, ProviderInventoryResourceKind}, + sandbox_workspace::{ + ProviderInventoryOwner, ProviderInventoryResource, ProviderInventoryResourceKind, + }, }; -use sqlx::Row; +use sqlx::{PgPool, Row}; +use uuid::Uuid; use super::sandbox_workspace_retention_db::{ create_workspace, maintenance_fixture, pools, seed_account, }; +async fn inventory_claim_generation(pool: &PgPool, account_id: ProviderAccountId) -> i64 { + let mut transaction = pool.begin().await.expect("begin maintenance claim read"); + sqlx::query("SET LOCAL ROLE moa_workspace_maintenance") + .execute(&mut *transaction) + .await + .expect("assume maintenance role for claim read"); + let generation = sqlx::query_scalar( + "SELECT claim_generation FROM moa.sandbox_provider_inventory_claims \ + WHERE provider_account_id = $1 AND provider_account_generation = 1", + ) + .bind(account_id) + .fetch_one(&mut *transaction) + .await + .expect("load inventory claim generation"); + transaction + .commit() + .await + .expect("commit maintenance claim read"); + generation +} + +async fn inventory_claim_row_version(pool: &PgPool, account_id: ProviderAccountId) -> String { + let mut transaction = pool.begin().await.expect("begin maintenance claim read"); + sqlx::query("SET LOCAL ROLE moa_workspace_maintenance") + .execute(&mut *transaction) + .await + .expect("assume maintenance role for claim read"); + let version = sqlx::query_scalar( + "SELECT xmin::text FROM moa.sandbox_provider_inventory_claims \ + WHERE provider_account_id = $1 AND provider_account_generation = 1", + ) + .bind(account_id) + .fetch_one(&mut *transaction) + .await + .expect("load inventory claim row version"); + transaction + .commit() + .await + .expect("commit maintenance claim read"); + version +} + +async fn expire_inventory_claim(pool: &PgPool, account_id: ProviderAccountId) { + let mut transaction = pool + .begin() + .await + .expect("begin maintenance claim mutation"); + sqlx::query("SET LOCAL ROLE moa_workspace_maintenance") + .execute(&mut *transaction) + .await + .expect("assume maintenance role for claim mutation"); + sqlx::query( + r#" + UPDATE moa.sandbox_provider_inventory_claims + SET claim_generation = claim_generation + 1, + claim_owner = $2, claim_token = $3, + claimed_at = now() - interval '2 minutes', + claim_expires_at = now() - interval '1 minute' + WHERE provider_account_id = $1 AND provider_account_generation = 1 + AND claim_token IS NULL AND last_succeeded_at IS NOT NULL + "#, + ) + .bind(account_id) + .bind(Uuid::now_v7()) + .bind(Uuid::now_v7()) + .execute(&mut *transaction) + .await + .expect("simulate crashed maintenance owner with expired claim"); + transaction + .commit() + .await + .expect("commit expired maintenance claim"); +} + +#[tokio::test] +#[ignore = "requires a fresh V60 database and distinct runtime/workspace-maintenance logins"] +async fn provider_inventory_claim_is_exclusive_and_recovers_after_owner_restart_db() { + // Pins: two maintenance replicas cannot scan the same provider-account + // generation concurrently, refreshing the queue does not rewrite an + // unchanged claimed row, and an expired owner claim is reclaimed under a + // strictly newer generation after restart. + let (runtime, maintenance) = pools().await; + let account_id = ProviderAccountId::new(); + seed_account(&runtime, account_id).await; + sqlx::query( + "UPDATE moa.sandbox_provider_accounts SET health = 'healthy' WHERE provider_account_id = $1", + ) + .bind(account_id) + .execute(&runtime) + .await + .expect("enable exact account inventory"); + let fixture = maintenance_fixture( + &runtime, + &maintenance, + account_id, + CheckpointRetentionConfig::default(), + ) + .await; + let (started, release) = fixture.storage.gate_next_inventory().await; + let first_coordinator = fixture.coordinator.clone(); + let first = tokio::spawn(async move { + first_coordinator + .reconcile_claimed_provider_inventory_once(1) + .await + }); + started + .await + .expect("first replica reaches provider only after durable claim"); + let claimed_row_version = inventory_claim_row_version(&maintenance, account_id).await; + let concurrent = fixture + .coordinator + .reconcile_claimed_provider_inventory_once(1) + .await + .expect("second replica sees no claimable account"); + assert_eq!(concurrent.accounts, 0); + assert_eq!( + inventory_claim_row_version(&maintenance, account_id).await, + claimed_row_version, + "refreshing the claim queue must not rewrite an unchanged provider row" + ); + release + .send(()) + .expect("release first inventory provider call"); + let completed = first + .await + .expect("first inventory task joins") + .expect("first claimed inventory succeeds"); + assert_eq!(completed.accounts, 1); + + let generation_after_success = inventory_claim_generation(&maintenance, account_id).await; + expire_inventory_claim(&maintenance, account_id).await; + let stale_generation = inventory_claim_generation(&maintenance, account_id).await; + assert!(stale_generation > generation_after_success); + let recovered = fixture + .coordinator + .reconcile_claimed_provider_inventory_once(1) + .await + .expect("new replica recovers expired inventory claim"); + assert_eq!(recovered.accounts, 1); + let recovered_generation = inventory_claim_generation(&maintenance, account_id).await; + assert!(recovered_generation > stale_generation); +} + #[tokio::test] #[ignore = "requires a fresh V58 database and distinct runtime/workspace-maintenance logins"] async fn provider_inventory_drift_is_quarantined_then_resolved_only_after_clean_inventory_db() { // Pins: provider-account inventory is compared by the production // coordinator, unknown resources become durable maintenance-only quarantine - // findings, and a later complete clean pass resolves them with audit proof. + // findings, durable workspace ownership is loaded only from the exact + // account generation being scanned, and a later complete clean pass + // resolves the findings with audit proof. let (runtime, maintenance) = pools().await; let tenant_id = TenantId::new(); let account_id = ProviderAccountId::new(); + let foreign_tenant_id = TenantId::new(); + let foreign_account_id = ProviderAccountId::new(); seed_account(&runtime, account_id).await; + seed_account(&runtime, foreign_account_id).await; sqlx::query( "UPDATE moa.sandbox_provider_accounts SET health = 'healthy' \ WHERE provider_account_id = $1", @@ -36,6 +187,8 @@ async fn provider_inventory_drift_is_quarantined_then_resolved_only_after_clean_ .await .expect("count the exact fleet-wide provider-account scan set"); let _workspace_id = create_workspace(&runtime, tenant_id, account_id).await; + let foreign_workspace_id = + create_workspace(&runtime, foreign_tenant_id, foreign_account_id).await; let fixture = maintenance_fixture( &runtime, &maintenance, @@ -45,30 +198,60 @@ async fn provider_inventory_drift_is_quarantined_then_resolved_only_after_clean_ .await; let resource_fingerprint = format!("sha256:unknown-{account_id}"); let evidence_digest = format!("sha256:evidence-{account_id}"); + let foreign_workspace_fingerprint = format!("sha256:foreign-{foreign_workspace_id}"); fixture .storage - .set_inventory(vec![ProviderInventoryResource { - kind: ProviderInventoryResourceKind::MutableFilesystem, - provider_reference: format!("unknown-volume-{account_id}"), - resource_fingerprint: resource_fingerprint.clone(), - evidence_digest: evidence_digest.clone(), - verified_owner: None, - }]) + .set_inventory(vec![ + ProviderInventoryResource { + kind: ProviderInventoryResourceKind::MutableFilesystem, + provider_reference: format!("unknown-volume-{account_id}"), + resource_fingerprint: resource_fingerprint.clone(), + evidence_digest: evidence_digest.clone(), + verified_owner: None, + }, + ProviderInventoryResource { + kind: ProviderInventoryResourceKind::MutableFilesystem, + provider_reference: format!("foreign-volume-{foreign_workspace_id}"), + resource_fingerprint: foreign_workspace_fingerprint.clone(), + evidence_digest: format!("sha256:foreign-evidence-{foreign_workspace_id}"), + verified_owner: Some(ProviderInventoryOwner { + tenant_id: foreign_tenant_id, + workspace_id: foreign_workspace_id, + provisioning_operation_id: None, + writer_epoch: Some(0), + instance_generation: Some(0), + }), + }, + ]) .await; let drift = fixture .coordinator - .reconcile_provider_inventory_once() + .reconcile_claimed_provider_inventory_once(32) .await .expect("reconcile provider inventory with one unknown resource"); assert_eq!( (drift.accounts, drift.resources, drift.unresolved_findings), ( u64::try_from(active_account_count).expect("active account count is nonnegative"), - 1, - 1, + 2, + 2, ) ); + let foreign_workspace_kind: String = sqlx::query_scalar( + "SELECT finding_kind FROM moa.sandbox_provider_inventory_findings \ + WHERE provider_account_id = $1 AND provider_account_generation = 1 \ + AND resource_fingerprint = $2", + ) + .bind(account_id) + .bind(&foreign_workspace_fingerprint) + .fetch_one(&runtime) + .await + .expect("load account-scoped foreign workspace finding"); + assert_eq!( + foreign_workspace_kind, "unknown", + "a workspace owned by another provider account is outside this account's durable inventory" + ); let quarantined = sqlx::query( "SELECT finding_kind, evidence_digest, quarantine_state, first_seen_at, last_seen_at, \ resolved_at, resolved_by, resolution_evidence_digest \ @@ -116,7 +299,7 @@ async fn provider_inventory_drift_is_quarantined_then_resolved_only_after_clean_ fixture.storage.set_inventory(Vec::new()).await; let clean = fixture .coordinator - .reconcile_provider_inventory_once() + .reconcile_claimed_provider_inventory_once(32) .await .expect("reconcile a complete clean provider inventory"); assert_eq!( diff --git a/crates/moa-hands/tests/hands_db/sandbox_workspace/retention_db.rs b/crates/moa-hands/tests/hands_db/sandbox_workspace/retention_db.rs index 816c251ec..ae2e9cb92 100644 --- a/crates/moa-hands/tests/hands_db/sandbox_workspace/retention_db.rs +++ b/crates/moa-hands/tests/hands_db/sandbox_workspace/retention_db.rs @@ -40,6 +40,7 @@ use object_store::memory::InMemory; use sqlx::{PgPool, Row, postgres::PgPoolOptions}; use tempfile::TempDir; use tokio::sync::Mutex; +use tokio::sync::oneshot; /// Provider registry key shared by the three maintenance DB behavior modules. pub(super) const SCRIPTED_PROVIDER: &str = "scripted-maintenance-db"; @@ -123,6 +124,8 @@ pub(super) struct ScriptedMaintenanceStorageProvider { account_id: ProviderAccountId, runtime_pool: PgPool, inventory: Mutex>, + inventory_started: Mutex>>, + inventory_release: Mutex>>, delete_observations: Mutex>, } @@ -133,6 +136,8 @@ impl ScriptedMaintenanceStorageProvider { account_id, runtime_pool, inventory: Mutex::new(Vec::new()), + inventory_started: Mutex::new(None), + inventory_release: Mutex::new(None), delete_observations: Mutex::new(Vec::new()), } } @@ -142,6 +147,15 @@ impl ScriptedMaintenanceStorageProvider { *self.inventory.lock().await = resources; } + /// Gates the next inventory call so a second maintenance replica can race its claim. + pub(super) async fn gate_next_inventory(&self) -> (oneshot::Receiver<()>, oneshot::Sender<()>) { + let (started_tx, started_rx) = oneshot::channel(); + let (release_tx, release_rx) = oneshot::channel(); + *self.inventory_started.lock().await = Some(started_tx); + *self.inventory_release.lock().await = Some(release_rx); + (started_rx, release_tx) + } + /// Returns the exact tenant-purge delete calls observed by the provider. pub(super) async fn delete_observations(&self) -> Vec { self.delete_observations.lock().await.clone() @@ -170,6 +184,14 @@ impl SandboxStorageProvider for ScriptedMaintenanceStorageProvider { "scripted inventory crossed its provider-account generation".to_string(), )); } + if let Some(started) = self.inventory_started.lock().await.take() { + let _ = started.send(()); + } + if let Some(release) = self.inventory_release.lock().await.take() { + release.await.map_err(|_| { + MoaError::StorageError("inventory claim gate disappeared".to_string()) + })?; + } Ok(ProviderAccountStorageInventory { provider_account_id, provider_account_generation, diff --git a/crates/moa-hands/tests/hands_db/sandbox_workspace/rls_db.rs b/crates/moa-hands/tests/hands_db/sandbox_workspace/rls_db.rs index cf049f527..bf349b767 100644 --- a/crates/moa-hands/tests/hands_db/sandbox_workspace/rls_db.rs +++ b/crates/moa-hands/tests/hands_db/sandbox_workspace/rls_db.rs @@ -663,15 +663,15 @@ async fn maintenance_reaper_crosses_tenants_without_exposing_a_foreground_bypass .expect("expire both tenant leases"); let tenant_a_rows = store - .list_session(tenants[0], sessions[0]) + .list_live_session_page(tenants[0], sessions[0], None) .await .expect("tenant A lists its own session"); let tenant_a_cross_rows = store - .list_session(tenants[0], sessions[1]) + .list_live_session_page(tenants[0], sessions[1], None) .await .expect("tenant A cross-session query is filtered"); - assert_eq!(tenant_a_rows.len(), 1); - assert!(tenant_a_cross_rows.is_empty()); + assert_eq!(tenant_a_rows.leases.len(), 1); + assert!(tenant_a_cross_rows.leases.is_empty()); let claims = PostgresExpiredHandLeaseClaims::new(pool.clone()) .claim_expired(64, Duration::from_secs(300)) diff --git a/crates/moa-hands/tests/hands_offline.rs b/crates/moa-hands/tests/hands_offline.rs index a4a970030..52161a685 100644 --- a/crates/moa-hands/tests/hands_offline.rs +++ b/crates/moa-hands/tests/hands_offline.rs @@ -6,6 +6,8 @@ mod call_origin_offline; mod connector_router; #[path = "hands_offline/local_tools_offline.rs"] mod local_tools_offline; +#[path = "hands_offline/maintenance_provider_inventory_offline.rs"] +mod maintenance_provider_inventory_offline; #[path = "hands_offline/mcp_router.rs"] mod mcp_router; #[path = "hands_offline/provider_credentials_offline.rs"] diff --git a/crates/moa-hands/tests/hands_offline/maintenance_provider_inventory_offline.rs b/crates/moa-hands/tests/hands_offline/maintenance_provider_inventory_offline.rs new file mode 100644 index 000000000..6f22a1aae --- /dev/null +++ b/crates/moa-hands/tests/hands_offline/maintenance_provider_inventory_offline.rs @@ -0,0 +1,333 @@ +use std::os::unix::fs::{MetadataExt as _, PermissionsExt as _}; +use std::sync::Arc; + +use async_trait::async_trait; +use moa_config::{ + CloudHandProviderAccountConfig, CloudHandProviderKind, CloudHandsConfig, + DaytonaStorageAccountConfig, DaytonaStorageConfig, LocalHandProviderAccountConfig, MoaConfig, + ProviderSecretFileSelector, SandboxWorkspaceMode, SecurityProfile, +}; +use moa_core::error::MoaError; +use moa_crypto::{ + DataKeyDecryptRequest, GeneratedDataKey, KeyHandle, KeyManagementProvider, LocalKmsProvider, + PlaintextDek, +}; +use moa_hands::{ + SandboxProviderInventory, + core::sandbox_workspace::checkpoint::{ + archive::ArchiveLimits, + store::{CheckpointObjectStore, ObservedCheckpointBucketVersioning}, + }, +}; +use object_store::memory::InMemory; +use sqlx::postgres::PgPoolOptions; +use tempfile::{TempDir, tempdir}; +use uuid::Uuid; + +struct DurableTestKms(LocalKmsProvider); + +impl DurableTestKms { + fn new() -> Self { + Self(LocalKmsProvider::new()) + } +} + +#[async_trait] +impl KeyManagementProvider for DurableTestKms { + async fn generate_data_keys( + &self, + contexts: &[moa_crypto::EncryptionContext], + ) -> std::result::Result, moa_crypto::Error> { + self.0.generate_data_keys(contexts).await + } + + async fn decrypt_data_keys( + &self, + requests: &[DataKeyDecryptRequest], + ) -> std::result::Result, moa_crypto::Error> { + self.0.decrypt_data_keys(requests).await + } + + async fn destroy_key(&self, handle: &KeyHandle) -> std::result::Result<(), moa_crypto::Error> { + self.0.destroy_key(handle).await + } + + async fn destroy_subject_key( + &self, + tenant_id: Uuid, + subject_id: Uuid, + ) -> std::result::Result<(), moa_crypto::Error> { + self.0.destroy_subject_key(tenant_id, subject_id).await + } + + fn is_durable(&self) -> bool { + true + } +} + +fn lazy_pool() -> sqlx::PgPool { + PgPoolOptions::new() + .connect_lazy("postgresql://moa:moa@127.0.0.1:1/moa") + .expect("offline provider construction should accept a lazy Postgres pool") +} + +fn checkpoint_store(kms: Arc) -> Arc { + Arc::new( + CheckpointObjectStore::new( + Arc::new(InMemory::new()), + kms, + format!( + "maintenance-provider-inventory/{}/checkpoints", + Uuid::new_v4() + ), + ArchiveLimits::default(), + ObservedCheckpointBucketVersioning::Unversioned, + ) + .expect("offline checkpoint store should construct"), + ) +} + +fn credential_account( + provider: CloudHandProviderKind, + credential_dir: &TempDir, +) -> CloudHandProviderAccountConfig { + let provider_name = provider.as_str(); + let path = credential_dir.path().join(provider_name); + std::fs::write( + &path, + format!("MOA_TEST_{}_KEY", provider_name.to_uppercase()), + ) + .expect("write provider credential"); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o400)) + .expect("restrict provider credential permissions"); + let owner_uid = std::fs::metadata(&path) + .expect("read provider credential metadata") + .uid(); + CloudHandProviderAccountConfig { + provider_account_id: moa_core::types::identifiers::ProviderAccountId::new(), + generation: 1, + provider, + isolation_cell: format!("{provider_name}-offline"), + api_origin: match provider { + CloudHandProviderKind::Daytona => "https://api.daytona.io".to_string(), + CloudHandProviderKind::E2b => "https://api.e2b.dev".to_string(), + }, + toolbox_origin: (provider == CloudHandProviderKind::Daytona) + .then(|| "https://proxy.app.daytona.io".to_string()), + sandbox_domain: (provider == CloudHandProviderKind::E2b).then(|| "e2b.app".to_string()), + default_runtime: Some("base".to_string()), + project_fingerprint: Some(format!("sha256:{provider_name}-offline")), + credential: ProviderSecretFileSelector { path, owner_uid }, + } +} + +fn provider_names(inventory: &SandboxProviderInventory) -> (Vec, Vec) { + let hands = inventory + .hand_providers() + .iter() + .map(|provider| provider.provider_name().to_string()) + .collect(); + let storage = inventory + .storage_providers() + .iter() + .map(|provider| provider.storage_provider_name().to_string()) + .collect(); + (hands, storage) +} + +fn config_error_message(error: MoaError) -> String { + match error { + MoaError::ConfigError(message) => message, + other => panic!("expected a configuration error, got {other}"), + } +} + +#[tokio::test] +async fn cloud_maintenance_inventory_does_not_require_tool_policy_owners_offline() { + // Pins: a cloud maintenance process constructs provider inventory without + // inheriting ToolRouter's action-policy/session/connector dependencies. + let credentials = tempdir().expect("credential tempdir"); + let mut config = MoaConfig { + security_profile: SecurityProfile::Cloud, + ..MoaConfig::default() + }; + config.sandbox_workspaces.mode = SandboxWorkspaceMode::Maintenance; + config.cloud.hands = Some(CloudHandsConfig { + default_provider: Some("e2b".to_string()), + provider_accounts: vec![credential_account(CloudHandProviderKind::E2b, &credentials)], + ..CloudHandsConfig::default() + }); + let kms: Arc = Arc::new(DurableTestKms::new()); + + let inventory = SandboxProviderInventory::for_maintenance( + &config, + &lazy_pool(), + checkpoint_store(Arc::clone(&kms)), + kms, + ) + .await + .expect("cloud maintenance inventory should not require an action-policy owner"); + + assert_eq!( + provider_names(&inventory), + (vec!["e2b".to_string()], vec!["e2b".to_string()]) + ); +} + +#[tokio::test] +async fn maintenance_inventory_constructs_each_configured_provider_kind_once_offline() { + // Pins: maintenance follows configured durable account kinds, not the + // default/fallback tool route, and returns stable unique registries. + let credentials = tempdir().expect("credential tempdir"); + let local_root = tempdir().expect("local sandbox tempdir"); + let daytona = credential_account(CloudHandProviderKind::Daytona, &credentials); + let daytona_account_id = daytona.provider_account_id; + let e2b = credential_account(CloudHandProviderKind::E2b, &credentials); + let mut config = MoaConfig::default(); + config.sandbox_workspaces.mode = SandboxWorkspaceMode::Maintenance; + config.local.docker_enabled = false; + config.local.sandbox_dir = local_root.path().display().to_string(); + config.local.provider_account = Some(LocalHandProviderAccountConfig { + provider_account_id: moa_core::types::identifiers::ProviderAccountId::new(), + generation: 1, + isolation_cell: "local-offline".to_string(), + }); + config.cloud.hands = Some(CloudHandsConfig { + default_provider: Some("e2b".to_string()), + provider_accounts: vec![daytona, e2b], + ..CloudHandsConfig::default() + }); + config.cloud.daytona_storage = DaytonaStorageConfig { + accounts: vec![DaytonaStorageAccountConfig { + provider_account_id: daytona_account_id, + security_class: "tenant-isolated".to_string(), + volume_ceiling: 100, + admission_headroom: 1, + }], + consistency_window_seconds: 1, + }; + let kms: Arc = Arc::new(DurableTestKms::new()); + + let inventory = SandboxProviderInventory::for_maintenance( + &config, + &lazy_pool(), + checkpoint_store(Arc::clone(&kms)), + kms, + ) + .await + .expect("all configured maintenance providers should construct"); + + let expected = vec![ + "daytona".to_string(), + "e2b".to_string(), + "local".to_string(), + ]; + assert_eq!(provider_names(&inventory), (expected.clone(), expected)); +} + +#[tokio::test] +async fn cloud_maintenance_inventory_validates_credentials_before_registration_offline() { + // Pins: maintenance cannot register a cloud provider whose configured + // credential file is absent, even though construction performs no API call. + let credentials = tempdir().expect("credential tempdir"); + let account = credential_account(CloudHandProviderKind::E2b, &credentials); + std::fs::remove_file(&account.credential.path).expect("remove provider credential"); + let mut config = MoaConfig::default(); + config.sandbox_workspaces.mode = SandboxWorkspaceMode::Maintenance; + config.cloud.hands = Some(CloudHandsConfig { + provider_accounts: vec![account], + ..CloudHandsConfig::default() + }); + let kms: Arc = Arc::new(DurableTestKms::new()); + + let result = SandboxProviderInventory::for_maintenance( + &config, + &lazy_pool(), + checkpoint_store(Arc::clone(&kms)), + kms, + ) + .await; + let error = match result { + Ok(_) => panic!("missing cloud credentials must fail provider registration"), + Err(error) => error, + }; + + assert_eq!(config_error_message(error), "credential file is missing"); +} + +#[tokio::test] +async fn maintenance_inventory_rejects_ephemeral_kms_before_provider_side_effects_offline() { + // Pins: every maintenance inventory has durable checkpoint authority even + // when the selected provider itself is local and makes no cloud API call. + let parent = tempdir().expect("local sandbox parent tempdir"); + let sandbox_root = parent.path().join("must-not-exist"); + let mut config = MoaConfig::default(); + config.sandbox_workspaces.mode = SandboxWorkspaceMode::Maintenance; + config.local.docker_enabled = false; + config.local.sandbox_dir = sandbox_root.display().to_string(); + config.local.provider_account = Some(LocalHandProviderAccountConfig { + provider_account_id: moa_core::types::identifiers::ProviderAccountId::new(), + generation: 1, + isolation_cell: "ephemeral-kms-local".to_string(), + }); + let ephemeral_kms: Arc = Arc::new(LocalKmsProvider::new()); + + let result = SandboxProviderInventory::for_maintenance( + &config, + &lazy_pool(), + checkpoint_store(Arc::clone(&ephemeral_kms)), + ephemeral_kms, + ) + .await; + let error = match result { + Ok(_) => panic!("ephemeral KMS must fail maintenance provider construction"), + Err(error) => error, + }; + + assert_eq!( + config_error_message(error), + "sandbox provider maintenance inventory requires durable KMS authority" + ); + assert!( + !sandbox_root.exists(), + "KMS validation must run before local provider side effects" + ); +} + +#[tokio::test] +async fn disabled_mode_rejects_before_provider_side_effects_offline() { + // Pins: disabled mode remains dark and does not create the configured local + // sandbox directory while trying to assemble maintenance providers. + let parent = tempdir().expect("local sandbox parent tempdir"); + let sandbox_root = parent.path().join("must-not-exist"); + let mut config = MoaConfig::default(); + config.local.docker_enabled = false; + config.local.sandbox_dir = sandbox_root.display().to_string(); + config.local.provider_account = Some(LocalHandProviderAccountConfig { + provider_account_id: moa_core::types::identifiers::ProviderAccountId::new(), + generation: 1, + isolation_cell: "disabled-local".to_string(), + }); + let kms: Arc = Arc::new(DurableTestKms::new()); + + let result = SandboxProviderInventory::for_maintenance( + &config, + &lazy_pool(), + checkpoint_store(Arc::clone(&kms)), + kms, + ) + .await; + let error = match result { + Ok(_) => panic!("disabled mode must reject maintenance provider construction"), + Err(error) => error, + }; + + assert_eq!( + config_error_message(error), + "sandbox provider maintenance inventory requires maintenance or admit mode" + ); + assert!( + !sandbox_root.exists(), + "disabled construction must not create a local sandbox directory" + ); +} diff --git a/crates/moa-hands/tests/sandbox_workspace_docker.rs b/crates/moa-hands/tests/sandbox_workspace_docker.rs index 55005b370..835b1b011 100644 --- a/crates/moa-hands/tests/sandbox_workspace_docker.rs +++ b/crates/moa-hands/tests/sandbox_workspace_docker.rs @@ -357,6 +357,7 @@ async fn docker_compute_replacement_restores_committed_workspace_from_rustfs() { operation: commit_operation, hand: first.clone(), parent_revision: binding.current_revision.clone(), + release_compute: false, }) .await .expect("commit encrypted portable checkpoint"); diff --git a/crates/moa-migrations/migration-ownership.toml b/crates/moa-migrations/migration-ownership.toml index 48d78620c..735ed25ec 100644 --- a/crates/moa-migrations/migration-ownership.toml +++ b/crates/moa-migrations/migration-ownership.toml @@ -459,6 +459,102 @@ schema = "moa" owner = "moa-execution" readers = ["moa-orchestrator", "moa-edge", "moa-experiments", "moa-analytics"] +[[table]] +name = "execution_node_state" +schema = "moa" +owner = "moa-execution" +readers = ["moa-orchestrator"] + +[[table]] +name = "execution_completion_scan" +schema = "moa" +owner = "moa-execution" +readers = ["moa-orchestrator"] + +[[table]] +name = "execution_amendment_receipt" +schema = "moa" +owner = "moa-execution" +readers = ["moa-orchestrator"] + +[[table]] +name = "execution_replan_stop_intent" +schema = "moa" +owner = "moa-execution" +readers = ["moa-orchestrator"] + +[[table]] +name = "execution_schedule" +schema = "moa" +owner = "moa-execution" +readers = ["moa-orchestrator", "moa-edge"] + +[[table]] +name = "execution_external_job" +schema = "moa" +owner = "moa-execution" +readers = ["moa-orchestrator", "moa-edge"] + +[[table]] +name = "execution_external_job_callback_receipt" +schema = "moa" +owner = "moa-execution" +readers = ["moa-orchestrator"] + +[[table]] +name = "execution_trigger" +schema = "moa" +owner = "moa-execution" +readers = ["moa-orchestrator"] + +[[table]] +name = "execution_dispatch_outbox" +schema = "moa" +owner = "moa-execution" +readers = ["moa-orchestrator"] + +[[table]] +name = "execution_task_checkpoint" +schema = "moa" +owner = "moa-execution" +readers = ["moa-orchestrator", "moa-hands"] + +[[table]] +name = "execution_capacity_bucket" +schema = "moa" +owner = "moa-execution" +readers = ["moa-orchestrator"] + +[[table]] +name = "execution_tenant_dispatch_state" +schema = "moa" +owner = "moa-execution" +readers = ["moa-orchestrator"] + +[[table]] +name = "execution_capacity_reservation" +schema = "moa" +owner = "moa-execution" +readers = ["moa-orchestrator"] + +[[table]] +name = "execution_terminal_archive" +schema = "moa" +owner = "moa-execution" +readers = ["moa-orchestrator", "moa-analytics"] + +[[table]] +name = "execution_terminal_archive_segment" +schema = "moa" +owner = "moa-execution" +readers = ["moa-orchestrator", "moa-analytics"] + +[[table]] +name = "execution_maintenance_checkpoint" +schema = "moa" +owner = "moa-execution" +readers = ["moa-orchestrator"] + [[table]] name = "execution_template_admission" schema = "moa" @@ -538,6 +634,18 @@ schema = "moa" owner = "moa-hands" readers = ["moa-orchestrator"] +[[table]] +name = "sandbox_provider_inventory_claims" +schema = "moa" +owner = "moa-hands" +readers = ["moa-orchestrator"] + +[[table]] +name = "sandbox_execution_hand_release_receipts" +schema = "moa" +owner = "moa-hands" +readers = ["moa-orchestrator"] + [[table]] name = "tenant_sandbox_policy" schema = "moa" diff --git a/crates/moa-migrations/migrations/postgres/V000059__long_horizon_execution.sql b/crates/moa-migrations/migrations/postgres/V000059__long_horizon_execution.sql new file mode 100644 index 000000000..5bb25760e --- /dev/null +++ b/crates/moa-migrations/migrations/postgres/V000059__long_horizon_execution.sql @@ -0,0 +1,3071 @@ +-- Bounded activation persistence for execution runs that can remain durable for +-- days or weeks without retaining a live handler, task lease, or sandbox. +-- +-- This is a deliberate hard break. Legacy nonterminal runs are tied to the +-- lifetime-spanning ExecutionRun/ExecutionTask workflow protocol and cannot be +-- reinterpreted safely as bounded activations. + +DO $long_horizon_cutover$ +DECLARE + nonterminal_count BIGINT; +BEGIN + SELECT count(*) + INTO nonterminal_count + FROM moa.execution_run + WHERE status NOT IN ( + 'completed', 'partial', 'blocked', 'unsupported', 'failed', 'cancelled' + ); + + IF nonterminal_count <> 0 THEN + RAISE EXCEPTION + 'cannot install long-horizon execution while % legacy execution run(s) are nonterminal; terminalize or cancel them and deliberately reset the old Restate execution journals before retrying', + nonterminal_count + USING ERRCODE = 'check_violation'; + END IF; +END +$long_horizon_cutover$; + +CREATE TABLE moa.execution_maintenance_checkpoint ( + job_kind TEXT PRIMARY KEY CHECK (btrim(job_kind) <> ''), + generation BIGINT NOT NULL DEFAULT 0 CHECK (generation >= 0), + last_started_at TIMESTAMPTZ, + last_succeeded_at TIMESTAMPTZ, + last_failure_at TIMESTAMPTZ, + next_run_at TIMESTAMPTZ, + scheduled_generation BIGINT CHECK (scheduled_generation >= 1), + claim_owner TEXT, + claimed_generation BIGINT CHECK (claimed_generation >= 1), + claim_expires_at TIMESTAMPTZ, + last_error TEXT CHECK ( + last_error IS NULL OR octet_length(last_error) BETWEEN 1 AND 4096 + ), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT execution_maintenance_checkpoint_failure_pair_check CHECK ( + (last_failure_at IS NULL) = (last_error IS NULL) + ), + CONSTRAINT execution_maintenance_checkpoint_time_order_check CHECK ( + (last_succeeded_at IS NULL + OR last_started_at IS NOT NULL) + AND + (last_failure_at IS NULL + OR last_started_at IS NOT NULL) + ), + CONSTRAINT execution_maintenance_checkpoint_schedule_pair_check CHECK ( + (next_run_at IS NULL) = (scheduled_generation IS NULL) + AND (scheduled_generation IS NULL OR scheduled_generation <= generation + 1) + ), + CONSTRAINT execution_maintenance_checkpoint_claim_shape_check CHECK ( + (claim_owner IS NULL + AND claimed_generation IS NULL + AND claim_expires_at IS NULL) + OR + (claim_owner IS NOT NULL + AND btrim(claim_owner) <> '' + AND claimed_generation IS NOT NULL + AND claim_expires_at IS NOT NULL + AND claimed_generation = scheduled_generation) + ) +); + +CREATE INDEX execution_maintenance_checkpoint_due_idx + ON moa.execution_maintenance_checkpoint (next_run_at, job_kind) + WHERE next_run_at IS NOT NULL; + +REVOKE ALL ON TABLE moa.execution_maintenance_checkpoint FROM PUBLIC; +GRANT SELECT, INSERT, UPDATE ON TABLE moa.execution_maintenance_checkpoint TO moa_app; + +CREATE OR REPLACE FUNCTION moa.execution_admitted_identity_is_valid( + candidate JSONB, + expected_tenant_id UUID +) RETURNS BOOLEAN +LANGUAGE sql +IMMUTABLE +AS $$ + SELECT moa.execution_json_object_has_exact_keys( + candidate, + ARRAY[ + 'identity_type', 'id', 'tenant_id', 'api_key_id', + 'acting_on_behalf_of' + ] + ) + AND candidate ->> 'identity_type' IN ('operator', 'contact', 'agent', 'service') + AND candidate ->> 'id' ~ + '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + AND candidate ->> 'tenant_id' = expected_tenant_id::TEXT + AND ( + candidate -> 'api_key_id' = 'null'::JSONB + OR candidate ->> 'api_key_id' ~ + '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + ) + AND ( + candidate -> 'acting_on_behalf_of' = 'null'::JSONB + OR candidate ->> 'acting_on_behalf_of' ~ + '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + ) +$$; + +CREATE OR REPLACE FUNCTION moa.execution_schedule_origin_is_valid( + candidate JSONB, + expected_tenant_id UUID +) RETURNS BOOLEAN +LANGUAGE sql +IMMUTABLE +AS $$ + SELECT moa.execution_json_object_has_exact_keys( + candidate, ARRAY['request_uid', 'created_by', 'source'] + ) + AND candidate ->> 'request_uid' ~ + '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + AND moa.execution_admitted_identity_is_valid( + candidate -> 'created_by', expected_tenant_id + ) + AND CASE candidate #>> '{source,kind}' + WHEN 'tenant_api' THEN candidate -> 'source' = '{"kind":"tenant_api"}'::JSONB + WHEN 'session' THEN + moa.execution_json_object_has_exact_keys( + candidate -> 'source', + ARRAY['kind', 'session_id', 'originating_user_sequence_num'] + ) + AND candidate #>> '{source,session_id}' ~ + '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + AND candidate #>> '{source,originating_user_sequence_num}' ~ '^[0-9]+$' + ELSE FALSE + END +$$; + +CREATE OR REPLACE FUNCTION moa.execution_temporal_target_is_valid(candidate JSONB) +RETURNS BOOLEAN +LANGUAGE plpgsql +IMMUTABLE +AS $$ +BEGIN + RETURN CASE candidate ->> 'kind' + WHEN 'at' THEN + moa.execution_json_object_has_exact_keys(candidate, ARRAY['kind', 'at']) + AND jsonb_typeof(candidate -> 'at') = 'string' + AND (candidate ->> 'at')::TIMESTAMPTZ IS NOT NULL + WHEN 'after' THEN + moa.execution_json_object_has_exact_keys( + candidate, ARRAY['kind', 'delay_seconds'] + ) + AND jsonb_typeof(candidate -> 'delay_seconds') = 'number' + AND (candidate ->> 'delay_seconds') ~ '^[1-9][0-9]*$' + ELSE FALSE + END; +EXCEPTION + WHEN OTHERS THEN RETURN FALSE; +END; +$$; + +CREATE OR REPLACE FUNCTION moa.execution_wait_expiry_action_is_valid(candidate JSONB) +RETURNS BOOLEAN +LANGUAGE sql +IMMUTABLE +AS $$ + SELECT CASE candidate ->> 'kind' + WHEN 'fail_task' THEN + candidate = '{"kind":"fail_task"}'::JSONB + WHEN 'fail_run' THEN + candidate = '{"kind":"fail_run"}'::JSONB + WHEN 'continue_with' THEN + moa.execution_json_object_has_exact_keys(candidate, ARRAY['kind', 'output']) + ELSE FALSE + END +$$; + +CREATE OR REPLACE FUNCTION moa.execution_wait_policy_is_valid(candidate JSONB) +RETURNS BOOLEAN +LANGUAGE sql +IMMUTABLE +AS $$ + SELECT moa.execution_json_object_has_exact_keys( + candidate, ARRAY['expiry', 'on_expiry'] + ) + AND moa.execution_temporal_target_is_valid(candidate -> 'expiry') + AND moa.execution_wait_expiry_action_is_valid(candidate -> 'on_expiry') +$$; + +-- The old four-key definition remains valid only so retained terminal audit +-- rows can continue to satisfy their original check constraint. Every new run +-- and serving skill template is required to use the current five-key shape. +CREATE OR REPLACE FUNCTION moa.execution_plan_definition_is_current(candidate JSONB) +RETURNS BOOLEAN +LANGUAGE plpgsql +IMMUTABLE +AS $$ +DECLARE + node JSONB; + operation JSONB; +BEGIN + IF NOT moa.execution_json_object_has_exact_keys( + candidate, + ARRAY[ + 'cancel_policy', 'input_schema', 'output_schema', 'input_wait_policy', + 'nodes' + ] + ) + OR candidate ->> 'cancel_policy' NOT IN ( + 'retain_effects', 'compensate_committed' + ) + OR NOT moa.execution_wait_policy_is_valid(candidate -> 'input_wait_policy') + OR jsonb_typeof(candidate -> 'nodes') <> 'array' THEN + RETURN FALSE; + END IF; + FOR node IN SELECT value FROM jsonb_array_elements(candidate -> 'nodes') LOOP + IF NOT moa.execution_json_object_has_exact_keys( + node, + ARRAY[ + 'id', 'requirement_ids', 'depends_on', 'when', 'input', + 'output_schema', 'operation', 'compensation', 'retry', 'budget' + ] + ) THEN + RETURN FALSE; + END IF; + operation := node -> 'operation'; + IF operation ->> 'kind' IN ('review', 'wait_signal') + AND NOT moa.execution_wait_policy_is_valid(operation -> 'wait_policy') THEN + RETURN FALSE; + END IF; + IF operation ->> 'kind' = 'wait_until' + AND NOT ( + moa.execution_json_object_has_exact_keys( + operation, ARRAY['kind', 'wake', 'result'] + ) + AND moa.execution_temporal_target_is_valid(operation -> 'wake') + ) THEN + RETURN FALSE; + END IF; + END LOOP; + RETURN TRUE; +EXCEPTION + WHEN OTHERS THEN RETURN FALSE; +END; +$$; + +CREATE OR REPLACE FUNCTION moa.execution_plan_snapshot_is_current(candidate JSONB) +RETURNS BOOLEAN +LANGUAGE sql +IMMUTABLE +AS $$ + SELECT moa.execution_json_object_has_exact_keys( + candidate, + ARRAY['definition', 'plan_hash', 'catalog_hash', 'estimate', 'report'] + ) + AND moa.execution_plan_definition_is_current(candidate -> 'definition') + AND jsonb_typeof(candidate -> 'plan_hash') = 'string' + AND candidate ->> 'plan_hash' ~ '^[0-9a-f]{64}$' +$$; + +-- V55 bound both run snapshots to the old four-key plan validator. Replace +-- those constraints at the cutover boundary so every nonterminal/current run +-- uses the five-key long-horizon contract. Pre-cutover terminal rows remain +-- immutable audit evidence and are the only permitted legacy shape. +ALTER TABLE moa.execution_run + DROP CONSTRAINT execution_run_initial_plan_check, + DROP CONSTRAINT execution_run_active_plan_check, + ADD CONSTRAINT execution_run_initial_plan_check CHECK ( + status IN ( + 'completed', 'partial', 'blocked', 'unsupported', + 'failed', 'cancelled' + ) + OR moa.execution_plan_snapshot_is_current(initial_plan) + ), + ADD CONSTRAINT execution_run_active_plan_check CHECK ( + status IN ( + 'completed', 'partial', 'blocked', 'unsupported', + 'failed', 'cancelled' + ) + OR moa.execution_plan_snapshot_is_current(active_plan) + ); + +CREATE OR REPLACE FUNCTION moa.skill_execution_template_is_valid(candidate JSONB) +RETURNS BOOLEAN +LANGUAGE plpgsql +IMMUTABLE +AS $$ +DECLARE + plan JSONB; +BEGIN + IF candidate #>> '{definition,type}' IS DISTINCT FROM 'skill' + OR candidate #> '{definition,spec,execution_plan}' IS NULL THEN + RETURN TRUE; + END IF; + plan := candidate #> '{definition,spec,execution_plan,plan}'; + RETURN moa.execution_plan_definition_is_current(plan); +EXCEPTION + WHEN OTHERS THEN RETURN FALSE; +END; +$$; + +-- Existing rows are terminal by the cutover precondition. Initialize them as +-- inactive while retaining all immutable plans, outcomes, and audit evidence. +ALTER TABLE moa.execution_run + ADD COLUMN admitted_identity JSONB, + ADD COLUMN controller_generation BIGINT NOT NULL DEFAULT 1 + CHECK (controller_generation >= 1), + ADD COLUMN activation_state TEXT NOT NULL DEFAULT 'queued' + CHECK (activation_state IN ('idle', 'queued', 'advancing', 'paused', 'terminal')), + ADD COLUMN next_wake_at TIMESTAMPTZ, + ADD COLUMN waiting_since TIMESTAMPTZ, + ADD COLUMN last_progress_at TIMESTAMPTZ NOT NULL DEFAULT now(), + ADD COLUMN pause_requested_at TIMESTAMPTZ, + ADD COLUMN paused_at TIMESTAMPTZ, + ADD COLUMN ready_task_count BIGINT NOT NULL DEFAULT 0 + CHECK (ready_task_count >= 0), + ADD COLUMN active_task_count BIGINT NOT NULL DEFAULT 0 + CHECK (active_task_count >= 0), + ADD COLUMN waiting_task_count BIGINT NOT NULL DEFAULT 0 + CHECK (waiting_task_count >= 0), + ADD COLUMN waiting_input_task_count BIGINT NOT NULL DEFAULT 0 + CHECK (waiting_input_task_count >= 0), + ADD COLUMN waiting_input_user_task_count BIGINT NOT NULL DEFAULT 0 + CHECK (waiting_input_user_task_count >= 0), + ADD COLUMN waiting_input_tenant_admin_task_count BIGINT NOT NULL DEFAULT 0 + CHECK (waiting_input_tenant_admin_task_count >= 0), + ADD COLUMN waiting_input_external_task_count BIGINT NOT NULL DEFAULT 0 + CHECK (waiting_input_external_task_count >= 0), + ADD COLUMN waiting_review_task_count BIGINT NOT NULL DEFAULT 0 + CHECK (waiting_review_task_count >= 0), + ADD COLUMN waiting_signal_task_count BIGINT NOT NULL DEFAULT 0 + CHECK (waiting_signal_task_count >= 0), + ADD COLUMN waiting_timer_task_count BIGINT NOT NULL DEFAULT 0 + CHECK (waiting_timer_task_count >= 0), + ADD COLUMN waiting_external_task_count BIGINT NOT NULL DEFAULT 0 + CHECK (waiting_external_task_count >= 0), + ADD COLUMN waiting_replan_task_count BIGINT NOT NULL DEFAULT 0 + CHECK (waiting_replan_task_count >= 0), + ADD COLUMN waiting_reasons_truncated BOOLEAN NOT NULL DEFAULT FALSE, + ADD CONSTRAINT execution_run_waiting_task_counts_check CHECK ( + waiting_task_count = waiting_input_task_count + + waiting_review_task_count + + waiting_signal_task_count + + waiting_timer_task_count + + waiting_external_task_count + + waiting_replan_task_count + ), + ADD CONSTRAINT execution_run_waiting_input_audience_counts_check CHECK ( + waiting_input_task_count = waiting_input_user_task_count + + waiting_input_tenant_admin_task_count + + waiting_input_external_task_count + ), + ADD CONSTRAINT execution_run_pause_timestamp_order_check CHECK ( + paused_at IS NULL + OR (pause_requested_at IS NOT NULL AND paused_at >= pause_requested_at) + ); + +ALTER TABLE moa.execution_run + DROP CONSTRAINT execution_run_waiting_reasons_check, + ADD CONSTRAINT execution_run_waiting_reasons_bounded_check CHECK ( + jsonb_typeof(waiting_reasons) = 'array' + AND jsonb_array_length(waiting_reasons) <= 64 + AND pg_column_size(waiting_reasons) <= 65536 + AND (NOT waiting_reasons_truncated OR waiting_task_count > 0) + ); + +UPDATE moa.execution_run +SET admitted_identity = jsonb_build_object( + 'identity_type', CASE WHEN contact_id IS NULL THEN 'operator' ELSE 'contact' END, + 'id', COALESCE( + contact_id::TEXT, + CASE + WHEN owner_user_id ~ + '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + THEN owner_user_id + ELSE run_uid::TEXT + END + ), + 'tenant_id', tenant_id::TEXT, + 'api_key_id', NULL, + 'acting_on_behalf_of', NULL + ), + activation_state = 'terminal', + next_wake_at = NULL, + waiting_since = NULL, + last_progress_at = updated_at, + ready_task_count = 0, + active_task_count = 0; + +ALTER TABLE moa.execution_run + ALTER COLUMN admitted_identity SET NOT NULL, + ALTER COLUMN activation_state SET DEFAULT 'queued', + ADD CONSTRAINT execution_run_admitted_identity_check + CHECK (moa.execution_admitted_identity_is_valid(admitted_identity, tenant_id)); + +CREATE OR REPLACE FUNCTION moa.enforce_execution_run_insert_confirmation() +RETURNS TRIGGER +LANGUAGE plpgsql +AS $$ +BEGIN + IF NEW.status NOT IN ('awaiting_confirmation', 'queued') THEN + RAISE EXCEPTION 'execution runs must start awaiting confirmation or queued'; + END IF; + IF NOT moa.execution_plan_snapshot_is_current(NEW.initial_plan) + OR NOT moa.execution_plan_snapshot_is_current(NEW.active_plan) THEN + RAISE EXCEPTION 'new execution runs require the current long-horizon plan contract'; + END IF; + NEW.created_at := now(); + NEW.queued_at := CASE + WHEN NEW.status = 'queued' THEN NEW.created_at + WHEN NEW.status = 'awaiting_confirmation' THEN NULL + ELSE NEW.queued_at + END; + NEW.activation_state := CASE + WHEN NEW.status = 'queued' THEN 'queued' + ELSE 'idle' + END; + NEW.last_progress_at := NEW.created_at; + IF NEW.confirmed_plan_hash IS NOT NULL OR NEW.confirmed_at IS NOT NULL THEN + RAISE EXCEPTION 'execution run confirmation proof must be created by confirmation'; + END IF; + RETURN NEW; +END; +$$; + +ALTER TABLE moa.execution_run + DROP CONSTRAINT execution_run_status_check, + ADD CONSTRAINT execution_run_status_check CHECK (status IN ( + 'awaiting_confirmation', 'queued', 'running', 'waiting_input', + 'waiting_review', 'waiting_signal', 'waiting_timer', 'waiting_external', + 'waiting_replan', 'pause_requested', 'pausing', 'paused', 'compensating', + 'completed', 'partial', 'blocked', 'unsupported', 'failed', 'cancelled' + )); + +DROP INDEX moa.execution_run_nonterminal_idx; +CREATE INDEX execution_run_nonterminal_idx + ON moa.execution_run (status, updated_at, run_uid) + WHERE status IN ( + 'awaiting_confirmation', 'queued', 'running', 'waiting_input', + 'waiting_review', 'waiting_signal', 'waiting_timer', 'waiting_external', + 'waiting_replan', 'pause_requested', 'pausing', 'paused', 'compensating' + ); + +CREATE INDEX execution_run_activation_idx + ON moa.execution_run (activation_state, next_wake_at, updated_at, run_uid) + WHERE activation_state IN ('queued', 'advancing'); + +CREATE INDEX execution_run_terminal_retention_idx + ON moa.execution_run (completed_at, tenant_id, run_uid) + WHERE status IN ('completed', 'partial', 'blocked', 'unsupported', 'failed', 'cancelled'); + +-- Compact, immutable terminal evidence is committed before bulky run detail is +-- paged away. The restrictive run FK deliberately keeps the run identity and +-- this receipt present while task/audit detail is being retained or deleted. +CREATE TABLE moa.execution_terminal_archive ( + archive_uid UUID PRIMARY KEY, + tenant_id UUID NOT NULL, + run_uid UUID NOT NULL, + contact_id UUID, + format_version BIGINT NOT NULL CHECK (format_version >= 1), + terminal_status TEXT NOT NULL CHECK (terminal_status IN ( + 'completed', 'partial', 'blocked', 'unsupported', 'failed', 'cancelled' + )), + terminal_completed_at TIMESTAMPTZ NOT NULL, + goal_hash TEXT NOT NULL CHECK (goal_hash ~ '^[0-9a-f]{64}$'), + initial_plan_hash TEXT NOT NULL CHECK (initial_plan_hash ~ '^[0-9a-f]{64}$'), + active_plan_hash TEXT NOT NULL CHECK (active_plan_hash ~ '^[0-9a-f]{64}$'), + source_record_count BIGINT NOT NULL DEFAULT 0 CHECK (source_record_count >= 0), + source_logical_bytes BIGINT NOT NULL DEFAULT 0 CHECK (source_logical_bytes >= 0), + segment_count BIGINT NOT NULL DEFAULT 0 CHECK (segment_count >= 0), + source_cursor JSONB NOT NULL DEFAULT '{}'::JSONB CHECK ( + jsonb_typeof(source_cursor) = 'object' + AND pg_column_size(source_cursor) <= 65536 + ), + rolling_chain_digest TEXT CHECK ( + rolling_chain_digest IS NULL OR rolling_chain_digest ~ '^[0-9a-f]{64}$' + ), + root_digest TEXT CHECK (root_digest IS NULL OR root_digest ~ '^[0-9a-f]{64}$'), + archive_generation BIGINT NOT NULL DEFAULT 1 CHECK (archive_generation >= 1), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + finalized_at TIMESTAMPTZ, + details_deleted_at TIMESTAMPTZ, + CONSTRAINT execution_terminal_archive_run_key UNIQUE (tenant_id, run_uid), + CONSTRAINT execution_terminal_archive_id_tenant_run_key + UNIQUE (archive_uid, tenant_id, run_uid), + CONSTRAINT execution_terminal_archive_run_tenant_fk + FOREIGN KEY (run_uid, tenant_id) + REFERENCES moa.execution_run (run_uid, tenant_id) ON DELETE RESTRICT, + CONSTRAINT execution_terminal_archive_finalization_pair_check CHECK ( + (root_digest IS NULL) = (finalized_at IS NULL) + AND (finalized_at IS NULL OR finalized_at >= created_at) + AND ( + finalized_at IS NULL + OR (source_record_count > 0 + AND source_logical_bytes > 0 + AND segment_count > 0) + ) + AND ( + ( + segment_count = 0 + AND source_record_count = 0 + AND source_logical_bytes = 0 + AND rolling_chain_digest IS NULL + ) + OR + ( + segment_count > 0 + AND source_record_count > 0 + AND source_logical_bytes > 0 + AND rolling_chain_digest IS NOT NULL + ) + ) + AND ( + details_deleted_at IS NULL + OR (finalized_at IS NOT NULL AND details_deleted_at >= finalized_at) + ) + ) +); + +CREATE INDEX execution_terminal_archive_retention_idx + ON moa.execution_terminal_archive ( + terminal_completed_at, tenant_id, run_uid + ); + +CREATE TABLE moa.execution_terminal_archive_segment ( + archive_uid UUID NOT NULL, + tenant_id UUID NOT NULL, + run_uid UUID NOT NULL, + segment_kind TEXT NOT NULL CHECK (btrim(segment_kind) <> ''), + segment_sequence BIGINT NOT NULL CHECK (segment_sequence >= 1), + format_version BIGINT NOT NULL CHECK (format_version >= 1), + record_count BIGINT NOT NULL CHECK (record_count > 0), + payload BYTEA NOT NULL CHECK ( + octet_length(payload) BETWEEN 1 AND 4194304 + ), + content_digest BYTEA NOT NULL CHECK (octet_length(content_digest) = 32), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT execution_terminal_archive_segment_key + PRIMARY KEY (archive_uid, segment_kind, segment_sequence), + CONSTRAINT execution_terminal_archive_segment_sequence_key + UNIQUE (archive_uid, segment_sequence), + CONSTRAINT execution_terminal_archive_segment_tenant_key + UNIQUE (archive_uid, tenant_id, segment_kind, segment_sequence), + CONSTRAINT execution_terminal_archive_segment_manifest_tenant_fk + FOREIGN KEY (archive_uid, tenant_id, run_uid) + REFERENCES moa.execution_terminal_archive (archive_uid, tenant_id, run_uid) + ON DELETE CASCADE +); + +CREATE INDEX execution_terminal_archive_segment_scan_idx + ON moa.execution_terminal_archive_segment ( + tenant_id, archive_uid, segment_kind, segment_sequence + ); + +ALTER TABLE moa.execution_run + ADD COLUMN terminal_archive_uid UUID, + ADD COLUMN terminal_archive_hash TEXT + CHECK (terminal_archive_hash IS NULL OR terminal_archive_hash ~ '^[0-9a-f]{64}$'), + ADD COLUMN terminal_details_archived_at TIMESTAMPTZ, + ADD CONSTRAINT execution_run_terminal_archive_pair_check CHECK ( + ( + (terminal_archive_uid IS NULL + AND terminal_archive_hash IS NULL + AND terminal_details_archived_at IS NULL) + OR + (terminal_archive_uid IS NOT NULL + AND terminal_archive_hash IS NOT NULL + AND terminal_details_archived_at IS NOT NULL) + ) + ); + +ALTER TABLE moa.execution_task + ADD COLUMN attempt_generation BIGINT, + ADD COLUMN attempt_state TEXT, + ADD COLUMN attempt_started_at TIMESTAMPTZ, + ADD COLUMN last_progress_at TIMESTAMPTZ, + ADD COLUMN attempt_deadline_at TIMESTAMPTZ, + ADD COLUMN waiting_since TIMESTAMPTZ, + ADD COLUMN ready_at TIMESTAMPTZ, + ADD COLUMN active_dispatch_uid UUID, + ADD COLUMN dispatch_sequence BIGINT NOT NULL DEFAULT 0 + CHECK (dispatch_sequence >= 0), + ADD COLUMN external_job_uid UUID, + ADD COLUMN failure_fingerprint TEXT CHECK ( + failure_fingerprint IS NULL OR failure_fingerprint ~ '^[0-9a-f]{64}$' + ), + ADD CONSTRAINT execution_task_id_run_tenant_key + UNIQUE (task_id, run_uid, tenant_id); + +UPDATE moa.execution_task +SET attempt_generation = generation, + attempt_state = CASE status + WHEN 'running' THEN 'running' + WHEN 'unknown_outcome' THEN 'unknown_outcome' + WHEN 'completed' THEN 'terminal' + WHEN 'skipped' THEN 'terminal' + WHEN 'failed' THEN 'terminal' + WHEN 'cancelled' THEN 'terminal' + WHEN 'reserved' THEN 'dispatching' + WHEN 'ready' THEN 'idle' + WHEN 'dispatching' THEN 'dispatching' + WHEN 'waiting_input' THEN 'waiting' + WHEN 'waiting_review' THEN 'waiting' + WHEN 'waiting_signal' THEN 'waiting' + WHEN 'waiting_timer' THEN 'waiting' + WHEN 'waiting_external' THEN 'waiting' + ELSE 'idle' + END, + last_progress_at = updated_at, + ready_at = CASE WHEN status IN ('pending', 'ready') THEN updated_at END; + +ALTER TABLE moa.execution_task + ALTER COLUMN attempt_generation SET NOT NULL, + ALTER COLUMN attempt_generation SET DEFAULT 1, + ALTER COLUMN attempt_state SET NOT NULL, + ALTER COLUMN attempt_state SET DEFAULT 'idle', + ALTER COLUMN last_progress_at SET NOT NULL, + ALTER COLUMN last_progress_at SET DEFAULT now(), + ADD CONSTRAINT execution_task_attempt_generation_check + CHECK (attempt_generation >= 1), + ADD CONSTRAINT execution_task_attempt_state_check CHECK ( + attempt_state IN ( + 'idle', 'dispatching', 'running', 'cancelling', 'waiting', 'terminal', + 'unknown_outcome' + ) + ), + ADD CONSTRAINT execution_task_attempt_time_order_check CHECK ( + attempt_deadline_at IS NULL + OR (attempt_started_at IS NOT NULL AND attempt_deadline_at > attempt_started_at) + ); + +ALTER TABLE moa.execution_task + DROP CONSTRAINT execution_task_status_check, + ADD CONSTRAINT execution_task_status_check CHECK (status IN ( + 'pending', 'ready', 'reserved', 'dispatching', 'running', + 'waiting_input', 'waiting_review', 'waiting_signal', 'waiting_timer', + 'waiting_external', 'waiting_replan', 'completed', 'skipped', 'failed', + 'cancelled', 'unknown_outcome' + )), + ADD CONSTRAINT execution_task_output_inline_size_check CHECK ( + output IS NULL OR pg_column_size(output) <= 65536 + ); + +DROP INDEX moa.execution_task_ready_idx; +CREATE INDEX execution_task_ready_idx + ON moa.execution_task (tenant_id, run_uid, ready_at, node_id, item_key, task_id) + WHERE status = 'ready'; + +CREATE INDEX execution_task_active_attempt_watchdog_idx + ON moa.execution_task (attempt_deadline_at, tenant_id, run_uid, task_id) + WHERE status = 'running' AND attempt_state = 'running'; + +CREATE INDEX execution_task_cancelling_reconciliation_idx + ON moa.execution_task (last_progress_at, tenant_id, run_uid, task_id) + WHERE attempt_state = 'cancelling'; + +CREATE INDEX execution_task_terminal_retention_idx + ON moa.execution_task (tenant_id, completed_at, run_uid, task_id) + WHERE status IN ('completed', 'skipped', 'failed', 'cancelled', 'unknown_outcome'); + +CREATE INDEX execution_task_nonterminal_run_idx + ON moa.execution_task (run_uid) + WHERE status NOT IN ('completed', 'skipped', 'failed', 'cancelled', 'unknown_outcome'); + +CREATE INDEX execution_task_waiting_projection_idx + ON moa.execution_task (run_uid, waiting_since, task_id) + WHERE status IN ( + 'waiting_input', 'waiting_review', 'waiting_signal', 'waiting_timer', + 'waiting_external', 'waiting_replan' + ) AND waiting_since IS NOT NULL; + +CREATE INDEX execution_task_failure_fingerprint_idx + ON moa.execution_task (run_uid, failure_fingerprint, task_id) + WHERE failure_fingerprint IS NOT NULL; + +ALTER TABLE moa.execution_compensation + ADD COLUMN attempt_generation BIGINT NOT NULL DEFAULT 1 + CHECK (attempt_generation >= 1), + ADD COLUMN attempt_state TEXT NOT NULL DEFAULT 'idle' CHECK (attempt_state IN ( + 'idle', 'dispatching', 'running', 'cancelling', 'waiting_review', + 'waiting_external', 'terminal', 'unknown_outcome' + )), + ADD COLUMN attempt_started_at TIMESTAMPTZ, + ADD COLUMN last_progress_at TIMESTAMPTZ NOT NULL DEFAULT now(), + ADD COLUMN attempt_deadline_at TIMESTAMPTZ, + ADD COLUMN waiting_since TIMESTAMPTZ, + ADD COLUMN active_dispatch_uid UUID, + ADD COLUMN external_job_uid UUID, + ADD COLUMN release_intent TEXT CHECK (release_intent IN ( + 'outcome', 'retry', 'review', 'external_job', 'pause', 'watchdog', + 'deadline', 'run_terminal' + )), + ADD COLUMN dispatch_sequence BIGINT NOT NULL DEFAULT 0 + CHECK (dispatch_sequence >= 0), + ADD CONSTRAINT execution_compensation_id_run_tenant_key + UNIQUE (compensation_id, run_uid, tenant_id), + ADD CONSTRAINT execution_compensation_attempt_time_order_check CHECK ( + attempt_deadline_at IS NULL + OR (attempt_started_at IS NOT NULL AND attempt_deadline_at > attempt_started_at) + ), + ADD CONSTRAINT execution_compensation_release_intent_shape_check CHECK ( + (attempt_state = 'cancelling') = (release_intent IS NOT NULL) + ); + +UPDATE moa.execution_compensation +SET attempt_generation = generation, + attempt_state = CASE status + WHEN 'running' THEN 'running' + WHEN 'completed' THEN 'terminal' + WHEN 'failed' THEN 'terminal' + WHEN 'unknown_outcome' THEN 'unknown_outcome' + ELSE 'idle' + END, + last_progress_at = updated_at; + +CREATE INDEX execution_compensation_active_watchdog_idx + ON moa.execution_compensation ( + attempt_deadline_at, tenant_id, run_uid, compensation_id + ) + WHERE status = 'running' AND attempt_state = 'running'; + +CREATE INDEX execution_compensation_cancelling_reconciliation_idx + ON moa.execution_compensation ( + last_progress_at, tenant_id, run_uid, compensation_id + ) + WHERE attempt_state = 'cancelling'; + +CREATE INDEX execution_compensation_terminal_retention_idx + ON moa.execution_compensation (tenant_id, completed_at, run_uid, compensation_id) + WHERE status IN ('completed', 'failed', 'unknown_outcome'); + +CREATE TABLE moa.execution_node_state ( + node_state_uid UUID PRIMARY KEY, + tenant_id UUID NOT NULL, + run_uid UUID NOT NULL, + node_id TEXT NOT NULL CHECK (btrim(node_id) <> ''), + node_order BIGINT NOT NULL CHECK (node_order >= 0), + node_status TEXT NOT NULL DEFAULT 'pending' CHECK (node_status IN ( + 'pending', 'ready', 'running', 'waiting', 'completed', 'skipped', + 'failed', 'cancelled' + )), + materialization_cursor BIGINT NOT NULL DEFAULT 0 CHECK (materialization_cursor >= 0), + materialization_complete BOOLEAN NOT NULL DEFAULT FALSE, + reduce_round BIGINT NOT NULL DEFAULT 1 CHECK (reduce_round >= 1), + reduce_batch_cursor BIGINT NOT NULL DEFAULT 0 CHECK (reduce_batch_cursor >= 0), + reduce_round_input_count BIGINT CHECK (reduce_round_input_count >= 0), + reduce_round_task_count BIGINT NOT NULL DEFAULT 0 + CHECK (reduce_round_task_count >= 0), + reduce_round_terminal_task_count BIGINT NOT NULL DEFAULT 0 + CHECK (reduce_round_terminal_task_count >= 0), + dependency_count BIGINT NOT NULL DEFAULT 0 CHECK (dependency_count >= 0), + remaining_dependency_count BIGINT NOT NULL DEFAULT 0 + CHECK (remaining_dependency_count >= 0), + aggregate_output JSONB, + aggregate_output_hash TEXT, + aggregate_cursor_item_key TEXT CHECK ( + aggregate_cursor_item_key IS NULL + OR ( + btrim(aggregate_cursor_item_key) <> '' + AND octet_length(aggregate_cursor_item_key) <= 1024 + ) + ), + aggregate_complete BOOLEAN NOT NULL DEFAULT FALSE, + total_task_count BIGINT NOT NULL DEFAULT 0 CHECK (total_task_count >= 0), + ready_task_count BIGINT NOT NULL DEFAULT 0 CHECK (ready_task_count >= 0), + active_task_count BIGINT NOT NULL DEFAULT 0 CHECK (active_task_count >= 0), + waiting_task_count BIGINT NOT NULL DEFAULT 0 CHECK (waiting_task_count >= 0), + terminal_task_count BIGINT NOT NULL DEFAULT 0 CHECK (terminal_task_count >= 0), + succeeded_task_count BIGINT NOT NULL DEFAULT 0 CHECK (succeeded_task_count >= 0), + failed_task_count BIGINT NOT NULL DEFAULT 0 CHECK (failed_task_count >= 0), + cancelled_task_count BIGINT NOT NULL DEFAULT 0 CHECK (cancelled_task_count >= 0), + reduce_ready BOOLEAN NOT NULL DEFAULT FALSE, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT execution_node_state_run_tenant_fk + FOREIGN KEY (run_uid, tenant_id) + REFERENCES moa.execution_run (run_uid, tenant_id) ON DELETE CASCADE, + CONSTRAINT execution_node_state_run_node_key UNIQUE (run_uid, node_id), + CONSTRAINT execution_node_state_id_tenant_key UNIQUE (node_state_uid, tenant_id), + CONSTRAINT execution_node_state_dependency_bounds_check CHECK ( + remaining_dependency_count <= dependency_count + ), + CONSTRAINT execution_node_state_aggregate_output_shape_check CHECK ( + (aggregate_output IS NULL AND aggregate_output_hash IS NULL) + OR ( + aggregate_output IS NOT NULL + AND aggregate_output_hash ~ '^[0-9a-f]{64}$' + AND pg_column_size(aggregate_output) <= 1048576 + ) + ), + CONSTRAINT execution_node_state_reduce_cursor_check CHECK ( + reduce_round_input_count IS NULL + OR reduce_batch_cursor <= reduce_round_input_count + ), + CONSTRAINT execution_node_state_reduce_round_totals_check CHECK ( + reduce_round_terminal_task_count <= reduce_round_task_count + ), + CONSTRAINT execution_node_state_task_totals_check CHECK ( + ready_task_count + active_task_count + waiting_task_count + terminal_task_count + <= total_task_count + AND succeeded_task_count + failed_task_count + cancelled_task_count + <= terminal_task_count + ) +); + +CREATE INDEX execution_node_state_drive_idx + ON moa.execution_node_state ( + tenant_id, run_uid, node_status, remaining_dependency_count, + node_order, node_state_uid + ) + WHERE node_status IN ('pending', 'ready', 'running', 'waiting'); + +CREATE INDEX execution_node_state_actionable_idx + ON moa.execution_node_state (run_uid, updated_at, node_order, node_state_uid) + WHERE remaining_dependency_count = 0 + AND NOT materialization_complete + AND node_status NOT IN ('completed', 'skipped', 'failed', 'cancelled'); + +CREATE INDEX execution_node_state_aggregate_actionable_idx + ON moa.execution_node_state (run_uid, updated_at, node_order, node_state_uid) + WHERE materialization_complete + AND NOT aggregate_complete + AND node_status NOT IN ('completed', 'skipped', 'failed', 'cancelled'); + +CREATE OR REPLACE FUNCTION moa.enforce_execution_node_aggregate_cursor_update() +RETURNS TRIGGER +LANGUAGE plpgsql +AS $$ +BEGIN + IF OLD.aggregate_cursor_item_key IS NOT NULL + AND ( + NEW.aggregate_cursor_item_key IS NULL + OR NEW.aggregate_cursor_item_key < OLD.aggregate_cursor_item_key + ) THEN + RAISE EXCEPTION 'execution node aggregate cursor must be monotonic'; + END IF; + IF OLD.aggregate_complete AND NOT NEW.aggregate_complete THEN + RAISE EXCEPTION 'execution node aggregate completion is one-way'; + END IF; + RETURN NEW; +END; +$$; + +CREATE TRIGGER execution_node_aggregate_cursor_update_guard +BEFORE UPDATE ON moa.execution_node_state +FOR EACH ROW EXECUTE FUNCTION moa.enforce_execution_node_aggregate_cursor_update(); + +-- Completion evaluation is itself a bounded, restartable scan. The hot row +-- holds only the current cursor and a capped accumulator, never task history. +CREATE TABLE moa.execution_completion_scan ( + tenant_id UUID NOT NULL, + run_uid UUID NOT NULL, + plan_revision BIGINT NOT NULL CHECK (plan_revision >= 1), + controller_generation BIGINT NOT NULL CHECK (controller_generation >= 1), + scan_kind TEXT NOT NULL DEFAULT 'ordinary' CHECK ( + scan_kind IN ('ordinary', 'replan_stop') + ), + excluded_task_id UUID, + source_progress_at TIMESTAMPTZ NOT NULL, + task_cursor UUID, + node_cursor BIGINT CHECK (node_cursor >= 0), + scanned_task_count BIGINT NOT NULL DEFAULT 0 CHECK (scanned_task_count >= 0), + task_evidence JSONB NOT NULL DEFAULT '{}'::JSONB CHECK ( + jsonb_typeof(task_evidence) = 'object' + AND pg_column_size(task_evidence) <= 1048576 + ), + scan_complete BOOLEAN NOT NULL DEFAULT FALSE, + node_scan_complete BOOLEAN NOT NULL DEFAULT FALSE, + completion_evidence JSONB NOT NULL DEFAULT '{}'::JSONB CHECK ( + jsonb_typeof(completion_evidence) = 'object' + AND pg_column_size(completion_evidence) <= 1048576 + ), + verifiers_materialized BOOLEAN NOT NULL DEFAULT FALSE, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT execution_completion_scan_key PRIMARY KEY (tenant_id, run_uid), + CONSTRAINT execution_completion_scan_run_tenant_key UNIQUE (run_uid, tenant_id), + CONSTRAINT execution_completion_scan_run_tenant_fk + FOREIGN KEY (run_uid, tenant_id) + REFERENCES moa.execution_run (run_uid, tenant_id) ON DELETE CASCADE, + CONSTRAINT execution_completion_scan_excluded_task_tenant_fk + FOREIGN KEY (excluded_task_id, run_uid, tenant_id) + REFERENCES moa.execution_task (task_id, run_uid, tenant_id) ON DELETE CASCADE, + CONSTRAINT execution_completion_scan_kind_shape_check CHECK ( + (scan_kind = 'ordinary' AND excluded_task_id IS NULL) + OR (scan_kind = 'replan_stop' AND excluded_task_id IS NOT NULL) + ), + CONSTRAINT execution_completion_scan_cursor_check CHECK ( + task_cursor IS NOT NULL OR scanned_task_count = 0 OR scan_complete + ), + CONSTRAINT execution_completion_scan_verifier_check CHECK ( + NOT verifiers_materialized OR (scan_complete AND node_scan_complete) + ) +); + +CREATE INDEX execution_completion_scan_actionable_idx + ON moa.execution_completion_scan (updated_at, tenant_id, run_uid) + WHERE NOT scan_complete OR NOT verifiers_materialized; + +CREATE UNIQUE INDEX execution_node_state_run_order_uidx + ON moa.execution_node_state (run_uid, node_order); + +CREATE OR REPLACE FUNCTION moa.enforce_execution_completion_scan_update() +RETURNS TRIGGER +LANGUAGE plpgsql +AS $$ +BEGIN + IF NEW.tenant_id IS DISTINCT FROM OLD.tenant_id + OR NEW.run_uid IS DISTINCT FROM OLD.run_uid + OR NEW.created_at IS DISTINCT FROM OLD.created_at THEN + RAISE EXCEPTION 'execution completion scan identity is immutable'; + END IF; + IF NEW.plan_revision < OLD.plan_revision + OR NEW.controller_generation < OLD.controller_generation + OR NEW.source_progress_at < OLD.source_progress_at + OR NEW.updated_at < OLD.updated_at THEN + RAISE EXCEPTION 'execution completion scan progress must be monotonic'; + END IF; + IF NEW.plan_revision IS DISTINCT FROM OLD.plan_revision + OR NEW.controller_generation IS DISTINCT FROM OLD.controller_generation + OR NEW.source_progress_at IS DISTINCT FROM OLD.source_progress_at THEN + IF NEW.task_cursor IS NOT NULL + OR NEW.node_cursor IS NOT NULL + OR NEW.scanned_task_count <> 0 + OR NEW.task_evidence <> '{}'::JSONB + OR NEW.completion_evidence <> '{}'::JSONB + OR NEW.scan_complete + OR NEW.node_scan_complete + OR NEW.verifiers_materialized THEN + RAISE EXCEPTION 'execution completion scan source change requires full reset'; + END IF; + ELSE + IF NEW.scan_kind IS DISTINCT FROM OLD.scan_kind + OR NEW.excluded_task_id IS DISTINCT FROM OLD.excluded_task_id THEN + RAISE EXCEPTION 'execution completion scan kind is immutable within its source'; + END IF; + IF NEW.scanned_task_count < OLD.scanned_task_count + OR (NEW.task_cursor IS DISTINCT FROM OLD.task_cursor + AND NEW.scanned_task_count <= OLD.scanned_task_count) + OR (OLD.node_cursor IS NOT NULL + AND (NEW.node_cursor IS NULL OR NEW.node_cursor < OLD.node_cursor)) + OR (OLD.scan_complete AND NOT NEW.scan_complete) + OR (OLD.node_scan_complete AND NOT NEW.node_scan_complete) + OR (OLD.verifiers_materialized AND NOT NEW.verifiers_materialized) THEN + RAISE EXCEPTION 'execution completion scan progress must be monotonic'; + END IF; + END IF; + RETURN NEW; +END; +$$; + +CREATE TRIGGER execution_completion_scan_update_guard +BEFORE UPDATE ON moa.execution_completion_scan +FOR EACH ROW EXECUTE FUNCTION moa.enforce_execution_completion_scan_update(); + +-- One immutable replay receipt replaces recovery-time scans of every task and +-- its JSON audit history. The current amendment contract releases at most the +-- one superseded task; replan-stop records release none. +CREATE TABLE moa.execution_amendment_receipt ( + tenant_id UUID NOT NULL, + run_uid UUID NOT NULL, + base_plan_revision BIGINT NOT NULL CHECK (base_plan_revision >= 1), + amendment_hash TEXT NOT NULL CHECK (amendment_hash ~ '^[0-9a-f]{64}$'), + receipt_kind TEXT NOT NULL CHECK (receipt_kind IN ('applied', 'replan_stop')), + superseded_task_id UUID NOT NULL, + task_generation BIGINT NOT NULL CHECK (task_generation >= 1), + task_ids_to_release UUID[] NOT NULL DEFAULT '{}'::UUID[] CHECK ( + cardinality(task_ids_to_release) <= 1 + ), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT execution_amendment_receipt_key + PRIMARY KEY (tenant_id, run_uid, base_plan_revision), + CONSTRAINT execution_amendment_receipt_run_tenant_key + UNIQUE (run_uid, tenant_id, base_plan_revision), + CONSTRAINT execution_amendment_receipt_run_tenant_fk + FOREIGN KEY (run_uid, tenant_id) + REFERENCES moa.execution_run (run_uid, tenant_id) ON DELETE CASCADE, + CONSTRAINT execution_amendment_receipt_task_tenant_fk + FOREIGN KEY (superseded_task_id, run_uid, tenant_id) + REFERENCES moa.execution_task (task_id, run_uid, tenant_id) ON DELETE CASCADE, + CONSTRAINT execution_amendment_receipt_release_shape_check CHECK ( + ( + receipt_kind = 'applied' + AND cardinality(task_ids_to_release) = 1 + AND task_ids_to_release[1] = superseded_task_id + ) + OR + ( + receipt_kind = 'replan_stop' + AND cardinality(task_ids_to_release) = 0 + ) + ) +); + +CREATE INDEX execution_amendment_receipt_retention_idx + ON moa.execution_amendment_receipt ( + tenant_id, created_at, run_uid, base_plan_revision + ); + +-- Replan-stop evaluation persists one bounded controller handoff. The exact +-- compensation fence consumes this row atomically; no controller activation +-- rescans task history to reconstruct the decision. +CREATE TABLE moa.execution_replan_stop_intent ( + tenant_id UUID NOT NULL, + run_uid UUID NOT NULL, + controller_generation BIGINT NOT NULL CHECK (controller_generation >= 1), + wake_epoch BIGINT NOT NULL CHECK (wake_epoch >= 1), + origin_task_id UUID NOT NULL, + task_generation BIGINT NOT NULL CHECK (task_generation >= 1), + base_plan_revision BIGINT NOT NULL CHECK (base_plan_revision >= 1), + stop_reason TEXT NOT NULL CHECK (stop_reason IN ( + 'duplicate_plan', 'duplicate_amendment', 'repeated_failure', + 'no_progress', 'deadline_exceeded', 'budget_exhausted' + )), + detail TEXT NOT NULL CHECK (octet_length(detail) BETWEEN 1 AND 4096), + amendment_hash TEXT NOT NULL CHECK (amendment_hash ~ '^[0-9a-f]{64}$'), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT execution_replan_stop_intent_key PRIMARY KEY (tenant_id, run_uid), + CONSTRAINT execution_replan_stop_intent_run_tenant_key UNIQUE (run_uid, tenant_id), + CONSTRAINT execution_replan_stop_intent_generation_key + UNIQUE (run_uid, tenant_id, controller_generation, wake_epoch), + CONSTRAINT execution_replan_stop_intent_run_tenant_fk + FOREIGN KEY (run_uid, tenant_id) + REFERENCES moa.execution_run (run_uid, tenant_id) ON DELETE CASCADE, + CONSTRAINT execution_replan_stop_intent_task_tenant_fk + FOREIGN KEY (origin_task_id, run_uid, tenant_id) + REFERENCES moa.execution_task (task_id, run_uid, tenant_id) ON DELETE CASCADE +); + +CREATE INDEX execution_replan_stop_intent_current_idx + ON moa.execution_replan_stop_intent ( + tenant_id, run_uid, controller_generation, wake_epoch + ); + +CREATE TABLE moa.execution_schedule ( + schedule_uid UUID PRIMARY KEY, + tenant_id UUID NOT NULL, + owner_user_id TEXT NOT NULL CHECK (btrim(owner_user_id) <> ''), + name TEXT NOT NULL CHECK (btrim(name) <> ''), + timezone TEXT NOT NULL CHECK (btrim(timezone) <> ''), + calendar_expression TEXT NOT NULL CHECK (btrim(calendar_expression) <> ''), + template_revision_uid UUID NOT NULL, + template_snapshot JSONB NOT NULL CHECK (jsonb_typeof(template_snapshot) = 'object'), + template_hash TEXT NOT NULL CHECK (template_hash ~ '^[0-9a-f]{64}$'), + run_as_identity JSONB NOT NULL, + creation_origin JSONB NOT NULL, + status TEXT NOT NULL DEFAULT 'active' + CHECK (status IN ('active', 'paused', 'completed', 'cancelled')), + missed_fire_policy TEXT NOT NULL + CHECK (missed_fire_policy IN ('skip', 'fire_once')), + overlap_policy TEXT NOT NULL + CHECK (overlap_policy IN ('skip', 'queue_one', 'allow')), + dst_policy TEXT NOT NULL + CHECK (dst_policy IN ('earliest', 'latest', 'skip')), + maximum_concurrent_runs BIGINT NOT NULL DEFAULT 1 + CHECK (maximum_concurrent_runs > 0), + occurrence_budget JSONB NOT NULL CHECK (jsonb_typeof(occurrence_budget) = 'object'), + schedule_incarnation BIGINT NOT NULL DEFAULT 1 CHECK (schedule_incarnation >= 1), + start_at TIMESTAMPTZ NOT NULL, + next_occurrence_at TIMESTAMPTZ, + next_occurrence_local TIMESTAMP WITHOUT TIME ZONE, + last_occurrence_sequence BIGINT NOT NULL DEFAULT 0 + CHECK (last_occurrence_sequence >= 0), + end_at TIMESTAMPTZ, + paused_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT execution_schedule_id_tenant_key UNIQUE (schedule_uid, tenant_id), + CONSTRAINT execution_schedule_run_as_identity_check CHECK ( + moa.execution_admitted_identity_is_valid(run_as_identity, tenant_id) + ), + CONSTRAINT execution_schedule_creation_origin_check CHECK ( + moa.execution_schedule_origin_is_valid(creation_origin, tenant_id) + ), + CONSTRAINT execution_schedule_next_occurrence_pair_check CHECK ( + (next_occurrence_at IS NULL) = (next_occurrence_local IS NULL) + ), + CONSTRAINT execution_schedule_end_order_check CHECK ( + end_at IS NULL OR end_at > start_at + ) +); + +CREATE INDEX execution_schedule_due_idx + ON moa.execution_schedule ( + next_occurrence_at, tenant_id, schedule_uid, schedule_incarnation + ) + WHERE status = 'active' AND next_occurrence_at IS NOT NULL; + +ALTER TABLE moa.execution_run + ADD COLUMN schedule_uid UUID, + ADD COLUMN schedule_incarnation BIGINT CHECK (schedule_incarnation >= 1), + ADD COLUMN schedule_occurrence_sequence BIGINT + CHECK (schedule_occurrence_sequence >= 1), + ADD CONSTRAINT execution_run_schedule_tenant_fk + FOREIGN KEY (schedule_uid, tenant_id) + REFERENCES moa.execution_schedule (schedule_uid, tenant_id) ON DELETE CASCADE, + ADD CONSTRAINT execution_run_schedule_occurrence_shape_check CHECK ( + (schedule_uid IS NULL AND schedule_incarnation IS NULL + AND schedule_occurrence_sequence IS NULL) + OR + (schedule_uid IS NOT NULL AND schedule_incarnation IS NOT NULL + AND schedule_occurrence_sequence IS NOT NULL) + ); + +CREATE UNIQUE INDEX execution_run_schedule_occurrence_uidx + ON moa.execution_run ( + tenant_id, schedule_uid, schedule_incarnation, schedule_occurrence_sequence + ) + WHERE schedule_uid IS NOT NULL; + +CREATE INDEX execution_run_schedule_nonterminal_idx + ON moa.execution_run (tenant_id, schedule_uid) + WHERE schedule_uid IS NOT NULL + AND status NOT IN ( + 'completed', 'partial', 'blocked', 'unsupported', 'failed', 'cancelled' + ); + +CREATE INDEX execution_run_schedule_queued_idx + ON moa.execution_run (tenant_id, schedule_uid) + WHERE schedule_uid IS NOT NULL AND status = 'queued'; + +CREATE TABLE moa.execution_external_job ( + external_job_uid UUID PRIMARY KEY, + tenant_id UUID NOT NULL, + run_uid UUID NOT NULL, + task_id UUID, + attempt_generation BIGINT CHECK (attempt_generation >= 1), + compensation_id UUID, + compensation_generation BIGINT CHECK (compensation_generation >= 1), + compensation_attempt_generation BIGINT + CHECK (compensation_attempt_generation >= 1), + job_generation BIGINT NOT NULL DEFAULT 1 CHECK (job_generation >= 1), + declared_provider TEXT NOT NULL CHECK (btrim(declared_provider) <> ''), + provider TEXT CHECK (provider IS NULL OR btrim(provider) <> ''), + provider_job_id TEXT CHECK (provider_job_id IS NULL OR btrim(provider_job_id) <> ''), + idempotency_key TEXT NOT NULL CHECK (octet_length(idempotency_key) BETWEEN 1 AND 256), + callback_auth_reference TEXT CHECK ( + callback_auth_reference IS NULL OR btrim(callback_auth_reference) <> '' + ), + state TEXT NOT NULL CHECK (state IN ( + 'unbound', 'starting', 'running', 'waiting_reconcile', 'cancel_requested', + 'completed', 'failed', 'cancelled', 'unknown_outcome' + )), + progress_phase TEXT, + cancel_supported BOOLEAN NOT NULL DEFAULT FALSE, + next_reconcile_at TIMESTAMPTZ, + last_provider_event_id TEXT, + output JSONB, + error JSONB, + provider_contract_violation JSONB, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + completed_at TIMESTAMPTZ, + CONSTRAINT execution_external_job_task_tenant_fk + FOREIGN KEY (task_id, run_uid, tenant_id) + REFERENCES moa.execution_task (task_id, run_uid, tenant_id) ON DELETE CASCADE, + CONSTRAINT execution_external_job_compensation_tenant_fk + FOREIGN KEY (compensation_id, run_uid, tenant_id) + REFERENCES moa.execution_compensation ( + compensation_id, run_uid, tenant_id + ) ON DELETE CASCADE, + CONSTRAINT execution_external_job_id_tenant_key + UNIQUE (external_job_uid, tenant_id), + CONSTRAINT execution_external_job_owner_shape_check CHECK ( + ( + task_id IS NOT NULL AND attempt_generation IS NOT NULL + AND compensation_id IS NULL + AND compensation_generation IS NULL + AND compensation_attempt_generation IS NULL + ) + OR + ( + task_id IS NULL AND attempt_generation IS NULL + AND compensation_id IS NOT NULL + AND compensation_generation IS NOT NULL + AND compensation_attempt_generation IS NOT NULL + ) + ), + CONSTRAINT execution_external_job_terminal_shape_check CHECK ( + (state IN ('completed', 'failed', 'cancelled', 'unknown_outcome')) + = (completed_at IS NOT NULL) + ), + CONSTRAINT execution_external_job_binding_shape_check CHECK ( + ( + state = 'unbound' + AND provider IS NULL + AND provider_job_id IS NULL + AND callback_auth_reference IS NULL + AND progress_phase IS NULL + AND NOT cancel_supported + AND next_reconcile_at IS NULL + AND last_provider_event_id IS NULL + AND output IS NULL + AND error IS NULL + AND completed_at IS NULL + ) + OR + ( + state <> 'unbound' + AND provider IS NOT NULL + AND provider = declared_provider + AND provider_job_id IS NOT NULL + AND callback_auth_reference IS NOT NULL + ) + ), + CONSTRAINT execution_external_job_contract_violation_shape_check CHECK ( + provider_contract_violation IS NULL + OR ( + state IN ( + 'cancel_requested', 'completed', 'failed', 'cancelled', + 'unknown_outcome' + ) + AND jsonb_typeof(provider_contract_violation) = 'object' + AND pg_column_size(provider_contract_violation) <= 16384 + AND moa.execution_json_object_has_exact_keys( + provider_contract_violation, ARRAY['kind', 'observed_at', 'detail'] + ) + AND provider_contract_violation ->> 'kind' = 'provider_contract_mismatch' + AND btrim(provider_contract_violation ->> 'observed_at') <> '' + AND octet_length(provider_contract_violation ->> 'detail') BETWEEN 1 AND 4096 + AND (state <> 'cancel_requested' OR next_reconcile_at IS NOT NULL) + ) + ) +); + +CREATE UNIQUE INDEX execution_external_job_provider_identity_key + ON moa.execution_external_job ( + tenant_id, provider, provider_job_id, job_generation + ) + WHERE provider IS NOT NULL; + +CREATE UNIQUE INDEX execution_external_job_task_attempt_uidx + ON moa.execution_external_job ( + tenant_id, run_uid, task_id, attempt_generation + ) + WHERE task_id IS NOT NULL; + +CREATE UNIQUE INDEX execution_external_job_compensation_attempt_uidx + ON moa.execution_external_job ( + tenant_id, run_uid, compensation_id, + compensation_generation, compensation_attempt_generation + ) + WHERE compensation_id IS NOT NULL; + +CREATE UNIQUE INDEX execution_external_job_callback_dedupe_uidx + ON moa.execution_external_job ( + tenant_id, provider, last_provider_event_id, job_generation + ) + WHERE last_provider_event_id IS NOT NULL; + +CREATE INDEX execution_external_job_reconcile_idx + ON moa.execution_external_job (next_reconcile_at, tenant_id, external_job_uid) + WHERE state IN ('starting', 'running', 'waiting_reconcile', 'cancel_requested') + AND next_reconcile_at IS NOT NULL; + +CREATE OR REPLACE FUNCTION moa.enforce_execution_external_job_update() +RETURNS TRIGGER +LANGUAGE plpgsql +AS $$ +DECLARE + transition_allowed BOOLEAN; +BEGIN + IF NEW.external_job_uid IS DISTINCT FROM OLD.external_job_uid + OR NEW.tenant_id IS DISTINCT FROM OLD.tenant_id + OR NEW.run_uid IS DISTINCT FROM OLD.run_uid + OR NEW.task_id IS DISTINCT FROM OLD.task_id + OR NEW.attempt_generation IS DISTINCT FROM OLD.attempt_generation + OR NEW.compensation_id IS DISTINCT FROM OLD.compensation_id + OR NEW.compensation_generation IS DISTINCT FROM OLD.compensation_generation + OR NEW.compensation_attempt_generation + IS DISTINCT FROM OLD.compensation_attempt_generation + OR NEW.job_generation IS DISTINCT FROM OLD.job_generation + OR NEW.declared_provider IS DISTINCT FROM OLD.declared_provider + OR NEW.idempotency_key IS DISTINCT FROM OLD.idempotency_key + OR NEW.created_at IS DISTINCT FROM OLD.created_at THEN + RAISE EXCEPTION 'execution external job owner and generation are immutable'; + END IF; + IF OLD.state <> 'unbound' + AND ( + NEW.provider IS DISTINCT FROM OLD.provider + OR NEW.provider_job_id IS DISTINCT FROM OLD.provider_job_id + OR NEW.callback_auth_reference IS DISTINCT FROM OLD.callback_auth_reference + ) THEN + RAISE EXCEPTION 'bound execution external job provider identity is immutable'; + END IF; + IF OLD.provider_contract_violation IS NOT NULL + AND NEW.provider_contract_violation IS DISTINCT FROM OLD.provider_contract_violation THEN + RAISE EXCEPTION 'execution external job contract violation evidence is immutable'; + END IF; + IF OLD.state IN ('completed', 'failed', 'cancelled', 'unknown_outcome') + AND NEW IS DISTINCT FROM OLD THEN + RAISE EXCEPTION 'terminal execution external job is immutable'; + END IF; + transition_allowed := CASE OLD.state + WHEN 'unbound' THEN NEW.state IN ( + 'unbound', 'starting', 'running', 'waiting_reconcile', + 'completed', 'failed', 'cancelled', 'unknown_outcome' + ) + WHEN 'starting' THEN NEW.state IN ( + 'starting', 'running', 'waiting_reconcile', 'cancel_requested', + 'completed', 'failed', 'cancelled', 'unknown_outcome' + ) + WHEN 'running' THEN NEW.state IN ( + 'running', 'waiting_reconcile', 'cancel_requested', + 'completed', 'failed', 'cancelled', 'unknown_outcome' + ) + WHEN 'waiting_reconcile' THEN NEW.state IN ( + 'running', 'waiting_reconcile', 'cancel_requested', + 'completed', 'failed', 'cancelled', 'unknown_outcome' + ) + WHEN 'cancel_requested' THEN NEW.state IN ( + 'cancel_requested', 'completed', 'failed', 'cancelled', 'unknown_outcome' + ) + ELSE NEW.state = OLD.state + END; + IF NOT transition_allowed THEN + RAISE EXCEPTION 'invalid execution external job state transition: % -> %', + OLD.state, NEW.state; + END IF; + IF NEW.updated_at < OLD.updated_at THEN + RAISE EXCEPTION 'execution external job updated_at must be monotonic'; + END IF; + RETURN NEW; +END; +$$; + +CREATE TRIGGER execution_external_job_update_guard +BEFORE UPDATE ON moa.execution_external_job +FOR EACH ROW EXECUTE FUNCTION moa.enforce_execution_external_job_update(); + +CREATE TABLE moa.execution_external_job_callback_receipt ( + tenant_id UUID NOT NULL, + external_job_uid UUID NOT NULL, + provider TEXT NOT NULL CHECK (btrim(provider) <> ''), + provider_event_id TEXT NOT NULL CHECK ( + octet_length(provider_event_id) BETWEEN 1 AND 512 + ), + job_generation BIGINT NOT NULL CHECK (job_generation >= 1), + received_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT execution_external_job_callback_receipt_identity_key + PRIMARY KEY ( + tenant_id, external_job_uid, job_generation, provider, provider_event_id + ), + CONSTRAINT execution_external_job_callback_receipt_job_tenant_fk + FOREIGN KEY (external_job_uid, tenant_id) + REFERENCES moa.execution_external_job (external_job_uid, tenant_id) + ON DELETE CASCADE +); + +CREATE INDEX execution_external_job_callback_receipt_retention_idx + ON moa.execution_external_job_callback_receipt ( + tenant_id, received_at, external_job_uid, job_generation + ); + +ALTER TABLE moa.execution_task + ADD CONSTRAINT execution_task_external_job_tenant_fk + FOREIGN KEY (external_job_uid, tenant_id) + REFERENCES moa.execution_external_job (external_job_uid, tenant_id) + ON DELETE SET NULL (external_job_uid); + +ALTER TABLE moa.execution_compensation + ADD CONSTRAINT execution_compensation_external_job_tenant_fk + FOREIGN KEY (external_job_uid, tenant_id) + REFERENCES moa.execution_external_job (external_job_uid, tenant_id) + ON DELETE SET NULL (external_job_uid); + +CREATE TABLE moa.execution_trigger ( + trigger_uid UUID PRIMARY KEY, + tenant_id UUID NOT NULL, + run_uid UUID, + task_id UUID, + compensation_id UUID, + schedule_uid UUID, + schedule_incarnation BIGINT CHECK (schedule_incarnation >= 1), + trigger_kind TEXT NOT NULL CHECK (trigger_kind IN ( + 'run_deadline', 'task_timer', 'wait_expiry', 'task_watchdog', + 'external_reconcile', 'external_start_recovery', 'schedule_occurrence', + 'compensation_watchdog' + )), + state TEXT NOT NULL DEFAULT 'pending' CHECK (state IN ( + 'pending', 'dispatching', 'delivered', 'superseded', 'cancelled', 'dead_letter' + )), + controller_generation BIGINT CHECK (controller_generation >= 1), + attempt_generation BIGINT CHECK (attempt_generation >= 1), + compensation_generation BIGINT CHECK (compensation_generation >= 1), + compensation_attempt_generation BIGINT + CHECK (compensation_attempt_generation >= 1), + occurrence_sequence BIGINT CHECK (occurrence_sequence >= 1), + due_at TIMESTAMPTZ NOT NULL, + payload JSONB NOT NULL DEFAULT '{}'::JSONB CHECK (jsonb_typeof(payload) = 'object'), + claim_owner TEXT, + claimed_at TIMESTAMPTZ, + claim_expires_at TIMESTAMPTZ, + delivery_attempts INTEGER NOT NULL DEFAULT 0 CHECK (delivery_attempts >= 0), + delivered_at TIMESTAMPTZ, + last_error TEXT CHECK (last_error IS NULL OR octet_length(last_error) <= 4096), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT execution_trigger_id_tenant_key UNIQUE (trigger_uid, tenant_id), + CONSTRAINT execution_trigger_run_tenant_fk + FOREIGN KEY (run_uid, tenant_id) + REFERENCES moa.execution_run (run_uid, tenant_id) ON DELETE CASCADE, + CONSTRAINT execution_trigger_task_tenant_fk + FOREIGN KEY (task_id, run_uid, tenant_id) + REFERENCES moa.execution_task (task_id, run_uid, tenant_id) ON DELETE CASCADE, + CONSTRAINT execution_trigger_compensation_tenant_fk + FOREIGN KEY (compensation_id, run_uid, tenant_id) + REFERENCES moa.execution_compensation ( + compensation_id, run_uid, tenant_id + ) ON DELETE CASCADE, + CONSTRAINT execution_trigger_schedule_tenant_fk + FOREIGN KEY (schedule_uid, tenant_id) + REFERENCES moa.execution_schedule (schedule_uid, tenant_id) ON DELETE CASCADE, + CONSTRAINT execution_trigger_target_shape_check CHECK ( + ( + trigger_kind = 'schedule_occurrence' + AND schedule_uid IS NOT NULL + AND schedule_incarnation IS NOT NULL + AND occurrence_sequence IS NOT NULL + AND run_uid IS NULL + AND task_id IS NULL + AND compensation_id IS NULL + AND controller_generation IS NULL + AND attempt_generation IS NULL + AND compensation_generation IS NULL + AND compensation_attempt_generation IS NULL + ) OR ( + trigger_kind = 'run_deadline' + AND run_uid IS NOT NULL + AND task_id IS NULL + AND compensation_id IS NULL + AND schedule_uid IS NULL + AND schedule_incarnation IS NULL + AND controller_generation IS NOT NULL + AND attempt_generation IS NULL + AND compensation_generation IS NULL + AND compensation_attempt_generation IS NULL + AND occurrence_sequence IS NULL + ) OR ( + trigger_kind IN ( + 'task_timer', 'wait_expiry', 'task_watchdog' + ) + AND run_uid IS NOT NULL + AND task_id IS NOT NULL + AND compensation_id IS NULL + AND schedule_uid IS NULL + AND schedule_incarnation IS NULL + AND controller_generation IS NOT NULL + AND attempt_generation IS NOT NULL + AND compensation_generation IS NULL + AND compensation_attempt_generation IS NULL + AND occurrence_sequence IS NULL + ) OR ( + trigger_kind IN ('external_reconcile', 'external_start_recovery') + AND run_uid IS NOT NULL + AND schedule_uid IS NULL + AND schedule_incarnation IS NULL + AND controller_generation IS NOT NULL + AND occurrence_sequence IS NULL + AND ( + ( + task_id IS NOT NULL + AND attempt_generation IS NOT NULL + AND compensation_id IS NULL + AND compensation_generation IS NULL + AND compensation_attempt_generation IS NULL + ) + OR + ( + task_id IS NULL + AND attempt_generation IS NULL + AND compensation_id IS NOT NULL + AND compensation_generation IS NOT NULL + AND compensation_attempt_generation IS NOT NULL + ) + ) + ) OR ( + trigger_kind = 'compensation_watchdog' + AND run_uid IS NOT NULL + AND task_id IS NULL + AND compensation_id IS NOT NULL + AND schedule_uid IS NULL + AND schedule_incarnation IS NULL + AND controller_generation IS NOT NULL + AND attempt_generation IS NULL + AND compensation_generation IS NOT NULL + AND compensation_attempt_generation IS NOT NULL + AND occurrence_sequence IS NULL + ) + ), + CONSTRAINT execution_trigger_claim_pair_check CHECK ( + (claim_owner IS NULL AND claimed_at IS NULL AND claim_expires_at IS NULL) + OR ( + claim_owner IS NOT NULL AND btrim(claim_owner) <> '' + AND claimed_at IS NOT NULL AND claim_expires_at IS NOT NULL + AND claim_expires_at > claimed_at + ) + ), + CONSTRAINT execution_trigger_delivery_pair_check CHECK ( + (state = 'delivered') = (delivered_at IS NOT NULL) + ), + CONSTRAINT execution_trigger_start_recovery_shape_check CHECK ( + trigger_kind <> 'external_start_recovery' + OR ( + moa.execution_json_object_has_exact_keys( + payload, ARRAY[ + 'external_job_uid', 'job_generation', 'declared_provider', + 'idempotency_key' + ] + ) + AND payload ->> 'external_job_uid' ~ + '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + AND payload ->> 'job_generation' ~ '^[1-9][0-9]*$' + AND octet_length(payload ->> 'declared_provider') BETWEEN 1 AND 256 + AND octet_length(payload ->> 'idempotency_key') BETWEEN 1 AND 256 + ) + ) +); + +CREATE UNIQUE INDEX execution_trigger_current_run_generation_uidx + ON moa.execution_trigger ( + tenant_id, run_uid, + COALESCE(task_id, compensation_id, '00000000-0000-0000-0000-000000000000'::UUID), + trigger_kind, controller_generation, COALESCE(attempt_generation, 0), + COALESCE(compensation_generation, 0), + COALESCE(compensation_attempt_generation, 0) + ) + WHERE state IN ('pending', 'dispatching') AND run_uid IS NOT NULL; + +CREATE UNIQUE INDEX execution_trigger_schedule_occurrence_uidx + ON moa.execution_trigger ( + tenant_id, schedule_uid, schedule_incarnation, occurrence_sequence + ) + WHERE trigger_kind = 'schedule_occurrence'; + +CREATE INDEX execution_trigger_due_idx + ON moa.execution_trigger (due_at, tenant_id, trigger_uid) + WHERE state = 'pending'; + +CREATE INDEX execution_trigger_claim_expiry_idx + ON moa.execution_trigger (claim_expires_at, due_at, trigger_uid) + WHERE state = 'dispatching'; + +CREATE INDEX execution_trigger_dead_letter_idx + ON moa.execution_trigger (created_at, tenant_id, trigger_uid) + WHERE state = 'dead_letter'; + +CREATE INDEX execution_trigger_run_wake_idx + ON moa.execution_trigger (run_uid, due_at, trigger_uid) + WHERE run_uid IS NOT NULL AND state IN ('pending', 'dispatching'); + +CREATE OR REPLACE FUNCTION moa.execution_attempt_cancel_payload_is_valid( + candidate JSONB, + expected_kind TEXT, + expected_dispatch_uid UUID, + expected_tenant_id UUID, + expected_run_uid UUID, + expected_owner_uid UUID, + expected_controller_generation BIGINT, + expected_attempt_generation BIGINT, + expected_compensation_generation BIGINT +) RETURNS BOOLEAN +LANGUAGE sql +IMMUTABLE +AS $$ + SELECT CASE expected_kind + WHEN 'task_attempt_cancel' THEN + moa.execution_json_object_has_exact_keys(candidate, ARRAY[ + 'dispatch_uid', 'tenant_id', 'run_uid', 'task_id', + 'controller_generation', 'attempt_controller_generation', + 'task_generation', 'attempt_generation', + 'active_dispatch_uid', 'capacity_reservation_uid', + 'watchdog_trigger_uid', 'reason' + ]) + AND candidate ->> 'task_id' = expected_owner_uid::TEXT + AND (candidate ->> 'task_generation') ~ '^[1-9][0-9]*$' + AND (candidate ->> 'attempt_generation')::BIGINT + = expected_attempt_generation + WHEN 'compensation_attempt_cancel' THEN + moa.execution_json_object_has_exact_keys(candidate, ARRAY[ + 'dispatch_uid', 'tenant_id', 'run_uid', 'compensation_id', + 'controller_generation', 'attempt_controller_generation', + 'compensation_generation', + 'compensation_attempt_generation', 'active_dispatch_uid', + 'capacity_reservation_uid', 'watchdog_trigger_uid', 'intent' + ]) + AND candidate ->> 'compensation_id' = expected_owner_uid::TEXT + AND (candidate ->> 'compensation_generation')::BIGINT + = expected_compensation_generation + AND (candidate ->> 'compensation_attempt_generation')::BIGINT + = expected_attempt_generation + ELSE FALSE + END + AND candidate ->> 'dispatch_uid' = expected_dispatch_uid::TEXT + AND candidate ->> 'tenant_id' = expected_tenant_id::TEXT + AND candidate ->> 'run_uid' = expected_run_uid::TEXT + AND (candidate ->> 'controller_generation')::BIGINT + = expected_controller_generation + AND candidate ->> 'attempt_controller_generation' ~ '^[1-9][0-9]*$' + AND candidate ->> 'active_dispatch_uid' ~ + '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + AND candidate ->> 'capacity_reservation_uid' ~ + '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + AND candidate ->> 'watchdog_trigger_uid' ~ + '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + AND octet_length( + candidate ->> CASE expected_kind + WHEN 'task_attempt_cancel' THEN 'reason' + WHEN 'compensation_attempt_cancel' THEN 'intent' + END + ) BETWEEN 1 AND 512 +$$; + +CREATE TABLE moa.execution_dispatch_outbox ( + dispatch_uid UUID PRIMARY KEY, + tenant_id UUID NOT NULL, + run_uid UUID, + task_id UUID, + compensation_id UUID, + trigger_uid UUID, + external_job_uid UUID, + dispatch_kind TEXT NOT NULL CHECK (dispatch_kind IN ( + 'run_activation', 'task_attempt', 'compensation_attempt', + 'task_attempt_cancel', 'compensation_attempt_cancel', + 'trigger_delivery', 'external_cancel' + )), + state TEXT NOT NULL DEFAULT 'pending' CHECK (state IN ( + 'pending', 'dispatching', 'delivered', 'superseded', 'cancelled', 'dead_letter' + )), + controller_generation BIGINT CHECK (controller_generation >= 1), + wake_epoch BIGINT CHECK (wake_epoch >= 1), + attempt_generation BIGINT CHECK (attempt_generation >= 1), + compensation_generation BIGINT CHECK (compensation_generation >= 1), + compensation_attempt_generation BIGINT + CHECK (compensation_attempt_generation >= 1), + not_before_at TIMESTAMPTZ NOT NULL DEFAULT now(), + payload JSONB NOT NULL DEFAULT '{}'::JSONB CHECK (jsonb_typeof(payload) = 'object'), + claim_owner TEXT, + claimed_at TIMESTAMPTZ, + claim_expires_at TIMESTAMPTZ, + delivery_attempts INTEGER NOT NULL DEFAULT 0 CHECK (delivery_attempts >= 0), + delivered_at TIMESTAMPTZ, + last_error TEXT CHECK (last_error IS NULL OR octet_length(last_error) <= 4096), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT execution_dispatch_outbox_id_tenant_key + UNIQUE (dispatch_uid, tenant_id), + CONSTRAINT execution_dispatch_outbox_task_identity_key + UNIQUE (dispatch_uid, tenant_id, run_uid, task_id), + CONSTRAINT execution_dispatch_outbox_compensation_identity_key + UNIQUE (dispatch_uid, tenant_id, run_uid, compensation_id), + CONSTRAINT execution_dispatch_outbox_run_tenant_fk + FOREIGN KEY (run_uid, tenant_id) + REFERENCES moa.execution_run (run_uid, tenant_id) ON DELETE CASCADE, + CONSTRAINT execution_dispatch_outbox_task_tenant_fk + FOREIGN KEY (task_id, run_uid, tenant_id) + REFERENCES moa.execution_task (task_id, run_uid, tenant_id) ON DELETE CASCADE, + CONSTRAINT execution_dispatch_outbox_compensation_tenant_fk + FOREIGN KEY (compensation_id, run_uid, tenant_id) + REFERENCES moa.execution_compensation ( + compensation_id, run_uid, tenant_id + ) ON DELETE CASCADE, + CONSTRAINT execution_dispatch_outbox_trigger_tenant_fk + FOREIGN KEY (trigger_uid, tenant_id) + REFERENCES moa.execution_trigger (trigger_uid, tenant_id) ON DELETE CASCADE, + CONSTRAINT execution_dispatch_outbox_external_job_tenant_fk + FOREIGN KEY (external_job_uid, tenant_id) + REFERENCES moa.execution_external_job (external_job_uid, tenant_id) ON DELETE CASCADE, + CONSTRAINT execution_dispatch_outbox_target_shape_check CHECK ( + ( + dispatch_kind = 'run_activation' + AND run_uid IS NOT NULL AND task_id IS NULL AND trigger_uid IS NULL + AND compensation_id IS NULL AND external_job_uid IS NULL + AND controller_generation IS NOT NULL + AND wake_epoch IS NOT NULL AND attempt_generation IS NULL + AND compensation_generation IS NULL + AND compensation_attempt_generation IS NULL + ) OR ( + dispatch_kind = 'task_attempt' + AND run_uid IS NOT NULL AND task_id IS NOT NULL AND trigger_uid IS NULL + AND compensation_id IS NULL AND external_job_uid IS NULL + AND controller_generation IS NOT NULL + AND wake_epoch IS NULL AND attempt_generation IS NOT NULL + AND compensation_generation IS NULL + AND compensation_attempt_generation IS NULL + ) OR ( + dispatch_kind = 'compensation_attempt' + AND run_uid IS NOT NULL AND task_id IS NULL AND trigger_uid IS NULL + AND compensation_id IS NOT NULL AND external_job_uid IS NULL + AND controller_generation IS NOT NULL AND wake_epoch IS NULL + AND attempt_generation IS NULL AND compensation_generation IS NOT NULL + AND compensation_attempt_generation IS NOT NULL + ) OR ( + dispatch_kind = 'task_attempt_cancel' + AND run_uid IS NOT NULL AND task_id IS NOT NULL AND trigger_uid IS NULL + AND compensation_id IS NULL AND external_job_uid IS NULL + AND controller_generation IS NOT NULL + AND wake_epoch IS NULL AND attempt_generation IS NOT NULL + AND compensation_generation IS NULL + AND compensation_attempt_generation IS NULL + AND moa.execution_attempt_cancel_payload_is_valid( + payload, dispatch_kind, dispatch_uid, tenant_id, run_uid, task_id, + controller_generation, attempt_generation, NULL + ) + ) OR ( + dispatch_kind = 'compensation_attempt_cancel' + AND run_uid IS NOT NULL AND task_id IS NULL AND trigger_uid IS NULL + AND compensation_id IS NOT NULL AND external_job_uid IS NULL + AND controller_generation IS NOT NULL AND wake_epoch IS NULL + AND attempt_generation IS NULL AND compensation_generation IS NOT NULL + AND compensation_attempt_generation IS NOT NULL + AND moa.execution_attempt_cancel_payload_is_valid( + payload, dispatch_kind, dispatch_uid, tenant_id, run_uid, + compensation_id, controller_generation, + compensation_attempt_generation, compensation_generation + ) + ) OR ( + dispatch_kind = 'trigger_delivery' + AND trigger_uid IS NOT NULL AND run_uid IS NULL AND task_id IS NULL + AND compensation_id IS NULL AND external_job_uid IS NULL + AND controller_generation IS NULL AND wake_epoch IS NULL + AND attempt_generation IS NULL AND compensation_generation IS NULL + AND compensation_attempt_generation IS NULL + ) OR ( + dispatch_kind = 'external_cancel' + AND run_uid IS NOT NULL AND trigger_uid IS NULL + AND external_job_uid IS NOT NULL + AND controller_generation IS NOT NULL AND wake_epoch IS NULL + AND ( + ( + task_id IS NOT NULL AND attempt_generation IS NOT NULL + AND compensation_id IS NULL + AND compensation_generation IS NULL + AND compensation_attempt_generation IS NULL + ) + OR + ( + task_id IS NULL AND attempt_generation IS NULL + AND compensation_id IS NOT NULL + AND compensation_generation IS NOT NULL + AND compensation_attempt_generation IS NOT NULL + ) + ) + ) + ), + CONSTRAINT execution_dispatch_outbox_claim_pair_check CHECK ( + (claim_owner IS NULL AND claimed_at IS NULL AND claim_expires_at IS NULL) + OR ( + claim_owner IS NOT NULL AND btrim(claim_owner) <> '' + AND claimed_at IS NOT NULL AND claim_expires_at IS NOT NULL + AND claim_expires_at > claimed_at + ) + ), + CONSTRAINT execution_dispatch_outbox_delivery_pair_check CHECK ( + (state = 'delivered') = (delivered_at IS NOT NULL) + ) +); + +CREATE UNIQUE INDEX execution_dispatch_outbox_run_activation_uidx + ON moa.execution_dispatch_outbox ( + tenant_id, run_uid, controller_generation, wake_epoch, dispatch_kind + ) + WHERE dispatch_kind = 'run_activation'; + +CREATE UNIQUE INDEX execution_dispatch_outbox_task_attempt_uidx + ON moa.execution_dispatch_outbox ( + tenant_id, run_uid, task_id, attempt_generation, dispatch_kind + ) + WHERE dispatch_kind = 'task_attempt'; + +CREATE UNIQUE INDEX execution_dispatch_outbox_compensation_attempt_uidx + ON moa.execution_dispatch_outbox ( + tenant_id, run_uid, compensation_id, compensation_generation, + compensation_attempt_generation, dispatch_kind + ) + WHERE dispatch_kind = 'compensation_attempt'; + +CREATE UNIQUE INDEX execution_dispatch_outbox_task_attempt_cancel_uidx + ON moa.execution_dispatch_outbox ( + tenant_id, run_uid, task_id, controller_generation, + attempt_generation, dispatch_kind, + (payload ->> 'active_dispatch_uid'), (payload ->> 'reason') + ) + WHERE dispatch_kind = 'task_attempt_cancel'; + +CREATE UNIQUE INDEX execution_dispatch_outbox_compensation_attempt_cancel_uidx + ON moa.execution_dispatch_outbox ( + tenant_id, run_uid, compensation_id, controller_generation, + compensation_generation, compensation_attempt_generation, dispatch_kind, + (payload ->> 'active_dispatch_uid'), (payload ->> 'intent') + ) + WHERE dispatch_kind = 'compensation_attempt_cancel'; + +CREATE UNIQUE INDEX execution_dispatch_outbox_trigger_delivery_uidx + ON moa.execution_dispatch_outbox (tenant_id, trigger_uid, dispatch_kind) + WHERE dispatch_kind = 'trigger_delivery'; + +CREATE UNIQUE INDEX execution_dispatch_outbox_external_cancel_uidx + ON moa.execution_dispatch_outbox (tenant_id, external_job_uid, dispatch_kind) + WHERE dispatch_kind = 'external_cancel'; + +CREATE INDEX execution_dispatch_outbox_pending_idx + ON moa.execution_dispatch_outbox (not_before_at, created_at, dispatch_uid) + WHERE state = 'pending'; + +CREATE INDEX execution_dispatch_outbox_claim_expiry_idx + ON moa.execution_dispatch_outbox (claim_expires_at, created_at, dispatch_uid) + WHERE state = 'dispatching'; + +CREATE INDEX execution_dispatch_outbox_dead_letter_idx + ON moa.execution_dispatch_outbox (created_at, tenant_id, dispatch_uid) + WHERE state = 'dead_letter'; + +ALTER TABLE moa.execution_task + ADD CONSTRAINT execution_task_active_dispatch_tenant_fk + FOREIGN KEY (active_dispatch_uid, tenant_id, run_uid, task_id) + REFERENCES moa.execution_dispatch_outbox ( + dispatch_uid, tenant_id, run_uid, task_id + ) + ON DELETE SET NULL (active_dispatch_uid); + +ALTER TABLE moa.execution_compensation + ADD CONSTRAINT execution_compensation_active_dispatch_tenant_fk + FOREIGN KEY (active_dispatch_uid, tenant_id, run_uid, compensation_id) + REFERENCES moa.execution_dispatch_outbox ( + dispatch_uid, tenant_id, run_uid, compensation_id + ) + ON DELETE SET NULL (active_dispatch_uid); + +-- Agent and direct capability-review continuations are bounded snapshots, not +-- live workflow journals. Replacing the current checkpoint only supersedes the +-- prior immutable row; historical rows remain page-retainable. +CREATE TABLE moa.execution_task_checkpoint ( + checkpoint_uid UUID PRIMARY KEY, + tenant_id UUID NOT NULL, + run_uid UUID NOT NULL, + task_id UUID NOT NULL, + checkpoint_sequence BIGINT NOT NULL CHECK (checkpoint_sequence >= 1), + controller_generation BIGINT CHECK (controller_generation >= 1), + task_generation BIGINT NOT NULL CHECK (task_generation >= 1), + attempt_generation BIGINT NOT NULL CHECK (attempt_generation >= 1), + dispatch_uid UUID NOT NULL, + checkpoint_kind TEXT NOT NULL CHECK (checkpoint_kind IN ( + 'agent_continuation', 'capability_review', 'capability_external_start' + )), + schema_version BIGINT NOT NULL CHECK (schema_version >= 1), + payload JSONB NOT NULL CHECK ( + jsonb_typeof(payload) = 'object' + AND pg_column_size(payload) <= 1048576 + ), + payload_hash TEXT NOT NULL CHECK (payload_hash ~ '^[0-9a-f]{64}$'), + workspace_release_receipt JSONB CHECK ( + workspace_release_receipt IS NULL + OR ( + jsonb_typeof(workspace_release_receipt) = 'object' + AND pg_column_size(workspace_release_receipt) <= 262144 + ) + ), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + superseded_at TIMESTAMPTZ, + CONSTRAINT execution_task_checkpoint_id_tenant_key + UNIQUE (checkpoint_uid, tenant_id), + CONSTRAINT execution_task_checkpoint_sequence_key + UNIQUE (tenant_id, run_uid, task_id, checkpoint_sequence), + CONSTRAINT execution_task_checkpoint_task_tenant_fk + FOREIGN KEY (task_id, run_uid, tenant_id) + REFERENCES moa.execution_task (task_id, run_uid, tenant_id) + ON DELETE CASCADE, + CONSTRAINT execution_task_checkpoint_dispatch_tenant_fk + FOREIGN KEY (dispatch_uid, tenant_id, run_uid, task_id) + REFERENCES moa.execution_dispatch_outbox ( + dispatch_uid, tenant_id, run_uid, task_id + ) ON DELETE RESTRICT, + CONSTRAINT execution_task_checkpoint_supersession_order_check CHECK ( + superseded_at IS NULL OR superseded_at >= created_at + ) +); + +CREATE UNIQUE INDEX execution_task_checkpoint_current_uidx + ON moa.execution_task_checkpoint (tenant_id, run_uid, task_id) + WHERE superseded_at IS NULL; + +CREATE INDEX execution_task_checkpoint_retention_idx + ON moa.execution_task_checkpoint ( + superseded_at, created_at, tenant_id, run_uid, task_id, checkpoint_sequence + ) + WHERE superseded_at IS NOT NULL; + +CREATE OR REPLACE FUNCTION moa.enforce_execution_task_checkpoint_update() +RETURNS TRIGGER +LANGUAGE plpgsql +AS $$ +BEGIN + IF NEW.checkpoint_uid IS DISTINCT FROM OLD.checkpoint_uid + OR NEW.tenant_id IS DISTINCT FROM OLD.tenant_id + OR NEW.run_uid IS DISTINCT FROM OLD.run_uid + OR NEW.task_id IS DISTINCT FROM OLD.task_id + OR NEW.checkpoint_sequence IS DISTINCT FROM OLD.checkpoint_sequence + OR NEW.controller_generation IS DISTINCT FROM OLD.controller_generation + OR NEW.task_generation IS DISTINCT FROM OLD.task_generation + OR NEW.attempt_generation IS DISTINCT FROM OLD.attempt_generation + OR NEW.dispatch_uid IS DISTINCT FROM OLD.dispatch_uid + OR NEW.checkpoint_kind IS DISTINCT FROM OLD.checkpoint_kind + OR NEW.schema_version IS DISTINCT FROM OLD.schema_version + OR NEW.payload IS DISTINCT FROM OLD.payload + OR NEW.payload_hash IS DISTINCT FROM OLD.payload_hash + OR NEW.workspace_release_receipt IS DISTINCT FROM OLD.workspace_release_receipt + OR NEW.created_at IS DISTINCT FROM OLD.created_at + OR OLD.superseded_at IS NOT NULL + OR NEW.superseded_at IS NULL THEN + RAISE EXCEPTION 'execution task checkpoints are append-only and may only be superseded once'; + END IF; + RETURN NEW; +END; +$$; + +CREATE TRIGGER execution_task_checkpoint_update_guard +BEFORE UPDATE ON moa.execution_task_checkpoint +FOR EACH ROW EXECUTE FUNCTION moa.enforce_execution_task_checkpoint_update(); + +CREATE TABLE moa.execution_capacity_bucket ( + capacity_bucket_uid UUID PRIMARY KEY, + scope_kind TEXT NOT NULL CHECK (scope_kind IN ('fleet', 'tenant')), + tenant_id UUID, + resource_dimension TEXT NOT NULL CHECK (resource_dimension IN ( + 'active_runs', 'active_tasks', 'parked_runs', 'scheduled_triggers', + 'external_jobs' + )), + limit_value BIGINT NOT NULL CHECK (limit_value > 0), + reserved_quantity BIGINT NOT NULL DEFAULT 0 + CHECK (reserved_quantity >= 0), + version BIGINT NOT NULL DEFAULT 1 CHECK (version >= 1), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT execution_capacity_bucket_scope_check CHECK ( + (scope_kind = 'fleet' AND tenant_id IS NULL) + OR (scope_kind = 'tenant' AND tenant_id IS NOT NULL) + ), + CONSTRAINT execution_capacity_bucket_tenant_resource_key + UNIQUE (scope_kind, tenant_id, resource_dimension) +); + +CREATE UNIQUE INDEX execution_capacity_bucket_fleet_resource_uidx + ON moa.execution_capacity_bucket (resource_dimension) + WHERE scope_kind = 'fleet'; + +CREATE INDEX execution_capacity_bucket_lock_order_idx + ON moa.execution_capacity_bucket ( + resource_dimension, scope_kind, tenant_id, capacity_bucket_uid + ); + +CREATE FUNCTION moa.enforce_execution_capacity_bucket_owner_immutable() +RETURNS TRIGGER +LANGUAGE plpgsql +AS $$ +BEGIN + IF NEW.capacity_bucket_uid IS DISTINCT FROM OLD.capacity_bucket_uid + OR NEW.scope_kind IS DISTINCT FROM OLD.scope_kind + OR NEW.tenant_id IS DISTINCT FROM OLD.tenant_id + OR NEW.resource_dimension IS DISTINCT FROM OLD.resource_dimension THEN + RAISE EXCEPTION 'execution capacity bucket owner coordinates are immutable'; + END IF; + RETURN NEW; +END; +$$; + +CREATE TRIGGER execution_capacity_bucket_owner_immutable +BEFORE UPDATE OF capacity_bucket_uid, scope_kind, tenant_id, resource_dimension +ON moa.execution_capacity_bucket +FOR EACH ROW EXECUTE FUNCTION moa.enforce_execution_capacity_bucket_owner_immutable(); + +CREATE TABLE moa.execution_tenant_dispatch_state ( + tenant_id UUID PRIMARY KEY, + weight NUMERIC(20, 6) NOT NULL DEFAULT 1 CHECK (weight > 0), + virtual_finish NUMERIC(30, 6) NOT NULL DEFAULT 0 CHECK (virtual_finish >= 0), + deficit NUMERIC(30, 6) NOT NULL DEFAULT 0, + last_dispatched_at TIMESTAMPTZ, + version BIGINT NOT NULL DEFAULT 1 CHECK (version >= 1), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX execution_tenant_dispatch_fairness_idx + ON moa.execution_tenant_dispatch_state ( + virtual_finish, last_dispatched_at, tenant_id + ); + +CREATE TABLE moa.execution_capacity_reservation ( + reservation_uid UUID PRIMARY KEY, + tenant_id UUID NOT NULL, + run_uid UUID, + task_id UUID, + compensation_id UUID, + trigger_uid UUID, + external_job_uid UUID, + controller_generation BIGINT CHECK (controller_generation >= 1), + attempt_generation BIGINT CHECK (attempt_generation >= 1), + compensation_generation BIGINT CHECK (compensation_generation >= 1), + compensation_attempt_generation BIGINT + CHECK (compensation_attempt_generation >= 1), + resource_dimension TEXT NOT NULL CHECK (resource_dimension IN ( + 'active_runs', 'active_tasks', 'parked_runs', 'scheduled_triggers', + 'external_jobs' + )), + quantity BIGINT NOT NULL CHECK (quantity > 0), + state TEXT NOT NULL DEFAULT 'reserved' + CHECK (state IN ('reserved', 'released', 'reconciling')), + expires_at TIMESTAMPTZ, + released_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT execution_capacity_reservation_id_tenant_key + UNIQUE (reservation_uid, tenant_id), + CONSTRAINT execution_capacity_reservation_run_tenant_fk + FOREIGN KEY (run_uid, tenant_id) + REFERENCES moa.execution_run (run_uid, tenant_id) ON DELETE CASCADE, + CONSTRAINT execution_capacity_reservation_task_tenant_fk + FOREIGN KEY (task_id, run_uid, tenant_id) + REFERENCES moa.execution_task (task_id, run_uid, tenant_id) ON DELETE CASCADE, + CONSTRAINT execution_capacity_reservation_compensation_tenant_fk + FOREIGN KEY (compensation_id, run_uid, tenant_id) + REFERENCES moa.execution_compensation ( + compensation_id, run_uid, tenant_id + ) ON DELETE CASCADE, + CONSTRAINT execution_capacity_reservation_trigger_tenant_fk + FOREIGN KEY (trigger_uid, tenant_id) + REFERENCES moa.execution_trigger (trigger_uid, tenant_id) + ON DELETE CASCADE, + CONSTRAINT execution_capacity_reservation_external_job_tenant_fk + FOREIGN KEY (external_job_uid, tenant_id) + REFERENCES moa.execution_external_job (external_job_uid, tenant_id) + ON DELETE CASCADE, + CONSTRAINT execution_capacity_reservation_owner_shape_check CHECK ( + (resource_dimension = 'active_tasks' + AND run_uid IS NOT NULL AND controller_generation IS NOT NULL + AND ( + ( + task_id IS NOT NULL AND attempt_generation IS NOT NULL + AND compensation_id IS NULL + AND compensation_generation IS NULL + AND compensation_attempt_generation IS NULL + AND trigger_uid IS NULL AND external_job_uid IS NULL + ) + OR ( + task_id IS NULL AND attempt_generation IS NULL + AND compensation_id IS NOT NULL + AND compensation_generation IS NOT NULL + AND compensation_attempt_generation IS NOT NULL + AND trigger_uid IS NULL AND external_job_uid IS NULL + ) + )) + OR + (resource_dimension IN ('active_runs', 'parked_runs') + AND run_uid IS NOT NULL AND controller_generation IS NOT NULL + AND task_id IS NULL AND attempt_generation IS NULL + AND compensation_id IS NULL + AND compensation_generation IS NULL + AND compensation_attempt_generation IS NULL + AND trigger_uid IS NULL AND external_job_uid IS NULL) + OR + (resource_dimension = 'scheduled_triggers' + AND trigger_uid IS NOT NULL AND external_job_uid IS NULL + AND (run_uid IS NULL) = (controller_generation IS NULL) + AND task_id IS NULL AND attempt_generation IS NULL + AND compensation_id IS NULL + AND compensation_generation IS NULL + AND compensation_attempt_generation IS NULL) + OR + (resource_dimension = 'external_jobs' + AND external_job_uid IS NOT NULL AND trigger_uid IS NULL + AND run_uid IS NOT NULL AND controller_generation IS NOT NULL + AND task_id IS NULL AND attempt_generation IS NULL + AND compensation_id IS NULL + AND compensation_generation IS NULL + AND compensation_attempt_generation IS NULL) + ), + CONSTRAINT execution_capacity_reservation_release_pair_check CHECK ( + (state = 'released') = (released_at IS NOT NULL) + ) +); + +CREATE UNIQUE INDEX execution_capacity_reservation_active_run_owner_uidx + ON moa.execution_capacity_reservation ( + tenant_id, run_uid, resource_dimension + ) + WHERE resource_dimension = 'active_runs'; + +CREATE UNIQUE INDEX execution_capacity_reservation_parked_run_owner_uidx + ON moa.execution_capacity_reservation ( + tenant_id, run_uid, resource_dimension + ) + WHERE resource_dimension = 'parked_runs' + AND state IN ('reserved', 'reconciling'); + +CREATE UNIQUE INDEX execution_capacity_reservation_task_owner_uidx + ON moa.execution_capacity_reservation ( + tenant_id, run_uid, task_id, resource_dimension, + controller_generation, attempt_generation + ) + WHERE task_id IS NOT NULL; + +CREATE UNIQUE INDEX execution_capacity_reservation_compensation_owner_uidx + ON moa.execution_capacity_reservation ( + tenant_id, run_uid, compensation_id, resource_dimension, + controller_generation, compensation_generation, + compensation_attempt_generation + ) + WHERE compensation_id IS NOT NULL; + +CREATE UNIQUE INDEX execution_capacity_reservation_trigger_owner_uidx + ON moa.execution_capacity_reservation ( + tenant_id, trigger_uid, resource_dimension + ) + WHERE resource_dimension = 'scheduled_triggers'; + +CREATE UNIQUE INDEX execution_capacity_reservation_external_job_owner_uidx + ON moa.execution_capacity_reservation ( + tenant_id, external_job_uid, resource_dimension + ) + WHERE resource_dimension = 'external_jobs'; + +CREATE INDEX execution_capacity_reservation_active_idx + ON moa.execution_capacity_reservation ( + tenant_id, resource_dimension, expires_at, reservation_uid + ) + WHERE state IN ('reserved', 'reconciling'); + +-- Provider calls are forbidden until the exact external-job capacity receipt +-- exists, and every nonterminal bound job retains that receipt. Deferral permits +-- intent insertion followed by reservation in one transaction and exact +-- terminalization followed by release in another. +CREATE OR REPLACE FUNCTION moa.enforce_execution_external_job_intent_capacity() +RETURNS TRIGGER +LANGUAGE plpgsql +AS $$ +BEGIN + IF EXISTS ( + SELECT 1 + FROM moa.execution_external_job AS job + WHERE job.external_job_uid = NEW.external_job_uid + AND job.tenant_id = NEW.tenant_id + AND job.state IN ( + 'unbound', 'starting', 'running', 'waiting_reconcile', + 'cancel_requested' + ) + ) AND NOT EXISTS ( + SELECT 1 + FROM moa.execution_capacity_reservation AS reservation + WHERE reservation.tenant_id = NEW.tenant_id + AND reservation.external_job_uid = NEW.external_job_uid + AND reservation.resource_dimension = 'external_jobs' + AND reservation.state IN ('reserved', 'reconciling') + ) THEN + RAISE EXCEPTION 'nonterminal execution external job requires active capacity'; + END IF; + RETURN NEW; +END; +$$; + +CREATE CONSTRAINT TRIGGER execution_external_job_intent_capacity_guard +AFTER INSERT OR UPDATE OF state ON moa.execution_external_job +DEFERRABLE INITIALLY DEFERRED +FOR EACH ROW EXECUTE FUNCTION moa.enforce_execution_external_job_intent_capacity(); + +-- Preserve the existing immutable-field and evidence guards while extending +-- only their finite transition tables for the hard-break state model. +DO $execution_run_long_horizon_transitions$ +DECLARE + definition TEXT; + old_block TEXT := $old$ + transition_allowed := CASE OLD.status + WHEN 'awaiting_confirmation' THEN NEW.status IN ('queued', 'cancelled') + WHEN 'queued' THEN NEW.status IN ( + 'running', 'compensating', 'blocked', 'unsupported', 'failed', 'cancelled' + ) + WHEN 'running' THEN NEW.status IN ( + 'waiting_input', 'waiting_review', 'waiting_replan', 'compensating', + 'completed', 'partial', 'blocked', 'unsupported', 'failed', 'cancelled' + ) + WHEN 'waiting_input' THEN NEW.status IN ( + 'running', 'compensating', 'partial', 'blocked', 'unsupported', + 'failed', 'cancelled' + ) + WHEN 'waiting_review' THEN NEW.status IN ( + 'running', 'compensating', 'partial', 'blocked', 'unsupported', + 'failed', 'cancelled' + ) + WHEN 'waiting_replan' THEN NEW.status IN ( + 'running', 'compensating', 'partial', 'blocked', 'unsupported', + 'failed', 'cancelled' + ) + WHEN 'compensating' THEN NEW.status IN ( + 'completed', 'partial', 'blocked', 'unsupported', 'failed', 'cancelled' + ) + ELSE FALSE + END; +$old$; + new_block TEXT := $new$ + transition_allowed := CASE OLD.status + WHEN 'awaiting_confirmation' THEN NEW.status IN ('queued', 'cancelled') + WHEN 'queued' THEN NEW.status IN ( + 'running', 'waiting_review', 'waiting_signal', 'waiting_timer', + 'pause_requested', 'compensating', 'blocked', 'unsupported', + 'failed', 'cancelled' + ) + WHEN 'running' THEN NEW.status IN ( + 'waiting_input', 'waiting_review', 'waiting_signal', 'waiting_timer', + 'waiting_external', 'waiting_replan', 'pause_requested', 'compensating', + 'completed', 'partial', 'blocked', 'unsupported', 'failed', 'cancelled' + ) + WHEN 'waiting_input' THEN NEW.status IN ( + 'running', 'waiting_review', 'waiting_signal', 'waiting_timer', + 'waiting_external', 'waiting_replan', 'pause_requested', 'compensating', + 'partial', 'blocked', 'unsupported', 'failed', 'cancelled' + ) + WHEN 'waiting_review' THEN NEW.status IN ( + 'running', 'waiting_input', 'waiting_signal', 'waiting_timer', + 'waiting_external', 'waiting_replan', 'pause_requested', 'compensating', + 'partial', 'blocked', 'unsupported', 'failed', 'cancelled' + ) + WHEN 'waiting_signal' THEN NEW.status IN ( + 'running', 'waiting_input', 'waiting_review', 'waiting_timer', + 'waiting_external', 'waiting_replan', 'pause_requested', 'compensating', + 'partial', 'blocked', 'unsupported', 'failed', 'cancelled' + ) + WHEN 'waiting_timer' THEN NEW.status IN ( + 'running', 'waiting_input', 'waiting_review', 'waiting_signal', + 'waiting_external', 'waiting_replan', 'pause_requested', 'compensating', + 'partial', 'blocked', 'unsupported', 'failed', 'cancelled' + ) + WHEN 'waiting_external' THEN NEW.status IN ( + 'running', 'waiting_input', 'waiting_review', 'waiting_signal', + 'waiting_timer', 'waiting_replan', 'pause_requested', 'compensating', + 'partial', 'blocked', 'unsupported', 'failed', 'cancelled' + ) + WHEN 'waiting_replan' THEN NEW.status IN ( + 'running', 'waiting_input', 'waiting_review', 'waiting_signal', + 'waiting_timer', 'waiting_external', 'pause_requested', 'compensating', + 'partial', 'blocked', 'unsupported', 'failed', 'cancelled' + ) + WHEN 'pause_requested' THEN NEW.status IN ( + 'pausing', 'paused', 'running', 'cancelled' + ) + WHEN 'pausing' THEN NEW.status IN ('paused', 'failed', 'cancelled') + WHEN 'paused' THEN NEW.status IN ('queued', 'cancelled') + WHEN 'compensating' THEN NEW.status IN ( + 'completed', 'partial', 'blocked', 'unsupported', 'failed', 'cancelled' + ) + ELSE FALSE + END; +$new$; +BEGIN + SELECT pg_get_functiondef('moa.enforce_execution_run_update()'::REGPROCEDURE) + INTO definition; + IF position(old_block IN definition) = 0 THEN + RAISE EXCEPTION 'execution run transition table drifted before V59' + USING ERRCODE = '55000'; + END IF; + EXECUTE replace(definition, old_block, new_block); +END +$execution_run_long_horizon_transitions$; + +CREATE OR REPLACE FUNCTION moa.enforce_execution_task_update() +RETURNS TRIGGER +LANGUAGE plpgsql +AS $$ +DECLARE + transition_allowed BOOLEAN; +BEGIN + IF NEW.task_id IS DISTINCT FROM OLD.task_id + OR NEW.run_uid IS DISTINCT FROM OLD.run_uid + OR NEW.tenant_id IS DISTINCT FROM OLD.tenant_id + OR NEW.contact_id IS DISTINCT FROM OLD.contact_id + OR NEW.node_id IS DISTINCT FROM OLD.node_id + OR NEW.item_key IS DISTINCT FROM OLD.item_key + OR NEW.requirement_ids IS DISTINCT FROM OLD.requirement_ids + OR NEW.plan_revision IS DISTINCT FROM OLD.plan_revision + OR NEW.input IS DISTINCT FROM OLD.input + OR NEW.task_kind IS DISTINCT FROM OLD.task_kind + OR NEW.compensation_contract IS DISTINCT FROM OLD.compensation_contract + OR NEW.retry_policy IS DISTINCT FROM OLD.retry_policy + OR NEW.estimate_cost_microusd IS DISTINCT FROM OLD.estimate_cost_microusd + OR NEW.estimate_tokens IS DISTINCT FROM OLD.estimate_tokens + OR NEW.estimate_tasks IS DISTINCT FROM OLD.estimate_tasks + OR NEW.estimate_tool_calls IS DISTINCT FROM OLD.estimate_tool_calls + OR NEW.estimate_retrieved_bytes IS DISTINCT FROM OLD.estimate_retrieved_bytes + OR NEW.created_at IS DISTINCT FROM OLD.created_at THEN + RAISE EXCEPTION 'execution task immutable fields cannot change'; + END IF; + + IF NOT moa.execution_jsonb_array_has_prefix( + NEW.resume_input_history, OLD.resume_input_history + ) + OR NOT moa.execution_jsonb_array_has_prefix( + NEW.generation_history, OLD.generation_history + ) + OR NOT moa.execution_jsonb_array_has_prefix(NEW.outcome_audit, OLD.outcome_audit) THEN + RAISE EXCEPTION 'execution task histories are append-only'; + END IF; + + IF OLD.status = 'waiting_input' AND NEW.status = 'ready' THEN + IF NEW.attempt <> OLD.attempt + OR NEW.generation <> OLD.generation + 1 + OR NEW.attempt_generation <> OLD.attempt_generation + 1 THEN + RAISE EXCEPTION 'execution input resume must advance generation fences exactly once'; + END IF; + ELSIF OLD.status = 'running' + AND NEW.status = 'ready' + AND ( + NEW.attempt IS DISTINCT FROM OLD.attempt + OR NEW.generation IS DISTINCT FROM OLD.generation + ) THEN + IF NEW.attempt <> OLD.attempt + 1 + OR NEW.generation <> OLD.generation + 1 + OR NEW.attempt_generation <> OLD.attempt_generation + 1 THEN + RAISE EXCEPTION 'execution retry must advance attempt and generation fences exactly once'; + END IF; + ELSIF NEW.attempt IS DISTINCT FROM OLD.attempt + OR NEW.generation IS DISTINCT FROM OLD.generation THEN + RAISE EXCEPTION 'execution task counters changed outside retry or input resume'; + END IF; + + IF NEW.attempt_generation IS DISTINCT FROM OLD.attempt_generation + AND ( + NEW.attempt_generation <> OLD.attempt_generation + 1 + OR OLD.status = 'ready' + OR NEW.status <> 'ready' + OR NEW.attempt_state <> 'idle' + ) THEN + RAISE EXCEPTION 'execution task attempt generation must advance once into ready idle'; + END IF; + IF OLD.attempt_state = 'cancelling' + AND NEW.attempt_state NOT IN ( + 'cancelling', 'idle', 'waiting', 'terminal', 'unknown_outcome' + ) THEN + RAISE EXCEPTION 'execution task cancelling state cannot become dispatchable'; + END IF; + IF NEW.dispatch_sequence < OLD.dispatch_sequence THEN + RAISE EXCEPTION 'execution task dispatch sequence must be monotonic'; + END IF; + IF OLD.last_progress_at IS NOT NULL + AND NEW.last_progress_at < OLD.last_progress_at THEN + RAISE EXCEPTION 'execution task last progress timestamp must be monotonic'; + END IF; + + IF NEW.status IS NOT DISTINCT FROM OLD.status THEN + RETURN NEW; + END IF; + + transition_allowed := CASE OLD.status + WHEN 'pending' THEN NEW.status IN ( + 'ready', 'reserved', 'waiting_review', 'waiting_signal', + 'waiting_timer', 'skipped', 'cancelled' + ) + WHEN 'ready' THEN NEW.status IN ('dispatching', 'reserved', 'cancelled') + WHEN 'reserved' THEN NEW.status IN ('dispatching', 'running', 'cancelled') + WHEN 'dispatching' THEN NEW.status IN ('running', 'ready', 'failed', 'cancelled') + WHEN 'running' THEN NEW.status IN ( + 'ready', 'waiting_input', 'waiting_review', 'waiting_signal', + 'waiting_timer', 'waiting_external', 'waiting_replan', 'completed', + 'failed', 'cancelled', 'unknown_outcome' + ) + WHEN 'waiting_input' THEN NEW.status IN ('ready', 'cancelled') + WHEN 'waiting_review' THEN NEW.status IN ('running', 'ready', 'cancelled') + WHEN 'waiting_signal' THEN NEW.status IN ('running', 'ready', 'cancelled') + WHEN 'waiting_timer' THEN NEW.status IN ('running', 'ready', 'cancelled') + WHEN 'waiting_external' THEN NEW.status IN ( + 'ready', 'completed', 'failed', 'cancelled', 'unknown_outcome' + ) + WHEN 'waiting_replan' THEN NEW.status IN ('ready', 'cancelled') + ELSE FALSE + END; + IF NOT transition_allowed THEN + RAISE EXCEPTION 'invalid execution task status transition: % -> %', + OLD.status, NEW.status; + END IF; + RETURN NEW; +END; +$$; + +CREATE OR REPLACE FUNCTION moa.enforce_execution_run_long_horizon_update() +RETURNS TRIGGER +LANGUAGE plpgsql +AS $$ +BEGIN + IF NEW.admitted_identity IS DISTINCT FROM OLD.admitted_identity THEN + RAISE EXCEPTION 'execution run admitted identity is immutable'; + END IF; + IF OLD.terminal_archive_uid IS NOT NULL + AND ( + NEW.terminal_archive_uid IS DISTINCT FROM OLD.terminal_archive_uid + OR NEW.terminal_archive_hash IS DISTINCT FROM OLD.terminal_archive_hash + OR NEW.terminal_details_archived_at + IS DISTINCT FROM OLD.terminal_details_archived_at + ) THEN + RAISE EXCEPTION 'execution run terminal archive binding is immutable'; + END IF; + IF NEW.terminal_archive_uid IS NOT NULL + AND NOT EXISTS ( + SELECT 1 + FROM moa.execution_terminal_archive AS archive + WHERE archive.archive_uid = NEW.terminal_archive_uid + AND archive.tenant_id = NEW.tenant_id + AND archive.run_uid = NEW.run_uid + AND archive.root_digest = NEW.terminal_archive_hash + AND archive.finalized_at IS NOT NULL + ) THEN + RAISE EXCEPTION 'execution run terminal archive binding is not exact'; + END IF; + IF NEW.schedule_uid IS DISTINCT FROM OLD.schedule_uid + OR NEW.schedule_incarnation IS DISTINCT FROM OLD.schedule_incarnation + OR NEW.schedule_occurrence_sequence IS DISTINCT FROM OLD.schedule_occurrence_sequence THEN + RAISE EXCEPTION 'execution run schedule occurrence identity is immutable'; + END IF; + IF NEW.active_plan IS DISTINCT FROM OLD.active_plan + AND NOT moa.execution_plan_snapshot_is_current(NEW.active_plan) THEN + RAISE EXCEPTION 'execution run amendment must use the current plan contract'; + END IF; + IF NEW.controller_generation < OLD.controller_generation THEN + RAISE EXCEPTION 'execution run controller generation must be monotonic'; + END IF; + IF NEW.ready_task_count < 0 OR NEW.active_task_count < 0 THEN + RAISE EXCEPTION 'execution run task counters cannot be negative'; + END IF; + IF OLD.status = 'pausing' AND NEW.active_task_count = 0 THEN + NEW.status := 'paused'; + NEW.activation_state := 'paused'; + NEW.paused_at := COALESCE(NEW.paused_at, now()); + END IF; + IF OLD.last_progress_at IS NOT NULL + AND NEW.last_progress_at < OLD.last_progress_at THEN + RAISE EXCEPTION 'execution run last progress timestamp must be monotonic'; + END IF; + RETURN NEW; +END; +$$; + +CREATE TRIGGER execution_run_long_horizon_update_guard +BEFORE UPDATE ON moa.execution_run +FOR EACH ROW EXECUTE FUNCTION moa.enforce_execution_run_long_horizon_update(); + +CREATE OR REPLACE FUNCTION moa.enforce_execution_compensation_long_horizon_update() +RETURNS TRIGGER +LANGUAGE plpgsql +AS $$ +BEGIN + IF NEW.attempt_generation < OLD.attempt_generation THEN + RAISE EXCEPTION 'execution compensation attempt generation must be monotonic'; + END IF; + IF OLD.attempt_state = 'cancelling' + AND NEW.attempt_state NOT IN ( + 'cancelling', 'idle', 'waiting_review', 'terminal', 'unknown_outcome' + ) THEN + RAISE EXCEPTION 'execution compensation cancelling state cannot become dispatchable'; + END IF; + IF OLD.attempt_state = 'cancelling' + AND NEW.attempt_state = 'cancelling' + AND NEW.release_intent IS DISTINCT FROM OLD.release_intent THEN + RAISE EXCEPTION 'execution compensation release intent is immutable while cancelling'; + END IF; + IF NEW.dispatch_sequence < OLD.dispatch_sequence THEN + RAISE EXCEPTION 'execution compensation dispatch sequence must be monotonic'; + END IF; + IF OLD.last_progress_at IS NOT NULL + AND NEW.last_progress_at < OLD.last_progress_at THEN + RAISE EXCEPTION 'execution compensation last progress timestamp must be monotonic'; + END IF; + RETURN NEW; +END; +$$; + +CREATE TRIGGER execution_compensation_long_horizon_update_guard +BEFORE UPDATE ON moa.execution_compensation +FOR EACH ROW EXECUTE FUNCTION moa.enforce_execution_compensation_long_horizon_update(); + +CREATE OR REPLACE FUNCTION moa.enforce_execution_dispatch_fairness_update() +RETURNS TRIGGER +LANGUAGE plpgsql +AS $$ +BEGIN + IF NEW.virtual_finish < OLD.virtual_finish THEN + RAISE EXCEPTION 'execution tenant virtual finish must be monotonic'; + END IF; + IF NEW.version < OLD.version THEN + RAISE EXCEPTION 'execution tenant dispatch version must be monotonic'; + END IF; + RETURN NEW; +END; +$$; + +CREATE TRIGGER execution_tenant_dispatch_fairness_update_guard +BEFORE UPDATE ON moa.execution_tenant_dispatch_state +FOR EACH ROW EXECUTE FUNCTION moa.enforce_execution_dispatch_fairness_update(); + +CREATE OR REPLACE FUNCTION moa.enforce_execution_schedule_update() +RETURNS TRIGGER +LANGUAGE plpgsql +AS $$ +BEGIN + IF NEW.template_revision_uid IS DISTINCT FROM OLD.template_revision_uid + OR NEW.template_snapshot IS DISTINCT FROM OLD.template_snapshot + OR NEW.template_hash IS DISTINCT FROM OLD.template_hash + OR NEW.run_as_identity IS DISTINCT FROM OLD.run_as_identity + OR NEW.creation_origin IS DISTINCT FROM OLD.creation_origin THEN + RAISE EXCEPTION 'execution schedule template, run-as identity, and origin are immutable'; + END IF; + IF NEW.schedule_incarnation < OLD.schedule_incarnation THEN + RAISE EXCEPTION 'execution schedule incarnation must be monotonic'; + END IF; + IF NEW.last_occurrence_sequence < 0 THEN + RAISE EXCEPTION 'execution schedule occurrence sequence cannot be negative'; + END IF; + RETURN NEW; +END; +$$; + +CREATE TRIGGER execution_schedule_update_guard +BEFORE UPDATE ON moa.execution_schedule +FOR EACH ROW EXECUTE FUNCTION moa.enforce_execution_schedule_update(); + +CREATE OR REPLACE FUNCTION moa.reject_execution_terminal_archive_mutation() +RETURNS TRIGGER +LANGUAGE plpgsql +AS $$ +BEGIN + IF TG_OP = 'UPDATE' + AND OLD.finalized_at IS NOT NULL + AND OLD.details_deleted_at IS NULL + AND NEW.details_deleted_at IS NOT NULL + AND NEW.archive_uid = OLD.archive_uid + AND NEW.tenant_id = OLD.tenant_id + AND NEW.run_uid = OLD.run_uid + AND NEW.contact_id IS NOT DISTINCT FROM OLD.contact_id + AND NEW.format_version = OLD.format_version + AND NEW.terminal_status = OLD.terminal_status + AND NEW.terminal_completed_at = OLD.terminal_completed_at + AND NEW.goal_hash = OLD.goal_hash + AND NEW.initial_plan_hash = OLD.initial_plan_hash + AND NEW.active_plan_hash = OLD.active_plan_hash + AND NEW.source_record_count = OLD.source_record_count + AND NEW.source_logical_bytes = OLD.source_logical_bytes + AND NEW.segment_count = OLD.segment_count + AND NEW.source_cursor = OLD.source_cursor + AND NEW.rolling_chain_digest = OLD.rolling_chain_digest + AND NEW.root_digest = OLD.root_digest + AND NEW.archive_generation = OLD.archive_generation + AND NEW.created_at = OLD.created_at + AND NEW.finalized_at = OLD.finalized_at + AND NOT EXISTS ( + SELECT 1 FROM moa.legal_hold AS hold + WHERE hold.tenant_id = OLD.tenant_id + AND hold.released_at IS NULL + AND (hold.subject_id IS NULL OR hold.subject_id = OLD.contact_id) + ) THEN + RETURN NEW; + END IF; + IF TG_OP = 'UPDATE' + AND OLD.finalized_at IS NULL + AND NEW.finalized_at IS NULL + AND NEW.root_digest IS NULL + AND NEW.details_deleted_at IS NULL + AND NEW.archive_uid = OLD.archive_uid + AND NEW.tenant_id = OLD.tenant_id + AND NEW.run_uid = OLD.run_uid + AND NEW.contact_id IS NOT DISTINCT FROM OLD.contact_id + AND NEW.format_version = OLD.format_version + AND NEW.terminal_status = OLD.terminal_status + AND NEW.terminal_completed_at = OLD.terminal_completed_at + AND NEW.goal_hash = OLD.goal_hash + AND NEW.initial_plan_hash = OLD.initial_plan_hash + AND NEW.active_plan_hash = OLD.active_plan_hash + AND NEW.archive_generation = OLD.archive_generation + AND NEW.created_at = OLD.created_at + AND ( + ( + NEW.segment_count = OLD.segment_count + 1 + AND NEW.source_record_count > OLD.source_record_count + AND NEW.source_logical_bytes > OLD.source_logical_bytes + AND NEW.rolling_chain_digest IS NOT NULL + AND NEW.rolling_chain_digest IS DISTINCT FROM OLD.rolling_chain_digest + ) + OR + ( + NEW.segment_count = OLD.segment_count + AND NEW.source_record_count = OLD.source_record_count + AND NEW.source_logical_bytes = OLD.source_logical_bytes + AND NEW.rolling_chain_digest IS NOT DISTINCT FROM OLD.rolling_chain_digest + AND NEW.source_cursor IS DISTINCT FROM OLD.source_cursor + ) + ) + AND NOT EXISTS ( + SELECT 1 FROM moa.legal_hold AS hold + WHERE hold.tenant_id = OLD.tenant_id + AND hold.released_at IS NULL + AND (hold.subject_id IS NULL OR hold.subject_id = OLD.contact_id) + ) THEN + RETURN NEW; + END IF; + IF TG_OP = 'UPDATE' + AND OLD.finalized_at IS NULL + AND NEW.finalized_at IS NOT NULL + AND NEW.root_digest IS NOT NULL + AND NEW.details_deleted_at IS NULL + AND NEW.archive_uid = OLD.archive_uid + AND NEW.tenant_id = OLD.tenant_id + AND NEW.run_uid = OLD.run_uid + AND NEW.contact_id IS NOT DISTINCT FROM OLD.contact_id + AND NEW.format_version = OLD.format_version + AND NEW.terminal_status = OLD.terminal_status + AND NEW.terminal_completed_at = OLD.terminal_completed_at + AND NEW.goal_hash = OLD.goal_hash + AND NEW.initial_plan_hash = OLD.initial_plan_hash + AND NEW.active_plan_hash = OLD.active_plan_hash + AND NEW.source_record_count = OLD.source_record_count + AND NEW.source_logical_bytes = OLD.source_logical_bytes + AND NEW.segment_count = OLD.segment_count + AND NEW.source_cursor = OLD.source_cursor + AND NEW.rolling_chain_digest = OLD.rolling_chain_digest + AND NEW.root_digest = OLD.rolling_chain_digest + AND NEW.archive_generation = OLD.archive_generation + AND NEW.created_at = OLD.created_at + AND NOT EXISTS ( + SELECT 1 FROM moa.legal_hold AS hold + WHERE hold.tenant_id = OLD.tenant_id + AND hold.released_at IS NULL + AND (hold.subject_id IS NULL OR hold.subject_id = OLD.contact_id) + ) THEN + RETURN NEW; + END IF; + IF TG_OP = 'DELETE' AND EXISTS ( + SELECT 1 + FROM moa.destruction_operation_fence + WHERE tenant_id = OLD.tenant_id + AND subject_id IS NULL + ) THEN + RETURN OLD; + END IF; + RAISE EXCEPTION 'execution terminal archive rows are immutable'; +END; +$$; + +CREATE OR REPLACE FUNCTION moa.enforce_execution_terminal_archive_insert() +RETURNS TRIGGER +LANGUAGE plpgsql +AS $$ +BEGIN + IF EXISTS ( + SELECT 1 + FROM moa.legal_hold AS hold + WHERE hold.tenant_id = NEW.tenant_id + AND hold.released_at IS NULL + AND (hold.subject_id IS NULL OR hold.subject_id = NEW.contact_id) + ) THEN + RAISE EXCEPTION 'execution terminal archive blocked by active legal hold'; + END IF; + RETURN NEW; +END; +$$; + +CREATE TRIGGER execution_terminal_archive_insert_guard +BEFORE INSERT ON moa.execution_terminal_archive +FOR EACH ROW EXECUTE FUNCTION moa.enforce_execution_terminal_archive_insert(); + +CREATE TRIGGER execution_terminal_archive_immutable_guard +BEFORE UPDATE OR DELETE ON moa.execution_terminal_archive +FOR EACH ROW EXECUTE FUNCTION moa.reject_execution_terminal_archive_mutation(); + +CREATE OR REPLACE FUNCTION moa.enforce_execution_terminal_archive_segment_mutation() +RETURNS TRIGGER +LANGUAGE plpgsql +AS $$ +BEGIN + IF TG_OP = 'INSERT' AND EXISTS ( + SELECT 1 + FROM moa.execution_terminal_archive AS archive + WHERE archive.archive_uid = NEW.archive_uid + AND archive.tenant_id = NEW.tenant_id + AND archive.run_uid = NEW.run_uid + AND archive.finalized_at IS NOT NULL + ) THEN + RAISE EXCEPTION 'cannot append to a finalized execution terminal archive'; + END IF; + IF TG_OP = 'INSERT' THEN + RETURN NEW; + END IF; + IF TG_OP = 'DELETE' AND EXISTS ( + SELECT 1 + FROM moa.destruction_operation_fence + WHERE tenant_id = OLD.tenant_id + AND subject_id IS NULL + ) THEN + RETURN OLD; + END IF; + RAISE EXCEPTION 'execution terminal archive segments are immutable'; +END; +$$; + +CREATE TRIGGER execution_terminal_archive_segment_mutation_guard +BEFORE INSERT OR UPDATE OR DELETE ON moa.execution_terminal_archive_segment +FOR EACH ROW EXECUTE FUNCTION moa.enforce_execution_terminal_archive_segment_mutation(); + +CREATE OR REPLACE FUNCTION moa.reject_execution_archived_detail_write() +RETURNS TRIGGER +LANGUAGE plpgsql +AS $$ +BEGIN + IF NEW.run_uid IS NOT NULL AND EXISTS ( + SELECT 1 + FROM moa.execution_terminal_archive AS archive + WHERE archive.run_uid = NEW.run_uid + AND archive.tenant_id = NEW.tenant_id + ) THEN + RAISE EXCEPTION 'cannot write execution detail after terminal archival has started'; + END IF; + RETURN NEW; +END; +$$; + +DO $execution_archived_detail_write_fences$ +DECLARE + table_name TEXT; +BEGIN + FOREACH table_name IN ARRAY ARRAY[ + 'execution_planner_call_audit', + 'execution_compile_audit', + 'execution_node_materialization', + 'execution_action_review_outbox', + 'execution_compensation', + 'execution_task', + 'execution_node_state', + 'execution_completion_scan', + 'execution_amendment_receipt', + 'execution_replan_stop_intent', + 'execution_external_job', + 'execution_trigger', + 'execution_dispatch_outbox', + 'execution_capacity_reservation', + 'execution_task_checkpoint' + ] + LOOP + EXECUTE format( + 'CREATE TRIGGER execution_archived_detail_write_fence ' + 'BEFORE INSERT OR UPDATE ON moa.%I ' + 'FOR EACH ROW EXECUTE FUNCTION moa.reject_execution_archived_detail_write()', + table_name + ); + END LOOP; +END +$execution_archived_detail_write_fences$; + +-- Once an exact compact archive is bound to the durable run, immutable bulky +-- analytics rows may be page-deleted unless a tenant/contact legal hold is +-- active. Tenant destruction retains its existing separate authority. +CREATE OR REPLACE FUNCTION moa.reject_execution_immutable_payload() +RETURNS TRIGGER +LANGUAGE plpgsql +AS $$ +BEGIN + IF TG_OP = 'DELETE' AND EXISTS ( + SELECT 1 + FROM moa.destruction_operation_fence + WHERE tenant_id = OLD.tenant_id + AND subject_id IS NULL + ) THEN + RETURN OLD; + END IF; + IF TG_OP = 'DELETE' + AND OLD.run_uid IS NOT NULL + AND EXISTS ( + SELECT 1 + FROM moa.execution_run AS run + JOIN moa.execution_terminal_archive AS archive + ON archive.archive_uid = run.terminal_archive_uid + AND archive.tenant_id = run.tenant_id + AND archive.run_uid = run.run_uid + AND archive.root_digest = run.terminal_archive_hash + AND archive.finalized_at IS NOT NULL + WHERE run.tenant_id = OLD.tenant_id + AND run.run_uid = OLD.run_uid + AND run.status IN ( + 'completed', 'partial', 'blocked', 'unsupported', + 'failed', 'cancelled' + ) + AND NOT EXISTS ( + SELECT 1 + FROM moa.legal_hold AS hold + WHERE hold.tenant_id = run.tenant_id + AND hold.released_at IS NULL + AND (hold.subject_id IS NULL OR hold.subject_id = run.contact_id) + ) + ) THEN + RETURN OLD; + END IF; + RAISE EXCEPTION 'execution analytics rows are immutable'; +END; +$$; + +CREATE OR REPLACE FUNCTION moa.reject_execution_replan_stop_intent_mutation() +RETURNS TRIGGER +LANGUAGE plpgsql +AS $$ +BEGIN + IF TG_OP = 'UPDATE' + AND NEW.tenant_id = OLD.tenant_id + AND NEW.run_uid = OLD.run_uid + AND NEW.controller_generation = OLD.controller_generation + AND NEW.wake_epoch > OLD.wake_epoch + AND NEW.origin_task_id = OLD.origin_task_id + AND NEW.task_generation = OLD.task_generation + AND NEW.base_plan_revision = OLD.base_plan_revision + AND NEW.stop_reason = OLD.stop_reason + AND NEW.detail = OLD.detail + AND NEW.amendment_hash = OLD.amendment_hash + AND NEW.created_at = OLD.created_at + AND NEW.updated_at > OLD.updated_at THEN + RETURN NEW; + END IF; + IF TG_OP = 'DELETE' AND EXISTS ( + SELECT 1 + FROM moa.destruction_operation_fence + WHERE tenant_id = OLD.tenant_id + AND subject_id IS NULL + ) THEN + RETURN OLD; + END IF; + IF TG_OP = 'DELETE' AND EXISTS ( + SELECT 1 + FROM moa.execution_amendment_receipt AS receipt + WHERE receipt.tenant_id = OLD.tenant_id + AND receipt.run_uid = OLD.run_uid + AND receipt.base_plan_revision = OLD.base_plan_revision + AND receipt.receipt_kind = 'replan_stop' + AND receipt.superseded_task_id = OLD.origin_task_id + AND receipt.task_generation = OLD.task_generation + AND receipt.amendment_hash = OLD.amendment_hash + ) THEN + RETURN OLD; + END IF; + IF TG_OP = 'DELETE' AND EXISTS ( + SELECT 1 + FROM moa.execution_run AS run + JOIN moa.execution_terminal_archive AS archive + ON archive.archive_uid = run.terminal_archive_uid + AND archive.tenant_id = run.tenant_id + AND archive.run_uid = run.run_uid + AND archive.root_digest = run.terminal_archive_hash + AND archive.finalized_at IS NOT NULL + WHERE run.tenant_id = OLD.tenant_id + AND run.run_uid = OLD.run_uid + AND run.status IN ( + 'completed', 'partial', 'blocked', 'unsupported', + 'failed', 'cancelled' + ) + AND NOT EXISTS ( + SELECT 1 + FROM moa.legal_hold AS hold + WHERE hold.tenant_id = run.tenant_id + AND hold.released_at IS NULL + AND (hold.subject_id IS NULL OR hold.subject_id = run.contact_id) + ) + ) THEN + RETURN OLD; + END IF; + RAISE EXCEPTION 'execution replan-stop intent is immutable until exact fencing'; +END; +$$; + +-- All new relations are tenant-owned. Composite foreign keys make it +-- impossible to attach an orchestration row to a differently scoped parent. +CREATE TRIGGER execution_node_state_tenant_immutable +BEFORE UPDATE OF tenant_id ON moa.execution_node_state +FOR EACH ROW EXECUTE FUNCTION moa.reject_tenant_id_change(); +CREATE TRIGGER execution_completion_scan_tenant_immutable +BEFORE UPDATE OF tenant_id ON moa.execution_completion_scan +FOR EACH ROW EXECUTE FUNCTION moa.reject_tenant_id_change(); +CREATE TRIGGER execution_amendment_receipt_immutable_guard +BEFORE UPDATE OR DELETE ON moa.execution_amendment_receipt +FOR EACH ROW EXECUTE FUNCTION moa.reject_execution_immutable_payload(); +CREATE TRIGGER execution_replan_stop_intent_immutable_guard +BEFORE UPDATE OR DELETE ON moa.execution_replan_stop_intent +FOR EACH ROW EXECUTE FUNCTION moa.reject_execution_replan_stop_intent_mutation(); +CREATE TRIGGER execution_trigger_tenant_immutable +BEFORE UPDATE OF tenant_id ON moa.execution_trigger +FOR EACH ROW EXECUTE FUNCTION moa.reject_tenant_id_change(); +CREATE TRIGGER execution_dispatch_outbox_tenant_immutable +BEFORE UPDATE OF tenant_id ON moa.execution_dispatch_outbox +FOR EACH ROW EXECUTE FUNCTION moa.reject_tenant_id_change(); +CREATE TRIGGER execution_external_job_tenant_immutable +BEFORE UPDATE OF tenant_id ON moa.execution_external_job +FOR EACH ROW EXECUTE FUNCTION moa.reject_tenant_id_change(); +CREATE TRIGGER execution_external_job_callback_receipt_tenant_immutable +BEFORE UPDATE OF tenant_id ON moa.execution_external_job_callback_receipt +FOR EACH ROW EXECUTE FUNCTION moa.reject_tenant_id_change(); +CREATE TRIGGER execution_capacity_reservation_tenant_immutable +BEFORE UPDATE OF tenant_id ON moa.execution_capacity_reservation +FOR EACH ROW EXECUTE FUNCTION moa.reject_tenant_id_change(); +CREATE TRIGGER execution_schedule_tenant_immutable +BEFORE UPDATE OF tenant_id ON moa.execution_schedule +FOR EACH ROW EXECUTE FUNCTION moa.reject_tenant_id_change(); +CREATE TRIGGER execution_capacity_bucket_tenant_immutable +BEFORE UPDATE OF tenant_id ON moa.execution_capacity_bucket +FOR EACH ROW EXECUTE FUNCTION moa.reject_tenant_id_change(); +CREATE TRIGGER execution_tenant_dispatch_state_tenant_immutable +BEFORE UPDATE OF tenant_id ON moa.execution_tenant_dispatch_state +FOR EACH ROW EXECUTE FUNCTION moa.reject_tenant_id_change(); +CREATE TRIGGER execution_task_checkpoint_tenant_immutable +BEFORE UPDATE OF tenant_id ON moa.execution_task_checkpoint +FOR EACH ROW EXECUTE FUNCTION moa.reject_tenant_id_change(); +CREATE TRIGGER execution_terminal_archive_tenant_immutable +BEFORE UPDATE OF tenant_id ON moa.execution_terminal_archive +FOR EACH ROW EXECUTE FUNCTION moa.reject_tenant_id_change(); +CREATE TRIGGER execution_terminal_archive_segment_tenant_immutable +BEFORE UPDATE OF tenant_id ON moa.execution_terminal_archive_segment +FOR EACH ROW EXECUTE FUNCTION moa.reject_tenant_id_change(); + +SELECT moa.apply_tenant_rls('moa.execution_node_state'); +SELECT moa.apply_tenant_rls('moa.execution_completion_scan'); +SELECT moa.apply_tenant_rls('moa.execution_amendment_receipt'); +SELECT moa.apply_tenant_rls('moa.execution_replan_stop_intent'); +SELECT moa.apply_tenant_rls('moa.execution_trigger'); +SELECT moa.apply_tenant_rls('moa.execution_dispatch_outbox'); +SELECT moa.apply_tenant_rls('moa.execution_external_job'); +SELECT moa.apply_tenant_rls('moa.execution_external_job_callback_receipt'); +SELECT moa.apply_tenant_rls('moa.execution_capacity_reservation'); +SELECT moa.apply_tenant_rls('moa.execution_schedule'); +SELECT moa.apply_tenant_rls('moa.execution_capacity_bucket'); +DROP POLICY tenant_isolation ON moa.execution_capacity_bucket; +CREATE POLICY execution_capacity_bucket_control_plane +ON moa.execution_capacity_bucket +FOR ALL TO moa_app +USING (moa.current_control_plane()) +WITH CHECK (moa.current_control_plane()); +CREATE POLICY execution_capacity_bucket_tenant_read +ON moa.execution_capacity_bucket +FOR SELECT TO moa_app +USING ( + (scope_kind = 'fleet' AND tenant_id IS NULL) + OR (scope_kind = 'tenant' AND tenant_id = moa.current_tenant_id()) +); +CREATE POLICY execution_capacity_bucket_tenant_insert +ON moa.execution_capacity_bucket +FOR INSERT TO moa_app +WITH CHECK ( + (scope_kind = 'fleet' AND tenant_id IS NULL) + OR (scope_kind = 'tenant' AND tenant_id = moa.current_tenant_id()) +); +CREATE POLICY execution_capacity_bucket_tenant_update +ON moa.execution_capacity_bucket +FOR UPDATE TO moa_app +USING ( + (scope_kind = 'fleet' AND tenant_id IS NULL) + OR (scope_kind = 'tenant' AND tenant_id = moa.current_tenant_id()) +) +WITH CHECK ( + (scope_kind = 'fleet' AND tenant_id IS NULL) + OR (scope_kind = 'tenant' AND tenant_id = moa.current_tenant_id()) +); +SELECT moa.apply_tenant_rls('moa.execution_tenant_dispatch_state'); +SELECT moa.apply_tenant_rls('moa.execution_task_checkpoint'); +SELECT moa.apply_tenant_rls('moa.execution_terminal_archive'); +SELECT moa.apply_tenant_rls('moa.execution_terminal_archive_segment'); + +DO $execution_long_horizon_purge_fences$ +DECLARE + table_name TEXT; +BEGIN + FOREACH table_name IN ARRAY ARRAY[ + 'execution_node_state', + 'execution_completion_scan', + 'execution_amendment_receipt', + 'execution_replan_stop_intent', + 'execution_trigger', + 'execution_dispatch_outbox', + 'execution_external_job', + 'execution_external_job_callback_receipt', + 'execution_capacity_reservation', + 'execution_schedule', + 'execution_capacity_bucket', + 'execution_tenant_dispatch_state', + 'execution_task_checkpoint', + 'execution_terminal_archive', + 'execution_terminal_archive_segment' + ] + LOOP + EXECUTE format( + 'CREATE TRIGGER moa_tenant_purge_fence_insert ' + 'AFTER INSERT ON moa.%I ' + 'REFERENCING NEW TABLE AS tenant_purge_new_rows ' + 'FOR EACH STATEMENT EXECUTE FUNCTION moa.guard_tenant_write_statement(''tenant_id'')', + table_name + ); + EXECUTE format( + 'CREATE TRIGGER moa_tenant_purge_fence_update ' + 'AFTER UPDATE ON moa.%I ' + 'REFERENCING OLD TABLE AS tenant_purge_old_rows ' + 'NEW TABLE AS tenant_purge_new_rows ' + 'FOR EACH STATEMENT EXECUTE FUNCTION moa.guard_tenant_write_statement(''tenant_id'')', + table_name + ); + END LOOP; +END +$execution_long_horizon_purge_fences$; + +-- Delete children before execution_compensation/task/run. Shift through a +-- remote range to preserve the catalog's unique stage ordering. +UPDATE moa.tenant_purge_catalog +SET stage_order = stage_order + 1000 +WHERE stage_order >= ( + SELECT stage_order FROM moa.tenant_purge_catalog + WHERE stage_name = 'moa.execution_compensation' +); + +UPDATE moa.tenant_purge_catalog +SET stage_order = stage_order - 985 +WHERE stage_order >= 1000; + +INSERT INTO moa.tenant_purge_catalog ( + stage_order, stage_name, table_schema, table_name, scope_mode, action_mode +) +SELECT compensation.stage_order - execution_stage.stage_offset, + execution_stage.stage_name, + 'moa', + execution_stage.table_name, + 'tenant_id', + 'delete' +FROM moa.tenant_purge_catalog AS compensation +CROSS JOIN (VALUES + (15, 'moa.execution_replan_stop_intent', 'execution_replan_stop_intent'), + (14, 'moa.execution_amendment_receipt', 'execution_amendment_receipt'), + (13, 'moa.execution_task_checkpoint', 'execution_task_checkpoint'), + (12, 'moa.execution_dispatch_outbox', 'execution_dispatch_outbox'), + (11, 'moa.execution_trigger', 'execution_trigger'), + (10, 'moa.execution_capacity_reservation', 'execution_capacity_reservation'), + (9, 'moa.execution_external_job_callback_receipt', + 'execution_external_job_callback_receipt'), + (8, 'moa.execution_external_job', 'execution_external_job'), + (7, 'moa.execution_completion_scan', 'execution_completion_scan'), + (6, 'moa.execution_node_state', 'execution_node_state'), + (5, 'moa.execution_schedule', 'execution_schedule'), + (4, 'moa.execution_tenant_dispatch_state', 'execution_tenant_dispatch_state'), + (3, 'moa.execution_capacity_bucket', 'execution_capacity_bucket'), + (2, 'moa.execution_terminal_archive_segment', + 'execution_terminal_archive_segment'), + (1, 'moa.execution_terminal_archive', 'execution_terminal_archive') +) AS execution_stage(stage_offset, stage_name, table_name) +WHERE compensation.stage_name = 'moa.execution_compensation'; + +COMMENT ON TABLE moa.tenant_purge_catalog IS + 'Closed 157-table tenant-offboarding residue surface. Fleet capacity-bucket rows, sandbox provider accounts, and inventory findings are global maintenance authority; the two nullable-scope simulator certification authority tables are also intentionally global and absent.'; + +DO $execution_long_horizon_purge_function$ +DECLARE + predecessor TEXT; + replacement TEXT; +BEGIN + SELECT pg_get_functiondef('moa.run_tenant_purge_batch(uuid,text)'::REGPROCEDURE) + INTO predecessor; + IF predecessor NOT LIKE '%catalog_count <> 142%' + OR predecessor NOT LIKE '%exactly 142 tables%' THEN + RAISE EXCEPTION 'unexpected V58 tenant purge function definition' + USING ERRCODE = '55000'; + END IF; + replacement := replace(predecessor, 'catalog_count <> 142', 'catalog_count <> 157'); + replacement := replace(replacement, 'exactly 142 tables', 'exactly 157 tables'); + EXECUTE replacement; +END +$execution_long_horizon_purge_function$; + +ALTER FUNCTION moa.run_tenant_purge_batch(UUID, TEXT) OWNER TO moa_owner; +REVOKE ALL ON FUNCTION moa.run_tenant_purge_batch(UUID, TEXT) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION moa.run_tenant_purge_batch(UUID, TEXT) + TO moa_app, moa_promoter, moa_workspace_maintenance; diff --git a/crates/moa-migrations/migrations/postgres/V000060__sandbox_active_compute_capacity.sql b/crates/moa-migrations/migrations/postgres/V000060__sandbox_active_compute_capacity.sql new file mode 100644 index 000000000..0d61a7f05 --- /dev/null +++ b/crates/moa-migrations/migrations/postgres/V000060__sandbox_active_compute_capacity.sql @@ -0,0 +1,527 @@ +-- Provider-neutral workspace, active-compute, and checkpoint capacity. +-- +-- This is a hard-break migration. Pre-V60 workspace rows were not guaranteed +-- to own a logical-workspace reservation, so carrying them forward would make +-- the new capacity totals incomplete. Operators must drain/reset that preview +-- state before installing this contract. + +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 + FROM moa.sandbox_workspaces + WHERE lifecycle_state <> 'deleted' + ) OR EXISTS ( + SELECT 1 + FROM moa.sandbox_capacity_reservations + ) THEN + RAISE EXCEPTION + 'cannot install provider-neutral sandbox capacity while pre-V60 workspace capacity state exists; destructively reset the preview workspace state' + USING ERRCODE = 'check_violation'; + END IF; +END; +$$; + +ALTER TABLE moa.sandbox_capacity_reservations + DROP CONSTRAINT sandbox_capacity_reservations_operation_fence_fk, + DROP CONSTRAINT sandbox_capacity_reservations_operation_kind_key, + DROP CONSTRAINT sandbox_capacity_reservations_lifetime_volume_shape_check, + DROP CONSTRAINT sandbox_capacity_reservations_resource_dimension_check, + ALTER COLUMN operation_id DROP NOT NULL, + ADD COLUMN expected_delete_generation BIGINT NOT NULL DEFAULT 0 + CHECK (expected_delete_generation >= 0), + ADD COLUMN hand_provisioning_operation_id UUID, + ADD COLUMN hand_lease_generation BIGINT + CHECK (hand_lease_generation IS NULL OR hand_lease_generation > 0), + ADD CONSTRAINT sandbox_capacity_reservations_resource_dimension_check CHECK ( + resource_dimension IN ( + 'workspaces', 'active_hands', 'volumes', 'checkpoints', 'logical_bytes' + ) + ), + ADD CONSTRAINT sandbox_capacity_reservations_operation_fence_fk + FOREIGN KEY ( + operation_id, tenant_id, workspace_id, + provider_account_id, provider_account_generation, + expected_writer_epoch, expected_instance_generation + ) REFERENCES moa.sandbox_workspace_operations ( + operation_id, tenant_id, workspace_id, + provider_account_id, provider_account_generation, + expected_writer_epoch, expected_instance_generation + ) ON DELETE RESTRICT, + ADD CONSTRAINT sandbox_capacity_reservations_dimension_shape_check CHECK ( + CASE resource_dimension + WHEN 'workspaces' THEN + operation_id IS NULL + AND storage_resource_id IS NULL + AND hand_provisioning_operation_id IS NULL + AND hand_lease_generation IS NULL + AND expected_writer_epoch = 0 + AND expected_instance_generation = 0 + AND expected_delete_generation = 0 + AND quantity = 1 + WHEN 'active_hands' THEN + operation_id IS NULL + AND storage_resource_id IS NULL + AND hand_provisioning_operation_id IS NOT NULL + AND hand_lease_generation IS NOT NULL + AND expected_delete_generation = 0 + AND quantity = 1 + WHEN 'volumes' THEN + operation_id IS NOT NULL + AND hand_provisioning_operation_id IS NULL + AND hand_lease_generation IS NULL + AND expected_delete_generation = 0 + AND quantity = 1 + WHEN 'checkpoints' THEN + operation_id IS NOT NULL + AND storage_resource_id IS NULL + AND hand_provisioning_operation_id IS NULL + AND hand_lease_generation IS NULL + AND expected_delete_generation = 0 + AND quantity = 1 + WHEN 'logical_bytes' THEN + operation_id IS NOT NULL + AND storage_resource_id IS NULL + AND hand_provisioning_operation_id IS NULL + AND hand_lease_generation IS NULL + AND expected_delete_generation = 0 + AND quantity > 0 + ELSE FALSE + END + ); + +CREATE UNIQUE INDEX sandbox_capacity_one_workspace_lifetime_key + ON moa.sandbox_capacity_reservations (tenant_id, workspace_id, resource_dimension) + WHERE resource_dimension = 'workspaces'; + +CREATE UNIQUE INDEX sandbox_capacity_one_hand_operation_key + ON moa.sandbox_capacity_reservations ( + tenant_id, hand_provisioning_operation_id, resource_dimension + ) + WHERE resource_dimension = 'active_hands'; + +CREATE UNIQUE INDEX sandbox_capacity_one_workspace_operation_dimension_key + ON moa.sandbox_capacity_reservations (tenant_id, operation_id, resource_dimension) + WHERE operation_id IS NOT NULL; + +CREATE INDEX sandbox_capacity_reclaimable_expiry_idx + ON moa.sandbox_capacity_reservations (expires_at, tenant_id, operation_id) + WHERE reservation_state IN ('pending', 'reconciling') + AND expires_at IS NOT NULL; + +-- Session-terminal cleanup keyset-pages only compute that can still consume +-- provider resources. Keep destroyed history out of both the page and its index. +CREATE INDEX hand_leases_tenant_live_owner_idx + ON moa.hand_leases (tenant_id, session_id, worker_id, provider) + WHERE status <> 'destroyed'; + +CREATE TABLE moa.sandbox_provider_inventory_claims ( + provider_account_id UUID NOT NULL, + provider_account_generation BIGINT NOT NULL CHECK (provider_account_generation > 0), + provider TEXT NOT NULL CHECK (btrim(provider) <> ''), + claim_generation BIGINT NOT NULL DEFAULT 0 CHECK (claim_generation >= 0), + claim_owner UUID, + claim_token UUID, + claimed_at TIMESTAMPTZ, + claim_expires_at TIMESTAMPTZ, + scan_cursor TEXT, + last_succeeded_at TIMESTAMPTZ, + last_error TEXT CHECK (last_error IS NULL OR btrim(last_error) <> ''), + last_error_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (provider_account_id, provider_account_generation), + CONSTRAINT sandbox_provider_inventory_claims_account_fk + FOREIGN KEY (provider_account_id, provider_account_generation) + REFERENCES moa.sandbox_provider_accounts (provider_account_id, generation) + ON UPDATE CASCADE ON DELETE CASCADE, + CONSTRAINT sandbox_provider_inventory_claims_owner_shape_check CHECK ( + (claim_owner IS NULL AND claim_token IS NULL + AND claimed_at IS NULL AND claim_expires_at IS NULL) + OR + (claim_owner IS NOT NULL AND claim_token IS NOT NULL + AND claimed_at IS NOT NULL AND claim_expires_at > claimed_at) + ), + CONSTRAINT sandbox_provider_inventory_claims_error_shape_check CHECK ( + (last_error IS NULL) = (last_error_at IS NULL) + ) +); + +CREATE INDEX sandbox_provider_inventory_claims_claimable_idx + ON moa.sandbox_provider_inventory_claims ( + claim_expires_at, last_succeeded_at, + provider, provider_account_id, provider_account_generation + ); + +REVOKE ALL ON moa.sandbox_provider_inventory_claims FROM moa_app; +GRANT SELECT, INSERT, UPDATE ON moa.sandbox_provider_inventory_claims + TO moa_workspace_maintenance; + +CREATE TABLE moa.sandbox_execution_hand_release_receipts ( + receipt_id UUID PRIMARY KEY, + tenant_id UUID NOT NULL, + run_uid UUID NOT NULL, + owner_kind TEXT NOT NULL CHECK (owner_kind IN ('task', 'compensation')), + task_id UUID, + compensation_id UUID, + logical_generation BIGINT NOT NULL CHECK (logical_generation >= 1), + attempt_generation BIGINT NOT NULL CHECK (attempt_generation >= 1), + workspace_id UUID, + writer_epoch BIGINT CHECK (writer_epoch >= 0), + instance_generation BIGINT CHECK (instance_generation >= 0), + hand_provisioning_operation_id UUID, + hand_lease_generation BIGINT CHECK (hand_lease_generation >= 1), + checkpoint_id UUID, + checkpoint_generation BIGINT CHECK (checkpoint_generation >= 1), + checkpoint_manifest_digest TEXT CHECK (btrim(checkpoint_manifest_digest) <> ''), + checkpoint_logical_bytes BIGINT CHECK (checkpoint_logical_bytes >= 0), + receipt_state TEXT NOT NULL CHECK (receipt_state IN ('pending', 'released')), + destroy_outcome TEXT CHECK (destroy_outcome = 'verified_absent'), + claim_token UUID, + claim_expires_at TIMESTAMPTZ, + requested_at TIMESTAMPTZ NOT NULL, + deadline_at TIMESTAMPTZ NOT NULL, + released_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT sandbox_execution_hand_release_receipts_owner_shape_check CHECK ( + (owner_kind = 'task' AND task_id IS NOT NULL AND compensation_id IS NULL) + OR + (owner_kind = 'compensation' AND task_id IS NULL AND compensation_id IS NOT NULL) + ), + CONSTRAINT sandbox_execution_hand_release_receipts_task_fk + FOREIGN KEY (task_id, run_uid, tenant_id) + REFERENCES moa.execution_task (task_id, run_uid, tenant_id) ON DELETE RESTRICT, + CONSTRAINT sandbox_execution_hand_release_receipts_compensation_fk + FOREIGN KEY (compensation_id, run_uid, tenant_id) + REFERENCES moa.execution_compensation ( + compensation_id, run_uid, tenant_id + ) ON DELETE RESTRICT, + CONSTRAINT sandbox_execution_hand_release_receipts_workspace_fk + FOREIGN KEY (workspace_id, tenant_id) + REFERENCES moa.sandbox_workspaces (workspace_id, tenant_id) ON DELETE RESTRICT, + CONSTRAINT sandbox_execution_hand_release_receipts_checkpoint_fk + FOREIGN KEY (checkpoint_id, workspace_id, tenant_id) + REFERENCES moa.sandbox_workspace_checkpoints ( + checkpoint_id, workspace_id, tenant_id + ) ON DELETE RESTRICT, + CONSTRAINT sandbox_execution_hand_release_receipts_state_shape_check CHECK ( + deadline_at >= requested_at + AND ((receipt_state = 'pending' + AND checkpoint_id IS NULL AND checkpoint_generation IS NULL + AND checkpoint_manifest_digest IS NULL AND checkpoint_logical_bytes IS NULL + AND destroy_outcome IS NULL AND released_at IS NULL + AND claim_token IS NOT NULL AND claim_expires_at IS NOT NULL + ) + OR + (receipt_state = 'released' + AND destroy_outcome = 'verified_absent' + AND claim_token IS NULL AND claim_expires_at IS NULL + AND released_at IS NOT NULL AND released_at >= requested_at)) + AND ((owner_kind = 'task' + AND ((workspace_id IS NOT NULL AND writer_epoch IS NOT NULL + AND instance_generation IS NOT NULL + AND hand_provisioning_operation_id IS NOT NULL + AND hand_lease_generation IS NOT NULL + AND (receipt_state = 'pending' + OR (checkpoint_id IS NOT NULL AND checkpoint_generation IS NOT NULL + AND checkpoint_manifest_digest IS NOT NULL + AND checkpoint_logical_bytes IS NOT NULL))) + OR (receipt_state = 'released' + AND workspace_id IS NULL AND writer_epoch IS NULL + AND instance_generation IS NULL + AND hand_provisioning_operation_id IS NULL + AND hand_lease_generation IS NULL + AND checkpoint_id IS NULL AND checkpoint_generation IS NULL + AND checkpoint_manifest_digest IS NULL + AND checkpoint_logical_bytes IS NULL))) + OR + (owner_kind = 'compensation' + AND workspace_id IS NULL AND writer_epoch IS NULL + AND instance_generation IS NULL + AND checkpoint_id IS NULL AND checkpoint_generation IS NULL + AND checkpoint_manifest_digest IS NULL AND checkpoint_logical_bytes IS NULL + AND ((hand_provisioning_operation_id IS NULL AND hand_lease_generation IS NULL) + OR (hand_provisioning_operation_id IS NOT NULL + AND hand_lease_generation IS NOT NULL)))) + ) +); + +CREATE UNIQUE INDEX sandbox_execution_hand_release_receipts_task_attempt_key + ON moa.sandbox_execution_hand_release_receipts ( + tenant_id, run_uid, task_id, logical_generation, attempt_generation + ) WHERE owner_kind = 'task'; + +CREATE UNIQUE INDEX sandbox_execution_hand_release_receipts_compensation_attempt_key + ON moa.sandbox_execution_hand_release_receipts ( + tenant_id, run_uid, compensation_id, logical_generation, attempt_generation + ) WHERE owner_kind = 'compensation'; + +CREATE INDEX sandbox_execution_hand_release_receipts_workspace_idx + ON moa.sandbox_execution_hand_release_receipts ( + tenant_id, workspace_id, instance_generation, hand_lease_generation + ) WHERE workspace_id IS NOT NULL; + +CREATE INDEX sandbox_execution_hand_release_receipts_pending_due_idx + ON moa.sandbox_execution_hand_release_receipts ( + claim_expires_at, deadline_at, tenant_id, receipt_id + ) + WHERE receipt_state = 'pending'; + +SELECT moa.apply_tenant_rls('moa.sandbox_execution_hand_release_receipts'); +GRANT SELECT, INSERT, UPDATE, DELETE ON moa.sandbox_execution_hand_release_receipts TO moa_app; +GRANT SELECT, INSERT, UPDATE ON moa.sandbox_execution_hand_release_receipts TO moa_workspace_maintenance; + +-- Release receipts are part of the terminal execution detail archive. Once the +-- exact finalized archive is bound, late writes would make the archive incomplete; +-- retention deletes remain legal-hold and destruction-fence guarded by V59. +CREATE TRIGGER sandbox_execution_hand_release_receipt_archived_write_guard +BEFORE INSERT OR UPDATE ON moa.sandbox_execution_hand_release_receipts +FOR EACH ROW EXECUTE FUNCTION moa.reject_execution_archived_detail_write(); + +CREATE TRIGGER sandbox_execution_hand_release_receipt_delete_guard +BEFORE DELETE ON moa.sandbox_execution_hand_release_receipts +FOR EACH ROW EXECUTE FUNCTION moa.reject_execution_immutable_payload(); + +CREATE FUNCTION moa.guard_pending_task_hand_release_attempt() +RETURNS TRIGGER +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, moa +SET row_security = off +AS $$ +BEGIN + IF (NEW.generation, NEW.attempt_generation) + IS DISTINCT FROM (OLD.generation, OLD.attempt_generation) + AND EXISTS ( + SELECT 1 + FROM moa.sandbox_execution_hand_release_receipts AS receipt + WHERE receipt.tenant_id = OLD.tenant_id + AND receipt.run_uid = OLD.run_uid + AND receipt.task_id = OLD.task_id + AND receipt.owner_kind = 'task' + AND receipt.logical_generation = OLD.generation + AND receipt.attempt_generation = OLD.attempt_generation + AND receipt.receipt_state = 'pending' + ) THEN + RAISE EXCEPTION 'execution task attempt cannot advance during sandbox hand release' + USING ERRCODE = 'check_violation'; + END IF; + RETURN NEW; +END; +$$; + +ALTER FUNCTION moa.guard_pending_task_hand_release_attempt() OWNER TO moa_owner; +REVOKE ALL ON FUNCTION moa.guard_pending_task_hand_release_attempt() FROM PUBLIC; + +CREATE TRIGGER execution_task_pending_hand_release_guard +BEFORE UPDATE OF generation, attempt_generation ON moa.execution_task +FOR EACH ROW EXECUTE FUNCTION moa.guard_pending_task_hand_release_attempt(); + +CREATE FUNCTION moa.guard_pending_compensation_hand_release_attempt() +RETURNS TRIGGER +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, moa +SET row_security = off +AS $$ +BEGIN + IF (NEW.generation, NEW.attempt_generation) + IS DISTINCT FROM (OLD.generation, OLD.attempt_generation) + AND EXISTS ( + SELECT 1 + FROM moa.sandbox_execution_hand_release_receipts AS receipt + WHERE receipt.tenant_id = OLD.tenant_id + AND receipt.run_uid = OLD.run_uid + AND receipt.compensation_id = OLD.compensation_id + AND receipt.owner_kind = 'compensation' + AND receipt.logical_generation = OLD.generation + AND receipt.attempt_generation = OLD.attempt_generation + AND receipt.receipt_state = 'pending' + ) THEN + RAISE EXCEPTION 'execution compensation cannot advance during sandbox hand release' + USING ERRCODE = 'check_violation'; + END IF; + RETURN NEW; +END; +$$; + +ALTER FUNCTION moa.guard_pending_compensation_hand_release_attempt() OWNER TO moa_owner; +REVOKE ALL ON FUNCTION moa.guard_pending_compensation_hand_release_attempt() FROM PUBLIC; + +CREATE TRIGGER execution_compensation_pending_hand_release_guard +BEFORE UPDATE OF generation, attempt_generation ON moa.execution_compensation +FOR EACH ROW EXECUTE FUNCTION moa.guard_pending_compensation_hand_release_attempt(); + +CREATE FUNCTION moa.guard_pending_hand_release_generation() +RETURNS TRIGGER +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, moa +SET row_security = off +AS $$ +BEGIN + IF (NEW.generation, NEW.provisioning_operation_id) + IS DISTINCT FROM (OLD.generation, OLD.provisioning_operation_id) + AND EXISTS ( + SELECT 1 + FROM moa.sandbox_execution_hand_release_receipts AS receipt + WHERE receipt.tenant_id = OLD.tenant_id + AND receipt.hand_provisioning_operation_id = OLD.provisioning_operation_id + AND receipt.hand_lease_generation = OLD.generation + AND receipt.receipt_state = 'pending' + ) THEN + RAISE EXCEPTION 'hand lease generation cannot rotate during execution hand release' + USING ERRCODE = 'check_violation'; + END IF; + RETURN NEW; +END; +$$; + +ALTER FUNCTION moa.guard_pending_hand_release_generation() OWNER TO moa_owner; +REVOKE ALL ON FUNCTION moa.guard_pending_hand_release_generation() FROM PUBLIC; + +CREATE TRIGGER hand_lease_pending_release_generation_guard +BEFORE UPDATE OF generation, provisioning_operation_id ON moa.hand_leases +FOR EACH ROW EXECUTE FUNCTION moa.guard_pending_hand_release_generation(); + +-- Workspace creation runs under tenant RLS but provider-account admission must +-- include other tenants. This narrowly-scoped definer function returns no +-- cross-tenant data and validates the inserted workspace before charging it. +CREATE FUNCTION moa.reserve_sandbox_workspace_capacity( + p_tenant_id UUID, + p_workspace_id UUID, + p_provider_account_id UUID, + p_provider_account_generation BIGINT, + p_expected_delete_generation BIGINT +) RETURNS UUID +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, pg_temp +AS $$ +DECLARE + reservation_id UUID := gen_random_uuid(); + tenant_limit BIGINT; + provider_limit BIGINT; + tenant_used BIGINT; + provider_used BIGINT; + tenant_label JSONB; + provider_label JSONB; +BEGIN + IF p_tenant_id IS NULL + OR p_workspace_id IS NULL + OR p_provider_account_id IS NULL + OR p_provider_account_generation <= 0 + OR p_expected_delete_generation < 0 + OR ( + current_setting('moa.control_plane', true) IS DISTINCT FROM 'true' + AND current_setting('moa.tenant_id', true) IS DISTINCT FROM p_tenant_id::TEXT + ) THEN + RAISE EXCEPTION 'invalid or cross-tenant workspace capacity request' + USING ERRCODE = 'check_violation'; + END IF; + + PERFORM pg_advisory_xact_lock( + hashtextextended('sandbox-capacity:tenant:' || p_tenant_id::TEXT, 0) + ); + PERFORM pg_advisory_xact_lock( + hashtextextended('sandbox-capacity:provider:' || p_provider_account_id::TEXT, 0) + ); + + IF NOT EXISTS ( + SELECT 1 + FROM moa.sandbox_workspaces AS workspace + WHERE workspace.tenant_id = p_tenant_id + AND workspace.workspace_id = p_workspace_id + AND workspace.provider_account_id = p_provider_account_id + AND workspace.provider_account_generation = p_provider_account_generation + AND workspace.lifecycle_state = 'creating' + AND workspace.writer_epoch = 0 + AND workspace.instance_generation = 0 + AND workspace.delete_generation = p_expected_delete_generation + AND workspace.access_fenced_at IS NULL + ) THEN + RAISE EXCEPTION 'workspace capacity request lost its exact creation fence' + USING ERRCODE = 'check_violation'; + END IF; + + SELECT limits.configured_limits -> 'workspaces' + INTO tenant_label + FROM moa.sandbox_tenant_capacity_limits AS limits + WHERE limits.tenant_id = p_tenant_id + FOR UPDATE; + + SELECT account.configured_limits -> 'workspaces' + INTO provider_label + FROM moa.sandbox_provider_accounts AS account + WHERE account.provider_account_id = p_provider_account_id + AND account.generation = p_provider_account_generation; + + IF NOT FOUND THEN + RAISE EXCEPTION 'provider-account generation not found' + USING ERRCODE = 'foreign_key_violation'; + END IF; + + IF tenant_label IS NOT NULL THEN + IF jsonb_typeof(tenant_label) <> 'number' + OR tenant_label::TEXT !~ '^[0-9]+$' THEN + RAISE EXCEPTION 'tenant workspaces capacity limit must be a nonnegative integer' + USING ERRCODE = 'check_violation'; + END IF; + tenant_limit := tenant_label::TEXT::BIGINT; + END IF; + + IF provider_label IS NOT NULL THEN + IF jsonb_typeof(provider_label) <> 'number' + OR provider_label::TEXT !~ '^[0-9]+$' THEN + RAISE EXCEPTION 'provider account workspaces capacity limit must be a nonnegative integer' + USING ERRCODE = 'check_violation'; + END IF; + provider_limit := provider_label::TEXT::BIGINT; + END IF; + + SELECT count(*) INTO tenant_used + FROM moa.sandbox_capacity_reservations AS reservation + WHERE reservation.tenant_id = p_tenant_id + AND reservation.resource_dimension = 'workspaces' + AND reservation.reservation_state IN ('pending', 'committed', 'reconciling'); + + SELECT count(*) INTO provider_used + FROM moa.sandbox_capacity_reservations AS reservation + WHERE reservation.provider_account_id = p_provider_account_id + AND reservation.resource_dimension = 'workspaces' + AND reservation.reservation_state IN ('pending', 'committed', 'reconciling'); + + IF tenant_limit IS NOT NULL AND tenant_used + 1 > tenant_limit THEN + RAISE EXCEPTION 'tenant workspaces capacity exceeded: % + 1 > %', + tenant_used, tenant_limit + USING ERRCODE = 'check_violation'; + END IF; + IF provider_limit IS NOT NULL AND provider_used + 1 > provider_limit THEN + RAISE EXCEPTION 'provider account workspaces capacity exceeded: % + 1 > %', + provider_used, provider_limit + USING ERRCODE = 'check_violation'; + END IF; + + INSERT INTO moa.sandbox_capacity_reservations ( + reservation_id, tenant_id, provider_account_id, + provider_account_generation, workspace_id, operation_id, + expected_writer_epoch, expected_instance_generation, + expected_delete_generation, resource_dimension, quantity, + reservation_state + ) VALUES ( + reservation_id, p_tenant_id, p_provider_account_id, + p_provider_account_generation, p_workspace_id, NULL, + 0, 0, p_expected_delete_generation, 'workspaces', 1, + 'committed' + ); + RETURN reservation_id; +END; +$$; + +ALTER FUNCTION moa.reserve_sandbox_workspace_capacity(UUID, UUID, UUID, BIGINT, BIGINT) + OWNER TO moa_owner; +REVOKE ALL ON FUNCTION moa.reserve_sandbox_workspace_capacity(UUID, UUID, UUID, BIGINT, BIGINT) + FROM PUBLIC; +GRANT EXECUTE ON FUNCTION moa.reserve_sandbox_workspace_capacity(UUID, UUID, UUID, BIGINT, BIGINT) + TO moa_app, moa_workspace_maintenance; diff --git a/crates/moa-migrations/tests/run_idempotency_db/execution_and_security_catalog.rs b/crates/moa-migrations/tests/run_idempotency_db/execution_and_security_catalog.rs index a7ab4a8d9..a01f4d93a 100644 --- a/crates/moa-migrations/tests/run_idempotency_db/execution_and_security_catalog.rs +++ b/crates/moa-migrations/tests/run_idempotency_db/execution_and_security_catalog.rs @@ -2,6 +2,35 @@ use super::support::*; +const LONG_HORIZON_EXECUTION_SQL: &str = + include_str!("../../migrations/postgres/V000059__long_horizon_execution.sql"); + +#[test] +fn long_horizon_task_guard_source_is_canonical_offline() { + // Pins: V59 owns one complete task-update guard instead of editing the + // inherited function body and layering a second ordinary trigger over it. + assert_eq!( + LONG_HORIZON_EXECUTION_SQL + .matches("CREATE OR REPLACE FUNCTION moa.enforce_execution_task_update()") + .count(), + 1 + ); + assert!(!LONG_HORIZON_EXECUTION_SQL.contains("$execution_task_long_horizon_transitions$")); + assert!(!LONG_HORIZON_EXECUTION_SQL.contains("enforce_execution_task_long_horizon_update")); + for required_clause in [ + "OLD.status = 'running'\n AND NEW.status = 'ready'", + "OLD.status = 'waiting_input' AND NEW.status = 'ready'", + "NEW.attempt_generation <> OLD.attempt_generation + 1", + "NEW.status <> 'ready'", + "NEW.attempt_state <> 'idle'", + ] { + assert!( + LONG_HORIZON_EXECUTION_SQL.contains(required_clause), + "canonical task guard is missing: {required_clause}" + ); + } +} + #[tokio::test] #[ignore = "requires a superuser-capable local Postgres via MOA_DATABASE_URL"] async fn privacy_export_auditor_final_catalog_reads_typed_surface_db() { @@ -799,6 +828,7 @@ async fn execution_analytics_fresh_cutover_and_exact_contract_db() { planning_context_hash,owner_user_id,goal_contract, initial_plan,active_plan,initial_plan_hash,active_plan_hash, capability_catalog,authorization_envelope,source_provenance,input, + admitted_identity, status,source_kind ) VALUES ( '00000000-0000-0000-0000-000000337041', @@ -806,7 +836,36 @@ async fn execution_analytics_fresh_cutover_and_exact_contract_db() { '00000000-0000-0000-0000-000000337010',11, '00000000-0000-0000-0000-000000337040',repeat('5',64), 'owner','{"requirements":[],"completion_checks":[]}', - '{}','{}',repeat('3',64),repeat('3',64),'{}','{}', + jsonb_build_object( + 'definition',jsonb_build_object( + 'cancel_policy','retain_effects','input_schema','{}'::JSONB, + 'output_schema','{}'::JSONB, + 'input_wait_policy',jsonb_build_object( + 'expiry',jsonb_build_object( + 'kind','after','delay_seconds',1 + ), + 'on_expiry',jsonb_build_object('kind','fail_run') + ), + 'nodes','[]'::JSONB + ), + 'plan_hash',repeat('3',64),'catalog_hash',repeat('0',64), + 'estimate','{}'::JSONB,'report','{}'::JSONB + ), + jsonb_build_object( + 'definition',jsonb_build_object( + 'cancel_policy','retain_effects','input_schema','{}'::JSONB, + 'output_schema','{}'::JSONB, + 'input_wait_policy',jsonb_build_object( + 'expiry',jsonb_build_object( + 'kind','after','delay_seconds',1 + ), + 'on_expiry',jsonb_build_object('kind','fail_run') + ), + 'nodes','[]'::JSONB + ), + 'plan_hash',repeat('3',64),'catalog_hash',repeat('0',64), + 'estimate','{}'::JSONB,'report','{}'::JSONB + ),repeat('3',64),repeat('3',64),'{}','{}', jsonb_build_object( 'kind','generated_plan', 'planner',jsonb_build_object( @@ -817,7 +876,13 @@ async fn execution_analytics_fresh_cutover_and_exact_contract_db() { 'repair_attempts',0 ) ), - '{}','queued','generated_plan' + '{}',jsonb_build_object( + 'identity_type','operator', + 'id','00000000-0000-0000-0000-000000337021', + 'tenant_id','00000000-0000-0000-0000-000000337020', + 'api_key_id',NULL, + 'acting_on_behalf_of',NULL + ),'queued','generated_plan' ); INSERT INTO moa.execution_task ( task_id,run_uid,tenant_id,contact_id,node_id,item_key, @@ -907,6 +972,7 @@ async fn execution_analytics_fresh_cutover_and_exact_contract_db() { planning_context_hash,owner_user_id,goal_contract,\ initial_plan,active_plan,initial_plan_hash,active_plan_hash,\ capability_catalog,authorization_envelope,source_provenance,input,\ + admitted_identity,\ status,source_kind\ ) VALUES (\ '00000000-0000-0000-0000-000000337052',\ @@ -914,7 +980,36 @@ async fn execution_analytics_fresh_cutover_and_exact_contract_db() { '00000000-0000-0000-0000-000000337010',12,\ '00000000-0000-0000-0000-000000337050',repeat('7',64),\ 'owner','{\"requirements\":[],\"completion_checks\":[]}',\ - '{}','{}',repeat('3',64),repeat('3',64),'{}','{}',\ + jsonb_build_object(\ + 'definition',jsonb_build_object(\ + 'cancel_policy','retain_effects',\ + 'input_schema','{}'::JSONB,'output_schema','{}'::JSONB,\ + 'input_wait_policy',jsonb_build_object(\ + 'expiry',jsonb_build_object(\ + 'kind','after','delay_seconds',1\ + ),\ + 'on_expiry',jsonb_build_object('kind','fail_run')\ + ),\ + 'nodes','[]'::JSONB\ + ),\ + 'plan_hash',repeat('3',64),'catalog_hash',repeat('0',64),\ + 'estimate','{}'::JSONB,'report','{}'::JSONB\ + ),\ + jsonb_build_object(\ + 'definition',jsonb_build_object(\ + 'cancel_policy','retain_effects',\ + 'input_schema','{}'::JSONB,'output_schema','{}'::JSONB,\ + 'input_wait_policy',jsonb_build_object(\ + 'expiry',jsonb_build_object(\ + 'kind','after','delay_seconds',1\ + ),\ + 'on_expiry',jsonb_build_object('kind','fail_run')\ + ),\ + 'nodes','[]'::JSONB\ + ),\ + 'plan_hash',repeat('3',64),'catalog_hash',repeat('0',64),\ + 'estimate','{}'::JSONB,'report','{}'::JSONB\ + ),repeat('3',64),repeat('3',64),'{}','{}',\ jsonb_build_object(\ 'kind','generated_plan',\ 'planner',jsonb_build_object(\ @@ -925,7 +1020,13 @@ async fn execution_analytics_fresh_cutover_and_exact_contract_db() { 'repair_attempts',0\ )\ ),\ - '{}','queued','generated_plan'\ + '{}',jsonb_build_object(\ + 'identity_type','operator',\ + 'id','00000000-0000-0000-0000-000000337021',\ + 'tenant_id','00000000-0000-0000-0000-000000337020',\ + 'api_key_id',NULL,\ + 'acting_on_behalf_of',NULL\ + ),'queued','generated_plan'\ )", ) .await @@ -1378,3 +1479,675 @@ async fn full_database_runner_installs_execution_schema_and_foreign_keys_db() { assert_eq!(session_fk_targets, vec!["public.sessions"; 2]); assert_eq!(execution_fk_targets, vec!["moa.execution_run"; 2]); } + +#[tokio::test] +#[ignore = "requires a superuser-capable local Postgres via MOA_DATABASE_URL"] +async fn long_horizon_execution_cutover_rejects_live_runs_and_installs_fenced_catalog_db() { + // Pins: V59 refuses to reinterpret a live legacy workflow, then preserves + // terminal evidence while installing the tenant-fenced activation catalog. + let admin_url = test_database_url(); + let db_name = unique_db_name(); + let admin = PgPoolOptions::new() + .max_connections(1) + .connect(&admin_url) + .await + .expect("connect long-horizon migration maintenance database"); + admin + .execute(format!("CREATE DATABASE \"{db_name}\"").as_str()) + .await + .expect("create long-horizon migration database"); + let target_url = with_database(&admin_url, &db_name); + + let outcome = async { + install_required_extensions(&target_url).await?; + apply_through_migration(&target_url, "sandbox_workspaces").await?; + let target = PgPoolOptions::new() + .max_connections(2) + .connect(&target_url) + .await?; + + let tenant_id = uuid::Uuid::new_v4(); + let session_id = uuid::Uuid::new_v4(); + let planning_context_uid = uuid::Uuid::new_v4(); + let run_uid = uuid::Uuid::new_v4(); + let plan_hash = "1".repeat(64); + let plan = serde_json::json!({ + "definition": { + "cancel_policy": "retain_effects", + "input_schema": {}, + "output_schema": {}, + "nodes": [{ + "id": "output", + "requirement_ids": [], + "depends_on": [], + "when": null, + "input": {}, + "output_schema": {}, + "operation": {"kind": "output", "value": {}}, + "compensation": null, + "retry": { + "max_attempts": 1, + "initial_backoff_ms": 1, + "max_backoff_ms": 1 + }, + "budget": null + }] + }, + "plan_hash": plan_hash, + "catalog_hash": "0".repeat(64), + "estimate": { + "cost_microusd": 0, + "tokens": 0, + "tool_calls": 0, + "retrieved_bytes": 0, + "tasks": 1 + }, + "report": {"issues": []} + }); + sqlx::query( + "INSERT INTO moa.execution_planning_context ( \ + planning_context_uid, tenant_id, session_id, \ + originating_user_sequence_num, originating_user_event_hash, \ + owner_user_id, planning_context_hash, snapshot \ + ) VALUES ($1, $2, $3, 0, $4, 'migration-test', $4, '{}'::JSONB)", + ) + .bind(planning_context_uid) + .bind(tenant_id) + .bind(session_id) + .bind("2".repeat(64)) + .execute(&target) + .await?; + sqlx::query( + "INSERT INTO moa.execution_run ( \ + run_uid, tenant_id, session_id, originating_user_sequence_num, \ + planning_context_uid, planning_context_hash, owner_user_id, goal_contract, \ + initial_plan, active_plan, initial_plan_hash, active_plan_hash, \ + capability_catalog, authorization_envelope, source_provenance, source_kind, \ + input, status \ + ) VALUES ( \ + $1, $2, $3, 0, $4, $5, 'migration-test', $6, $7, $7, $8, $8, \ + $9, $10, $11, 'generated_plan', '{}'::JSONB, 'queued' \ + )", + ) + .bind(run_uid) + .bind(tenant_id) + .bind(session_id) + .bind(planning_context_uid) + .bind("2".repeat(64)) + .bind(serde_json::json!({ + "objective": "migration", + "requirements": [], + "deliverables": [], + "coverage": [], + "constraints": [], + "completion_checks": [] + })) + .bind(&plan) + .bind(&plan_hash) + .bind(serde_json::json!({ + "capabilities": [], + "catalog_hash": "0".repeat(64) + })) + .bind(serde_json::json!({"capability_refs": [], "skill_refs": []})) + .bind(serde_json::json!({ + "kind": "generated_plan", + "planner": { + "model": "migration-test", + "prompt_version": "planner", + "candidate_hash": "3".repeat(64), + "compiler_report_hash": "4".repeat(64), + "final_plan_hash": plan_hash, + "repair_attempts": 0 + } + })) + .execute(&target) + .await?; + + let cutover_error = run_reporting_applied_serialized(&target_url) + .await + .expect_err("V59 must reject a nonterminal legacy execution run") + .to_string(); + let schema_not_partially_installed: bool = sqlx::query_scalar( + "SELECT to_regclass('moa.execution_trigger') IS NULL \ + AND NOT EXISTS ( \ + SELECT 1 FROM information_schema.columns \ + WHERE table_schema = 'moa' AND table_name = 'execution_run' \ + AND column_name = 'controller_generation' \ + )", + ) + .fetch_one(&target) + .await?; + + sqlx::query( + "UPDATE moa.execution_run \ + SET status = 'cancelled', cancellation_reason = 'cutover test', \ + terminal_cause = '{\"kind\":\"cancellation\"}'::JSONB, \ + terminal_reason = 'cancelled', \ + terminal_satisfied_requirement_count = 0, \ + terminal_requirement_count = 0, completed_at = now() \ + WHERE run_uid = $1", + ) + .bind(run_uid) + .execute(&target) + .await?; + + let applied = run_reporting_applied_serialized(&target_url).await?; + let second = run_reporting_applied_serialized(&target_url).await?; + + let retry_task_id = uuid::Uuid::new_v4(); + let input_task_id = uuid::Uuid::new_v4(); + let invalid_attempt_task_id = uuid::Uuid::new_v4(); + for (task_id, status, attempt_state) in [ + (retry_task_id, "running", "running"), + (input_task_id, "waiting_input", "waiting"), + (invalid_attempt_task_id, "waiting_review", "waiting"), + ] { + sqlx::query( + "INSERT INTO moa.execution_task ( \ + task_id, run_uid, tenant_id, node_id, item_key, plan_revision, status, \ + input, task_kind, retry_policy, estimate_cost_microusd, estimate_tokens, \ + estimate_tasks, estimate_tool_calls, estimate_retrieved_bytes, \ + attempt_state \ + ) VALUES ( \ + $1, $2, $3, $4, $4, 1, $5, '{}', \ + '{\"kind\":\"output\",\"value\":null}', \ + '{\"max_attempts\":2,\"initial_backoff_ms\":1,\"max_backoff_ms\":1}', \ + 0, 0, 1, 0, 0, $6 \ + )", + ) + .bind(task_id) + .bind(run_uid) + .bind(tenant_id) + .bind(format!("counter-guard-{task_id}")) + .bind(status) + .bind(attempt_state) + .execute(&target) + .await?; + } + + let retry_counters: (i32, i64, i64) = sqlx::query_as( + "UPDATE moa.execution_task \ + SET status='ready', attempt_state='idle', attempt=attempt+1, \ + generation=generation+1, attempt_generation=attempt_generation+1 \ + WHERE task_id=$1 RETURNING attempt, generation, attempt_generation", + ) + .bind(retry_task_id) + .fetch_one(&target) + .await?; + let input_resume_counters: (i32, i64, i64) = sqlx::query_as( + "UPDATE moa.execution_task \ + SET status='ready', attempt_state='idle', generation=generation+1, \ + attempt_generation=attempt_generation+1 \ + WHERE task_id=$1 RETURNING attempt, generation, attempt_generation", + ) + .bind(input_task_id) + .fetch_one(&target) + .await?; + let invalid_attempt_generation_rejected = sqlx::query( + "UPDATE moa.execution_task \ + SET status='ready', attempt_state='idle', \ + attempt_generation=attempt_generation+2 \ + WHERE task_id=$1", + ) + .bind(invalid_attempt_task_id) + .execute(&target) + .await + .is_err(); + + let catalog_shape: (bool, bool, bool, bool, bool, bool) = sqlx::query_as( + r#" + SELECT + (SELECT count(*) = 142 + FROM information_schema.columns + WHERE table_schema = 'moa' + AND ( + (table_name = 'execution_run' AND column_name IN ( + 'admitted_identity', 'controller_generation', 'activation_state', + 'next_wake_at', 'waiting_since', 'last_progress_at', + 'pause_requested_at', 'paused_at', 'ready_task_count', + 'active_task_count', 'waiting_task_count', + 'waiting_input_task_count', 'waiting_input_user_task_count', + 'waiting_input_tenant_admin_task_count', + 'waiting_input_external_task_count', 'waiting_review_task_count', + 'waiting_signal_task_count', 'waiting_timer_task_count', + 'waiting_external_task_count', 'waiting_replan_task_count', + 'waiting_reasons_truncated' + ,'schedule_uid', 'schedule_incarnation', + 'schedule_occurrence_sequence', 'terminal_archive_uid', + 'terminal_archive_hash', 'terminal_details_archived_at' + )) + OR + (table_name = 'execution_task' AND column_name IN ( + 'attempt_generation', 'attempt_state', 'attempt_started_at', + 'last_progress_at', 'attempt_deadline_at', 'waiting_since', + 'ready_at', 'active_dispatch_uid', 'dispatch_sequence', + 'external_job_uid', 'failure_fingerprint' + )) + OR + (table_name = 'execution_compensation' AND column_name IN ( + 'attempt_generation', 'attempt_state', 'attempt_started_at', + 'last_progress_at', 'attempt_deadline_at', 'waiting_since', + 'active_dispatch_uid', 'dispatch_sequence', 'external_job_uid', + 'release_intent' + )) + OR + (table_name = 'execution_external_job' AND column_name IN ( + 'compensation_id', 'compensation_generation', + 'compensation_attempt_generation', + 'declared_provider', 'provider_contract_violation' + )) + OR + (table_name = 'execution_node_state' AND column_name IN ( + 'aggregate_output', 'aggregate_output_hash', 'reduce_round', + 'reduce_batch_cursor', 'reduce_round_input_count', + 'reduce_round_task_count', 'reduce_round_terminal_task_count', + 'materialization_complete', 'aggregate_cursor_item_key', + 'aggregate_complete' + )) + OR + (table_name = 'execution_completion_scan' AND column_name IN ( + 'plan_revision', 'controller_generation', 'scan_kind', + 'excluded_task_id', 'source_progress_at', 'task_cursor', + 'node_cursor', 'scanned_task_count', 'task_evidence', 'scan_complete', + 'node_scan_complete', 'completion_evidence', + 'verifiers_materialized', 'created_at', 'updated_at' + )) + OR + (table_name = 'execution_amendment_receipt' AND column_name IN ( + 'base_plan_revision', 'amendment_hash', 'receipt_kind', + 'superseded_task_id', 'task_generation', + 'task_ids_to_release', 'created_at' + )) + OR + (table_name = 'execution_replan_stop_intent' AND column_name IN ( + 'controller_generation', 'wake_epoch', 'origin_task_id', + 'task_generation', 'base_plan_revision', 'stop_reason', + 'detail', 'amendment_hash', 'created_at', 'updated_at' + )) + OR + (table_name = 'execution_schedule' AND column_name IN ( + 'template_revision_uid', 'run_as_identity', 'creation_origin', + 'schedule_incarnation', 'start_at', 'next_occurrence_local' + )) + OR + (table_name = 'execution_trigger' AND column_name = 'schedule_incarnation') + OR + (table_name = 'execution_capacity_reservation' AND column_name IN ( + 'trigger_uid', 'external_job_uid' + )) + OR + (table_name = 'execution_task_checkpoint' AND column_name IN ( + 'checkpoint_sequence', 'controller_generation', + 'task_generation', 'attempt_generation', 'dispatch_uid', + 'checkpoint_kind', 'schema_version', 'payload', 'payload_hash', + 'workspace_release_receipt', 'superseded_at' + )) + OR + (table_name = 'execution_terminal_archive' AND column_name IN ( + 'format_version', 'terminal_status', 'terminal_completed_at', + 'goal_hash', 'initial_plan_hash', 'active_plan_hash', + 'source_record_count', 'source_logical_bytes', 'segment_count', + 'source_cursor', 'rolling_chain_digest', 'root_digest', + 'archive_generation', 'finalized_at', + 'details_deleted_at' + )) + OR + (table_name = 'execution_terminal_archive_segment' AND column_name IN ( + 'archive_uid', 'segment_kind', 'segment_sequence', + 'format_version', 'record_count', 'payload', 'content_digest' + )) + OR + (table_name = 'execution_maintenance_checkpoint' AND column_name IN ( + 'next_run_at', 'scheduled_generation', 'claim_owner', + 'claimed_generation', 'claim_expires_at' + )) + )), + (SELECT count(*) = 15 + FROM pg_class relation + JOIN pg_namespace namespace ON namespace.oid = relation.relnamespace + WHERE namespace.nspname = 'moa' + AND relation.relkind = 'r' + AND relation.relname IN ( + 'execution_node_state', 'execution_trigger', + 'execution_dispatch_outbox', 'execution_external_job', + 'execution_capacity_reservation', 'execution_schedule', + 'execution_capacity_bucket', 'execution_tenant_dispatch_state', + 'execution_external_job_callback_receipt', + 'execution_completion_scan', + 'execution_amendment_receipt', + 'execution_replan_stop_intent', + 'execution_task_checkpoint', 'execution_terminal_archive', + 'execution_terminal_archive_segment' + )), + (SELECT count(*) = 15 AND bool_and(relrowsecurity AND relforcerowsecurity) + FROM pg_class relation + JOIN pg_namespace namespace ON namespace.oid = relation.relnamespace + WHERE namespace.nspname = 'moa' + AND relation.relname IN ( + 'execution_node_state', 'execution_trigger', + 'execution_dispatch_outbox', 'execution_external_job', + 'execution_capacity_reservation', 'execution_schedule', + 'execution_capacity_bucket', 'execution_tenant_dispatch_state', + 'execution_external_job_callback_receipt', + 'execution_completion_scan', + 'execution_amendment_receipt', + 'execution_replan_stop_intent', + 'execution_task_checkpoint', 'execution_terminal_archive', + 'execution_terminal_archive_segment' + )), + (SELECT count(*) = 46 + FROM pg_indexes + WHERE schemaname = 'moa' + AND indexname IN ( + 'execution_run_terminal_retention_idx', + 'execution_task_ready_idx', + 'execution_task_active_attempt_watchdog_idx', + 'execution_trigger_due_idx', + 'execution_dispatch_outbox_pending_idx', + 'execution_dispatch_outbox_task_attempt_uidx', + 'execution_trigger_schedule_occurrence_uidx', + 'execution_schedule_due_idx', + 'execution_task_terminal_retention_idx', + 'execution_dispatch_outbox_compensation_attempt_uidx', + 'execution_compensation_active_watchdog_idx', + 'execution_capacity_bucket_lock_order_idx', + 'execution_tenant_dispatch_fairness_idx', + 'execution_dispatch_outbox_claim_expiry_idx', + 'execution_trigger_claim_expiry_idx' + ,'execution_run_schedule_occurrence_uidx' + ,'execution_external_job_callback_receipt_retention_idx' + ,'execution_trigger_dead_letter_idx' + ,'execution_dispatch_outbox_dead_letter_idx' + ,'execution_dispatch_outbox_task_attempt_cancel_uidx' + ,'execution_dispatch_outbox_compensation_attempt_cancel_uidx' + ,'execution_task_checkpoint_current_uidx' + ,'execution_task_checkpoint_retention_idx' + ,'execution_terminal_archive_retention_idx' + ,'execution_terminal_archive_segment_scan_idx' + ,'execution_terminal_archive_segment_sequence_key' + ,'execution_maintenance_checkpoint_due_idx' + ,'execution_run_activation_idx' + ,'execution_node_state_actionable_idx' + ,'execution_node_state_aggregate_actionable_idx' + ,'execution_capacity_reservation_active_run_owner_uidx' + ,'execution_capacity_reservation_parked_run_owner_uidx' + ,'execution_capacity_reservation_trigger_owner_uidx' + ,'execution_capacity_reservation_external_job_owner_uidx' + ,'execution_task_cancelling_reconciliation_idx' + ,'execution_compensation_cancelling_reconciliation_idx' + ,'execution_completion_scan_actionable_idx' + ,'execution_task_failure_fingerprint_idx' + ,'execution_amendment_receipt_retention_idx' + ,'execution_replan_stop_intent_current_idx' + ,'execution_node_state_run_order_uidx' + ,'execution_external_job_task_attempt_uidx' + ,'execution_external_job_compensation_attempt_uidx' + ,'execution_task_waiting_projection_idx' + ,'execution_trigger_run_wake_idx' + ,'execution_dispatch_outbox_external_cancel_uidx' + )), + moa.execution_admitted_identity_is_valid(admitted_identity, tenant_id) + AND activation_state = 'terminal' + AND status = 'cancelled', + (SELECT count(*) = 8 + FROM pg_constraint + WHERE conname IN ( + 'execution_trigger_compensation_tenant_fk', + 'execution_dispatch_outbox_compensation_tenant_fk', + 'execution_capacity_reservation_compensation_tenant_fk', + 'execution_capacity_reservation_trigger_tenant_fk', + 'execution_capacity_reservation_external_job_tenant_fk', + 'execution_external_job_compensation_tenant_fk', + 'execution_compensation_external_job_tenant_fk', + 'execution_completion_scan_excluded_task_tenant_fk' + ) + AND ( + (conname IN ( + 'execution_trigger_compensation_tenant_fk', + 'execution_dispatch_outbox_compensation_tenant_fk', + 'execution_capacity_reservation_compensation_tenant_fk', + 'execution_external_job_compensation_tenant_fk' + ) AND pg_get_constraintdef(oid) + LIKE '%compensation_id, run_uid, tenant_id%') + OR + (conname = 'execution_capacity_reservation_trigger_tenant_fk' + AND pg_get_constraintdef(oid) LIKE '%trigger_uid, tenant_id%') + OR + (conname = 'execution_capacity_reservation_external_job_tenant_fk' + AND pg_get_constraintdef(oid) LIKE '%external_job_uid, tenant_id%') + OR + (conname = 'execution_compensation_external_job_tenant_fk' + AND pg_get_constraintdef(oid) LIKE '%external_job_uid, tenant_id%') + OR + (conname = 'execution_completion_scan_excluded_task_tenant_fk' + AND pg_get_constraintdef(oid) + LIKE '%excluded_task_id, run_uid, tenant_id%') + )) + AND + (SELECT count(*) = 3 + AND bool_and(privilege_type IN ('SELECT', 'INSERT', 'UPDATE')) + FROM information_schema.role_table_grants + WHERE table_schema = 'moa' + AND table_name = 'execution_maintenance_checkpoint' + AND grantee = 'moa_app') + AND + (SELECT count(*) = 10 + FROM pg_constraint + WHERE conname IN ( + 'execution_compensation_release_intent_shape_check', + 'execution_run_waiting_task_counts_check', + 'execution_run_waiting_input_audience_counts_check', + 'execution_run_waiting_reasons_bounded_check', + 'execution_external_job_binding_shape_check', + 'execution_external_job_contract_violation_shape_check', + 'execution_amendment_receipt_release_shape_check', + 'execution_task_output_inline_size_check', + 'execution_trigger_start_recovery_shape_check', + 'execution_completion_scan_kind_shape_check' + )) + AND + (SELECT count(*) = 2 + AND bool_and(convalidated) + AND bool_and( + pg_get_constraintdef(oid) + LIKE '%execution_plan_snapshot_is_current%' + AND pg_get_constraintdef(oid) LIKE '%completed%' + AND pg_get_constraintdef(oid) LIKE '%cancelled%' + ) + FROM pg_constraint + WHERE conrelid = 'moa.execution_run'::regclass + AND conname IN ( + 'execution_run_initial_plan_check', + 'execution_run_active_plan_check' + )) + AND + (SELECT pg_get_constraintdef(oid) LIKE '%waiting_external%' + FROM pg_constraint + WHERE conname = 'execution_compensation_attempt_state_check') + AND + (SELECT pg_get_constraintdef(oid) + LIKE '%capability_external_start%' + FROM pg_constraint + WHERE conrelid = 'moa.execution_task_checkpoint'::regclass + AND conname = 'execution_task_checkpoint_checkpoint_kind_check') + AND + (SELECT count(*) = 4 + AND count(*) FILTER (WHERE cmd = 'ALL') = 1 + AND count(*) FILTER (WHERE cmd = 'SELECT') = 1 + AND count(*) FILTER (WHERE cmd = 'INSERT') = 1 + AND count(*) FILTER (WHERE cmd = 'UPDATE') = 1 + AND bool_and( + policyname = 'execution_capacity_bucket_control_plane' + OR COALESCE(qual, with_check, '') LIKE '%scope_kind%fleet%' + ) + AND bool_and( + policyname = 'execution_capacity_bucket_control_plane' + OR COALESCE(qual, with_check, '') LIKE '%current_tenant_id%' + ) + FROM pg_policies + WHERE schemaname = 'moa' + AND tablename = 'execution_capacity_bucket') + AND + EXISTS ( + SELECT 1 FROM pg_trigger + WHERE tgname = 'execution_capacity_bucket_owner_immutable' + AND NOT tgisinternal + ) + AND + (SELECT indexdef LIKE 'CREATE UNIQUE INDEX%' + FROM pg_indexes + WHERE schemaname = 'moa' + AND indexname = 'execution_node_state_run_order_uidx') + AND + (SELECT indexdef LIKE 'CREATE UNIQUE INDEX%' + FROM pg_indexes + WHERE schemaname = 'moa' + AND indexname = 'execution_terminal_archive_segment_sequence_key') + AND + (SELECT indexdef LIKE '%WHERE (provider IS NOT NULL)%' + FROM pg_indexes + WHERE schemaname = 'moa' + AND indexname = 'execution_external_job_provider_identity_key') + AND + (SELECT tgdeferrable AND tginitdeferred + FROM pg_trigger + WHERE tgname = 'execution_external_job_intent_capacity_guard') + AND + EXISTS ( + SELECT 1 FROM pg_trigger + WHERE tgname = 'execution_node_aggregate_cursor_update_guard' + AND NOT tgisinternal + ) + AND + EXISTS ( + SELECT 1 FROM pg_trigger + WHERE tgname = 'execution_replan_stop_intent_immutable_guard' + AND NOT tgisinternal + ) + AND + EXISTS ( + SELECT 1 FROM pg_trigger + WHERE tgname = 'execution_completion_scan_update_guard' + AND NOT tgisinternal + ) + AND + EXISTS ( + SELECT 1 FROM pg_trigger + WHERE tgname = 'execution_terminal_archive_segment_mutation_guard' + AND NOT tgisinternal + ) + AND + (SELECT count(*) = 1 + AND bool_and(trigger.tgname = 'execution_task_update_guard') + AND bool_and(proc.proname = 'enforce_execution_task_update') + FROM pg_trigger AS trigger + JOIN pg_proc AS proc ON proc.oid = trigger.tgfoid + WHERE trigger.tgrelid = 'moa.execution_task'::REGCLASS + AND NOT trigger.tgisinternal) + AND + to_regprocedure('moa.enforce_execution_task_long_horizon_update()') IS NULL + AND + (SELECT regexp_replace( + pg_get_functiondef( + 'moa.enforce_execution_task_update()'::REGPROCEDURE + ), + '[[:space:]]+', ' ', 'g' + ) LIKE '%OLD.status = ''running'' AND NEW.status = ''ready''%' + AND regexp_replace( + pg_get_functiondef( + 'moa.enforce_execution_task_update()'::REGPROCEDURE + ), + '[[:space:]]+', ' ', 'g' + ) LIKE '%OLD.status = ''waiting_input'' AND NEW.status = ''ready''%' + AND regexp_replace( + pg_get_functiondef( + 'moa.enforce_execution_task_update()'::REGPROCEDURE + ), + '[[:space:]]+', ' ', 'g' + ) LIKE '%NEW.attempt_generation <> OLD.attempt_generation + 1%') + FROM moa.execution_run + WHERE run_uid = $1 + "#, + ) + .bind(run_uid) + .fetch_one(&target) + .await?; + + sqlx::query( + "INSERT INTO moa.execution_dispatch_outbox ( \ + dispatch_uid, tenant_id, run_uid, dispatch_kind, \ + controller_generation, wake_epoch \ + ) VALUES ($1, $2, $3, 'run_activation', 1, 1)", + ) + .bind(uuid::Uuid::new_v4()) + .bind(tenant_id) + .bind(run_uid) + .execute(&target) + .await?; + let duplicate_activation_rejected = sqlx::query( + "INSERT INTO moa.execution_dispatch_outbox ( \ + dispatch_uid, tenant_id, run_uid, dispatch_kind, \ + controller_generation, wake_epoch \ + ) VALUES ($1, $2, $3, 'run_activation', 1, 1)", + ) + .bind(uuid::Uuid::new_v4()) + .bind(tenant_id) + .bind(run_uid) + .execute(&target) + .await + .is_err(); + + target.close().await; + Ok::<_, Box>(( + cutover_error, + schema_not_partially_installed, + applied, + second, + retry_counters, + input_resume_counters, + invalid_attempt_generation_rejected, + catalog_shape, + duplicate_activation_rejected, + )) + } + .await; + + drop_database_with_zero_connections(&admin, &db_name).await; + admin.close().await; + + let ( + cutover_error, + schema_not_partially_installed, + applied, + second, + retry_counters, + input_resume_counters, + invalid_attempt_generation_rejected, + catalog_shape, + duplicate_activation_rejected, + ) = outcome.expect("long-horizon migration assertions should complete"); + assert!( + cutover_error.contains("legacy execution run(s) are nonterminal"), + "cutover diagnostic must identify the live-run precondition: {cutover_error}" + ); + assert!( + schema_not_partially_installed, + "the failed migration must leave no partial V59 catalog" + ); + assert_eq!( + applied, + expected_migration_labels_from("long_horizon_execution") + ); + assert!(second.is_empty(), "V59 must not reapply: {second:?}"); + assert_eq!(retry_counters, (2, 2, 2)); + assert_eq!(input_resume_counters, (1, 2, 2)); + assert!( + invalid_attempt_generation_rejected, + "attempt generation may only advance one fence at a time" + ); + assert_eq!(catalog_shape, (true, true, true, true, true, true)); + assert!( + duplicate_activation_rejected, + "one run generation/wake epoch must have exactly one dispatch" + ); +} diff --git a/crates/moa-migrations/tests/run_idempotency_db/execution_compensation.rs b/crates/moa-migrations/tests/run_idempotency_db/execution_compensation.rs index 10bc1a8f0..2552be64b 100644 --- a/crates/moa-migrations/tests/run_idempotency_db/execution_compensation.rs +++ b/crates/moa-migrations/tests/run_idempotency_db/execution_compensation.rs @@ -145,6 +145,10 @@ async fn seed_execution_run(target: &PgPool) -> TestResult { "cancel_policy": "retain_effects", "input_schema": {}, "output_schema": {}, + "input_wait_policy": { + "expiry": {"kind": "after", "delay_seconds": 1}, + "on_expiry": {"kind": "fail_run"} + }, "nodes": [{ "id": "output", "requirement_ids": [], @@ -192,10 +196,10 @@ async fn seed_execution_run(target: &PgPool) -> TestResult { planning_context_uid, planning_context_hash, owner_user_id, goal_contract, \ initial_plan, active_plan, initial_plan_hash, active_plan_hash, \ capability_catalog, authorization_envelope, source_provenance, source_kind, \ - input, status \ + input, admitted_identity, status \ ) VALUES ( \ $1, $2, $3, 0, $4, $5, 'migration-test', $6, $7, $7, $8, $8, \ - $9, $10, $11, 'generated_plan', '{}'::JSONB, 'awaiting_confirmation' \ + $9, $10, $11, 'generated_plan', '{}'::JSONB, $12, 'awaiting_confirmation' \ )", ) .bind(run_uid) @@ -229,6 +233,13 @@ async fn seed_execution_run(target: &PgPool) -> TestResult { "repair_attempts": 0 } })) + .bind(json!({ + "identity_type": "operator", + "id": run_uid, + "tenant_id": tenant_id, + "api_key_id": null, + "acting_on_behalf_of": null + })) .execute(target) .await?; sqlx::query( diff --git a/crates/moa-migrations/tests/run_idempotency_db/hand_leases.rs b/crates/moa-migrations/tests/run_idempotency_db/hand_leases.rs index 943d315ef..0d1c02ca8 100644 --- a/crates/moa-migrations/tests/run_idempotency_db/hand_leases.rs +++ b/crates/moa-migrations/tests/run_idempotency_db/hand_leases.rs @@ -884,7 +884,8 @@ async fn hand_storage_v58_requires_legacy_drain_and_installs_tenant_schema_db() AND class.relname IN (\ 'hand_leases', 'sandbox_workspaces', 'sandbox_workspace_operations',\ 'sandbox_workspace_checkpoints', 'sandbox_workspace_grants',\ - 'sandbox_storage_resources', 'sandbox_capacity_reservations'\ + 'sandbox_storage_resources', 'sandbox_capacity_reservations',\ + 'sandbox_execution_hand_release_receipts'\ ) ORDER BY class.relname", ) .fetch_all(&pool) @@ -896,6 +897,34 @@ async fn hand_storage_v58_requires_legacy_drain_and_installs_tenant_schema_db() ) .fetch_one(&pool) .await?; + let release_receipt_fks = sqlx::query_scalar::<_, bool>( + "SELECT count(*) = 4 FROM pg_constraint \ + WHERE conrelid = 'moa.sandbox_execution_hand_release_receipts'::regclass \ + AND conname IN (\ + 'sandbox_execution_hand_release_receipts_task_fk',\ + 'sandbox_execution_hand_release_receipts_compensation_fk',\ + 'sandbox_execution_hand_release_receipts_workspace_fk',\ + 'sandbox_execution_hand_release_receipts_checkpoint_fk'\ + )", + ) + .fetch_one(&pool) + .await?; + let release_receipt_retention_guards = sqlx::query_scalar::<_, bool>( + "SELECT count(*) = 2 FROM pg_trigger AS trigger \ + JOIN pg_proc AS procedure ON procedure.oid = trigger.tgfoid \ + JOIN pg_namespace AS namespace ON namespace.oid = procedure.pronamespace \ + WHERE trigger.tgrelid = 'moa.sandbox_execution_hand_release_receipts'::regclass \ + AND NOT trigger.tgisinternal \ + AND namespace.nspname = 'moa' \ + AND (trigger.tgname, procedure.proname) IN (\ + ('sandbox_execution_hand_release_receipt_archived_write_guard',\ + 'reject_execution_archived_detail_write'),\ + ('sandbox_execution_hand_release_receipt_delete_guard',\ + 'reject_execution_immutable_payload')\ + )", + ) + .fetch_one(&pool) + .await?; pool.close().await; Ok::<_, Box>(( first_error, @@ -904,12 +933,22 @@ async fn hand_storage_v58_requires_legacy_drain_and_installs_tenant_schema_db() attachment, rls_tables, composite_fk, + release_receipt_fks, + release_receipt_retention_guards, )) } .await; let outcome = database.finish(outcome).await; - let (first_error, applied, workspace_count, attachment, rls_tables, composite_fk) = - outcome.expect("V58 should apply after legacy compute is drained"); + let ( + first_error, + applied, + workspace_count, + attachment, + rls_tables, + composite_fk, + release_receipt_fks, + release_receipt_retention_guards, + ) = outcome.expect("V58 should apply after legacy compute is drained"); assert!( first_error.contains("legacy hands remain live"), "preflight should require an explicit drain: {first_error}" @@ -922,7 +961,7 @@ async fn hand_storage_v58_requires_legacy_drain_and_installs_tenant_schema_db() ); assert_eq!(workspace_count, 0, "V58 must not fabricate workspace state"); assert_eq!(attachment, (None, None, None)); - assert_eq!(rls_tables.len(), 7); + assert_eq!(rls_tables.len(), 8); assert!( rls_tables .iter() @@ -933,4 +972,12 @@ async fn hand_storage_v58_requires_legacy_drain_and_installs_tenant_schema_db() composite_fk, "hand leases must reference session plus tenant" ); + assert!( + release_receipt_fks, + "task hand release receipts must retain task, workspace, and checkpoint ownership" + ); + assert!( + release_receipt_retention_guards, + "task hand release receipts must reject post-archive writes and fence retention deletes" + ); } diff --git a/crates/moa-observability/src/runtime_metrics.rs b/crates/moa-observability/src/runtime_metrics.rs index 72148dd5f..921a8ad7b 100644 --- a/crates/moa-observability/src/runtime_metrics.rs +++ b/crates/moa-observability/src/runtime_metrics.rs @@ -94,6 +94,102 @@ impl WorkerFanInSettledKind { } } +/// Bounded nonterminal execution phases exported for fleet run counts. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ExecutionRunMetricPhase { + /// Accepted work waiting for controller activation. + Queued, + /// Work currently advancing or running an attempt. + Running, + /// Storage-only wait for user or external input. + WaitingInput, + /// Storage-only wait for tenant review. + WaitingReview, + /// Storage-only wait for a named signal. + WaitingSignal, + /// Storage-only wait for an exact durable timer. + WaitingTimer, + /// Storage-only wait for an asynchronous external job. + WaitingExternal, + /// A pause has been requested but active work is still settling. + PauseRequested, + /// The run is checkpointing and releasing resources before pausing. + Pausing, + /// The run is fully parked by an operator request. + Paused, + /// The run is reversing committed compensatable effects. + Compensating, +} + +impl ExecutionRunMetricPhase { + /// Returns the stable low-cardinality phase label. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Queued => "queued", + Self::Running => "running", + Self::WaitingInput => "waiting_input", + Self::WaitingReview => "waiting_review", + Self::WaitingSignal => "waiting_signal", + Self::WaitingTimer => "waiting_timer", + Self::WaitingExternal => "waiting_external", + Self::PauseRequested => "pause_requested", + Self::Pausing => "pausing", + Self::Paused => "paused", + Self::Compensating => "compensating", + } + } +} + +/// Bounded long-horizon resources governed by execution admission. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ExecutionAdmissionResource { + /// Nonterminal runs that are not fully parked. + ActiveRuns, + /// Task attempts currently holding active-compute reservations. + ActiveAttempts, + /// Runs retained in storage-only waiting or paused states. + ParkedRuns, + /// Pending durable trigger rows. + ScheduledTriggers, + /// Nonterminal asynchronous provider jobs. + ExternalJobs, +} + +impl ExecutionAdmissionResource { + /// Returns the stable low-cardinality resource label. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::ActiveRuns => "active_runs", + Self::ActiveAttempts => "active_attempts", + Self::ParkedRuns => "parked_runs", + Self::ScheduledTriggers => "scheduled_triggers", + Self::ExternalJobs => "external_jobs", + } + } +} + +/// Bounded aggregation scopes for execution admission utilization. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ExecutionAdmissionScope { + /// Utilization of the shared fleet ceiling. + Fleet, + /// Highest utilization observed across tenant-scoped ceilings. + TenantPeak, +} + +impl ExecutionAdmissionScope { + /// Returns the stable low-cardinality aggregation-scope label. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Fleet => "fleet", + Self::TenantPeak => "tenant_peak", + } + } +} + /// Bounded sandbox-provider classes permitted on workspace metric labels. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum SandboxWorkspaceProviderKind { @@ -963,6 +1059,152 @@ pub fn record_execution_owned_in_flight_tasks(count: usize) { histogram!("moa_execution_owned_in_flight_tasks").record(count as f64); } +/// Sets the current fleet count for one bounded nonterminal execution phase. +pub fn record_execution_run_phase(phase: ExecutionRunMetricPhase, count: u64) { + gauge!("moa_execution_runs", "phase" => phase.as_str()).set(count as f64); +} + +/// Sets the age of the oldest task currently ready for dispatch. +pub fn record_execution_oldest_ready_age(age: Duration) { + gauge!("moa_execution_oldest_ready_age_seconds").set(age.as_secs_f64()); +} + +/// Sets the number of nonterminal runs whose absolute deadline has elapsed. +pub fn record_execution_overdue_deadlines(count: u64) { + gauge!("moa_execution_overdue_deadlines").set(count as f64); +} + +/// Sets trigger delivery lag, capped depth, and sample-completeness from one fleet snapshot. +pub fn record_execution_trigger_queue( + lag: Duration, + due_triggers: u64, + due_sample_saturated: bool, + dead_letters: u64, + dead_letter_sample_saturated: bool, +) { + gauge!("moa_execution_trigger_lag_seconds").set(lag.as_secs_f64()); + gauge!("moa_execution_trigger_due").set(due_triggers as f64); + gauge!("moa_execution_trigger_dead_letters").set(dead_letters as f64); + gauge!( + "moa_execution_queue_sample_saturated", + "queue" => "trigger", + "sample" => "due" + ) + .set(if due_sample_saturated { 1.0 } else { 0.0 }); + gauge!( + "moa_execution_queue_sample_saturated", + "queue" => "trigger", + "sample" => "dead_letter" + ) + .set(if dead_letter_sample_saturated { + 1.0 + } else { + 0.0 + }); +} + +/// Sets outbox delivery lag, capped depth, and sample-completeness from one fleet snapshot. +pub fn record_execution_outbox_queue( + lag: Duration, + claimable_dispatches: u64, + claimable_sample_saturated: bool, + dead_letters: u64, + dead_letter_sample_saturated: bool, +) { + gauge!("moa_execution_outbox_lag_seconds").set(lag.as_secs_f64()); + gauge!("moa_execution_outbox_claimable").set(claimable_dispatches as f64); + gauge!("moa_execution_outbox_dead_letters").set(dead_letters as f64); + gauge!( + "moa_execution_queue_sample_saturated", + "queue" => "outbox", + "sample" => "claimable" + ) + .set(if claimable_sample_saturated { 1.0 } else { 0.0 }); + gauge!( + "moa_execution_queue_sample_saturated", + "queue" => "outbox", + "sample" => "dead_letter" + ) + .set(if dead_letter_sample_saturated { + 1.0 + } else { + 0.0 + }); +} + +/// Sets the age of the oldest active task-attempt lease. +pub fn record_execution_active_attempt_oldest_age(age: Duration) { + gauge!("moa_execution_active_attempt_oldest_age_seconds").set(age.as_secs_f64()); +} + +/// Sets the age of the oldest nonterminal asynchronous external job. +pub fn record_execution_external_job_oldest_age(age: Duration) { + gauge!("moa_execution_external_job_oldest_age_seconds").set(age.as_secs_f64()); +} + +/// Sets admission utilization for one bounded resource and aggregation scope. +pub fn record_execution_admission_utilization( + resource: ExecutionAdmissionResource, + scope: ExecutionAdmissionScope, + ratio: f64, +) { + gauge!( + "moa_execution_admission_utilization_ratio", + "resource" => resource.as_str(), + "scope" => scope.as_str() + ) + .set(ratio.clamp(0.0, 1.0)); +} + +/// Sets the largest tenant share of one fleet execution resource. +/// +/// This exposes fairness pressure without tenant identifiers or one series per +/// tenant. Callers calculate the maximum share from a complete fleet snapshot. +pub fn record_execution_tenant_max_share(resource: ExecutionAdmissionResource, ratio: f64) { + gauge!( + "moa_execution_tenant_max_share_ratio", + "resource" => resource.as_str() + ) + .set(ratio.clamp(0.0, 1.0)); +} + +/// Sets maintenance health from the durable last-success reconciliation receipt. +/// +/// A missing receipt is exported as positive infinity so it is unambiguously +/// older than every finite staleness threshold. +pub fn record_execution_maintenance(ready: bool, last_success_age: Option) { + gauge!("moa_execution_maintenance_ready").set(if ready { 1.0 } else { 0.0 }); + gauge!("moa_execution_maintenance_last_success_age_seconds") + .set(last_success_age.map_or(f64::INFINITY, |duration| duration.as_secs_f64())); +} + +/// Sets execution-retention health from its independent durable success receipt. +/// +/// A missing receipt is exported as positive infinity so a process restart cannot +/// make retention appear fresh before a bounded retention pass has succeeded. +pub fn record_execution_retention(ready: bool, last_success_age: Option) { + gauge!("moa_execution_retention_ready").set(if ready { 1.0 } else { 0.0 }); + gauge!("moa_execution_retention_last_success_age_seconds") + .set(last_success_age.map_or(f64::INFINITY, |duration| duration.as_secs_f64())); +} + +/// Sets draining Restate deployment age and resource-cost snapshots. +/// +/// Replica-hours is a gauge because the maintenance owner derives the complete +/// current value from Kubernetes observations rather than incrementing it from +/// potentially duplicated process-local samples. +pub fn record_restate_draining_deployments( + deployments: u64, + replicas: u64, + oldest_age: Duration, + replica_hours: f64, +) { + gauge!("moa_restate_draining_deployments").set(deployments as f64); + gauge!("moa_restate_draining_deployment_replicas").set(replicas as f64); + gauge!("moa_restate_draining_deployment_oldest_age_seconds").set(oldest_age.as_secs_f64()); + gauge!("moa_restate_draining_deployment_replica_hours").set(replica_hours.max(0.0)); +} + /// Records whether one worker terminal delivery was accepted or deduplicated. pub fn record_worker_terminal_delivery(result: WorkerTerminalDeliveryResult) { counter!( @@ -1113,6 +1355,45 @@ pub fn record_sandbox_workspace_reaper( .set(oldest_work_age.as_secs_f64()); } +/// Sets active sandbox-hand compute by bounded provider class. +pub fn record_sandbox_workspace_active_hands(provider: SandboxWorkspaceProviderKind, count: u64) { + gauge!( + "moa_sandbox_workspace_active_hands", + "provider_kind" => provider.as_str() + ) + .set(count as f64); +} + +/// Sets the number of parked execution tasks that still own active compute. +/// +/// This must remain zero. The aggregate intentionally carries no run, task, or +/// tenant label so an invariant check cannot create an unbounded series. +pub fn record_sandbox_workspace_parked_tasks_with_active_hands(count: u64) { + gauge!("moa_sandbox_workspace_parked_tasks_with_active_hands").set(count as f64); +} + +/// Records one portable-checkpoint restore into fresh sandbox compute. +pub fn record_sandbox_workspace_restore(provider: SandboxWorkspaceProviderKind) { + counter!( + "moa_sandbox_workspace_restores_total", + "provider_kind" => provider.as_str() + ) + .increment(1); +} + +/// Records one checkpoint-and-release result at an execution yield boundary. +pub fn record_sandbox_workspace_release( + provider: SandboxWorkspaceProviderKind, + result: SandboxWorkspaceMetricResult, +) { + counter!( + "moa_sandbox_workspace_releases_total", + "provider_kind" => provider.as_str(), + "result" => result.as_str() + ) + .increment(1); +} + /// Records checkpoint bytes and latency for one bounded lifecycle outcome. pub fn record_sandbox_workspace_checkpoint( provider: SandboxWorkspaceProviderKind, @@ -1515,6 +1796,94 @@ fn register_metric_descriptions() { "moa_execution_owned_in_flight_tasks", "Task calls owned by an ExecutionRun after a bounded dispatch refill." ); + describe_gauge!( + "moa_execution_runs", + "Current nonterminal execution runs by bounded product phase." + ); + describe_gauge!( + "moa_execution_oldest_ready_age_seconds", + "Age in seconds of the oldest execution task ready for dispatch." + ); + describe_gauge!( + "moa_execution_overdue_deadlines", + "Nonterminal execution runs whose absolute deadline has elapsed." + ); + describe_gauge!( + "moa_execution_trigger_lag_seconds", + "Age in seconds of the oldest due undelivered execution trigger." + ); + describe_gauge!( + "moa_execution_trigger_due", + "Due execution triggers observed in the bounded fleet queue-health sample." + ); + describe_gauge!( + "moa_execution_trigger_dead_letters", + "Execution triggers currently held in dead-letter state." + ); + describe_gauge!( + "moa_execution_outbox_lag_seconds", + "Age in seconds of the oldest undispatched execution outbox row." + ); + describe_gauge!( + "moa_execution_outbox_claimable", + "Claimable dispatch-outbox rows observed in the bounded fleet queue-health sample." + ); + describe_gauge!( + "moa_execution_outbox_dead_letters", + "Execution dispatch-outbox rows currently held in dead-letter state." + ); + describe_gauge!( + "moa_execution_queue_sample_saturated", + "Whether a bounded execution queue-health sample reached its observation cap, by fixed queue and sample kind." + ); + describe_gauge!( + "moa_execution_active_attempt_oldest_age_seconds", + "Age in seconds of the oldest active execution task-attempt lease." + ); + describe_gauge!( + "moa_execution_external_job_oldest_age_seconds", + "Age in seconds of the oldest nonterminal asynchronous execution job." + ); + describe_gauge!( + "moa_execution_admission_utilization_ratio", + "Execution admission utilization by bounded resource and aggregation scope." + ); + describe_gauge!( + "moa_execution_tenant_max_share_ratio", + "Largest tenant share of one bounded fleet execution resource." + ); + describe_gauge!( + "moa_execution_maintenance_ready", + "Whether the singleton execution-maintenance owner is healthy." + ); + describe_gauge!( + "moa_execution_maintenance_last_success_age_seconds", + "Age in seconds of the durable last successful bounded execution reconciliation receipt." + ); + describe_gauge!( + "moa_execution_retention_ready", + "Whether execution retention has a healthy durable success receipt." + ); + describe_gauge!( + "moa_execution_retention_last_success_age_seconds", + "Age in seconds of the durable last successful bounded execution-retention receipt." + ); + describe_gauge!( + "moa_restate_draining_deployments", + "Restate service deployment revisions still draining active invocations." + ); + describe_gauge!( + "moa_restate_draining_deployment_replicas", + "Kubernetes replicas retained by draining Restate deployment revisions." + ); + describe_gauge!( + "moa_restate_draining_deployment_oldest_age_seconds", + "Age in seconds of the oldest draining Restate deployment revision." + ); + describe_gauge!( + "moa_restate_draining_deployment_replica_hours", + "Replica-hours currently attributable to draining Restate revisions." + ); describe_counter!( "moa_worker_terminal_deliveries_total", "Worker terminal deliveries by bounded acceptance result." @@ -1579,6 +1948,22 @@ fn register_metric_descriptions() { "moa_sandbox_workspace_reaper_oldest_work_age_seconds", "Age in seconds of the oldest workspace reaper item." ); + describe_gauge!( + "moa_sandbox_workspace_active_hands", + "Active sandbox-hand compute by bounded provider class." + ); + describe_gauge!( + "moa_sandbox_workspace_parked_tasks_with_active_hands", + "Parked execution tasks that incorrectly retain active sandbox compute." + ); + describe_counter!( + "moa_sandbox_workspace_restores_total", + "Portable-checkpoint restores into fresh compute by bounded provider class." + ); + describe_counter!( + "moa_sandbox_workspace_releases_total", + "Checkpoint-and-release outcomes at execution yield boundaries." + ); describe_counter!( "moa_sandbox_workspace_checkpoint_bytes_total", "Portable checkpoint bytes processed by bounded provider class, operation, and result." @@ -1874,6 +2259,148 @@ mod tests { } } + #[test] + fn long_horizon_metrics_export_descriptions_and_only_bounded_labels() { + // Pins: execution, drain, and sandbox-yield health reaches production + // exporters without tenant, run, task, deployment-version, or provider-account IDs. + let recorder = PrometheusBuilder::new().build_recorder(); + let handle = recorder.handle(); + metrics::with_local_recorder(&recorder, || { + register_metric_descriptions(); + for phase in [ + ExecutionRunMetricPhase::Queued, + ExecutionRunMetricPhase::Running, + ExecutionRunMetricPhase::WaitingInput, + ExecutionRunMetricPhase::WaitingReview, + ExecutionRunMetricPhase::WaitingSignal, + ExecutionRunMetricPhase::WaitingTimer, + ExecutionRunMetricPhase::WaitingExternal, + ExecutionRunMetricPhase::PauseRequested, + ExecutionRunMetricPhase::Pausing, + ExecutionRunMetricPhase::Paused, + ExecutionRunMetricPhase::Compensating, + ] { + record_execution_run_phase(phase, 1); + } + record_execution_oldest_ready_age(Duration::from_secs(31)); + record_execution_overdue_deadlines(2); + record_execution_trigger_queue(Duration::from_secs(7), 11, true, 1, false); + record_execution_outbox_queue(Duration::from_secs(8), 12, false, 1, true); + record_execution_active_attempt_oldest_age(Duration::from_secs(61)); + record_execution_external_job_oldest_age(Duration::from_secs(62)); + record_execution_admission_utilization( + ExecutionAdmissionResource::ActiveAttempts, + ExecutionAdmissionScope::Fleet, + 0.75, + ); + record_execution_admission_utilization( + ExecutionAdmissionResource::ParkedRuns, + ExecutionAdmissionScope::TenantPeak, + 0.5, + ); + record_execution_tenant_max_share(ExecutionAdmissionResource::ActiveRuns, 0.4); + record_execution_maintenance(true, Some(Duration::from_secs(3))); + record_execution_maintenance(false, None); + record_execution_retention(true, Some(Duration::from_secs(3_600))); + record_execution_retention(false, None); + record_restate_draining_deployments(2, 3, Duration::from_secs(3_600), 4.5); + record_sandbox_workspace_active_hands(SandboxWorkspaceProviderKind::E2b, 2); + record_sandbox_workspace_parked_tasks_with_active_hands(0); + record_sandbox_workspace_restore(SandboxWorkspaceProviderKind::E2b); + record_sandbox_workspace_release( + SandboxWorkspaceProviderKind::E2b, + SandboxWorkspaceMetricResult::Succeeded, + ); + }); + let rendered = handle.render(); + + let metrics = [ + "moa_execution_runs", + "moa_execution_oldest_ready_age_seconds", + "moa_execution_overdue_deadlines", + "moa_execution_trigger_lag_seconds", + "moa_execution_trigger_due", + "moa_execution_trigger_dead_letters", + "moa_execution_outbox_lag_seconds", + "moa_execution_outbox_claimable", + "moa_execution_outbox_dead_letters", + "moa_execution_queue_sample_saturated", + "moa_execution_active_attempt_oldest_age_seconds", + "moa_execution_external_job_oldest_age_seconds", + "moa_execution_admission_utilization_ratio", + "moa_execution_tenant_max_share_ratio", + "moa_execution_maintenance_ready", + "moa_execution_maintenance_last_success_age_seconds", + "moa_execution_retention_ready", + "moa_execution_retention_last_success_age_seconds", + "moa_restate_draining_deployments", + "moa_restate_draining_deployment_replicas", + "moa_restate_draining_deployment_oldest_age_seconds", + "moa_restate_draining_deployment_replica_hours", + "moa_sandbox_workspace_active_hands", + "moa_sandbox_workspace_parked_tasks_with_active_hands", + "moa_sandbox_workspace_restores_total", + "moa_sandbox_workspace_releases_total", + ]; + for metric in metrics { + assert!( + rendered.contains(&format!("# HELP {metric} ")), + "long-horizon metric {metric} should export a HELP description; rendered:\n{rendered}" + ); + } + + assert!( + rendered.contains("moa_execution_maintenance_ready 0"), + "a missing durable success receipt must make maintenance unready:\n{rendered}" + ); + assert!( + rendered.contains("moa_execution_maintenance_last_success_age_seconds inf"), + "a missing durable success receipt must be older than every finite SLO:\n{rendered}" + ); + assert!( + rendered.contains("moa_execution_retention_ready 0"), + "a missing durable retention receipt must make retention unready:\n{rendered}" + ); + assert!( + rendered.contains("moa_execution_retention_last_success_age_seconds inf"), + "a missing durable retention receipt must be older than every finite SLO:\n{rendered}" + ); + + for label in [ + "phase=\"waiting_timer\"", + "resource=\"active_attempts\"", + "scope=\"fleet\"", + "scope=\"tenant_peak\"", + "provider_kind=\"e2b\"", + "result=\"succeeded\"", + "queue=\"trigger\"", + "sample=\"due\"", + "queue=\"outbox\"", + "sample=\"dead_letter\"", + ] { + assert!( + rendered.contains(label), + "long-horizon metrics should include bounded label `{label}`:\n{rendered}" + ); + } + for forbidden in [ + "tenant_id", + "run_id", + "run_uid", + "task_id", + "task_uid", + "deployment_id", + "deployment_version", + "provider_account_id", + "external_job_id", + ] { + assert!( + !rendered.contains(forbidden), + "long-horizon metrics must not carry high-cardinality label `{forbidden}`:\n{rendered}" + ); + } + } + #[test] fn tool_name_label_buckets_unknown_tools_as_other() { // Pins: built-in tool names pass through as metric labels; tenant/MCP-defined names bucket diff --git a/crates/moa-orchestrator/src/action_reviews/app.rs b/crates/moa-orchestrator/src/action_reviews/app.rs index 00a31033c..3981de4b4 100644 --- a/crates/moa-orchestrator/src/action_reviews/app.rs +++ b/crates/moa-orchestrator/src/action_reviews/app.rs @@ -433,8 +433,9 @@ pub(crate) async fn mark_owner_registered( pool: sqlx::PgPool, storage_partition_id: StoragePartitionId, review_id: Uuid, + expected_owner: Option<&ActionReviewOwner>, ) -> Result<(), HandlerError> { - store::mark_owner_registered(pool, storage_partition_id, review_id).await + store::mark_owner_registered(pool, storage_partition_id, review_id, expected_owner).await } fn decision_from_request(request: &DecideActionReviewRequest) -> ActionReviewDecision { diff --git a/crates/moa-orchestrator/src/action_reviews/store.rs b/crates/moa-orchestrator/src/action_reviews/store.rs index 90f503b08..e5bbad9ae 100644 --- a/crates/moa-orchestrator/src/action_reviews/store.rs +++ b/crates/moa-orchestrator/src/action_reviews/store.rs @@ -115,6 +115,32 @@ pub(crate) struct PendingActionReviewRelease { pub(crate) release: ActionReviewRelease, } +/// Exact tenant action-review row requested by a durable timeout delivery. +pub(crate) struct ActionReviewTimeoutLookup { + /// Tenant that owns the review and supplies the control-plane isolation fence. + pub(crate) tenant_id: TenantId, + /// Stable review identifier carried by the delayed trigger. + pub(crate) review_id: Uuid, +} + +/// Persisted action-review state used to fence one durable timeout delivery. +pub(crate) struct ActionReviewTimeoutSnapshot { + /// Exact owner incarnation stored when the review was created. + pub(crate) owner: ActionReviewOwner, + /// Current typed review status. + pub(crate) status: ActionReviewStatus, + /// Whether the persisted expiry is due at the database clock. + pub(crate) is_due: bool, + /// Whether the owner durably acknowledged review registration. + pub(crate) owner_registered: bool, + /// Timestamp proving durable execution already claimed a clear decision. + pub(crate) execution_requested_at: Option>, + /// Persisted terminal decision timestamp. + pub(crate) decided_at: Option>, + /// Timestamp proving conversational owner release delivery completed. + pub(crate) owner_release_delivered_at: Option>, +} + /// Insert a pending tenant action review, or load the existing idempotent row. /// /// `review_timeout_secs` sets the row's `expires_at` relative to insertion so @@ -224,7 +250,7 @@ pub(crate) async fn list_pending_reviews( r#" SELECT id, tenant_id, storage_partition_id, session_id, worker_id, tool_call_id, tool_name, action_class, risk_level, input_summary, envelope, preview, status, - requested_by, decided_by, deny_reason, created_at, decided_at + requested_by, decided_by, deny_reason, created_at, expires_at, decided_at FROM tenant_action_reviews WHERE storage_partition_id = $1 AND status = 'pending' @@ -448,7 +474,12 @@ pub(crate) async fn mark_owner_registered( pool: sqlx::PgPool, storage_partition_id: StoragePartitionId, review_id: Uuid, + expected_owner: Option<&ActionReviewOwner>, ) -> Result<(), HandlerError> { + let expected_owner = expected_owner + .map(serde_json::to_value) + .transpose() + .map_err(|error| TerminalError::new(format!("serialize action review owner: {error}")))?; let result = sqlx::query( r#" UPDATE tenant_action_reviews @@ -456,10 +487,12 @@ pub(crate) async fn mark_owner_registered( WHERE storage_partition_id = $1 AND id = $2 AND status = 'pending' + AND ($3::JSONB IS NULL OR envelope -> 'owner' = $3) "#, ) .bind(storage_partition_id.to_string()) .bind(review_id) + .bind(expected_owner) .execute(&pool) .await .map_err(db_error)?; @@ -557,6 +590,51 @@ pub(crate) async fn timeout_expired_reviews( Ok(timed_out) } +/// Loads one exact action-review timeout snapshot under control-plane scope. +/// +/// Both tenant and review identity are matched before the stored owner is +/// returned, so the caller can compare the complete owner-generation fence +/// without crossing the action-review storage boundary. +pub(crate) async fn load_action_review_timeout_snapshot( + pool: &sqlx::PgPool, + request: ActionReviewTimeoutLookup, +) -> Result, sqlx::Error> { + let mut tx = pool.begin().await?; + install_control_plane_scope(&mut tx).await?; + let row = sqlx::query( + r#" + SELECT envelope, status, expires_at <= NOW() AS is_due, + owner_registered_at IS NOT NULL AS owner_registered, + execution_requested_at, decided_at, owner_release_delivered_at + FROM tenant_action_reviews + WHERE id = $1 AND tenant_id = $2 + "#, + ) + .bind(request.review_id) + .bind(request.tenant_id.0) + .fetch_optional(&mut *tx) + .await?; + tx.commit().await?; + row.map(|row| { + let envelope: ActionEnvelope = serde_json::from_value(row.try_get("envelope")?) + .map_err(|error| sqlx::Error::Decode(Box::new(error)))?; + let status = row + .try_get::("status")? + .parse::() + .map_err(|_| sqlx::Error::Decode("unknown action review status".into()))?; + Ok(ActionReviewTimeoutSnapshot { + owner: envelope.owner, + status, + is_due: row.try_get("is_due")?, + owner_registered: row.try_get("owner_registered")?, + execution_requested_at: row.try_get("execution_requested_at")?, + decided_at: row.try_get("decided_at")?, + owner_release_delivered_at: row.try_get("owner_release_delivered_at")?, + }) + }) + .transpose() +} + /// Loads one bounded batch of timed-out conversational owner releases. pub(crate) async fn pending_action_review_releases( pool: &sqlx::PgPool, @@ -945,7 +1023,7 @@ async fn load_review_state( r#" SELECT id, tenant_id, storage_partition_id, session_id, worker_id, tool_call_id, tool_name, action_class, risk_level, input_summary, envelope, preview, status, - requested_by, decided_by, deny_reason, created_at, decided_at, + requested_by, decided_by, deny_reason, created_at, expires_at, decided_at, owner_registered_at, tool_request FROM tenant_action_reviews WHERE storage_partition_id = $1 AND id = $2 @@ -1015,6 +1093,7 @@ fn summary_from_row(row: &sqlx::postgres::PgRow) -> Result Result, sqlx::Error> { + let delay_millis = sqlx::query_scalar::<_, i64>( + r#" + SELECT GREATEST( + FLOOR(EXTRACT(EPOCH FROM (expires_at - NOW())) * 1000), + 0 + )::BIGINT + FROM builtin_pending_approvals + WHERE id = $1 AND awakeable_id = $2 AND status = 'pending' + "#, + ) + .bind(request.challenge_id) + .bind(&request.awakeable_id) + .fetch_optional(pool) + .await?; + delay_millis + .map(|delay_millis| { + u64::try_from(delay_millis) + .map(|delay_millis| BuiltinChallengeTimeoutDelay { delay_millis }) + .map_err(|error| sqlx::Error::Decode(Box::new(error))) + }) + .transpose() +} + +/// Applies and claims one exact delayed builtin challenge timeout atomically. +/// +/// The pending-to-timeout transition, resolution lease claim, and persisted +/// state inspection share one transaction so replays and competing reapers +/// observe a single durable delivery owner. +pub(crate) async fn apply_builtin_challenge_timeout( + pool: &sqlx::PgPool, + request: &BuiltinChallengeTimeoutLookup, +) -> Result { + let mut tx = pool.begin().await?; + let newly_timed_out = sqlx::query( + r#" + UPDATE builtin_pending_approvals + SET status = 'timeout', + decided_at = NOW() + WHERE id = $1 + AND awakeable_id = $2 + AND status = 'pending' + AND expires_at <= NOW() + AND resolved_at IS NULL + "#, + ) + .bind(request.challenge_id) + .bind(&request.awakeable_id) + .execute(&mut *tx) + .await? + .rows_affected() + == 1; + + let resolve_claim_token = Uuid::new_v4(); + let claimed: Option<(Uuid, String, Uuid)> = sqlx::query_as( + r#" + UPDATE builtin_pending_approvals + SET resolve_claim_token = $3, + resolve_claim_expires_at = NOW() + INTERVAL '2 minutes' + WHERE id = $1 + AND awakeable_id = $2 + AND status = 'timeout' + AND resolved_at IS NULL + AND ( + resolve_claim_expires_at IS NULL + OR resolve_claim_expires_at <= NOW() + ) + RETURNING id, awakeable_id, resolve_claim_token + "#, + ) + .bind(request.challenge_id) + .bind(&request.awakeable_id) + .bind(resolve_claim_token) + .fetch_optional(&mut *tx) + .await?; + + let state: Option<(String, Option>, Option)> = sqlx::query_as( + r#" + SELECT status, resolved_at, resolve_claim_token + FROM builtin_pending_approvals + WHERE id = $1 AND awakeable_id = $2 + "#, + ) + .bind(request.challenge_id) + .bind(&request.awakeable_id) + .fetch_optional(&mut *tx) + .await?; + tx.commit().await?; + + if let Some((challenge_id, awakeable_id, resolve_claim_token)) = claimed { + return Ok(BuiltinChallengeTimeoutClaim::Resolve { + challenge_id, + awakeable_id, + resolve_claim_token, + newly_timed_out, + }); + } + match state { + Some((status, resolved_at, claim)) + if status == "timeout" && (resolved_at.is_some() || claim.is_some()) => + { + Ok(BuiltinChallengeTimeoutClaim::AlreadyDelivered) + } + _ => Ok(BuiltinChallengeTimeoutClaim::Stale), + } +} + /// Result of one reaper sweep over builtin approvals. pub(crate) struct BuiltinChallengeSweep { /// Terminal rows still awaiting awakeable delivery. diff --git a/crates/moa-orchestrator/src/external_job_ingress.rs b/crates/moa-orchestrator/src/external_job_ingress.rs new file mode 100644 index 000000000..e90b31923 --- /dev/null +++ b/crates/moa-orchestrator/src/external_job_ingress.rs @@ -0,0 +1,1053 @@ +//! Private non-Restate boundary for asynchronous-provider callbacks. + +use std::collections::BTreeMap; +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; +use axum::Router; +use axum::body::Bytes; +use axum::extract::{DefaultBodyLimit, Path, State}; +use axum::http::{HeaderMap, StatusCode}; +use axum::response::{IntoResponse, Response}; +use axum::routing::post; +use moa_core::error::MoaError; +use moa_execution::repository::external_job::{ + ExecutionExternalJobCallback, ExecutionExternalJobCallbackOutcome, + ExecutionExternalJobCallbackWrite, ExecutionExternalJobRecord, +}; +use moa_execution::repository::{ExecutionRepository, ExecutionScope}; +use reqwest::{Client, Url}; +use serde::Deserialize; +use sha2::{Digest, Sha256}; +use uuid::Uuid; + +use crate::services::execution_dispatcher::DispatchExecutionsRequest; +use crate::services::tool_executor::{ + ExecutionExternalJobAdapterRegistry, ExecutionExternalJobCallbackAuthentication, +}; + +/// Exact private callback route served outside Restate. +pub const EXTERNAL_JOB_CALLBACK_INGRESS_ROUTE: &str = "/internal/v1/execution/external-jobs/{external_job_uid}/generations/{job_generation}/callbacks/{provider_event_id}"; +/// Maximum callback body accepted before adapter authentication or parsing. +pub const MAX_EXTERNAL_JOB_CALLBACK_INGRESS_BODY_BYTES: usize = 256 * 1024; +/// Maximum number of provider callback headers. +pub const MAX_EXTERNAL_JOB_CALLBACK_INGRESS_HEADERS: usize = 64; +/// Maximum aggregate provider callback-header bytes. +pub const MAX_EXTERNAL_JOB_CALLBACK_INGRESS_HEADER_BYTES: usize = 32 * 1024; +const MAX_PROVIDER_EVENT_ID_BYTES: usize = 512; + +/// Public-safe callback ingress failure. +#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)] +pub enum ExternalJobIngressError { + /// Selectors, headers, or parsed callback fields were invalid. + #[error("invalid external-job callback")] + InvalidRequest, + /// Raw callback evidence exceeded a fixed limit. + #[error("external-job callback exceeds the size limit")] + RequestTooLarge, + /// Provider authentication failed. + #[error("external-job callback authentication failed")] + Unauthorized, + /// Persistence or a provider authentication dependency was unavailable. + #[error("external-job callback service unavailable")] + Unavailable, +} + +/// Stable disposition returned after all transient callback bytes are dropped. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ExternalJobCallbackDisposition { + /// The exact generation advanced. + Applied, + /// The provider event was already durably accepted. + Duplicate, + /// The path generation or provider job identity was stale. + Stale, + /// The job had already reached a terminal state. + AlreadyTerminal, +} + +/// Persistence seam for the callback boundary. +#[async_trait] +pub trait ExternalJobCallbackStore: Send + Sync { + /// Loads canonical job metadata before adapter selection and authentication. + async fn load( + &self, + external_job_uid: Uuid, + ) -> Result, moa_execution::Error>; + + /// Atomically persists the callback receipt, transition, and controller wake. + async fn apply( + &self, + config: &moa_config::ExecutionConfig, + callback: ExecutionExternalJobCallback, + ) -> Result; +} + +#[async_trait] +impl ExternalJobCallbackStore for ExecutionRepository { + async fn load( + &self, + external_job_uid: Uuid, + ) -> Result, moa_execution::Error> { + self.load_external_job(ExecutionScope::ControlPlane, external_job_uid) + .await + } + + async fn apply( + &self, + config: &moa_config::ExecutionConfig, + callback: ExecutionExternalJobCallback, + ) -> Result { + self.apply_external_job_callback_and_activate( + ExecutionScope::ControlPlane, + config, + callback, + ) + .await + } +} + +/// Best-effort persist-then-send dispatcher wake. +#[async_trait] +pub trait ExternalJobDispatcherKick: Send + Sync { + /// Requests one bounded outbox drain using a stable idempotency key. + async fn kick(&self, idempotency_key: &str) -> Result<(), ExternalJobDispatcherKickError>; +} + +/// Sanitized dispatcher acceptance failure. +#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)] +#[error("execution dispatcher did not accept callback wake")] +pub struct ExternalJobDispatcherKickError; + +/// Restate HTTP implementation of the best-effort dispatcher wake. +pub struct RestateExternalJobDispatcherKick { + http: Client, + dispatch_url: Url, +} + +impl RestateExternalJobDispatcherKick { + /// Builds the dispatcher client from an origin-only Restate ingress URL. + pub fn new(restate_ingress_origin: impl AsRef) -> Result { + let mut dispatch_url = Url::parse(restate_ingress_origin.as_ref()) + .map_err(|_| MoaError::ConfigError("invalid Restate ingress origin".to_string()))?; + if !matches!(dispatch_url.scheme(), "http" | "https") + || dispatch_url.host_str().is_none() + || !dispatch_url.username().is_empty() + || dispatch_url.password().is_some() + || dispatch_url.query().is_some() + || dispatch_url.fragment().is_some() + || !matches!(dispatch_url.path(), "" | "/") + { + return Err(MoaError::ConfigError( + "Restate ingress must be an origin-only HTTP(S) URL".to_string(), + )); + } + dispatch_url.set_path("/restate/send/ExecutionDispatcher/dispatch"); + let http = Client::builder() + .timeout(Duration::from_secs(5)) + .connect_timeout(Duration::from_secs(2)) + .redirect(reqwest::redirect::Policy::none()) + .build() + .map_err(|error| MoaError::ConfigError(error.to_string()))?; + Ok(Self { http, dispatch_url }) + } +} + +#[async_trait] +impl ExternalJobDispatcherKick for RestateExternalJobDispatcherKick { + async fn kick(&self, idempotency_key: &str) -> Result<(), ExternalJobDispatcherKickError> { + let response = self + .http + .post(self.dispatch_url.clone()) + .header("idempotency-key", idempotency_key) + .json(&DispatchExecutionsRequest::default()) + .send() + .await + .map_err(|_| ExternalJobDispatcherKickError)?; + if response.status().is_success() { + Ok(()) + } else { + Err(ExternalJobDispatcherKickError) + } + } +} + +/// Host-local callback controller that owns all transient provider evidence. +#[derive(Clone)] +pub struct ExternalJobCallbackIngress { + store: Arc, + adapters: ExecutionExternalJobAdapterRegistry, + config: moa_config::ExecutionConfig, + dispatcher: Arc, +} + +impl ExternalJobCallbackIngress { + /// Builds a production callback boundary over Postgres and Restate HTTP. + pub fn new( + pool: sqlx::PgPool, + adapters: ExecutionExternalJobAdapterRegistry, + config: moa_config::ExecutionConfig, + restate_ingress_origin: impl AsRef, + ) -> Result { + Ok(Self { + store: Arc::new(ExecutionRepository::new(pool)), + adapters, + config, + dispatcher: Arc::new(RestateExternalJobDispatcherKick::new( + restate_ingress_origin, + )?), + }) + } + + #[cfg(test)] + fn with_dependencies( + store: Arc, + adapters: ExecutionExternalJobAdapterRegistry, + dispatcher: Arc, + ) -> Self { + Self { + store, + adapters, + config: moa_config::ExecutionConfig::default(), + dispatcher, + } + } + + /// Authenticates, parses, and atomically applies one bounded raw callback. + pub async fn handle( + &self, + external_job_uid: Uuid, + job_generation: u64, + provider_event_id: String, + headers: HeaderMap, + body: Bytes, + ) -> Result { + validate_selectors(external_job_uid, job_generation, &provider_event_id)?; + if body.is_empty() { + return Err(ExternalJobIngressError::InvalidRequest); + } + if body.len() > MAX_EXTERNAL_JOB_CALLBACK_INGRESS_BODY_BYTES { + return Err(ExternalJobIngressError::RequestTooLarge); + } + let authentication = callback_authentication(&headers, &body)?; + let Some(job) = self.store.load(external_job_uid).await.map_err(|_| { + tracing::warn!("callback authentication dependency failed"); + ExternalJobIngressError::Unauthorized + })? + else { + tracing::debug!("callback authentication rejected"); + return Err(ExternalJobIngressError::Unauthorized); + }; + let Some(provider) = job.provider.as_deref() else { + tracing::debug!("callback authentication rejected"); + return Err(ExternalJobIngressError::Unauthorized); + }; + let Some(callback_auth_reference) = job.callback_auth_reference.as_deref() else { + tracing::debug!("callback authentication rejected"); + return Err(ExternalJobIngressError::Unauthorized); + }; + let adapter = self.adapters.require(provider).map_err(|_| { + tracing::debug!("callback authentication rejected"); + ExternalJobIngressError::Unauthorized + })?; + let authenticated = adapter + .authenticate_callback(callback_auth_reference, &authentication, &body) + .await + .map_err(|_| { + tracing::warn!("callback authentication dependency failed"); + ExternalJobIngressError::Unauthorized + })?; + if !authenticated { + return Err(ExternalJobIngressError::Unauthorized); + } + let parsed = adapter + .parse_callback(&authentication, &body) + .await + .map_err(map_adapter_parse_error)?; + if parsed.provider_event_id != provider_event_id { + return Err(ExternalJobIngressError::InvalidRequest); + } + let dispatcher_idempotency_key = + dispatcher_idempotency_key(external_job_uid, job_generation, &provider_event_id); + let callback = ExecutionExternalJobCallback { + external_job_uid, + job_generation, + provider: provider.to_string(), + provider_job_id: parsed.provider_job_id, + provider_event_id, + update: parsed.outcome.into(), + }; + let write = self + .store + .apply(&self.config, callback) + .await + .map_err(map_execution_error)?; + let disposition = match write.outcome { + ExecutionExternalJobCallbackOutcome::Applied(job) => { + let needs_dispatch = write.activation.is_some() || job.next_reconcile_at.is_some(); + if needs_dispatch + && self + .dispatcher + .kick(&dispatcher_idempotency_key) + .await + .is_err() + { + tracing::warn!( + external_job_uid = %external_job_uid, + job_generation, + "execution dispatcher wake failed after durable external-job callback; reconciliation will repair delivery" + ); + } + ExternalJobCallbackDisposition::Applied + } + ExecutionExternalJobCallbackOutcome::Duplicate => { + ExternalJobCallbackDisposition::Duplicate + } + ExecutionExternalJobCallbackOutcome::StaleGeneration => { + ExternalJobCallbackDisposition::Stale + } + ExecutionExternalJobCallbackOutcome::AlreadyTerminal => { + ExternalJobCallbackDisposition::AlreadyTerminal + } + ExecutionExternalJobCallbackOutcome::NotFound => { + return Err(ExternalJobIngressError::Unauthorized); + } + }; + Ok(disposition) + } +} + +#[derive(Debug, Deserialize)] +struct ExternalJobCallbackPath { + external_job_uid: Uuid, + job_generation: u64, + provider_event_id: String, +} + +/// Builds the exact private non-Restate callback surface. +pub fn router(ingress: ExternalJobCallbackIngress) -> Router { + Router::new() + .route(EXTERNAL_JOB_CALLBACK_INGRESS_ROUTE, post(callback_handler)) + .layer(DefaultBodyLimit::max( + MAX_EXTERNAL_JOB_CALLBACK_INGRESS_BODY_BYTES, + )) + .with_state(ingress) +} + +// SAFETY: provider authentication uses the bounded raw headers and body before parsing or persistence. +async fn callback_handler( + State(ingress): State, + Path(path): Path, + headers: HeaderMap, + body: Bytes, +) -> Response { + match ingress + .handle( + path.external_job_uid, + path.job_generation, + path.provider_event_id, + headers, + body, + ) + .await + { + Ok(_) => StatusCode::NO_CONTENT.into_response(), + Err(error) => ingress_error_response(error), + } +} + +fn ingress_error_response(error: ExternalJobIngressError) -> Response { + let status = match error { + ExternalJobIngressError::InvalidRequest => StatusCode::BAD_REQUEST, + ExternalJobIngressError::RequestTooLarge => StatusCode::PAYLOAD_TOO_LARGE, + ExternalJobIngressError::Unauthorized => StatusCode::UNAUTHORIZED, + ExternalJobIngressError::Unavailable => StatusCode::SERVICE_UNAVAILABLE, + }; + status.into_response() +} + +fn validate_selectors( + external_job_uid: Uuid, + job_generation: u64, + provider_event_id: &str, +) -> Result<(), ExternalJobIngressError> { + if external_job_uid.is_nil() + || job_generation == 0 + || provider_event_id.trim().is_empty() + || provider_event_id.len() > MAX_PROVIDER_EVENT_ID_BYTES + || provider_event_id.chars().any(char::is_control) + { + return Err(ExternalJobIngressError::InvalidRequest); + } + Ok(()) +} + +fn callback_authentication( + headers: &HeaderMap, + body: &[u8], +) -> Result { + if headers.len() > MAX_EXTERNAL_JOB_CALLBACK_INGRESS_HEADERS { + return Err(ExternalJobIngressError::RequestTooLarge); + } + let mut selected = BTreeMap::new(); + let mut total_bytes = 0usize; + for name in headers.keys() { + let values = headers.get_all(name); + let mut values = values.iter(); + let value = values + .next() + .ok_or(ExternalJobIngressError::InvalidRequest)?; + if values.next().is_some() { + return Err(ExternalJobIngressError::InvalidRequest); + } + total_bytes = total_bytes + .checked_add(name.as_str().len()) + .and_then(|sum| sum.checked_add(value.as_bytes().len())) + .ok_or(ExternalJobIngressError::RequestTooLarge)?; + if total_bytes > MAX_EXTERNAL_JOB_CALLBACK_INGRESS_HEADER_BYTES { + return Err(ExternalJobIngressError::RequestTooLarge); + } + if authentication_header(name.as_str()) { + selected.insert( + name.as_str().to_ascii_lowercase(), + value + .to_str() + .map_err(|_| ExternalJobIngressError::InvalidRequest)? + .to_string(), + ); + } + } + Ok(ExecutionExternalJobCallbackAuthentication { + headers: selected, + body_sha256: Sha256::digest(body).into(), + }) +} + +fn authentication_header(name: &str) -> bool { + !name.eq_ignore_ascii_case("host") + && !name.eq_ignore_ascii_case("content-length") + && !name.eq_ignore_ascii_case("connection") + && !name.eq_ignore_ascii_case("keep-alive") + && !name.eq_ignore_ascii_case("proxy-authenticate") + && !name.eq_ignore_ascii_case("proxy-authorization") + && !name.eq_ignore_ascii_case("te") + && !name.eq_ignore_ascii_case("trailers") + && !name.eq_ignore_ascii_case("transfer-encoding") + && !name.eq_ignore_ascii_case("upgrade") + && !name.to_ascii_lowercase().starts_with("x-moa-") + && !name.eq_ignore_ascii_case(moa_observability::TRACEPARENT_HEADER) + && !name.eq_ignore_ascii_case(moa_observability::TRACESTATE_HEADER) +} + +fn dispatcher_idempotency_key( + external_job_uid: Uuid, + job_generation: u64, + provider_event_id: &str, +) -> String { + let mut digest = Sha256::new(); + digest.update(external_job_uid.as_bytes()); + digest.update(job_generation.to_be_bytes()); + digest.update(provider_event_id.as_bytes()); + format!( + "external-job-callback-dispatch:{}", + hex::encode(digest.finalize()) + ) +} + +fn map_adapter_parse_error(error: MoaError) -> ExternalJobIngressError { + match error { + MoaError::ValidationError(_) + | MoaError::SerializationError(_) + | MoaError::SerdeJson(_) + | MoaError::Uuid(_) => ExternalJobIngressError::InvalidRequest, + _ => ExternalJobIngressError::Unavailable, + } +} + +fn map_execution_error(error: moa_execution::Error) -> ExternalJobIngressError { + match error { + moa_execution::Error::InvalidRepositoryInput { .. } => { + ExternalJobIngressError::InvalidRequest + } + _ => ExternalJobIngressError::Unavailable, + } +} + +#[cfg(test)] +mod tests { + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + + use chrono::Utc; + use moa_core::types::identifiers::TenantId; + use moa_core::types::tools::{ + AsyncToolJobCallbackOutcome, AsyncToolJobCancelOutcome, AsyncToolJobTerminalOutcome, + ExternalJobStartContext, + }; + use moa_execution::repository::external_job::{ + ExecutionExternalJobCallbackWrite, ExecutionExternalJobOwner, ExecutionExternalJobState, + }; + use moa_execution::wire::{ + ExecutionExternalJobCancelRequest, ExecutionExternalJobReconcileRequest, + }; + use tokio::sync::Mutex; + + use super::*; + use crate::services::tool_executor::{ + ExecutionExternalJobAdapter, ExecutionExternalJobAdapterCallback, + ExecutionExternalJobStartOutcome, ExecutionExternalJobStartRecovery, + ExecutionExternalJobStartRequest, + }; + + #[test] + fn dispatcher_kick_targets_the_coalescing_service_router() { + // Pins: callbacks outside the SDK enter the same stateless head-coalescing router as + // generated clients; only that router addresses the ingress-private fleet drain object. + let kick = RestateExternalJobDispatcherKick::new("http://127.0.0.1:8080") + .expect("valid fixture Restate ingress origin"); + assert_eq!( + kick.dispatch_url.as_str(), + "http://127.0.0.1:8080/restate/send/ExecutionDispatcher/dispatch" + ); + } + + struct FixtureAdapter { + authenticated: bool, + authentication_error: bool, + parse_calls: AtomicUsize, + } + + #[async_trait] + impl ExecutionExternalJobAdapter for FixtureAdapter { + fn provider_key(&self) -> &'static str { + "fixture" + } + + async fn start( + &self, + _request: &ExecutionExternalJobStartRequest, + ) -> moa_core::error::Result { + Err(MoaError::Unsupported( + "callback fixture does not start provider jobs".to_string(), + )) + } + + async fn recover_start( + &self, + _context: &ExternalJobStartContext, + ) -> moa_core::error::Result { + Err(MoaError::Unsupported( + "callback fixture does not recover provider starts".to_string(), + )) + } + + async fn authenticate_callback( + &self, + callback_auth_reference: &str, + authentication: &ExecutionExternalJobCallbackAuthentication, + body: &[u8], + ) -> moa_core::error::Result { + if self.authentication_error { + return Err(MoaError::ProviderTransport( + "fixture authentication dependency failed".to_string(), + )); + } + Ok(self.authenticated + && callback_auth_reference == "auth-ref" + && body == b"callback-body" + && authentication + .headers + .get("x-fixture-signature") + .map(String::as_str) + == Some("valid")) + } + + async fn parse_callback( + &self, + _authentication: &ExecutionExternalJobCallbackAuthentication, + body: &[u8], + ) -> moa_core::error::Result { + self.parse_calls.fetch_add(1, Ordering::SeqCst); + if body != b"callback-body" { + return Err(MoaError::ValidationError( + "invalid callback body".to_string(), + )); + } + Ok(ExecutionExternalJobAdapterCallback { + provider_job_id: "provider-job".to_string(), + provider_event_id: "event-1".to_string(), + outcome: AsyncToolJobCallbackOutcome::Terminal { + outcome: AsyncToolJobTerminalOutcome::Cancelled, + }, + }) + } + + async fn cancel( + &self, + _request: &ExecutionExternalJobCancelRequest, + ) -> moa_core::error::Result { + Ok(AsyncToolJobCancelOutcome::Cancelled) + } + + async fn reconcile( + &self, + _request: &ExecutionExternalJobReconcileRequest, + ) -> moa_core::error::Result { + Err(MoaError::Unsupported("fixture reconcile".to_string())) + } + } + + struct FixtureStore { + job: Option, + outcome: Mutex>, + activation: bool, + apply_calls: AtomicUsize, + } + + #[async_trait] + impl ExternalJobCallbackStore for FixtureStore { + async fn load( + &self, + _external_job_uid: Uuid, + ) -> Result, moa_execution::Error> { + Ok(self.job.clone()) + } + + async fn apply( + &self, + _config: &moa_config::ExecutionConfig, + callback: ExecutionExternalJobCallback, + ) -> Result { + self.apply_calls.fetch_add(1, Ordering::SeqCst); + assert_eq!(callback.provider_event_id, "event-1"); + Ok(ExecutionExternalJobCallbackWrite { + outcome: self + .outcome + .lock() + .await + .take() + .expect("one fixture outcome"), + activation: self.activation.then(fixture_activation), + }) + } + } + + struct FixtureDispatcher { + called: AtomicBool, + } + + #[async_trait] + impl ExternalJobDispatcherKick for FixtureDispatcher { + async fn kick(&self, _idempotency_key: &str) -> Result<(), ExternalJobDispatcherKickError> { + self.called.store(true, Ordering::SeqCst); + Err(ExternalJobDispatcherKickError) + } + } + + #[tokio::test] + async fn bad_auth_stops_before_parse_or_persistence() { + // Pins: raw unauthenticated provider bytes never reach parsing or durable storage. + let adapter = Arc::new(FixtureAdapter { + authenticated: false, + authentication_error: false, + parse_calls: AtomicUsize::new(0), + }); + let store = Arc::new(fixture_store( + ExecutionExternalJobCallbackOutcome::Duplicate, + )); + let ingress = fixture_ingress(store.clone(), adapter.clone()); + assert_eq!( + ingress + .handle( + Uuid::from_u128(1), + 1, + "event-1".to_string(), + signed_headers(), + Bytes::from_static(b"callback-body"), + ) + .await, + Err(ExternalJobIngressError::Unauthorized) + ); + assert_eq!(adapter.parse_calls.load(Ordering::SeqCst), 0); + assert_eq!(store.apply_calls.load(Ordering::SeqCst), 0); + } + + #[tokio::test] + async fn authentication_dependency_error_is_the_same_unauthorized_boundary() { + // Pins: an adapter dependency failure before authentication cannot reveal that the + // supplied external-job and provider identities exist. + let adapter = Arc::new(FixtureAdapter { + authenticated: false, + authentication_error: true, + parse_calls: AtomicUsize::new(0), + }); + let store = Arc::new(fixture_store( + ExecutionExternalJobCallbackOutcome::Duplicate, + )); + let ingress = fixture_ingress(store.clone(), adapter.clone()); + + assert_eq!( + ingress + .handle( + Uuid::from_u128(1), + 1, + "event-1".to_string(), + signed_headers(), + Bytes::from_static(b"callback-body"), + ) + .await, + Err(ExternalJobIngressError::Unauthorized) + ); + assert_eq!(adapter.parse_calls.load(Ordering::SeqCst), 0); + assert_eq!(store.apply_calls.load(Ordering::SeqCst), 0); + } + + #[tokio::test] + async fn unknown_provider_fails_closed_before_authentication() { + // Pins: a persisted provider key cannot select a placeholder or generic parser. + let store = Arc::new(fixture_store( + ExecutionExternalJobCallbackOutcome::Duplicate, + )); + let ingress = ExternalJobCallbackIngress::with_dependencies( + store.clone(), + ExecutionExternalJobAdapterRegistry::default(), + Arc::new(FixtureDispatcher { + called: AtomicBool::new(false), + }), + ); + assert_eq!( + ingress + .handle( + Uuid::from_u128(1), + 1, + "event-1".to_string(), + signed_headers(), + Bytes::from_static(b"callback-body"), + ) + .await, + Err(ExternalJobIngressError::Unauthorized) + ); + assert_eq!(store.apply_calls.load(Ordering::SeqCst), 0); + } + + #[tokio::test] + async fn missing_job_and_unknown_provider_are_indistinguishable_from_bad_auth() { + // Pins: unauthenticated callers cannot enumerate durable jobs or installed adapters. + let mut missing = fixture_store(ExecutionExternalJobCallbackOutcome::Duplicate); + missing.job = None; + let ingress = ExternalJobCallbackIngress::with_dependencies( + Arc::new(missing), + ExecutionExternalJobAdapterRegistry::default(), + Arc::new(FixtureDispatcher { + called: AtomicBool::new(false), + }), + ); + assert_eq!( + ingress + .handle( + Uuid::from_u128(1), + 1, + "event-1".to_string(), + signed_headers(), + Bytes::from_static(b"callback-body"), + ) + .await, + Err(ExternalJobIngressError::Unauthorized) + ); + assert_eq!( + ingress_error_response(ExternalJobIngressError::Unauthorized).status(), + StatusCode::UNAUTHORIZED + ); + } + + #[tokio::test] + async fn raw_body_mutation_fails_authentication_before_parsing() { + // Pins: providers can authenticate the exact raw body; a one-byte + // mutation cannot pass merely because a derived evidence shape exists. + let adapter = Arc::new(FixtureAdapter { + authenticated: true, + authentication_error: false, + parse_calls: AtomicUsize::new(0), + }); + let store = Arc::new(fixture_store( + ExecutionExternalJobCallbackOutcome::Duplicate, + )); + let ingress = fixture_ingress(store.clone(), adapter.clone()); + assert_eq!( + ingress + .handle( + Uuid::from_u128(1), + 1, + "event-1".to_string(), + signed_headers(), + Bytes::from_static(b"callback-bodz"), + ) + .await, + Err(ExternalJobIngressError::Unauthorized) + ); + assert_eq!(adapter.parse_calls.load(Ordering::SeqCst), 0); + assert_eq!(store.apply_calls.load(Ordering::SeqCst), 0); + } + + #[tokio::test] + async fn stale_and_duplicate_callbacks_are_accepted_no_ops() { + // Pins: provider retries and older generations receive success without + // scheduling duplicate controller work. + for (outcome, expected) in [ + ( + ExecutionExternalJobCallbackOutcome::StaleGeneration, + ExternalJobCallbackDisposition::Stale, + ), + ( + ExecutionExternalJobCallbackOutcome::Duplicate, + ExternalJobCallbackDisposition::Duplicate, + ), + ] { + let adapter = Arc::new(FixtureAdapter { + authenticated: true, + authentication_error: false, + parse_calls: AtomicUsize::new(0), + }); + let store = Arc::new(fixture_store(outcome)); + let dispatcher = Arc::new(FixtureDispatcher { + called: AtomicBool::new(false), + }); + let ingress = ExternalJobCallbackIngress::with_dependencies( + store, + ExecutionExternalJobAdapterRegistry::new([ + adapter as Arc + ]) + .expect("fixture registry"), + dispatcher.clone(), + ); + assert_eq!( + ingress + .handle( + Uuid::from_u128(1), + 1, + "event-1".to_string(), + signed_headers(), + Bytes::from_static(b"callback-body"), + ) + .await, + Ok(expected) + ); + assert!(!dispatcher.called.load(Ordering::SeqCst)); + } + } + + #[tokio::test] + async fn applied_progress_kicks_dispatcher_for_rearmed_reconciliation() { + // Pins: applied provider progress wakes the central dispatcher so a newly earlier + // sparse-reconciliation deadline cannot remain pending until unrelated traffic. + let adapter = Arc::new(FixtureAdapter { + authenticated: true, + authentication_error: false, + parse_calls: AtomicUsize::new(0), + }); + let applied_job = fixture_store(ExecutionExternalJobCallbackOutcome::Duplicate) + .job + .expect("fixture job"); + let store = Arc::new(fixture_store(ExecutionExternalJobCallbackOutcome::Applied( + Box::new(applied_job), + ))); + let dispatcher = Arc::new(FixtureDispatcher { + called: AtomicBool::new(false), + }); + let ingress = ExternalJobCallbackIngress::with_dependencies( + store, + ExecutionExternalJobAdapterRegistry::new([ + adapter as Arc + ]) + .expect("fixture registry"), + dispatcher.clone(), + ); + assert_eq!( + ingress + .handle( + Uuid::from_u128(1), + 1, + "event-1".to_string(), + signed_headers(), + Bytes::from_static(b"callback-body"), + ) + .await, + Ok(ExternalJobCallbackDisposition::Applied) + ); + assert!(dispatcher.called.load(Ordering::SeqCst)); + } + + #[tokio::test] + async fn durable_applied_callback_succeeds_when_best_effort_dispatcher_kick_fails() { + // Pins: provider acknowledgement follows the committed callback receipt; + // a transient Restate outage leaves the outbox for bounded reconciliation. + let adapter = Arc::new(FixtureAdapter { + authenticated: true, + authentication_error: false, + parse_calls: AtomicUsize::new(0), + }); + let applied_job = fixture_store(ExecutionExternalJobCallbackOutcome::Duplicate) + .job + .expect("fixture job"); + let mut store = fixture_store(ExecutionExternalJobCallbackOutcome::Applied(Box::new( + applied_job, + ))); + store.activation = true; + let store = Arc::new(store); + let dispatcher = Arc::new(FixtureDispatcher { + called: AtomicBool::new(false), + }); + let ingress = ExternalJobCallbackIngress::with_dependencies( + store, + ExecutionExternalJobAdapterRegistry::new([ + adapter as Arc + ]) + .expect("fixture registry"), + dispatcher.clone(), + ); + + assert_eq!( + ingress + .handle( + Uuid::from_u128(1), + 1, + "event-1".to_string(), + signed_headers(), + Bytes::from_static(b"callback-body"), + ) + .await, + Ok(ExternalJobCallbackDisposition::Applied) + ); + assert!(dispatcher.called.load(Ordering::SeqCst)); + } + + #[tokio::test] + async fn terminal_callback_without_activation_or_reconcile_does_not_kick_dispatcher() { + // Pins: a terminal callback settled before the active attempt releases is already + // durable and has no delivery work, so it does not emit a spurious dispatcher wake. + let adapter = Arc::new(FixtureAdapter { + authenticated: true, + authentication_error: false, + parse_calls: AtomicUsize::new(0), + }); + let mut applied_job = fixture_store(ExecutionExternalJobCallbackOutcome::Duplicate) + .job + .expect("fixture job"); + applied_job.state = ExecutionExternalJobState::Completed; + applied_job.next_reconcile_at = None; + applied_job.completed_at = Some(Utc::now()); + let store = Arc::new(fixture_store(ExecutionExternalJobCallbackOutcome::Applied( + Box::new(applied_job), + ))); + let dispatcher = Arc::new(FixtureDispatcher { + called: AtomicBool::new(false), + }); + let ingress = ExternalJobCallbackIngress::with_dependencies( + store, + ExecutionExternalJobAdapterRegistry::new([ + adapter as Arc + ]) + .expect("fixture registry"), + dispatcher.clone(), + ); + + assert_eq!( + ingress + .handle( + Uuid::from_u128(1), + 1, + "event-1".to_string(), + signed_headers(), + Bytes::from_static(b"callback-body"), + ) + .await, + Ok(ExternalJobCallbackDisposition::Applied) + ); + assert!(!dispatcher.called.load(Ordering::SeqCst)); + } + + fn fixture_ingress( + store: Arc, + adapter: Arc, + ) -> ExternalJobCallbackIngress { + ExternalJobCallbackIngress::with_dependencies( + store, + ExecutionExternalJobAdapterRegistry::new([ + adapter as Arc + ]) + .expect("fixture registry"), + Arc::new(FixtureDispatcher { + called: AtomicBool::new(false), + }), + ) + } + + fn fixture_store(outcome: ExecutionExternalJobCallbackOutcome) -> FixtureStore { + FixtureStore { + job: Some(ExecutionExternalJobRecord { + external_job_uid: Uuid::from_u128(1), + tenant_id: TenantId::from(Uuid::from_u128(2)), + run_uid: Uuid::from_u128(3), + owner: ExecutionExternalJobOwner::Task { + task_id: Uuid::from_u128(4), + attempt_generation: 1, + }, + job_generation: 1, + declared_provider: "fixture".to_string(), + provider: Some("fixture".to_string()), + provider_job_id: Some("provider-job".to_string()), + idempotency_key: "idempotency".to_string(), + callback_auth_reference: Some("auth-ref".to_string()), + state: ExecutionExternalJobState::WaitingReconcile, + progress_phase: Some("waiting".to_string()), + cancel_supported: true, + next_reconcile_at: Some(Utc::now()), + last_provider_event_id: None, + output: None, + error: None, + created_at: Utc::now(), + updated_at: Utc::now(), + completed_at: None, + provider_contract_violation: None, + }), + outcome: Mutex::new(Some(outcome)), + activation: false, + apply_calls: AtomicUsize::new(0), + } + } + + fn fixture_activation() -> moa_execution::repository::outbox::ExecutionDispatchRecord { + moa_execution::repository::outbox::ExecutionDispatchRecord { + dispatch_uid: Uuid::from_u128(10), + tenant_id: TenantId::from(Uuid::from_u128(2)), + run_uid: Some(Uuid::from_u128(3)), + task_id: None, + compensation_id: None, + trigger_uid: None, + external_job_uid: None, + kind: moa_execution::repository::outbox::ExecutionDispatchKind::RunActivation, + state: moa_execution::repository::outbox::ExecutionDeliveryState::Pending, + controller_generation: Some(1), + wake_epoch: Some(1), + attempt_generation: None, + compensation_generation: None, + compensation_attempt_generation: None, + not_before_at: Utc::now(), + payload: serde_json::json!({}), + delivery_attempts: 0, + claim_owner: None, + claimed_at: None, + claim_expires_at: None, + delivered_at: None, + last_error: None, + created_at: Utc::now(), + updated_at: Utc::now(), + } + } + + fn signed_headers() -> HeaderMap { + let mut headers = HeaderMap::new(); + headers.insert("x-fixture-signature", "valid".parse().expect("header")); + headers + } +} diff --git a/crates/moa-orchestrator/src/lib.rs b/crates/moa-orchestrator/src/lib.rs index adc38a57d..d630eee4f 100644 --- a/crates/moa-orchestrator/src/lib.rs +++ b/crates/moa-orchestrator/src/lib.rs @@ -10,6 +10,8 @@ pub(crate) mod connector_catalog; pub mod credential_ingress; pub mod ctx; mod delegation; +/// Private non-Restate external-job callback boundary types. +pub mod external_job_ingress; pub mod guardrails; pub mod handlers; pub(crate) mod identity_admin; diff --git a/crates/moa-orchestrator/src/main.rs b/crates/moa-orchestrator/src/main.rs index dfd66acf5..4964b9044 100644 --- a/crates/moa-orchestrator/src/main.rs +++ b/crates/moa-orchestrator/src/main.rs @@ -12,13 +12,19 @@ use axum::http::StatusCode; use axum::response::IntoResponse; use axum::routing::get; use axum::{Router, serve}; +use chrono::{DateTime, Utc}; use clap::{Parser, Subcommand}; +use moa_execution::repository::{ + ExecutionRepository, ExecutionScope, + outbox::{ExecutionMaintenanceCheckpoint, ExecutionMaintenanceJobKind}, + retention::ExecutionRetentionCheckpoint, +}; use moa_observability::{TelemetryConfig, init_observability, metrics_endpoint_url}; use moa_orchestrator::objects::session_status_migrator::build_status_migration_endpoint; use moa_orchestrator::services::scim::{self, ScimState}; use moa_orchestrator::{ config::{ProvidersOverride, load_moa_config_from_env, restate_ingress_url, skip_fga_from_env}, - credential_ingress, + credential_ingress, external_job_ingress, runtime::{ bootstrap::{BootstrapOptions, run as run_bootstrap, wait_for_session_status_cutover}, channel_ingress::spawn_channel_ingress, @@ -26,9 +32,10 @@ use moa_orchestrator::{ deps::RuntimeDeps, endpoint::build_endpoint, jobs::{ + build_maintenance_dependencies, ensure_execution_maintenance_cron_jobs, start_action_review_reaper, start_authz_challenge_reaper_if_configured, - start_checkpoint_bucket_versioning_refresh, start_hand_lease_reaper, - start_mcp_catalog_refresh, start_workspace_reaper, + start_authz_outbox_poller, start_checkpoint_bucket_versioning_refresh, + start_hand_lease_reaper, start_mcp_catalog_refresh, start_workspace_reaper, }, kms::KmsRuntime, sandbox_workspace_rollout::validate_startup_state as validate_sandbox_workspace_rollout, @@ -40,17 +47,18 @@ use tokio::task::JoinHandle; use tokio_util::sync::CancellationToken; const DEFAULT_RESTATE_PORT: u16 = 10020; -const DEFAULT_HEALTH_PORT: u16 = 10021; +const DEFAULT_HEALTH_PORT: u16 = 9081; const DEFAULT_SCIM_PORT: u16 = 10022; const DEFAULT_CONNECTOR_CREDENTIAL_PORT: u16 = 10023; const SHUTDOWN_DRAIN_DELAY: Duration = Duration::from_secs(5); const SHUTDOWN_TASK_TIMEOUT: Duration = Duration::from_secs(15); +const EXECUTION_CRON_RECONCILE_MAX_DELAY: Duration = Duration::from_secs(300); const ORCHESTRATOR_WORKER_STACK_SIZE: usize = 16 * 1024 * 1024; /// Process arguments for the orchestrator process. #[derive(Debug, Parser)] struct Args { - /// Run database migrations and exit without starting Restate services. + /// Optional process role or one-shot administrative command. #[command(subcommand)] command: Option, /// HTTP port for the Restate handler endpoint. @@ -69,6 +77,8 @@ struct Args { #[derive(Debug, Clone, PartialEq, Eq, Subcommand)] enum Command { + /// Run the singleton correctness-maintenance process without serving product ingress. + Maintenance, /// Apply database migrations and exit. Migrate, /// Serve only the raw Session state cutover handlers. @@ -222,6 +232,17 @@ async fn async_main() -> anyhow::Result<()> { Some(Command::ServeStatusMigration) | Some(Command::WaitStatusCutover { .. }) => { unreachable!("pre-runtime command returned before runtime config") } + Some(Command::Maintenance) => { + let result = run_maintenance( + moa_config, + args.health_port, + &database_search_path, + skip_fga, + ) + .await; + telemetry.shutdown(); + return result; + } None => {} } @@ -269,7 +290,7 @@ async fn async_main() -> anyhow::Result<()> { } else { None }; - let mut runtime_deps = RuntimeDeps::build( + let runtime_deps = RuntimeDeps::build( moa_config.clone(), pool.clone(), background_pool, @@ -290,15 +311,6 @@ async fn async_main() -> anyhow::Result<()> { .clone() .map(start_checkpoint_bucket_versioning_refresh); - let mut workspace_reaper_handle = runtime_deps - .workspace_maintenance - .clone() - .map(|coordinator| start_workspace_reaper(coordinator, moa_config.as_ref())) - .transpose()?; - let workspace_reaper_readiness = workspace_reaper_handle - .as_ref() - .map(moa_hands::core::sandbox_workspace::reaper::WorkspaceReaperHandle::readiness); - let readiness = Arc::new(AtomicBool::new(false)); let probe_state = ProbeState::new( readiness.clone(), @@ -306,31 +318,9 @@ async fn async_main() -> anyhow::Result<()> { runtime_deps.kms.clone(), runtime_deps.lineage.writer.clone(), runtime_deps.checkpoint_versioning_observer.clone(), - workspace_reaper_readiness, + MaintenanceReadiness::default(), ); let shutdown = CancellationToken::new(); - let authz_challenge_reaper_handle = start_authz_challenge_reaper_if_configured( - &runtime_deps.background_pool, - moa_config.as_ref(), - runtime_deps.awakeable_resolver.clone(), - )?; - let action_review_reaper_handle = - start_action_review_reaper(&runtime_deps.background_pool, restate_ingress_url.clone()); - // The destruction owner for every bounded sandbox deadline. It is started - // before the servers accept traffic, and startup fails outright if no hand - // provider is registered, so no sandbox is ever provisioned under a policy - // this process cannot enforce. - let hand_lease_reaper_handle = moa_config - .sandbox_workspaces - .mode - .maintenance_enabled() - .then(|| { - start_hand_lease_reaper( - &runtime_deps.background_pool, - runtime_deps.tool_router.hand_providers(), - ) - }) - .transpose()?; // Optional connectors that failed discovery at startup are retried here, and // schema changes republished, without restarting the process. let mcp_catalog_refresh_handle = @@ -346,7 +336,14 @@ async fn async_main() -> anyhow::Result<()> { let mut scim_server = spawn_scim_server(scim_listener, scim_state, shutdown.clone()); let mut credential_server = spawn_credential_server( credential_listener, - credential_ingress::router(runtime_deps.connector_credential_ingress()), + credential_ingress::router(runtime_deps.connector_credential_ingress()).merge( + external_job_ingress::router(external_job_ingress::ExternalJobCallbackIngress::new( + pool.clone(), + runtime_deps.external_job_adapters.clone(), + moa_config.execution.clone(), + &restate_ingress_url, + )?), + ), shutdown.clone(), ); let mut channel_ingress = spawn_channel_ingress( @@ -433,22 +430,6 @@ async fn async_main() -> anyhow::Result<()> { result.context("join connector credential ingress server")??; bail!("connector credential ingress server exited unexpectedly"); } - result = await_workspace_reaper_exit(&mut workspace_reaper_handle) => { - readiness.store(false, Ordering::Release); - shutdown.cancel(); - restate_server.abort(); - health_server.abort(); - scim_server.abort(); - credential_server.abort(); - if let Some(handle) = channel_ingress.take() { - handle.abort(); - } - if let Some(handle) = analytics_export.take() { - handle.abort(); - } - result.context("durable workspace reaper exited")?; - bail!("durable workspace reaper exited unexpectedly"); - } result = await_checkpoint_versioning_refresh_exit(&mut checkpoint_versioning_refresh_handle) => { readiness.store(false, Ordering::Release); shutdown.cancel(); @@ -503,14 +484,6 @@ async fn async_main() -> anyhow::Result<()> { let _ = join_task_bounded("analytics export", handle).await; } - if let Some(handle) = workspace_reaper_handle.take() { - let _ = shutdown_future_bounded( - "durable workspace reaper", - handle.shutdown(), - ) - .await; - } - if let Some(handle) = checkpoint_versioning_refresh_handle.take() { let _ = shutdown_future_bounded( "checkpoint bucket versioning refresh", @@ -519,32 +492,9 @@ async fn async_main() -> anyhow::Result<()> { .await; } - if let Some(handle) = hand_lease_reaper_handle { - abort_and_join_task("hand lease reaper", handle).await; - } if let Some(handle) = mcp_catalog_refresh_handle { abort_and_join_task("MCP catalog refresh", handle).await; } - if let Some(poller_handle) = runtime_deps.authz_outbox_poller.take() { - let _ = shutdown_future_bounded( - "authz outbox poller", - poller_handle.shutdown(), - ) - .await; - } - if let Some(reaper_handle) = authz_challenge_reaper_handle { - let _ = shutdown_future_bounded( - "authz challenge reaper", - reaper_handle.shutdown(), - ) - .await; - } - let _ = shutdown_future_bounded( - "action review reaper", - action_review_reaper_handle.shutdown(), - ) - .await; - let audit = runtime_deps.audit.clone(); if let Some(writer) = runtime_deps.lineage.writer.clone() { @@ -587,6 +537,304 @@ async fn async_main() -> anyhow::Result<()> { Ok(()) } +/// Runs the singleton correctness-maintenance role without binding product ingress. +async fn run_maintenance( + config: Arc, + health_port: u16, + database_search_path: &str, + skip_fga: bool, +) -> anyhow::Result<()> { + config + .validate_sandbox_workspace_runtime(skip_fga) + .context("validate sandbox workspace maintenance rollout")?; + let restate_ingress_url = restate_ingress_url(config.as_ref())?; + let pool = build_database_pool( + config.database.runtime_url(), + database_search_path, + config.database.max_connections, + Duration::from_secs(config.database.connect_timeout_seconds), + ) + .await + .context("connect maintenance runtime database pool")?; + moa_migrations::validate_complete_history(&pool) + .await + .context("validate complete database migration history")?; + validate_sandbox_workspace_rollout(config.as_ref(), &pool).await?; + ensure_execution_maintenance_cron_jobs( + &restate_ingress_url, + config.execution.trigger_reconciliation_cadence_seconds, + ) + .await + .context("ensure durable execution dispatch repair CronJob")?; + + // The dedicated maintenance connection is constructed only when durable + // workspaces are enabled. Disabled deployments therefore keep one small + // runtime pool and never wake a sandbox provider or object-store client. + let maintenance_pool = if config.sandbox_workspaces.mode.maintenance_enabled() { + let url = config.database.maintenance_url().ok_or_else(|| { + anyhow::anyhow!("sandbox workspace maintenance database URL is unavailable") + })?; + Some( + build_database_pool( + url, + database_search_path, + config.database.background_max_connections, + Duration::from_secs(config.database.connect_timeout_seconds), + ) + .await + .context("connect dedicated sandbox workspace maintenance database pool")?, + ) + } else { + None + }; + + let dependencies = build_maintenance_dependencies( + config.as_ref(), + pool.clone(), + maintenance_pool, + &restate_ingress_url, + skip_fga, + ) + .await?; + + let mut checkpoint_versioning_refresh_handle = dependencies + .checkpoint_versioning_observer + .clone() + .map(start_checkpoint_bucket_versioning_refresh); + let mut workspace_reaper_handle = dependencies + .workspace_maintenance + .clone() + .map(|coordinator| start_workspace_reaper(coordinator, config.as_ref())) + .transpose()?; + let mut hand_lease_reaper_handle = config + .sandbox_workspaces + .mode + .maintenance_enabled() + .then(|| start_hand_lease_reaper(&pool, dependencies.hand_providers)) + .transpose()?; + let mut authz_outbox_poller_handle = dependencies + .fga_client + .map(|client| start_authz_outbox_poller(&pool, client)); + let mut authz_challenge_reaper_handle = match dependencies.awakeable_resolver { + Some(resolver) => { + start_authz_challenge_reaper_if_configured(&pool, config.as_ref(), resolver)? + } + None => None, + }; + let mut action_review_reaper_handle = Some(start_action_review_reaper( + &pool, + restate_ingress_url.clone(), + )); + + let readiness = Arc::new(AtomicBool::new(false)); + let execution_maintenance_repository = ExecutionRepository::new(pool.clone()); + let probe_state = ProbeState::new( + readiness.clone(), + pool, + dependencies.kms, + None, + dependencies.checkpoint_versioning_observer, + MaintenanceReadiness { + workspace_reaper: workspace_reaper_handle + .as_ref() + .map(|handle| handle.readiness()), + hand_lease_reaper: hand_lease_reaper_handle + .as_ref() + .map(|handle| handle.readiness()), + authz_outbox_poller: authz_outbox_poller_handle + .as_ref() + .map(|handle| handle.readiness()), + action_review_reaper: action_review_reaper_handle + .as_ref() + .map(|handle| handle.readiness()), + authz_challenge_reaper: authz_challenge_reaper_handle + .as_ref() + .map(|handle| handle.readiness()), + execution_repository: Some(execution_maintenance_repository), + }, + ); + let shutdown = CancellationToken::new(); + let mut execution_cron_reconciler = spawn_execution_cron_reconciler( + restate_ingress_url, + config.execution.trigger_reconciliation_cadence_seconds, + shutdown.clone(), + ); + let health_listener = bind_listener(health_port).await?; + let mut health_server = + spawn_health_server(health_listener, probe_state.clone(), shutdown.clone()); + tracing::info!( + health_port, + metrics_url = + metrics_endpoint_url(&config.metrics).unwrap_or_else(|| "disabled".to_string()), + "starting moa-maintenance" + ); + readiness.store(true, Ordering::Release); + + tokio::select! { + result = &mut health_server => { + close_maintenance_readiness(&readiness); + shutdown.cancel(); + result.context("join maintenance health probe server")??; + bail!("maintenance health probe server exited unexpectedly"); + } + result = await_checkpoint_versioning_refresh_exit(&mut checkpoint_versioning_refresh_handle) => { + close_maintenance_readiness(&readiness); + shutdown.cancel(); + result.context("checkpoint bucket versioning refresh exited")?; + bail!("checkpoint bucket versioning refresh exited unexpectedly"); + } + result = await_workspace_reaper_exit(&mut workspace_reaper_handle) => { + close_maintenance_readiness(&readiness); + shutdown.cancel(); + result.context("durable workspace reaper exited")?; + bail!("durable workspace reaper exited unexpectedly"); + } + result = await_hand_lease_reaper_exit(&mut hand_lease_reaper_handle) => { + close_maintenance_readiness(&readiness); + shutdown.cancel(); + result.context("durable hand-lease reaper exited")?; + bail!("durable hand-lease reaper exited unexpectedly"); + } + result = await_authz_outbox_poller_exit(&mut authz_outbox_poller_handle) => { + close_maintenance_readiness(&readiness); + shutdown.cancel(); + result.context("authorization outbox poller exited")?; + bail!("authorization outbox poller exited unexpectedly"); + } + result = await_authz_challenge_reaper_exit(&mut authz_challenge_reaper_handle) => { + close_maintenance_readiness(&readiness); + shutdown.cancel(); + result.context("authorization challenge reaper exited")?; + bail!("authorization challenge reaper exited unexpectedly"); + } + result = await_action_review_reaper_exit(&mut action_review_reaper_handle) => { + close_maintenance_readiness(&readiness); + shutdown.cancel(); + result.context("action-review reaper exited")?; + bail!("action-review reaper exited unexpectedly"); + } + result = &mut execution_cron_reconciler => { + close_maintenance_readiness(&readiness); + shutdown.cancel(); + result.context("join execution CronJob reconciler")??; + bail!("execution CronJob reconciler exited unexpectedly"); + } + signal = shutdown_signal() => signal?, + } + + tracing::info!("shutdown signal received, stopping maintenance owners"); + close_maintenance_readiness(&readiness); + shutdown.cancel(); + + let checkpoint_versioning_refresh_handle = checkpoint_versioning_refresh_handle.take(); + let workspace_reaper_handle = workspace_reaper_handle.take(); + let hand_lease_reaper_handle = hand_lease_reaper_handle.take(); + let authz_outbox_poller_handle = authz_outbox_poller_handle.take(); + let authz_challenge_reaper_handle = authz_challenge_reaper_handle.take(); + let action_review_reaper_handle = action_review_reaper_handle.take(); + tokio::join!( + async move { + if let Some(result) = + join_task_bounded("maintenance health probe server", health_server).await + && let Err(error) = result + { + tracing::warn!(%error, "maintenance health probe server failed during shutdown"); + } + }, + async move { + if let Some(handle) = checkpoint_versioning_refresh_handle { + let _ = shutdown_future_bounded( + "checkpoint bucket versioning refresh", + handle.shutdown(), + ) + .await; + } + }, + async move { + if let Some(handle) = workspace_reaper_handle { + let _ = + shutdown_future_bounded("durable workspace reaper", handle.shutdown()).await; + } + }, + async move { + if let Some(handle) = hand_lease_reaper_handle { + let _ = + shutdown_future_bounded("durable hand-lease reaper", handle.shutdown()).await; + } + }, + async move { + if let Some(handle) = authz_outbox_poller_handle { + let _ = + shutdown_future_bounded("authorization outbox poller", handle.shutdown()).await; + } + }, + async move { + if let Some(handle) = authz_challenge_reaper_handle { + let _ = + shutdown_future_bounded("authorization challenge reaper", handle.shutdown()) + .await; + } + }, + async move { + if let Some(handle) = action_review_reaper_handle { + let _ = shutdown_future_bounded("action-review reaper", handle.shutdown()).await; + } + }, + async move { + let _ = + join_task_bounded("execution CronJob reconciler", execution_cron_reconciler).await; + }, + ); + + Ok(()) +} + +fn spawn_execution_cron_reconciler( + restate_ingress_url: String, + cadence_seconds: u64, + shutdown: CancellationToken, +) -> JoinHandle> { + tokio::spawn(async move { + let mut consecutive_failures = 0; + loop { + let retry_delay = execution_cron_reconcile_delay(cadence_seconds, consecutive_failures); + tokio::select! { + () = shutdown.cancelled() => return Ok(()), + () = tokio::time::sleep(retry_delay) => {} + } + match ensure_execution_maintenance_cron_jobs(&restate_ingress_url, cadence_seconds) + .await + { + Ok(()) => consecutive_failures = 0, + Err(error) => { + consecutive_failures = consecutive_failures.saturating_add(1); + let next_retry_delay = + execution_cron_reconcile_delay(cadence_seconds, consecutive_failures); + tracing::warn!( + %error, + retry_delay_secs = next_retry_delay.as_secs(), + "execution CronJob reconciliation failed; retrying with bounded backoff" + ); + } + } + } + }) +} + +fn execution_cron_reconcile_delay(cadence_seconds: u64, consecutive_failures: u32) -> Duration { + let steady_delay = Duration::from_secs(cadence_seconds.clamp(1, 60)); + let multiplier = 1_u32 + .checked_shl(consecutive_failures.min(31)) + .unwrap_or(u32::MAX); + steady_delay + .saturating_mul(multiplier) + .min(EXECUTION_CRON_RECONCILE_MAX_DELAY) +} + +fn close_maintenance_readiness(readiness: &AtomicBool) { + readiness.store(false, Ordering::Release); +} + async fn await_workspace_reaper_exit( handle: &mut Option, ) -> moa_core::error::Result<()> { @@ -596,6 +844,46 @@ async fn await_workspace_reaper_exit( } } +async fn await_authz_outbox_poller_exit( + handle: &mut Option, +) -> Result<(), moa_authz::poller::PollerTaskError> { + match handle { + Some(handle) => handle.task_result().await, + None => std::future::pending().await, + } +} + +async fn await_authz_challenge_reaper_exit( + handle: &mut Option< + moa_orchestrator::services::authz_challenges_reaper::AuthzChallengeReaperHandle, + >, +) -> Result<(), moa_orchestrator::services::authz_challenges_reaper::ReaperError> { + match handle { + Some(handle) => handle.task_result().await, + None => std::future::pending().await, + } +} + +async fn await_action_review_reaper_exit( + handle: &mut Option< + moa_orchestrator::services::action_reviews_reaper::ActionReviewReaperHandle, + >, +) -> Result<(), moa_orchestrator::services::action_reviews_reaper::ActionReviewReaperError> { + match handle { + Some(handle) => handle.task_result().await, + None => std::future::pending().await, + } +} + +async fn await_hand_lease_reaper_exit( + handle: &mut Option, +) -> moa_core::error::Result<()> { + match handle { + Some(handle) => handle.task_result().await, + None => std::future::pending().await, + } +} + async fn await_checkpoint_versioning_refresh_exit( handle: &mut Option, ) -> anyhow::Result<()> { @@ -668,9 +956,25 @@ struct ProbeState { checkpoint_versioning: Option< moa_hands::core::sandbox_workspace::checkpoint::versioning::CheckpointBucketVersioningObserver, >, + maintenance: MaintenanceReadiness, +} + +#[derive(Clone, Default)] +struct MaintenanceReadiness { /// Supervised workspace-maintenance readiness, when maintenance is enabled. - workspace_reaper: - Option, + workspace_reaper: Option, + /// Supervised hand-lease cleanup readiness, when maintenance is enabled. + hand_lease_reaper: Option, + /// Supervised authorization-outbox readiness, in the maintenance role. + authz_outbox_poller: Option, + /// Supervised tenant action-review timeout readiness, in the maintenance role. + action_review_reaper: + Option, + /// Supervised builtin-authz timeout readiness, when that provider is configured. + authz_challenge_reaper: + Option, + /// Durable receipt reader for the Cron-owned execution reconciliation pass. + execution_repository: Option, } impl ProbeState { @@ -682,9 +986,7 @@ impl ProbeState { checkpoint_versioning: Option< moa_hands::core::sandbox_workspace::checkpoint::versioning::CheckpointBucketVersioningObserver, >, - workspace_reaper: Option< - moa_hands::core::sandbox_workspace::reaper::WorkspaceReaperReadiness, - >, + maintenance: MaintenanceReadiness, ) -> Self { Self { readiness, @@ -692,11 +994,49 @@ impl ProbeState { kms, lineage_writer, checkpoint_versioning, - workspace_reaper, + maintenance, } } async fn check_ready(&self) -> anyhow::Result<()> { + let result = self.check_ready_inner().await; + if let Some(repository) = &self.maintenance.execution_repository { + let (receipt_ready, last_success_age) = match repository + .load_execution_maintenance_checkpoint( + ExecutionScope::ControlPlane, + ExecutionMaintenanceJobKind::DispatchReconciliation, + ) + .await + { + Ok(checkpoint) => execution_maintenance_status(checkpoint.as_ref(), Utc::now()), + Err(error) => { + tracing::warn!(%error, "failed to observe durable execution maintenance receipt"); + (false, None) + } + }; + moa_observability::runtime_metrics::record_execution_maintenance( + result.is_ok() && receipt_ready, + last_success_age, + ); + let (retention_ready, retention_last_success_age) = match repository + .load_execution_retention_checkpoint(ExecutionScope::ControlPlane) + .await + { + Ok(checkpoint) => execution_retention_status(checkpoint.as_ref(), Utc::now()), + Err(error) => { + tracing::warn!(%error, "failed to observe durable execution retention receipt"); + (false, None) + } + }; + moa_observability::runtime_metrics::record_execution_retention( + result.is_ok() && retention_ready, + retention_last_success_age, + ); + } + result + } + + async fn check_ready_inner(&self) -> anyhow::Result<()> { if !self.readiness.load(Ordering::Acquire) { bail!("readiness disabled"); } @@ -720,16 +1060,80 @@ impl ProbeState { bail!("checkpoint bucket versioning observation is missing or stale"); } - if let Some(reaper) = &self.workspace_reaper + if let Some(reaper) = &self.maintenance.workspace_reaper && let Some(reason) = reaper.unready_reason() { bail!("durable workspace maintenance not ready: {reason}"); } + if let Some(reaper) = &self.maintenance.hand_lease_reaper + && let Some(reason) = reaper.unready_reason() + { + bail!("durable hand lease cleanup not ready: {reason}"); + } + + if let Some(poller) = &self.maintenance.authz_outbox_poller + && let Some(reason) = poller.unready_reason() + { + bail!("authorization outbox not ready: {reason}"); + } + + if let Some(reaper) = &self.maintenance.action_review_reaper + && let Some(reason) = reaper.unready_reason() + { + bail!("action-review reconciliation not ready: {reason}"); + } + + if let Some(reaper) = &self.maintenance.authz_challenge_reaper + && let Some(reason) = reaper.unready_reason() + { + bail!("authorization challenge reconciliation not ready: {reason}"); + } + Ok(()) } } +fn execution_maintenance_status( + checkpoint: Option<&ExecutionMaintenanceCheckpoint>, + observed_at: DateTime, +) -> (bool, Option) { + let Some(checkpoint) = checkpoint else { + return (false, None); + }; + let Some(last_succeeded_at) = checkpoint.last_succeeded_at else { + return (false, None); + }; + let failed_since_success = checkpoint + .last_failure_at + .is_some_and(|last_failure_at| last_failure_at > last_succeeded_at); + let age = observed_at + .signed_duration_since(last_succeeded_at) + .to_std() + .unwrap_or(Duration::ZERO); + (!failed_since_success, Some(age)) +} + +fn execution_retention_status( + checkpoint: Option<&ExecutionRetentionCheckpoint>, + observed_at: DateTime, +) -> (bool, Option) { + let Some(checkpoint) = checkpoint else { + return (false, None); + }; + let Some(last_succeeded_at) = checkpoint.last_succeeded_at else { + return (false, None); + }; + let last_success_age = observed_at + .signed_duration_since(last_succeeded_at) + .to_std() + .unwrap_or_default(); + let newer_failure = checkpoint + .last_failure_at + .is_some_and(|failed_at| failed_at > last_succeeded_at); + (!newer_failure, Some(last_success_age)) +} + async fn live_handler() -> impl IntoResponse { StatusCode::OK } @@ -852,3 +1256,109 @@ async fn shutdown_signal() -> anyhow::Result<()> { ctrl_c.await } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn maintenance_command_uses_the_dedicated_health_port_default() { + // Pins: the fixture and deployment can give the maintenance role a health-only port + // without starting a Restate handler endpoint. + let args = + Args::try_parse_from(["moa-orchestrator", "--health-port", "19081", "maintenance"]) + .expect("maintenance command must parse"); + + assert_eq!(args.command, Some(Command::Maintenance)); + assert_eq!(args.health_port, 19081); + } + + #[test] + fn execution_cron_reconciliation_sleeps_and_backs_off_with_a_hard_cap() { + // Pins: the maintenance owner never hot-polls Restate, refreshes a very long configured + // cadence at least once per minute, and applies bounded exponential failure backoff. + assert_eq!( + execution_cron_reconcile_delay(3_600, 0), + Duration::from_secs(60) + ); + assert_eq!(execution_cron_reconcile_delay(1, 0), Duration::from_secs(1)); + assert_eq!(execution_cron_reconcile_delay(1, 3), Duration::from_secs(8)); + assert_eq!( + execution_cron_reconcile_delay(60, 32), + EXECUTION_CRON_RECONCILE_MAX_DELAY + ); + } + + #[test] + fn execution_maintenance_metric_uses_the_durable_success_receipt() { + // Pins: the maintenance gauge ages the durable successful Cron receipt, + // rather than a process-local timer that resets on pod restart. + let observed_at = Utc::now(); + let succeeded_at = observed_at - chrono::Duration::seconds(37); + let checkpoint = ExecutionMaintenanceCheckpoint { + job_kind: ExecutionMaintenanceJobKind::DispatchReconciliation, + generation: 4, + last_started_at: Some(succeeded_at), + last_succeeded_at: Some(succeeded_at), + last_failure_at: None, + last_error: None, + updated_at: succeeded_at, + }; + + assert_eq!( + execution_maintenance_status(Some(&checkpoint), observed_at), + (true, Some(Duration::from_secs(37))) + ); + } + + #[test] + fn execution_maintenance_metric_is_unready_without_a_success_or_after_failure() { + // Pins: a missing receipt and a failure newer than the last success are + // both observable as unready, even while the maintenance pod is alive. + let observed_at = Utc::now(); + assert_eq!( + execution_maintenance_status(None, observed_at), + (false, None) + ); + + let succeeded_at = observed_at - chrono::Duration::seconds(60); + let checkpoint = ExecutionMaintenanceCheckpoint { + job_kind: ExecutionMaintenanceJobKind::DispatchReconciliation, + generation: 5, + last_started_at: Some(observed_at - chrono::Duration::seconds(10)), + last_succeeded_at: Some(succeeded_at), + last_failure_at: Some(observed_at - chrono::Duration::seconds(5)), + last_error: Some("bounded repair failed".to_string()), + updated_at: observed_at - chrono::Duration::seconds(5), + }; + + assert_eq!( + execution_maintenance_status(Some(&checkpoint), observed_at), + (false, Some(Duration::from_secs(60))) + ); + } + + #[test] + fn execution_retention_metric_uses_only_its_durable_receipt() { + // Pins: terminal-detail retention has an independent SLO and cannot + // inherit readiness from the dispatch-reconciliation receipt. + let observed_at = Utc::now(); + let succeeded_at = observed_at - chrono::Duration::minutes(17); + let checkpoint = ExecutionRetentionCheckpoint { + generation: 9, + last_started_at: Some(succeeded_at), + last_succeeded_at: Some(succeeded_at), + last_failure_at: None, + next_run_at: Some(observed_at + chrono::Duration::minutes(30)), + scheduled_generation: Some(10), + last_error: None, + updated_at: succeeded_at, + }; + + assert_eq!( + execution_retention_status(Some(&checkpoint), observed_at), + (true, Some(Duration::from_secs(17 * 60))) + ); + assert_eq!(execution_retention_status(None, observed_at), (false, None)); + } +} diff --git a/crates/moa-orchestrator/src/objects/cron_job.rs b/crates/moa-orchestrator/src/objects/cron_job.rs index 1b890548b..8cc1d5c0d 100644 --- a/crates/moa-orchestrator/src/objects/cron_job.rs +++ b/crates/moa-orchestrator/src/objects/cron_job.rs @@ -82,7 +82,7 @@ pub trait CronJob { /// Resume firing and reschedule from the current wall clock. async fn resume() -> Result<(), HandlerError>; - /// Stop and clear all state for this job key. + /// Stop and clear the schedule while retaining its version tombstone. async fn stop() -> Result<(), HandlerError>; /// Internal handler fired by delayed sends. @@ -109,13 +109,9 @@ impl CronJob for CronJobImpl { validate(&config)?; let mut state = load_state(&ctx).await?; - if state.config.as_ref() == Some(&config) && !state.paused { + if !install_config(&mut state, config)? { return Ok(()); } - - state.config = Some(config); - state.paused = false; - state.version = state.version.wrapping_add(1); persist_state(&ctx, &state); schedule_next_tick(&ctx, &mut state).await?; @@ -129,7 +125,7 @@ impl CronJob for CronJobImpl { annotate_restate_handler_span("CronJob", "pause"); let mut state = load_state(&ctx).await?; state.paused = true; - state.version = state.version.wrapping_add(1); + advance_version(&mut state)?; persist_state(&ctx, &state); Ok(()) } @@ -144,7 +140,7 @@ impl CronJob for CronJobImpl { } state.paused = false; - state.version = state.version.wrapping_add(1); + advance_version(&mut state)?; persist_state(&ctx, &state); schedule_next_tick(&ctx, &mut state).await?; @@ -156,7 +152,9 @@ impl CronJob for CronJobImpl { async fn stop(&self, ctx: ObjectContext<'_>) -> Result<(), HandlerError> { crate::ctx::adopt_incoming_trace_parent(&ctx); annotate_restate_handler_span("CronJob", "stop"); - ctx.clear_all(); + let mut state = load_state(&ctx).await?; + stop_schedule(&mut state); + persist_state(&ctx, &state); Ok(()) } @@ -171,10 +169,7 @@ impl CronJob for CronJobImpl { let payload = payload.into_inner(); let state = load_state(&ctx).await?; - if payload.version != state.version || state.paused { - return Ok(()); - } - let Some(config) = state.config.clone() else { + let Some(config) = config_for_tick(&state, &payload).cloned() else { return Ok(()); }; @@ -241,6 +236,42 @@ fn persist_state(ctx: &ObjectContext<'_>, state: &CronJobState) { ctx.set(K_STATE, Json::from(state.clone())); } +fn advance_version(state: &mut CronJobState) -> Result<(), TerminalError> { + state.version = state + .version + .checked_add(1) + .ok_or_else(|| TerminalError::new("cron job version exhausted"))?; + Ok(()) +} + +fn install_config(state: &mut CronJobState, config: CronJobConfig) -> Result { + if state.config.as_ref() == Some(&config) && !state.paused { + return Ok(false); + } + + advance_version(state)?; + state.config = Some(config); + state.last_scheduled_fire = None; + state.paused = false; + Ok(true) +} + +fn stop_schedule(state: &mut CronJobState) { + state.config = None; + state.last_scheduled_fire = None; + state.paused = false; +} + +fn config_for_tick<'a>( + state: &'a CronJobState, + payload: &TickPayload, +) -> Option<&'a CronJobConfig> { + if state.paused || payload.version != state.version { + return None; + } + state.config.as_ref() +} + fn validate(config: &CronJobConfig) -> Result<(), HandlerError> { parse_cron(&config.schedule) .map_err(|error| TerminalError::new(format!("invalid cron schedule: {error}")))?; @@ -386,4 +417,76 @@ mod tests { assert_eq!(next.nanosecond(), 0); assert!(next > Utc::now()); } + + #[test] + fn stop_reconfigure_rejects_late_tick_from_old_incarnation() { + // Pins: stopping and reconfiguring a CronJob never lets a delayed tick from the + // previous configuration dispatch against the new target. + let original_config = valid_config(); + let mut state = CronJobState::default(); + assert!( + install_config(&mut state, original_config) + .expect("initial configuration should advance the incarnation") + ); + let stale_tick = TickPayload { + version: state.version, + scheduled_for: DateTime::::from_timestamp(1_700_000_000, 0) + .expect("fixture timestamp should be valid"), + }; + assert_eq!(state.version, 1); + assert_eq!(config_for_tick(&state, &stale_tick), state.config.as_ref()); + + stop_schedule(&mut state); + assert_eq!(state.version, 1); + assert_eq!(state.config, None); + assert_eq!(state.last_scheduled_fire, None); + assert!(!state.paused); + assert_eq!(config_for_tick(&state, &stale_tick), None); + + let replacement_config = CronJobConfig { + target_handler: "replacement".to_string(), + ..valid_config() + }; + assert!( + install_config(&mut state, replacement_config.clone()) + .expect("replacement configuration should advance the incarnation") + ); + assert_eq!(state.version, 2); + assert_eq!(state.config, Some(replacement_config)); + assert_eq!(config_for_tick(&state, &stale_tick), None); + + let replacement_tick = TickPayload { + version: state.version, + scheduled_for: stale_tick.scheduled_for, + }; + assert_eq!( + config_for_tick(&state, &replacement_tick), + state.config.as_ref() + ); + } + + #[test] + fn version_exhaustion_preserves_the_existing_incarnation() { + // Pins: version rollover cannot make an ancient delayed tick current again. + let mut state = CronJobState { + config: Some(valid_config()), + version: u64::MAX, + ..CronJobState::default() + }; + + let replacement_config = CronJobConfig { + target_handler: "replacement".to_string(), + ..valid_config() + }; + let error = install_config(&mut state, replacement_config) + .expect_err("an exhausted version must fail closed"); + + assert_eq!(state.version, u64::MAX); + assert_eq!(state.config, Some(valid_config())); + assert_eq!( + error.message(), + "cron job version exhausted", + "exhaustion should return the exact terminal reason" + ); + } } diff --git a/crates/moa-orchestrator/src/objects/execution_run_controller.rs b/crates/moa-orchestrator/src/objects/execution_run_controller.rs new file mode 100644 index 000000000..3a6d2385b --- /dev/null +++ b/crates/moa-orchestrator/src/objects/execution_run_controller.rs @@ -0,0 +1,131 @@ +//! Bounded, generation-fenced activations for one durable execution run. +//! +//! A controller activation performs a finite amount of database-backed scheduler +//! work and then returns. Parked runs retain only Postgres state and exact +//! delayed triggers; they never retain a promise, child join, or polling loop. + +mod advance; +mod progress; +mod settlement; + +#[cfg(test)] +mod tests; + +use moa_core::types::identifiers::TenantId; +use restate_sdk::prelude::*; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +/// Exact durable dispatch accepted by one controller activation. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ExecutionRunAdvanceRequest { + /// Immutable dispatch-outbox identity. + pub dispatch_uid: Uuid, + /// Tenant that owns the execution run. + pub tenant_id: TenantId, + /// Durable execution-run identifier and virtual-object key. + pub run_uid: Uuid, + /// Exact controller generation fenced by the dispatch. + pub controller_generation: u64, + /// Exact persisted scheduling wake claimed by this activation. + pub wake_epoch: u64, +} + +/// Durable disposition of one controller activation request. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ExecutionRunAdvanceOutcome { + /// The exact wake was claimed and bounded scheduler work committed. + Advanced, + /// The exact wake had already committed and was replayed as a no-op. + Replayed, + /// A newer generation or wake superseded this dispatch. + Stale, + /// The run was already terminal and could not be advanced. + Terminal, +} + +/// Bounded acknowledgement returned to the dispatch owner. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ExecutionRunAdvanceResponse { + /// Durable request disposition. + pub outcome: ExecutionRunAdvanceOutcome, + /// Controller generation observed after the transaction. + pub controller_generation: u64, + /// Wake epoch observed after the transaction. + pub wake_epoch: u64, + /// Number of scheduler transitions performed by this activation. + pub activation_steps: usize, + /// Number of newly materialized logical tasks. + pub materialized_tasks: usize, + /// Whether the same transaction enqueued one bounded continuation. + pub continuation_enqueued: bool, +} + +/// Restate virtual object that serializes bounded advancement by execution run. +#[restate_sdk::object] +#[name = "ExecutionRunController"] +pub trait ExecutionRunController { + /// Claims and advances one exact persisted scheduling wake. + async fn advance( + request: Json, + ) -> Result, HandlerError>; +} + +/// PostgreSQL-backed bounded execution-run controller. +#[derive(Clone)] +pub struct ExecutionRunControllerImpl { + repository: moa_execution::repository::ExecutionRepository, + config: moa_config::ExecutionConfig, +} + +impl ExecutionRunControllerImpl { + /// Creates a bounded controller over the shared execution repository. + #[must_use] + pub fn new(pool: sqlx::PgPool, config: moa_config::ExecutionConfig) -> Self { + Self { + repository: moa_execution::repository::ExecutionRepository::new(pool), + config, + } + } +} + +impl ExecutionRunController for ExecutionRunControllerImpl { + #[tracing::instrument(skip(self, ctx, request), fields(run_uid = %request.0.run_uid))] + // SAFETY: ingress-private dispatch; the exact key and admitted owner scope are revalidated from Postgres. + async fn advance( + &self, + ctx: ObjectContext<'_>, + request: Json, + ) -> Result, HandlerError> { + crate::ctx::adopt_incoming_trace_parent(&ctx); + moa_observability::restate_observability::annotate_restate_handler_span( + "ExecutionRunController", + "advance", + ); + let request = request.into_inner(); + advance::validate_request(ctx.key(), &request)?; + + let repository = self.repository.clone(); + let config = self.config.clone(); + let operation = request.clone(); + let committed = ctx + .run(|| async move { + advance::advance(repository, config, operation) + .await + .map(Json::from) + .map_err(crate::workflows::errors::execution_error_to_handler_error) + }) + .name(format!( + "execution_controller_advance_{}_{}", + request.controller_generation, request.wake_epoch + )) + .await? + .into_inner(); + + progress::deliver(&ctx, &self.repository, &request, &committed).await?; + Ok(Json::from(committed.response)) + } +} diff --git a/crates/moa-orchestrator/src/objects/execution_run_controller/advance.rs b/crates/moa-orchestrator/src/objects/execution_run_controller/advance.rs new file mode 100644 index 000000000..635f24923 --- /dev/null +++ b/crates/moa-orchestrator/src/objects/execution_run_controller/advance.rs @@ -0,0 +1,1360 @@ +//! Exact wake claiming and bounded node-page advancement. + +use std::collections::BTreeMap; + +use super::{ + ExecutionRunAdvanceOutcome, ExecutionRunAdvanceRequest, ExecutionRunAdvanceResponse, settlement, +}; +use chrono::{DateTime, Utc}; +use moa_execution::{ + NodeMaterializationPage, ReduceMaterializationPageInput, ScheduleRequest, + budget::BudgetLedger, + materialize_node_page, + repository::{ + ExecutionRepository, ExecutionScope, RunControllerClaimOutcome, + RunControllerCompletionOutcome, RunControllerCompletionRequest, RunDeadlineArmOutcome, + completion::{CompletionAdvanceOutcome, CompletionAdvanceRequest}, + outbox::{ExecutionDispatchKind, ExecutionDispatchRecord}, + ready::{ + ExecutionReduceMaterializationCursor, MapAggregatePageOutcome, MapAggregatePageRequest, + ReadyMaterializationOutcome, ReadyMaterializationRequest, ReduceRoundInputPageRequest, + }, + terminal::{ + FinalizationOutcome, PendingTerminalAdvanceOutcome, PendingTerminalAdvanceStage, + RunTriggerDrainOutcome, RunTriggerDrainRequest, + }, + }, + state::{ExecutionProjection, ExecutionTaskStatus}, +}; +use serde::{Deserialize, Serialize}; +use serde_json::json; + +/// Journaled database commit and the bounded side effects selected by it. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub(super) struct ControllerAdvanceCommit { + pub(super) response: ExecutionRunAdvanceResponse, + pub(super) publish_progress: bool, + pub(super) terminal_delivery: Option, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct ActivationLimits { + remaining_steps: usize, + remaining_tasks: usize, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum ActivationPreflight { + PendingTerminal, + DueDeadline, + Ordinary, +} + +fn completion_scan_steps(scanned_tasks: u32, scanned_nodes: u32) -> moa_execution::Result { + usize::try_from(scanned_tasks) + .ok() + .and_then(|tasks| { + usize::try_from(scanned_nodes) + .ok() + .and_then(|nodes| tasks.checked_add(nodes)) + }) + .ok_or_else(|| moa_execution::Error::ArithmeticOverflow { + context: "controller completion scan count".to_string(), + }) +} + +fn terminal_trigger_page_limit(remaining_steps: usize) -> moa_execution::Result> { + if remaining_steps == 0 { + return Ok(None); + } + u32::try_from(remaining_steps.min(1_000)) + .map(Some) + .map_err(|_| moa_execution::Error::ArithmeticOverflow { + context: "controller terminal trigger-drain page limit".to_string(), + }) +} + +fn pending_terminal_step_count( + settled_task_count: u64, + drained_trigger_count: u32, + cancellation_dispatch_count: usize, + compensation_admitted: bool, +) -> moa_execution::Result { + usize::try_from(settled_task_count) + .ok() + .and_then(|settled| { + usize::try_from(drained_trigger_count) + .ok() + .and_then(|drained| settled.checked_add(drained)) + }) + .and_then(|count| count.checked_add(cancellation_dispatch_count)) + .and_then(|count| count.checked_add(usize::from(compensation_admitted))) + .ok_or_else(|| moa_execution::Error::ArithmeticOverflow { + context: "controller pending-terminal step count".to_string(), + }) +} + +fn map_aggregate_requires_continuation( + outcome: &MapAggregatePageOutcome, +) -> moa_execution::Result { + match outcome { + MapAggregatePageOutcome::Applied { + aggregate_complete, .. + } + | MapAggregatePageOutcome::Replayed { + aggregate_complete, .. + } => Ok(!aggregate_complete), + MapAggregatePageOutcome::Overflow | MapAggregatePageOutcome::Conflict => Ok(true), + MapAggregatePageOutcome::NotFound => Err(moa_execution::Error::InvalidRepositoryData { + message: "execution run disappeared during bounded map aggregation".to_string(), + }), + } +} + +fn validate_resumed_recovery_commit( + prior_wake_epoch: u64, + current_wake_epoch: u64, + continuation_enqueued: bool, +) -> moa_execution::Result<()> { + let expected_wake_epoch = prior_wake_epoch.checked_add(1).ok_or_else(|| { + moa_execution::Error::ArithmeticOverflow { + context: "resumed activation continuation wake epoch".to_string(), + } + })?; + if !continuation_enqueued || current_wake_epoch != expected_wake_epoch { + return Err(moa_execution::Error::InvalidRepositoryData { + message: "resumed activation recovery must enqueue exactly one fresh wake".to_string(), + }); + } + Ok(()) +} + +fn validate_trigger_drain_continuation( + prior_wake_epoch: u64, + current_wake_epoch: u64, + drained_trigger_count: u32, +) -> moa_execution::Result<()> { + let expected_wake_epoch = prior_wake_epoch.checked_add(1).ok_or_else(|| { + moa_execution::Error::ArithmeticOverflow { + context: "terminal trigger-drain continuation wake epoch".to_string(), + } + })?; + if drained_trigger_count == 0 || current_wake_epoch != expected_wake_epoch { + return Err(moa_execution::Error::InvalidRepositoryData { + message: + "terminal trigger drain must settle a nonempty page and enqueue one fresh wake" + .to_string(), + }); + } + Ok(()) +} + +fn validate_replan_stop_continuation( + run: &moa_execution::repository::ExecutionRunRecord, + continuation: &ExecutionDispatchRecord, + continuation_wake_epoch: u64, +) -> moa_execution::Result<()> { + let exact_owner = continuation.kind == ExecutionDispatchKind::RunActivation + && continuation.tenant_id == run.tenant_id + && continuation.run_uid == Some(run.run_uid) + && continuation.controller_generation == Some(run.controller_generation) + && continuation.wake_epoch == Some(continuation_wake_epoch); + validate_replan_stop_continuation_fields(run.wake_epoch, continuation_wake_epoch, exact_owner) +} + +fn validate_replan_stop_continuation_fields( + prior_wake_epoch: u64, + continuation_wake_epoch: u64, + exact_owner: bool, +) -> moa_execution::Result<()> { + validate_resumed_recovery_commit(prior_wake_epoch, continuation_wake_epoch, true)?; + if !exact_owner { + return Err(moa_execution::Error::InvalidRepositoryData { + message: "replan-stop continuation does not own the exact fresh run wake".to_string(), + }); + } + Ok(()) +} + +fn activation_preflight( + has_pending_terminal: bool, + deadline_at: Option>, + now: DateTime, +) -> ActivationPreflight { + if has_pending_terminal { + ActivationPreflight::PendingTerminal + } else if deadline_at.is_some_and(|deadline_at| deadline_at <= now) { + ActivationPreflight::DueDeadline + } else { + ActivationPreflight::Ordinary + } +} + +impl ActivationLimits { + fn new(maximum_steps: usize, dispatch_batch_size: usize) -> moa_execution::Result { + if maximum_steps == 0 || dispatch_batch_size == 0 { + return Err(moa_execution::Error::InvalidRepositoryInput { + message: "controller activation bounds must both be greater than zero".to_string(), + }); + } + Ok(Self { + remaining_steps: maximum_steps, + remaining_tasks: dispatch_batch_size, + }) + } + + fn inspect_nodes(&mut self, count: usize) -> usize { + let inspected = self.remaining_steps.min(count); + self.remaining_steps -= inspected; + inspected + } + + fn record_steps(&mut self, count: usize) -> moa_execution::Result<()> { + self.remaining_steps = self.remaining_steps.checked_sub(count).ok_or_else(|| { + moa_execution::Error::InvalidRepositoryData { + message: "controller completion scan exceeded its activation bound".to_string(), + } + })?; + Ok(()) + } + + fn task_page_limit(&self) -> moa_execution::Result { + u32::try_from(self.remaining_tasks.min(1_000)).map_err(|_| { + moa_execution::Error::ArithmeticOverflow { + context: "controller task page limit".to_string(), + } + }) + } + + fn record_tasks(&mut self, count: usize) -> moa_execution::Result<()> { + self.remaining_tasks = self.remaining_tasks.checked_sub(count).ok_or_else(|| { + moa_execution::Error::InvalidRepositoryData { + message: "controller materialized beyond its dispatch bound".to_string(), + } + })?; + Ok(()) + } +} + +pub(super) fn validate_request( + object_key: &str, + request: &ExecutionRunAdvanceRequest, +) -> Result<(), restate_sdk::prelude::HandlerError> { + if request.dispatch_uid.is_nil() || request.run_uid.is_nil() { + return Err(crate::workflows::errors::bad_request( + "execution controller identifiers must not be nil", + )); + } + if object_key != request.run_uid.to_string() { + return Err(crate::workflows::errors::bad_request( + "execution controller key does not match run_uid", + )); + } + Ok(()) +} + +pub(super) async fn advance( + repository: ExecutionRepository, + config: moa_config::ExecutionConfig, + request: ExecutionRunAdvanceRequest, +) -> moa_execution::Result { + let Some(admitted) = repository + .load_run(ExecutionScope::ControlPlane, request.run_uid) + .await? + else { + return Err(moa_execution::Error::InvalidRepositoryInput { + message: "execution run not found".to_string(), + }); + }; + if admitted.tenant_id != request.tenant_id + || admitted.admitted_identity.tenant_id != request.tenant_id + { + return Err(moa_execution::Error::InvalidRepositoryInput { + message: "activation tenant does not own the admitted execution run".to_string(), + }); + } + let scope = admitted.contact_id.map_or( + ExecutionScope::Tenant { + tenant_id: admitted.tenant_id, + }, + |contact_id| ExecutionScope::Contact { + tenant_id: admitted.tenant_id, + contact_id, + }, + ); + let claim = repository + .claim_controller_wake( + scope, + request.run_uid, + request.controller_generation, + request.wake_epoch, + ) + .await?; + let run = match claim { + RunControllerClaimOutcome::Claimed(run) => run, + RunControllerClaimOutcome::Resumed(run) => { + return resume_with_bounded_continuation(&repository, scope, &config, &request, &run) + .await; + } + RunControllerClaimOutcome::Replayed(run) => { + return Ok(noop_commit(ExecutionRunAdvanceOutcome::Replayed, &run)); + } + RunControllerClaimOutcome::Terminal(run) => { + return terminal_commit(&repository, scope, &run).await; + } + RunControllerClaimOutcome::StaleGeneration { current_generation } => { + return Ok(stale_commit(current_generation, admitted.wake_epoch)); + } + RunControllerClaimOutcome::StaleWake { + current_wake_epoch, .. + } => { + return Ok(stale_commit( + admitted.controller_generation, + current_wake_epoch, + )); + } + RunControllerClaimOutcome::NotFound => { + return Err(moa_execution::Error::InvalidRepositoryInput { + message: "execution run disappeared during activation claim".to_string(), + }); + } + RunControllerClaimOutcome::InvalidState => { + return Ok(stale_commit( + admitted.controller_generation, + admitted.wake_epoch, + )); + } + }; + + let mut limits = + ActivationLimits::new(config.maximum_activation_steps, config.dispatch_batch_size)?; + let now = Utc::now(); + let terminal_page_limit = u32::try_from( + limits + .remaining_steps + .min(limits.remaining_tasks) + .min(1_000), + ) + .map_err(|_| moa_execution::Error::ArithmeticOverflow { + context: "controller pending-terminal page limit".to_string(), + })?; + match activation_preflight( + run.pending_terminal.is_some(), + run.approved_budget.deadline_at, + now, + ) { + ActivationPreflight::PendingTerminal => { + let outcome = repository + .advance_pending_terminal_settlement( + &config, + scope, + run.run_uid, + run.controller_generation, + run.wake_epoch, + now, + terminal_page_limit, + ) + .await?; + return pending_terminal_commit(&repository, scope, &run, outcome).await; + } + ActivationPreflight::DueDeadline => { + let outcome = repository + .fence_deadline_and_enqueue_settlement( + &config, + scope, + run.run_uid, + run.controller_generation, + run.wake_epoch, + now, + terminal_page_limit, + ) + .await?; + return pending_terminal_commit(&repository, scope, &run, outcome).await; + } + ActivationPreflight::Ordinary => {} + } + + match repository + .arm_run_deadline(scope, run.run_uid, run.controller_generation, &config) + .await? + { + RunDeadlineArmOutcome::Armed(_) + | RunDeadlineArmOutcome::NoDeadline + | RunDeadlineArmOutcome::Terminal => {} + RunDeadlineArmOutcome::NotFound => { + return Err(moa_execution::Error::InvalidRepositoryInput { + message: "execution run disappeared while arming deadline".to_string(), + }); + } + RunDeadlineArmOutcome::StaleGeneration { .. } => { + return Ok(stale_commit(run.controller_generation, run.wake_epoch)); + } + } + + let mut activation_steps = 0usize; + let mut materialized_tasks = 0usize; + let mut bounded_work_remains = false; + + if let Some(intent) = repository + .load_replan_stop_intent( + scope, + run.run_uid, + run.controller_generation, + run.wake_epoch, + ) + .await? + { + let page_size = u32::try_from(limits.remaining_steps.min(1_000)).map_err(|_| { + moa_execution::Error::ArithmeticOverflow { + context: "controller replan-stop completion page limit".to_string(), + } + })?; + let completion = repository + .advance_replan_stop_completion_projection( + scope, + &config, + CompletionAdvanceRequest { + run_uid: run.run_uid, + controller_generation: run.controller_generation, + wake_epoch: run.wake_epoch, + page_size, + now, + }, + &intent, + ) + .await?; + match completion { + CompletionAdvanceOutcome::ReplanStopContinue { + scanned_tasks, + scanned_nodes, + continuation, + } => { + let scanned = completion_scan_steps(scanned_tasks, scanned_nodes)?; + if scanned == 0 { + return Err(moa_execution::Error::InvalidRepositoryData { + message: "replan-stop continuation made no bounded progress".to_string(), + }); + } + limits.record_steps(scanned)?; + activation_steps = activation_steps.checked_add(scanned).ok_or_else(|| { + moa_execution::Error::ArithmeticOverflow { + context: "controller activation step count".to_string(), + } + })?; + let continuation_wake_epoch = continuation.wake_epoch.ok_or_else(|| { + moa_execution::Error::InvalidRepositoryData { + message: "replan-stop continuation is missing its wake epoch".to_string(), + } + })?; + validate_replan_stop_continuation(&run, &continuation, continuation_wake_epoch)?; + return Ok(ControllerAdvanceCommit { + response: ExecutionRunAdvanceResponse { + outcome: ExecutionRunAdvanceOutcome::Advanced, + controller_generation: run.controller_generation, + wake_epoch: continuation_wake_epoch, + activation_steps, + materialized_tasks: 0, + continuation_enqueued: true, + }, + publish_progress: true, + terminal_delivery: None, + }); + } + CompletionAdvanceOutcome::ReplanStopReady { + pending_terminal, + receipt, + } => { + let outcome = repository + .fence_replan_stop_and_enqueue_settlement( + &config, + scope, + run.run_uid, + run.controller_generation, + run.plan_revision, + run.wake_epoch, + pending_terminal, + receipt, + now, + terminal_page_limit, + ) + .await?; + return pending_terminal_commit(&repository, scope, &run, outcome).await; + } + CompletionAdvanceOutcome::NotReady => { + return Ok(stale_commit(run.controller_generation, run.wake_epoch)); + } + CompletionAdvanceOutcome::Continue { .. } + | CompletionAdvanceOutcome::VerifiersMaterialized { .. } + | CompletionAdvanceOutcome::WaitingForVerifiers + | CompletionAdvanceOutcome::FinalizationReady(_) + | CompletionAdvanceOutcome::NonSuccessTerminal { .. } => { + return Err(moa_execution::Error::InvalidRepositoryData { + message: "replan-stop completion returned an ordinary completion outcome" + .to_string(), + }); + } + } + } + + loop { + if limits.remaining_steps == 0 { + bounded_work_remains = true; + break; + } + let Some(candidate) = repository + .load_map_aggregate_candidate( + scope, + run.run_uid, + run.controller_generation, + run.wake_epoch, + ) + .await? + else { + break; + }; + let outcome = repository + .advance_map_aggregate_page( + scope, + MapAggregatePageRequest { + run_uid: run.run_uid, + plan_revision: run.plan_revision, + controller_generation: run.controller_generation, + wake_epoch: run.wake_epoch, + node_id: candidate.node_id, + expected_cursor_item_key: candidate.cursor_item_key, + }, + ) + .await?; + limits.record_steps(1)?; + activation_steps = activation_steps.checked_add(1).ok_or_else(|| { + moa_execution::Error::ArithmeticOverflow { + context: "controller activation step count".to_string(), + } + })?; + if map_aggregate_requires_continuation(&outcome)? { + bounded_work_remains = true; + break; + } + } + + 'pages: while !bounded_work_remains { + if limits.remaining_steps == 0 { + bounded_work_remains = true; + break; + } + let page_limit = u32::try_from(limits.remaining_steps.min(1_000)).map_err(|_| { + moa_execution::Error::ArithmeticOverflow { + context: "controller activation node page limit".to_string(), + } + })?; + let Some(projection) = repository + .load_activation_projection(scope, run.run_uid, page_limit) + .await? + else { + return Err(moa_execution::Error::InvalidRepositoryData { + message: "claimed execution run has no activation projection".to_string(), + }); + }; + if projection.nodes.is_empty() { + break; + } + let inspected = limits.inspect_nodes(projection.nodes.len()); + activation_steps = activation_steps.checked_add(inspected).ok_or_else(|| { + moa_execution::Error::ArithmeticOverflow { + context: "controller activation step count".to_string(), + } + })?; + for node in projection.nodes.iter().take(inspected) { + if limits.remaining_tasks == 0 { + bounded_work_remains = true; + break 'pages; + } + let schedule = ScheduleRequest { + run_uid: projection.run.run_uid, + goal: projection.run.goal.clone(), + plan: projection.run.active_plan.clone(), + catalog: projection.run.catalog.clone(), + run_input: projection.run.input.clone(), + projection: ExecutionProjection { + plan_revision: projection.run.plan_revision, + node_statuses: BTreeMap::new(), + tasks: Vec::new(), + }, + config: config.clone(), + budget_ledger: BudgetLedger { + limit: projection.run.approved_budget.clone(), + reserved: projection.run.reserved, + consumed: projection.run.consumed, + overrun: projection.run.budget_overrun, + }, + now, + }; + let task_page_limit = limits.task_page_limit()?; + let plan_node = schedule + .plan + .definition + .nodes + .iter() + .find(|plan_node| plan_node.id == node.node_id) + .ok_or_else(|| moa_execution::Error::InvalidRepositoryData { + message: format!( + "activation node `{}` is missing from the active plan", + node.node_id + ), + })?; + let reduce_input = match &plan_node.operation { + moa_artifacts::execution_plan::ExecutionOperation::Reduce { + batch_size, .. + } => { + let page_inputs = if node.reduce_round == 1 { + Vec::new() + } else { + let round_input_count = node.reduce_round_input_count.ok_or_else(|| { + moa_execution::Error::InvalidRepositoryData { + message: format!( + "reduce node `{}` round {} is missing its input count", + node.node_id, node.reduce_round + ), + } + })?; + repository + .load_reduce_round_inputs( + scope, + ReduceRoundInputPageRequest { + run_uid: run.run_uid, + node_id: node.node_id.clone(), + source_round: node.reduce_round.checked_sub(1).ok_or_else( + || moa_execution::Error::ArithmeticOverflow { + context: format!( + "reduce node {} source round", + node.node_id + ), + }, + )?, + cursor: ExecutionReduceMaterializationCursor { + round: node.reduce_round, + batch_cursor: node.reduce_batch_cursor, + round_input_count, + }, + batch_size: *batch_size, + target_batch_limit: task_page_limit, + }, + ) + .await? + }; + Some(ReduceMaterializationPageInput { + round: node.reduce_round, + batch_cursor: node.reduce_batch_cursor, + round_input_count: node.reduce_round_input_count, + page_inputs, + }) + } + _ => None, + }; + let NodeMaterializationPage { + tasks, + source_exhausted, + reduce_cursor, + terminal_output, + .. + } = materialize_node_page( + &schedule, + &node.node_id, + &projection.referenced_outputs, + node.materialization_cursor, + task_page_limit, + reduce_input.as_ref(), + )?; + let task_count = tasks.len(); + match repository + .materialize_ready_page( + scope, + &config, + ReadyMaterializationRequest { + run_uid: run.run_uid, + plan_revision: run.plan_revision, + node_id: node.node_id.clone(), + expected_cursor: node.materialization_cursor, + reduce_cursor: reduce_cursor.map(|cursor| { + ExecutionReduceMaterializationCursor { + round: cursor.round, + batch_cursor: cursor.batch_cursor, + round_input_count: cursor.round_input_count, + } + }), + source_exhausted, + terminal_output, + tasks, + }, + ) + .await? + { + ReadyMaterializationOutcome::Applied { tasks, .. } + | ReadyMaterializationOutcome::Replayed { tasks, .. } => { + if tasks.len() != task_count { + return Err(moa_execution::Error::InvalidRepositoryData { + message: "ready materialization returned a different task page" + .to_string(), + }); + } + limits.record_tasks(task_count)?; + materialized_tasks = + materialized_tasks.checked_add(task_count).ok_or_else(|| { + moa_execution::Error::ArithmeticOverflow { + context: "controller materialized task count".to_string(), + } + })?; + } + ReadyMaterializationOutcome::Conflict => { + bounded_work_remains = true; + break 'pages; + } + } + bounded_work_remains |= !source_exhausted; + if limits.remaining_tasks == 0 { + break 'pages; + } + } + if inspected < projection.nodes.len() + || projection.has_more_actionable + || limits.remaining_steps == 0 + { + bounded_work_remains = true; + break; + } + } + + let readiness = repository + .load_activation_readiness(scope, run.run_uid) + .await? + .ok_or_else(|| moa_execution::Error::InvalidRepositoryData { + message: "execution run disappeared while loading activation readiness".to_string(), + })?; + bounded_work_remains |= readiness.has_actionable_nodes; + + if readiness.terminal_ready() && !bounded_work_remains { + if limits.remaining_steps == 0 || limits.remaining_tasks == 0 { + bounded_work_remains = true; + } else { + let completion_page_limit = u32::try_from( + limits + .remaining_steps + .min(limits.remaining_tasks) + .min(1_000), + ) + .map_err(|_| moa_execution::Error::ArithmeticOverflow { + context: "controller completion page limit".to_string(), + })?; + match repository + .advance_completion_projection( + scope, + &config, + CompletionAdvanceRequest { + run_uid: run.run_uid, + controller_generation: request.controller_generation, + wake_epoch: request.wake_epoch, + page_size: completion_page_limit, + now, + }, + ) + .await? + { + CompletionAdvanceOutcome::Continue { + scanned_tasks, + scanned_nodes, + } => { + let scanned = completion_scan_steps(scanned_tasks, scanned_nodes)?; + if scanned == 0 { + return Err(moa_execution::Error::InvalidRepositoryData { + message: "completion continuation made no bounded progress".to_string(), + }); + } + limits.record_steps(scanned)?; + activation_steps = activation_steps.checked_add(scanned).ok_or_else(|| { + moa_execution::Error::ArithmeticOverflow { + context: "controller activation step count".to_string(), + } + })?; + bounded_work_remains = true; + } + CompletionAdvanceOutcome::ReplanStopContinue { .. } + | CompletionAdvanceOutcome::ReplanStopReady { .. } => { + return Err(moa_execution::Error::InvalidRepositoryData { + message: "ordinary completion returned a replan-stop outcome".to_string(), + }); + } + CompletionAdvanceOutcome::VerifiersMaterialized { tasks } => { + if tasks.is_empty() + || tasks + .iter() + .any(|task| task.status != ExecutionTaskStatus::Ready) + { + return Err(moa_execution::Error::InvalidRepositoryData { + message: + "completion verifier page was empty or contained non-ready work" + .to_string(), + }); + } + let task_count = tasks.len(); + limits.record_steps(task_count)?; + limits.record_tasks(task_count)?; + activation_steps = + activation_steps.checked_add(task_count).ok_or_else(|| { + moa_execution::Error::ArithmeticOverflow { + context: "controller activation step count".to_string(), + } + })?; + materialized_tasks = + materialized_tasks.checked_add(task_count).ok_or_else(|| { + moa_execution::Error::ArithmeticOverflow { + context: "controller materialized task count".to_string(), + } + })?; + bounded_work_remains = true; + } + CompletionAdvanceOutcome::WaitingForVerifiers => {} + CompletionAdvanceOutcome::FinalizationReady(finalization) => { + if let Some(trigger_page_limit) = + terminal_trigger_page_limit(limits.remaining_steps)? + { + let mut may_finalize = true; + match repository + .drain_run_triggers_page( + scope, + &config, + RunTriggerDrainRequest { + run_uid: run.run_uid, + controller_generation: run.controller_generation, + wake_epoch: run.wake_epoch, + page_limit: trigger_page_limit, + now, + }, + ) + .await? + { + RunTriggerDrainOutcome::PageDrained(commit) => { + let drained = usize::try_from(commit.drained_trigger_count) + .map_err(|_| moa_execution::Error::ArithmeticOverflow { + context: "controller drained trigger count".to_string(), + })?; + limits.record_steps(drained)?; + activation_steps = activation_steps + .checked_add(drained) + .ok_or_else(|| moa_execution::Error::ArithmeticOverflow { + context: "controller activation step count".to_string(), + })?; + validate_trigger_drain_continuation( + run.wake_epoch, + commit.run.wake_epoch, + commit.drained_trigger_count, + )?; + return Ok(ControllerAdvanceCommit { + response: ExecutionRunAdvanceResponse { + outcome: ExecutionRunAdvanceOutcome::Advanced, + controller_generation: commit.run.controller_generation, + wake_epoch: commit.run.wake_epoch, + activation_steps, + materialized_tasks, + continuation_enqueued: true, + }, + publish_progress: true, + terminal_delivery: None, + }); + } + RunTriggerDrainOutcome::ReadyToFinalize { + run: drained_run, + drained_trigger_count, + } => { + let drained = + usize::try_from(drained_trigger_count).map_err(|_| { + moa_execution::Error::ArithmeticOverflow { + context: "controller drained trigger count".to_string(), + } + })?; + limits.record_steps(drained)?; + activation_steps = activation_steps + .checked_add(drained) + .ok_or_else(|| moa_execution::Error::ArithmeticOverflow { + context: "controller activation step count".to_string(), + })?; + if drained_run.controller_generation != run.controller_generation + || drained_run.wake_epoch != run.wake_epoch + { + return Err(moa_execution::Error::InvalidRepositoryData { + message: "terminal trigger drain changed the claimed wake" + .to_string(), + }); + } + if limits.remaining_steps == 0 { + bounded_work_remains = true; + may_finalize = false; + } else { + limits.record_steps(1)?; + activation_steps = + activation_steps.checked_add(1).ok_or_else(|| { + moa_execution::Error::ArithmeticOverflow { + context: "controller activation step count" + .to_string(), + } + })?; + } + } + RunTriggerDrainOutcome::Replayed(replayed) => { + return Ok(noop_commit( + ExecutionRunAdvanceOutcome::Replayed, + &replayed, + )); + } + RunTriggerDrainOutcome::StaleGeneration { current_generation } => { + return Ok(stale_commit(current_generation, run.wake_epoch)); + } + RunTriggerDrainOutcome::StaleWake { + current_wake_epoch, .. + } => { + return Ok(stale_commit( + run.controller_generation, + current_wake_epoch, + )); + } + RunTriggerDrainOutcome::NotFound => { + return Err(moa_execution::Error::InvalidRepositoryData { + message: + "execution run disappeared during terminal trigger drain" + .to_string(), + }); + } + RunTriggerDrainOutcome::InvalidState => { + return Ok(stale_commit(run.controller_generation, run.wake_epoch)); + } + } + if may_finalize { + match repository.finalize_run(scope, *finalization).await? { + FinalizationOutcome::Finalized(terminal) + | FinalizationOutcome::Replayed(terminal) => { + let mut commit = + terminal_commit(&repository, scope, &terminal).await?; + commit.response.activation_steps = activation_steps; + commit.response.materialized_tasks = materialized_tasks; + return Ok(commit); + } + FinalizationOutcome::Conflict => bounded_work_remains = true, + FinalizationOutcome::NotFound => { + return Err(moa_execution::Error::InvalidRepositoryData { + message: "execution run disappeared during finalization" + .to_string(), + }); + } + } + } + } else { + bounded_work_remains = true; + } + } + CompletionAdvanceOutcome::NonSuccessTerminal { pending_terminal } => { + let outcome = repository + .fence_completion_terminal_and_enqueue_settlement( + &config, + scope, + run.run_uid, + run.controller_generation, + run.wake_epoch, + pending_terminal, + now, + completion_page_limit, + ) + .await?; + let mut commit = + pending_terminal_commit(&repository, scope, &run, outcome).await?; + commit.response.activation_steps = commit + .response + .activation_steps + .checked_add(activation_steps) + .ok_or_else(|| moa_execution::Error::ArithmeticOverflow { + context: "controller terminal activation step count".to_string(), + })?; + commit.response.materialized_tasks = materialized_tasks; + return Ok(commit); + } + CompletionAdvanceOutcome::NotReady => bounded_work_remains = true, + } + } + } + + let current = repository + .load_run(scope, run.run_uid) + .await? + .ok_or_else(|| moa_execution::Error::InvalidRepositoryData { + message: "execution run disappeared before controller checkpoint".to_string(), + })?; + let checkpoint = if bounded_work_remains { + settlement::continuation_checkpoint(¤t) + } else { + settlement::parked_checkpoint(¤t, now) + }; + let completion = repository + .complete_controller_wake( + scope, + &config, + run.run_uid, + RunControllerCompletionRequest { + controller_generation: request.controller_generation, + wake_epoch: request.wake_epoch, + checkpoint, + continuation_payload: bounded_work_remains.then(|| { + json!({ + "cause": "bounded_controller_continuation", + "prior_dispatch_uid": request.dispatch_uid, + }) + }), + continuation_not_before_at: now, + }, + ) + .await?; + let (current, continuation_enqueued, outcome) = match completion { + RunControllerCompletionOutcome::Applied { run, continuation } => { + let enqueued = continuation.is_some(); + (*run, enqueued, ExecutionRunAdvanceOutcome::Advanced) + } + RunControllerCompletionOutcome::Replayed(run) => { + (*run, false, ExecutionRunAdvanceOutcome::Replayed) + } + RunControllerCompletionOutcome::CapacitySaturated { dimension } => { + return Err(moa_execution::Error::CapacitySaturated { + dimension: dimension.as_str(), + }); + } + RunControllerCompletionOutcome::StaleGeneration { current_generation } => { + return Ok(stale_commit(current_generation, current.wake_epoch)); + } + RunControllerCompletionOutcome::StaleWake { + current_wake_epoch, .. + } => { + return Ok(stale_commit( + current.controller_generation, + current_wake_epoch, + )); + } + RunControllerCompletionOutcome::NotFound => { + return Err(moa_execution::Error::InvalidRepositoryData { + message: "execution run disappeared during controller completion".to_string(), + }); + } + RunControllerCompletionOutcome::InvalidState => { + return Ok(stale_commit( + current.controller_generation, + current.wake_epoch, + )); + } + }; + Ok(ControllerAdvanceCommit { + response: ExecutionRunAdvanceResponse { + outcome, + controller_generation: current.controller_generation, + wake_epoch: current.wake_epoch, + activation_steps, + materialized_tasks, + continuation_enqueued, + }, + publish_progress: true, + terminal_delivery: None, + }) +} + +async fn resume_with_bounded_continuation( + repository: &ExecutionRepository, + scope: ExecutionScope, + config: &moa_config::ExecutionConfig, + request: &ExecutionRunAdvanceRequest, + run: &moa_execution::repository::ExecutionRunRecord, +) -> moa_execution::Result { + let completion = repository + .complete_controller_wake( + scope, + config, + run.run_uid, + RunControllerCompletionRequest { + controller_generation: request.controller_generation, + wake_epoch: request.wake_epoch, + checkpoint: settlement::continuation_checkpoint(run), + continuation_payload: Some(json!({ + "cause": "resumed_activation_recovery", + "prior_dispatch_uid": request.dispatch_uid, + })), + continuation_not_before_at: Utc::now(), + }, + ) + .await?; + match completion { + RunControllerCompletionOutcome::Applied { run, continuation } => { + let continuation_enqueued = continuation.is_some(); + validate_resumed_recovery_commit( + request.wake_epoch, + run.wake_epoch, + continuation_enqueued, + )?; + Ok(ControllerAdvanceCommit { + response: ExecutionRunAdvanceResponse { + outcome: ExecutionRunAdvanceOutcome::Advanced, + controller_generation: run.controller_generation, + wake_epoch: run.wake_epoch, + activation_steps: 0, + materialized_tasks: 0, + continuation_enqueued, + }, + publish_progress: true, + terminal_delivery: None, + }) + } + RunControllerCompletionOutcome::Replayed(run) => { + Ok(noop_commit(ExecutionRunAdvanceOutcome::Replayed, &run)) + } + RunControllerCompletionOutcome::CapacitySaturated { dimension } => { + Err(moa_execution::Error::CapacitySaturated { + dimension: dimension.as_str(), + }) + } + RunControllerCompletionOutcome::StaleGeneration { current_generation } => { + Ok(stale_commit(current_generation, run.wake_epoch)) + } + RunControllerCompletionOutcome::StaleWake { + current_wake_epoch, .. + } => Ok(stale_commit(run.controller_generation, current_wake_epoch)), + RunControllerCompletionOutcome::NotFound => { + Err(moa_execution::Error::InvalidRepositoryData { + message: "execution run disappeared during resumed activation recovery".to_string(), + }) + } + RunControllerCompletionOutcome::InvalidState => { + Ok(stale_commit(run.controller_generation, run.wake_epoch)) + } + } +} + +async fn pending_terminal_commit( + repository: &ExecutionRepository, + scope: ExecutionScope, + claimed_run: &moa_execution::repository::ExecutionRunRecord, + outcome: PendingTerminalAdvanceOutcome, +) -> moa_execution::Result { + let (commit, response_outcome) = match outcome { + PendingTerminalAdvanceOutcome::Applied(commit) => { + (commit, ExecutionRunAdvanceOutcome::Advanced) + } + PendingTerminalAdvanceOutcome::Replayed(commit) => { + (commit, ExecutionRunAdvanceOutcome::Replayed) + } + PendingTerminalAdvanceOutcome::Conflict => { + return Ok(stale_commit( + claimed_run.controller_generation, + claimed_run.wake_epoch, + )); + } + PendingTerminalAdvanceOutcome::NotFound => { + return Err(moa_execution::Error::InvalidRepositoryData { + message: "execution run disappeared during pending-terminal settlement".to_string(), + }); + } + }; + let activation_steps = pending_terminal_step_count( + commit.settled_task_count, + commit.drained_trigger_count, + commit.cancellation_dispatches.len(), + commit.compensation_admission.is_some(), + )?; + if matches!( + commit.stage, + PendingTerminalAdvanceStage::Finalized | PendingTerminalAdvanceStage::ManualRepairRequired + ) { + let mut terminal = terminal_commit(repository, scope, &commit.run).await?; + terminal.response.activation_steps = activation_steps; + return Ok(terminal); + } + if commit.compensation_admission.is_some() && commit.continuation.is_some() { + return Err(moa_execution::Error::InvalidRepositoryData { + message: "admitted compensation must park until its exact attempt settles".to_string(), + }); + } + let continuation_enqueued = commit.continuation.is_some(); + Ok(ControllerAdvanceCommit { + response: ExecutionRunAdvanceResponse { + outcome: response_outcome, + controller_generation: commit.run.controller_generation, + wake_epoch: commit.run.wake_epoch, + activation_steps, + materialized_tasks: 0, + continuation_enqueued, + }, + publish_progress: true, + terminal_delivery: None, + }) +} + +async fn terminal_commit( + repository: &ExecutionRepository, + scope: ExecutionScope, + run: &moa_execution::repository::ExecutionRunRecord, +) -> moa_execution::Result { + let terminal_delivery = repository + .load_bounded_terminal_delivery(scope, run.run_uid) + .await? + .ok_or_else(|| moa_execution::Error::InvalidRepositoryData { + message: "terminal execution run has no bounded Session delivery".to_string(), + })?; + Ok(ControllerAdvanceCommit { + response: ExecutionRunAdvanceResponse { + outcome: ExecutionRunAdvanceOutcome::Terminal, + controller_generation: run.controller_generation, + wake_epoch: run.wake_epoch, + activation_steps: 0, + materialized_tasks: 0, + continuation_enqueued: false, + }, + publish_progress: true, + terminal_delivery: Some(terminal_delivery), + }) +} + +fn noop_commit( + outcome: ExecutionRunAdvanceOutcome, + run: &moa_execution::repository::ExecutionRunRecord, +) -> ControllerAdvanceCommit { + ControllerAdvanceCommit { + response: ExecutionRunAdvanceResponse { + outcome, + controller_generation: run.controller_generation, + wake_epoch: run.wake_epoch, + activation_steps: 0, + materialized_tasks: 0, + continuation_enqueued: false, + }, + publish_progress: false, + terminal_delivery: None, + } +} + +fn stale_commit(controller_generation: u64, wake_epoch: u64) -> ControllerAdvanceCommit { + ControllerAdvanceCommit { + response: ExecutionRunAdvanceResponse { + outcome: ExecutionRunAdvanceOutcome::Stale, + controller_generation, + wake_epoch, + activation_steps: 0, + materialized_tasks: 0, + continuation_enqueued: false, + }, + publish_progress: false, + terminal_delivery: None, + } +} + +#[cfg(test)] +pub(super) fn stale_commit_for_test( + controller_generation: u64, + wake_epoch: u64, +) -> ControllerAdvanceCommit { + stale_commit(controller_generation, wake_epoch) +} + +#[cfg(test)] +pub(super) fn consume_limits_for_test( + maximum_steps: usize, + dispatch_batch_size: usize, + node_counts: &[usize], + task_counts: &[usize], +) -> moa_execution::Result<(usize, usize, usize, usize)> { + let mut limits = ActivationLimits::new(maximum_steps, dispatch_batch_size)?; + let mut inspected = 0usize; + let mut tasks = 0usize; + for count in node_counts { + inspected += limits.inspect_nodes(*count); + } + for count in task_counts { + let accepted = limits.remaining_tasks.min(*count); + limits.record_tasks(accepted)?; + tasks += accepted; + } + Ok(( + inspected, + tasks, + limits.remaining_steps, + limits.remaining_tasks, + )) +} + +#[cfg(test)] +pub(super) fn completion_scan_steps_for_test( + scanned_tasks: u32, + scanned_nodes: u32, +) -> moa_execution::Result { + completion_scan_steps(scanned_tasks, scanned_nodes) +} + +#[cfg(test)] +pub(super) fn validate_resumed_recovery_for_test( + prior_wake_epoch: u64, + current_wake_epoch: u64, + continuation_enqueued: bool, +) -> moa_execution::Result<()> { + validate_resumed_recovery_commit(prior_wake_epoch, current_wake_epoch, continuation_enqueued) +} + +#[cfg(test)] +pub(super) fn validate_trigger_drain_for_test( + prior_wake_epoch: u64, + current_wake_epoch: u64, + drained_trigger_count: u32, +) -> moa_execution::Result<()> { + validate_trigger_drain_continuation(prior_wake_epoch, current_wake_epoch, drained_trigger_count) +} + +#[cfg(test)] +pub(super) fn terminal_trigger_page_limit_for_test( + remaining_steps: usize, +) -> moa_execution::Result> { + terminal_trigger_page_limit(remaining_steps) +} + +#[cfg(test)] +pub(super) fn pending_terminal_step_count_for_test( + settled_task_count: u64, + drained_trigger_count: u32, + cancellation_dispatch_count: usize, + compensation_admitted: bool, +) -> moa_execution::Result { + pending_terminal_step_count( + settled_task_count, + drained_trigger_count, + cancellation_dispatch_count, + compensation_admitted, + ) +} + +#[cfg(test)] +pub(super) fn map_aggregate_requires_continuation_for_test( + outcome: &MapAggregatePageOutcome, +) -> moa_execution::Result { + map_aggregate_requires_continuation(outcome) +} + +#[cfg(test)] +pub(super) fn validate_replan_stop_continuation_for_test( + prior_wake_epoch: u64, + continuation_wake_epoch: u64, + exact_owner: bool, +) -> moa_execution::Result<()> { + validate_replan_stop_continuation_fields(prior_wake_epoch, continuation_wake_epoch, exact_owner) +} + +#[cfg(test)] +pub(super) fn activation_preflight_for_test( + has_pending_terminal: bool, + deadline_at: Option>, + now: DateTime, +) -> &'static str { + match activation_preflight(has_pending_terminal, deadline_at, now) { + ActivationPreflight::PendingTerminal => "pending_terminal", + ActivationPreflight::DueDeadline => "due_deadline", + ActivationPreflight::Ordinary => "ordinary", + } +} diff --git a/crates/moa-orchestrator/src/objects/execution_run_controller/progress.rs b/crates/moa-orchestrator/src/objects/execution_run_controller/progress.rs new file mode 100644 index 000000000..12a8f1b56 --- /dev/null +++ b/crates/moa-orchestrator/src/objects/execution_run_controller/progress.rs @@ -0,0 +1,106 @@ +//! Product progress projection and Session delivery for controller activations. + +use moa_core::{events::ExecutionProgress, traits::Identity, types::identifiers::SessionId}; +use moa_execution::{ + repository::{ExecutionRepository, ExecutionScope}, + wire::execution_progress_from_run, +}; +use restate_sdk::prelude::*; +use serde::{Deserialize, Serialize}; + +use super::{ExecutionRunAdvanceRequest, advance::ControllerAdvanceCommit}; +use crate::objects::session::SessionClient; + +pub(super) async fn deliver( + ctx: &ObjectContext<'_>, + repository: &ExecutionRepository, + request: &ExecutionRunAdvanceRequest, + committed: &ControllerAdvanceCommit, +) -> Result<(), HandlerError> { + if !committed.publish_progress { + return Ok(()); + } + let repository = repository.clone(); + let run_uid = request.run_uid; + let tenant_id = request.tenant_id; + let terminal = committed.terminal_delivery.clone(); + let delivery = ctx + .run(|| async move { + let run = repository + .load_run(ExecutionScope::ControlPlane, run_uid) + .await + .map_err(crate::workflows::errors::execution_error_to_handler_error)? + .ok_or_else(|| TerminalError::new_with_code(404, "execution run not found"))?; + if run.tenant_id != tenant_id || run.admitted_identity.tenant_id != tenant_id { + return Err(TerminalError::new_with_code( + 409, + "execution progress owner does not match activation", + ) + .into()); + } + let progress = execution_progress_from_run(&run) + .map_err(crate::workflows::errors::execution_error_to_handler_error)?; + Ok::<_, HandlerError>(Json::from(ControllerProgressDeliveryWire { + identity: run.admitted_identity, + session_id: run.session_id, + progress, + terminal, + })) + }) + .name(format!( + "execution_controller_progress_{}_{}", + request.controller_generation, request.wake_epoch + )) + .await? + .into_inner(); + + tracing::info!( + session_id = %delivery.session_id, + summary = %crate::workflows::progress_delivery::execution_controller_summary( + &delivery.progress.status, + delivery.progress.ready_tasks, + delivery.progress.active_tasks, + delivery.progress.parked_tasks, + delivery.progress.completed, + delivery.progress.next_wake_at, + ), + "execution controller progress committed" + ); + + let call = ctx + .object_client::(delivery.session_id.to_string()) + .execution_progress(Json::from(delivery.progress)); + let handle = crate::restate_identity::replay_safe_request( + crate::restate_identity::with_identity_headers(call, &delivery.identity), + ) + .send(); + let _progress_invocation_id = handle.invocation_id().await?; + if let Some(terminal) = delivery.terminal { + if !matches!( + terminal.status, + moa_execution::state::ExecutionRunStatus::Completed + | moa_execution::state::ExecutionRunStatus::Cancelled + ) { + moa_execution::wire::execution_failure_disposition(terminal.status) + .map_err(crate::workflows::errors::execution_error_to_handler_error)?; + } + let call = ctx + .object_client::(delivery.session_id.to_string()) + .execution_terminal(Json::from(terminal)); + let handle = crate::restate_identity::replay_safe_request( + crate::restate_identity::with_identity_headers(call, &delivery.identity), + ) + .send(); + let _terminal_invocation_id = handle.invocation_id().await?; + } + Ok(()) +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct ControllerProgressDeliveryWire { + identity: Identity, + session_id: SessionId, + progress: ExecutionProgress, + terminal: Option, +} diff --git a/crates/moa-orchestrator/src/objects/execution_run_controller/settlement.rs b/crates/moa-orchestrator/src/objects/execution_run_controller/settlement.rs new file mode 100644 index 000000000..fe8f897a9 --- /dev/null +++ b/crates/moa-orchestrator/src/objects/execution_run_controller/settlement.rs @@ -0,0 +1,99 @@ +//! Pure checkpoint derivation for bounded controller activations. + +use chrono::{DateTime, Utc}; +use moa_execution::{ + repository::{ExecutionActivationState, ExecutionRunActivationCheckpoint, ExecutionRunRecord}, + state::{ExecutionRunStatus, WaitingReason}, +}; + +pub(super) fn parked_checkpoint( + run: &ExecutionRunRecord, + now: DateTime, +) -> ExecutionRunActivationCheckpoint { + ExecutionRunActivationCheckpoint { + status: checkpoint_status(run.status, &run.waiting_reasons), + activation_state: ExecutionActivationState::Idle, + next_wake_at: earliest_wake(run.next_wake_at, run.approved_budget.deadline_at), + waiting_since: (run.waiting_task_count > 0).then_some(run.waiting_since.unwrap_or(now)), + ready_task_count: run.ready_task_count, + active_task_count: run.active_task_count, + } +} + +pub(super) fn checkpoint_status( + persisted_status: ExecutionRunStatus, + bounded_waiting_reasons: &[WaitingReason], +) -> ExecutionRunStatus { + if matches!( + persisted_status, + ExecutionRunStatus::WaitingInput + | ExecutionRunStatus::WaitingReview + | ExecutionRunStatus::WaitingSignal + | ExecutionRunStatus::WaitingTimer + | ExecutionRunStatus::WaitingExternal + | ExecutionRunStatus::WaitingReplan + ) { + persisted_status + } else { + waiting_status(bounded_waiting_reasons) + } +} + +pub(super) fn continuation_checkpoint( + run: &ExecutionRunRecord, +) -> ExecutionRunActivationCheckpoint { + ExecutionRunActivationCheckpoint { + status: ExecutionRunStatus::Running, + activation_state: ExecutionActivationState::Queued, + next_wake_at: run.next_wake_at, + waiting_since: run.waiting_since, + ready_task_count: run.ready_task_count, + active_task_count: run.active_task_count, + } +} + +pub(super) fn waiting_status(waiting: &[WaitingReason]) -> ExecutionRunStatus { + if waiting + .iter() + .any(|reason| matches!(reason, WaitingReason::Input { .. })) + { + return ExecutionRunStatus::WaitingInput; + } + if waiting + .iter() + .any(|reason| matches!(reason, WaitingReason::Review { .. })) + { + return ExecutionRunStatus::WaitingReview; + } + if waiting + .iter() + .any(|reason| matches!(reason, WaitingReason::Signal { .. })) + { + return ExecutionRunStatus::WaitingSignal; + } + if waiting + .iter() + .any(|reason| matches!(reason, WaitingReason::Timer { .. })) + { + return ExecutionRunStatus::WaitingTimer; + } + if waiting + .iter() + .any(|reason| matches!(reason, WaitingReason::External { .. })) + { + return ExecutionRunStatus::WaitingExternal; + } + ExecutionRunStatus::Running +} + +pub(super) fn earliest_wake( + persisted_wait_wake: Option>, + run_deadline: Option>, +) -> Option> { + match (persisted_wait_wake, run_deadline) { + (Some(wait), Some(deadline)) => Some(wait.min(deadline)), + (Some(wait), None) => Some(wait), + (None, Some(deadline)) => Some(deadline), + (None, None) => None, + } +} diff --git a/crates/moa-orchestrator/src/objects/execution_run_controller/tests.rs b/crates/moa-orchestrator/src/objects/execution_run_controller/tests.rs new file mode 100644 index 000000000..f48af3556 --- /dev/null +++ b/crates/moa-orchestrator/src/objects/execution_run_controller/tests.rs @@ -0,0 +1,339 @@ +use moa_core::types::identifiers::TenantId; +use uuid::Uuid; + +use chrono::{TimeDelta, TimeZone, Utc}; +use moa_artifacts::execution_plan::{ + ExecutionTemporalTarget, ExecutionWaitExpiryAction, ExecutionWaitPolicy, +}; +use moa_execution::{ + repository::ready::MapAggregatePageOutcome, + state::{ExecutionRunStatus, ExecutionTaskId, WaitingReason}, +}; + +use super::{ + ExecutionRunAdvanceOutcome, ExecutionRunAdvanceRequest, ExecutionRunAdvanceResponse, advance, + settlement, +}; + +fn request(run_uid: Uuid) -> ExecutionRunAdvanceRequest { + ExecutionRunAdvanceRequest { + dispatch_uid: Uuid::from_u128(7), + tenant_id: TenantId::from(Uuid::from_u128(8)), + run_uid, + controller_generation: 3, + wake_epoch: 5, + } +} + +#[test] +fn activation_limits_are_independent_hard_ceilings() { + // Pins: a large node page cannot consume more scheduler transitions than the activation + // bound, and a large ready page cannot borrow that unused budget to exceed dispatch_batch_size. + let observed = advance::consume_limits_for_test(3, 2, &[2, 4], &[1, 9]) + .expect("positive limits are valid"); + + assert_eq!(observed, (3, 2, 0, 0)); +} + +#[test] +fn activation_limits_reject_zero_instead_of_creating_a_busy_loop() { + // Pins: invalid zero bounds fail before the controller can enqueue an endless continuation. + let error = advance::consume_limits_for_test(0, 2, &[1], &[1]) + .expect_err("zero activation steps must fail closed"); + + assert_eq!( + error.to_string(), + "invalid execution repository request: controller activation bounds must both be greater than zero" + ); +} + +#[test] +fn completion_projection_counts_every_bounded_row_against_activation_work() { + // Pins: task-evidence and node-evidence scans share the activation-step ceiling; neither + // page can be omitted from accounting and turn terminal evaluation into unbounded work. + assert_eq!( + advance::completion_scan_steps_for_test(7, 11) + .expect("bounded completion counts fit in usize"), + 18 + ); +} + +#[test] +fn stale_or_paused_activation_has_no_controller_side_effects() { + // Pins: a stale delivery—including a defensive activation delivered while the run is + // paused—must acknowledge successfully without polling, trigger, or progress work. + let commit = advance::stale_commit_for_test(9, 14); + + assert_eq!( + commit.response, + ExecutionRunAdvanceResponse { + outcome: ExecutionRunAdvanceOutcome::Stale, + controller_generation: 9, + wake_epoch: 14, + activation_steps: 0, + materialized_tasks: 0, + continuation_enqueued: false, + } + ); + assert!(!commit.publish_progress); + assert!(commit.terminal_delivery.is_none()); +} + +#[test] +fn resumed_activation_recovery_enqueues_exactly_one_fresh_wake() { + // Pins: after a crash following any committed page, the resumed wake performs no second page; + // it can only ACK wake 5 and create wake 6 as the bounded continuation. + advance::validate_resumed_recovery_for_test(5, 6, true) + .expect("one exact fresh continuation is valid"); + + let skipped_wake = advance::validate_resumed_recovery_for_test(5, 7, true) + .expect_err("recovery cannot skip to a second continuation wake"); + assert_eq!( + skipped_wake.to_string(), + "invalid execution repository data: resumed activation recovery must enqueue exactly one fresh wake" + ); + let missing = advance::validate_resumed_recovery_for_test(5, 6, false) + .expect_err("recovery cannot ACK without a continuation"); + assert_eq!( + missing.to_string(), + "invalid execution repository data: resumed activation recovery must enqueue exactly one fresh wake" + ); +} + +#[test] +fn replan_stop_page_transfers_exactly_one_fresh_wake_to_its_run_activation() { + // Pins: a bounded replan-stop scan commits its cursor, ACKs the source wake, rebinds the + // durable intent, and creates exactly one new RunActivation in the same transaction. The + // controller must reject both a skipped epoch and a dispatch owned by another run boundary. + advance::validate_replan_stop_continuation_for_test(12, 13, true) + .expect("one exact replan-stop continuation is valid"); + + let skipped = advance::validate_replan_stop_continuation_for_test(12, 14, true) + .expect_err("a replan-stop page cannot skip a fresh wake"); + assert_eq!( + skipped.to_string(), + "invalid execution repository data: resumed activation recovery must enqueue exactly one fresh wake" + ); + + let wrong_owner = advance::validate_replan_stop_continuation_for_test(12, 13, false) + .expect_err("a replan-stop continuation must own the exact run wake"); + assert_eq!( + wrong_owner.to_string(), + "invalid execution repository data: replan-stop continuation does not own the exact fresh run wake" + ); +} + +#[test] +fn successful_terminal_trigger_drain_is_nonempty_and_owns_one_fresh_wake() { + // Pins: successful finalization drains active deadline/wait triggers in bounded pages. A + // committed page must settle real work and transfer ownership to exactly the next wake; a + // zero-row page or skipped epoch could otherwise hot-loop or strand terminal finalization. + advance::validate_trigger_drain_for_test(8, 9, 2) + .expect("one nonempty drain page and one exact continuation are valid"); + + let empty = advance::validate_trigger_drain_for_test(8, 9, 0) + .expect_err("a page continuation cannot be committed without trigger progress"); + assert_eq!( + empty.to_string(), + "invalid execution repository data: terminal trigger drain must settle a nonempty page and enqueue one fresh wake" + ); + let skipped = advance::validate_trigger_drain_for_test(8, 10, 2) + .expect_err("a drain page cannot skip a controller wake"); + assert_eq!( + skipped.to_string(), + "invalid execution repository data: terminal trigger drain must settle a nonempty page and enqueue one fresh wake" + ); +} + +#[test] +fn exhausted_activation_budget_defers_terminal_trigger_drain() { + // Pins: if completion evaluation consumes the last activation step, the controller must not + // call the repository's nonzero-page drain API. It checkpoints one continuation so the fresh + // wake starts with a real drain budget instead of failing or spinning on page_limit=0. + assert_eq!( + advance::terminal_trigger_page_limit_for_test(0) + .expect("zero remaining work is a valid deferral"), + None + ); + assert_eq!( + advance::terminal_trigger_page_limit_for_test(1) + .expect("one remaining step permits one trigger"), + Some(1) + ); + assert_eq!( + advance::terminal_trigger_page_limit_for_test(2_000) + .expect("repository page size is bounded"), + Some(1_000) + ); +} + +#[test] +fn pending_terminal_pages_charge_every_bounded_transition() { + // Pins: forward storage settlement, trigger cleanup, cancellation dispatch, and the single + // reverse-order compensation admission all share maximum_activation_steps. A compensation + // success/retry wake may admit only one slice, while review/external waits charge no phantom + // work and remain parked until their persisted resolution enqueues a fresh wake. + assert_eq!( + advance::pending_terminal_step_count_for_test(2, 3, 4, true) + .expect("bounded terminal page accounting fits"), + 10 + ); + assert_eq!( + advance::pending_terminal_step_count_for_test(0, 0, 0, false) + .expect("a parked review or external wait performs no controller work"), + 0 + ); +} + +#[test] +fn bounded_map_aggregate_pages_continue_only_after_a_completed_page() { + // Pins: partial/replayed partial pages, overflow, and a cursor conflict end this activation and + // enqueue one continuation. Only a completed page may spend remaining steps on another node; + // a missing run is corruption rather than a retry loop. + assert!( + advance::map_aggregate_requires_continuation_for_test(&MapAggregatePageOutcome::Applied { + next_cursor_item_key: Some("item-16".to_string()), + aggregated_tasks: 16, + aggregate_complete: false, + },) + .expect("partial aggregate page is valid") + ); + assert!( + !advance::map_aggregate_requires_continuation_for_test( + &MapAggregatePageOutcome::Replayed { + next_cursor_item_key: Some("item-32".to_string()), + aggregate_complete: true, + }, + ) + .expect("completed replay is valid") + ); + assert!( + advance::map_aggregate_requires_continuation_for_test(&MapAggregatePageOutcome::Overflow,) + .expect("overflow persists a failed node for the next wake") + ); + assert!( + advance::map_aggregate_requires_continuation_for_test(&MapAggregatePageOutcome::Conflict,) + .expect("cursor conflict yields to a fresh wake") + ); + let missing = + advance::map_aggregate_requires_continuation_for_test(&MapAggregatePageOutcome::NotFound) + .expect_err("a claimed run cannot disappear"); + assert_eq!( + missing.to_string(), + "invalid execution repository data: execution run disappeared during bounded map aggregation" + ); +} + +#[test] +fn parked_wait_phase_prioritizes_human_review_over_a_timer() { + // Pins: a run with both a timed wake and an unresolved tenant decision remains visibly + // WaitingReview; the timer must not hide the human blocker in product progress. + let waiting = vec![ + WaitingReason::Timer { + task_id: ExecutionTaskId::from_uuid(Uuid::from_u128(21)), + wake: ExecutionTemporalTarget::After { delay_seconds: 60 }, + }, + WaitingReason::Review { + task_id: ExecutionTaskId::from_uuid(Uuid::from_u128(22)), + prompt: "approve release".to_string(), + wait_policy: ExecutionWaitPolicy { + expiry: ExecutionTemporalTarget::After { + delay_seconds: 3_600, + }, + on_expiry: ExecutionWaitExpiryAction::FailRun, + }, + }, + ]; + + assert_eq!( + settlement::waiting_status(&waiting), + ExecutionRunStatus::WaitingReview + ); + assert_eq!( + settlement::checkpoint_status(ExecutionRunStatus::WaitingReview, &waiting[..1],), + ExecutionRunStatus::WaitingReview, + "a truncated timer sample cannot downgrade the exact persisted review phase" + ); +} + +#[test] +fn parked_replan_phase_survives_a_bounded_empty_reason_sample() { + // Pins: WaitingReplan is represented by an exact run scalar rather than a WaitingReason; + // parking the controller must preserve it so ParkedRuns capacity is reserved and the product + // phase does not incorrectly regress to Running. + assert_eq!( + settlement::checkpoint_status(ExecutionRunStatus::WaitingReplan, &[]), + ExecutionRunStatus::WaitingReplan + ); +} + +#[test] +fn checkpoint_preserves_the_persisted_exact_wait_wake() { + // Pins: controller replay never re-resolves an After target from a new wall clock; the exact + // due time persisted by wait materialization wins unless the run deadline is earlier. + let persisted_wait = Utc + .with_ymd_and_hms(2026, 8, 11, 12, 0, 0) + .single() + .expect("test timestamp is valid"); + let later_deadline = persisted_wait + TimeDelta::hours(4); + let earlier_deadline = persisted_wait - TimeDelta::minutes(1); + + assert_eq!( + settlement::earliest_wake(Some(persisted_wait), Some(later_deadline)), + Some(persisted_wait) + ); + assert_eq!( + settlement::earliest_wake(Some(persisted_wait), Some(earlier_deadline)), + Some(earlier_deadline) + ); +} + +#[test] +fn terminal_fences_run_before_any_ordinary_scheduler_work() { + // Pins: an already-fenced terminal intent wins over a due deadline, and an exact due deadline + // wins over ordinary materialization; neither path can launch new forward work. + let now = Utc + .with_ymd_and_hms(2026, 8, 11, 12, 0, 0) + .single() + .expect("test timestamp is valid"); + + assert_eq!( + advance::activation_preflight_for_test(true, Some(now), now), + "pending_terminal" + ); + assert_eq!( + advance::activation_preflight_for_test(false, Some(now), now), + "due_deadline" + ); + assert_eq!( + advance::activation_preflight_for_test(false, Some(now + TimeDelta::seconds(1)), now,), + "ordinary" + ); +} + +#[test] +fn controller_key_must_equal_the_persisted_run_uid() { + // Pins: Restate object serialization cannot be bypassed by carrying a different run in JSON. + let run_uid = Uuid::from_u128(11); + let error = advance::validate_request(&Uuid::from_u128(12).to_string(), &request(run_uid)) + .expect_err("mismatched controller key must fail"); + + assert_eq!( + crate::workflows::errors::handler_error_message(&error), + "Terminal error [400]: execution controller key does not match run_uid" + ); +} + +#[test] +fn controller_request_rejects_nil_durable_identifiers() { + // Pins: nil dispatch/run IDs never enter a durable claim transaction. + let mut request = request(Uuid::nil()); + request.dispatch_uid = Uuid::nil(); + let error = advance::validate_request(&Uuid::nil().to_string(), &request) + .expect_err("nil durable identifiers must fail"); + + assert_eq!( + crate::workflows::errors::handler_error_message(&error), + "Terminal error [400]: execution controller identifiers must not be nil" + ); +} diff --git a/crates/moa-orchestrator/src/objects/mod.rs b/crates/moa-orchestrator/src/objects/mod.rs index cccda8789..932dd43f8 100644 --- a/crates/moa-orchestrator/src/objects/mod.rs +++ b/crates/moa-orchestrator/src/objects/mod.rs @@ -1,6 +1,7 @@ //! Restate virtual objects hosted by the orchestrator binary. pub mod cron_job; +pub mod execution_run_controller; pub mod ingestion; pub mod session; pub mod session_status_migrator; diff --git a/crates/moa-orchestrator/src/objects/session/execution_runs.rs b/crates/moa-orchestrator/src/objects/session/execution_runs.rs index 1e66ad7a0..d2d9abc1c 100644 --- a/crates/moa-orchestrator/src/objects/session/execution_runs.rs +++ b/crates/moa-orchestrator/src/objects/session/execution_runs.rs @@ -2,6 +2,7 @@ use std::sync::Arc; +use super::*; use moa_brain::execution_planning::{ ExecutionPlanningRequest, ExecutionPlanningResultKind, ExecutionRoutingInput, plan_execution, route_execution, @@ -15,15 +16,14 @@ use moa_core::types::execution_planning::{ }; use moa_core::types::identifiers::ModelId; use moa_core::types::model::ModelCapabilities; +use moa_execution::repository::audit::{ + CompileAuditWriteOutcome, PlannerCallAuditWriteOutcome, RouteAuditWriteOutcome, +}; use moa_execution::repository::{ - CompileAuditWriteOutcome, ExecutionRepository, ExecutionScope, - ExecutionTemplateAdmissionRecord, PlannerCallAuditWriteOutcome, RouteAuditWriteOutcome, + ExecutionRepository, ExecutionScope, ExecutionTemplateAdmissionRecord, }; use moa_session::PostgresSessionStore; -use super::*; -use crate::workflows::execution_run::ExecutionRunClient; - const EXECUTION_SYNTHESIS_TURN_NAMESPACE: uuid::Uuid = uuid::Uuid::from_u128(0xf61c_9bb0_e9a7_5793_80f5_6a38_5d6e_8eb2); const EXECUTION_SYNTHESIS_TURN_DOMAIN: &str = "moa.execution.synthesis-turn"; @@ -337,6 +337,13 @@ async fn start_external_template_execution( ) .await?; + let horizon_seconds = i64::try_from(config.execution.maximum_horizon_seconds) + .map_err(|_| TerminalError::new("execution maximum horizon does not fit i64"))?; + let horizon = chrono::TimeDelta::try_seconds(horizon_seconds) + .ok_or_else(|| TerminalError::new("execution maximum horizon does not fit chrono"))?; + let deadline_at = accepted_at + .checked_add_signed(horizon) + .ok_or_else(|| TerminalError::new("execution maximum horizon exceeds timestamp range"))?; let planning_call = ctx .service_client::() .planning_context(Json::from( @@ -345,6 +352,7 @@ async fn start_external_template_execution( contact_id: request.contact_id, session_id, originating_user_sequence_num, + deadline_at, requested_template: Some(request.template.clone()), }, )); @@ -420,33 +428,6 @@ async fn start_external_template_execution( Ok(started.run.run_uid) } -/// Durably launches one committed run after the owning Session has activated it. -pub(super) fn dispatch_execution_run( - ctx: &ObjectContext<'_>, - state: &SessionVoState, - run_uid: uuid::Uuid, - identity: moa_core::traits::Identity, -) -> Result<(), HandlerError> { - let session_id = parse_session_key(ctx.key())?; - let meta = state - .ensure_initialized() - .map_err(crate::workflows::errors::moa_error_to_status_handler_error)?; - crate::restate_identity::replay_safe_request( - ctx.workflow_client::(run_uid.to_string()) - .run(Json::from( - moa_execution::wire::ExecutionRunWorkflowRequest { - run_uid, - tenant_id: meta.tenant_id, - contact_id: meta.contact.as_ref().map(|contact| contact.contact_id), - session_id, - identity, - }, - )), - ) - .send(); - Ok(()) -} - /// Persists one normalized planning audit after validating its Session origin. pub(super) async fn persist_execution_planning_audit( ctx: &ObjectContext<'_>, @@ -695,21 +676,22 @@ pub(super) async fn accept_execution_progress( { return Ok(()); } - append_exact_execution_event( - ctx, - Event::ExecutionProgress(progress.clone()), - format!( - "execution-progress:{}:{}:{}:{}:{}:{}:{}", - progress.run_uid, - progress.plan_revision, - progress.status, - progress.total, - progress.completed, - progress.failed, - progress.cancelled, - ), - ) - .await + let dedupe_key = execution_progress_event_dedupe_key(&progress)?; + append_exact_execution_event(ctx, Event::ExecutionProgress(progress), dedupe_key).await +} + +fn execution_progress_event_dedupe_key( + progress: &ExecutionProgress, +) -> Result { + let canonical = moa_core::canonical_json::canonical_json_bytes(progress).map_err(|error| { + TerminalError::new(format!( + "execution progress canonicalization failed: {error}" + )) + })?; + Ok(format!( + "execution-progress:{}", + blake3::hash(&canonical).to_hex() + )) } /// Publishes and activates one exact waiting execution task input request. @@ -899,7 +881,59 @@ pub(super) async fn accept_execution_run_started( #[cfg(test)] mod tests { - use super::stable_execution_synthesis_turn_id; + use super::{execution_progress_event_dedupe_key, stable_execution_synthesis_turn_id}; + + #[test] + fn progress_event_dedupe_key_covers_wait_pause_and_external_projection_offline() { + // Pins: two published projections cannot collide merely because their legacy status and + // counters match while the typed parked-state evidence changed. + let waiting_since = chrono::DateTime::parse_from_rfc3339("2026-08-11T12:00:00Z") + .expect("fixture timestamp parses") + .with_timezone(&chrono::Utc); + let baseline = moa_core::events::ExecutionProgress { + run_uid: uuid::Uuid::from_u128(91), + originating_user_sequence_num: 7, + plan_revision: 2, + status: "waiting_external".to_string(), + phase: moa_core::events::ExecutionProgressPhase::WaitingExternal, + waiting_since: Some(waiting_since), + next_wake_at: Some(waiting_since + chrono::TimeDelta::hours(1)), + last_progress_at: waiting_since, + external_job_uid: None, + ready_tasks: 1, + active_tasks: 1, + parked_tasks: 1, + blocker_audience: Some(moa_core::events::ExecutionBlockerAudience::External), + remaining_budget: moa_core::events::ExecutionRemainingBudget { + cost_microusd: Some(20), + tokens: Some(200), + tasks: Some(3), + tool_calls: Some(4), + retrieved_bytes: Some(2_000), + deadline_at: Some(waiting_since + chrono::TimeDelta::hours(4)), + }, + total: 5, + completed: 2, + failed: 0, + cancelled: 0, + }; + let baseline_key = + execution_progress_event_dedupe_key(&baseline).expect("baseline key hashes"); + + let mut phase_changed = baseline.clone(); + phase_changed.phase = moa_core::events::ExecutionProgressPhase::Paused; + let mut wake_changed = baseline.clone(); + wake_changed.next_wake_at = Some(waiting_since + chrono::TimeDelta::hours(2)); + let mut external_job_changed = baseline.clone(); + external_job_changed.external_job_uid = Some(uuid::Uuid::from_u128(92)); + + for changed in [phase_changed, wake_changed, external_job_changed] { + assert_ne!( + execution_progress_event_dedupe_key(&changed).expect("changed key hashes"), + baseline_key + ); + } + } #[test] fn synthesis_turn_id_is_deterministic_uuid_scoped_to_run_and_origin() { diff --git a/crates/moa-orchestrator/src/objects/session/handlers.rs b/crates/moa-orchestrator/src/objects/session/handlers.rs index 3636b3808..dc7bf51c5 100644 --- a/crates/moa-orchestrator/src/objects/session/handlers.rs +++ b/crates/moa-orchestrator/src/objects/session/handlers.rs @@ -2,7 +2,7 @@ use super::execution_runs::{ accept_execution_input_required, accept_execution_progress, accept_execution_run_started, - accept_execution_terminal, admit_execution_template, dispatch_execution_run, + accept_execution_terminal, admit_execution_template, }; use super::state::resume::signal_kind_is_resume_eligible; use super::*; diff --git a/crates/moa-orchestrator/src/objects/session/handlers/execution_bridge.rs b/crates/moa-orchestrator/src/objects/session/handlers/execution_bridge.rs index 37688aa1f..76e1deb2a 100644 --- a/crates/moa-orchestrator/src/objects/session/handlers/execution_bridge.rs +++ b/crates/moa-orchestrator/src/objects/session/handlers/execution_bridge.rs @@ -9,7 +9,7 @@ impl SessionImpl { delivery: Json, ) -> Result<(), HandlerError> { annotate_restate_handler_span("Session", "execution_run_started"); - let identity = require_identity(&ctx)?; + let _identity = require_identity(&ctx)?; let mut state = Tracked::::load(&ctx).await?; let delivery = delivery.into_inner(); let run_uid = delivery.started.run_uid; @@ -31,7 +31,6 @@ impl SessionImpl { if !terminal_replay { let session_id = parse_session_key(ctx.key())?; sync_status(&ctx, session_id, &state).await?; - dispatch_execution_run(&ctx, &state, run_uid, identity)?; } Ok(()) } diff --git a/crates/moa-orchestrator/src/objects/session/handlers/lifecycle.rs b/crates/moa-orchestrator/src/objects/session/handlers/lifecycle.rs index 3d534fd67..5c1b1c525 100644 --- a/crates/moa-orchestrator/src/objects/session/handlers/lifecycle.rs +++ b/crates/moa-orchestrator/src/objects/session/handlers/lifecycle.rs @@ -211,6 +211,7 @@ impl SessionImpl { .release_session_hands(Json::from(ReleaseSessionHandsRequest { tenant_id: meta.tenant_id, session_id, + continuation_attempt: 0, })), ) .send(); diff --git a/crates/moa-orchestrator/src/objects/session/state.rs b/crates/moa-orchestrator/src/objects/session/state.rs index c9ce5b134..3b24a3b59 100644 --- a/crates/moa-orchestrator/src/objects/session/state.rs +++ b/crates/moa-orchestrator/src/objects/session/state.rs @@ -162,13 +162,33 @@ pub struct ResumeTurnContext { pub consumed_signal_ids: Vec, } -/// Exact aggregate tuple used for execution-progress delta gating. +/// Exact public projection signature used for execution-progress delta gating. #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct ExecutionProgressSignature { /// Active immutable plan revision. pub plan_revision: u64, /// Exhaustively mapped durable run status. pub status: String, + /// Public active, parked, or pause-state distinction. + pub phase: moa_core::events::ExecutionProgressPhase, + /// Time at which the current storage-only wait began. + pub waiting_since: Option>, + /// Earliest durable controller reactivation time. + pub next_wake_at: Option>, + /// Latest durable scheduler progress time. + pub last_progress_at: DateTime, + /// Exact provider job when this projection is task-qualified. + pub external_job_uid: Option, + /// Exact ready logical task count. + pub ready_tasks: u64, + /// Exact active task-attempt count. + pub active_tasks: u64, + /// Exact durably parked logical task count. + pub parked_tasks: u64, + /// Highest-priority audience expected to resolve the current blocker. + pub blocker_audience: Option, + /// Exact unconsumed and unreserved execution budget. + pub remaining_budget: moa_core::events::ExecutionRemainingBudget, /// Materialized logical task count. pub total: u64, /// Successfully completed logical task count. @@ -184,6 +204,16 @@ impl From<&moa_core::events::ExecutionProgress> for ExecutionProgressSignature { Self { plan_revision: progress.plan_revision, status: progress.status.clone(), + phase: progress.phase, + waiting_since: progress.waiting_since, + next_wake_at: progress.next_wake_at, + last_progress_at: progress.last_progress_at, + external_job_uid: progress.external_job_uid, + ready_tasks: progress.ready_tasks, + active_tasks: progress.active_tasks, + parked_tasks: progress.parked_tasks, + blocker_audience: progress.blocker_audience, + remaining_budget: progress.remaining_budget.clone(), total: progress.total, completed: progress.completed, failed: progress.failed, @@ -201,7 +231,7 @@ pub struct ActiveExecutionRunState { pub originating_user_sequence_num: u64, /// Last aggregate progress published by the Session VO. pub progress: Option, - /// Exact aggregate tuple corresponding to the last progress publication. + /// Exact public projection signature corresponding to the last progress publication. pub last_progress_signature: Option, /// Durable time of the last progress publication. pub last_progress_at: Option>, diff --git a/crates/moa-orchestrator/src/objects/session/state/execution.rs b/crates/moa-orchestrator/src/objects/session/state/execution.rs index eb7794817..32b1fa5b8 100644 --- a/crates/moa-orchestrator/src/objects/session/state/execution.rs +++ b/crates/moa-orchestrator/src/objects/session/state/execution.rs @@ -49,15 +49,17 @@ impl SessionVoState { let signature = ExecutionProgressSignature::from(&progress); let changed = run.last_progress_signature.as_ref() != Some(&signature); + let immediate_transition = + progress_transition_requires_immediate_publication(run.progress.as_ref(), &progress); let cadence_due = run.last_progress_at.is_none_or(|last| { let elapsed_ms = now.signed_duration_since(last).num_milliseconds(); elapsed_ms >= i64::try_from(progress_interval_ms).unwrap_or(i64::MAX) }); - if !(changed && cadence_due) { + run.progress = Some(progress); + if !(changed && (immediate_transition || cadence_due)) { return Ok(false); } - run.progress = Some(progress); run.last_progress_signature = Some(signature); run.last_progress_at = Some(now); Ok(true) @@ -117,6 +119,22 @@ impl SessionVoState { } } +fn progress_transition_requires_immediate_publication( + previous: Option<&moa_core::events::ExecutionProgress>, + next: &moa_core::events::ExecutionProgress, +) -> bool { + previous.is_none_or(|previous| { + previous.plan_revision != next.plan_revision + || previous.status != next.status + || previous.phase != next.phase + || previous.waiting_since != next.waiting_since + || previous.next_wake_at != next.next_wake_at + || previous.external_job_uid != next.external_job_uid + || previous.parked_tasks != next.parked_tasks + || previous.blocker_audience != next.blocker_audience + }) +} + fn pending_reply_belongs_to_run(target: &PendingUserReplyTarget, run_uid: uuid::Uuid) -> bool { matches!( target, diff --git a/crates/moa-orchestrator/src/runtime/deps.rs b/crates/moa-orchestrator/src/runtime/deps.rs index 4f52fff57..fd3f79cbe 100644 --- a/crates/moa-orchestrator/src/runtime/deps.rs +++ b/crates/moa-orchestrator/src/runtime/deps.rs @@ -67,13 +67,15 @@ use crate::services::{ }; use moa_artifacts::registry::ArtifactRegistry; +#[cfg(all(feature = "provider-overrides", feature = "integration"))] +use crate::services::tool_executor::{ + ExecutionExternalJobAdapter, FixtureExternalJobTool, FixtureHttpExecutionExternalJobAdapter, +}; use crate::{ config::ProvidersOverride, lineage::{LineageSinkRuntime, build_lineage_sink}, - runtime::{ - jobs::{restate_ingress_base_url, start_authz_outbox_poller}, - kms::KmsRuntime, - }, + runtime::{jobs::restate_ingress_base_url, kms::KmsRuntime}, + services::tool_executor::ExecutionExternalJobAdapterRegistry, }; /// Constructed dependencies shared by Restate handlers and process services. @@ -106,6 +108,8 @@ pub struct RuntimeDeps { pub embedding_provider: Option>, /// Tool router used by ToolExecutor and runtime services. pub tool_router: Arc, + /// Shared asynchronous-provider adapter registry used by execution and raw callbacks. + pub external_job_adapters: ExecutionExternalJobAdapterRegistry, /// Authenticated checkpoint-bucket versioning observer shared with the store gate. pub checkpoint_versioning_observer: Option< moa_hands::core::sandbox_workspace::checkpoint::versioning::CheckpointBucketVersioningObserver, @@ -141,8 +145,6 @@ pub struct RuntimeDeps { pub lineage: LineageSinkRuntime, /// Awakeable resolver used by builtin async authorization. pub awakeable_resolver: Arc, - /// Optional OpenFGA outbox poller handle. - pub authz_outbox_poller: Option, /// Owned security-audit writer. /// /// Held so shutdown can drain it. Dropping these deps instead aborts it, @@ -218,9 +220,6 @@ impl RuntimeDeps { emit_allows: config.audit_security.emit_authz_allows, })) }; - let authz_outbox_poller = fga_client - .clone() - .map(|fga_client| start_authz_outbox_poller(&background_pool, fga_client)); let session_store = Arc::new( PostgresSessionStore::from_existing_pool_with_config(config.as_ref(), pool.clone()) .await?, @@ -239,6 +238,7 @@ impl RuntimeDeps { .context("build operator-message delivery sink")?; let egress_classifier = (!config.mcp_servers.is_empty() || config.llm_dlp.tokenize_enabled) .then(|| build_egress_pii_classifier(config.as_ref())); + let provider_override_active = providers_override.is_active(); let providers = Arc::new(build_provider_registry( config.as_ref(), Arc::clone(&runtime_cache), @@ -307,7 +307,7 @@ impl RuntimeDeps { mcp_egress_guard, Some(session_store.clone()), checkpoint_store.clone(), - workspace_runtime_enabled.then(|| pool.clone()), + Some(pool.clone()), workspace_runtime_enabled.then(|| kms.provider()), workspace_runtime_enabled, ) @@ -334,6 +334,8 @@ impl RuntimeDeps { .with_memory_tool_executor(Arc::new( moa_memory_ingest::FastMemoryToolExecutor::new(ingest_runtime.clone()), )); + let external_job_adapters = build_external_job_adapter_registry(provider_override_active)?; + let tool_router = register_fixture_external_job_tool(tool_router, &external_job_adapters)?; // Both sandbox owners are attached by the builder chain above, so the // cloud requirement can only be checked once the router is complete. if workspace_runtime_enabled { @@ -386,7 +388,6 @@ impl RuntimeDeps { lineage.handle.clone(), )); let channel_adapters = build_channel_adapters(config.as_ref(), runtime_cache.clone())?; - Ok(Self { config, pool, @@ -400,6 +401,7 @@ impl RuntimeDeps { providers, embedding_provider, tool_router, + external_job_adapters, checkpoint_versioning_observer, workspace_maintenance, sandbox_workspace_fenced_tenants, @@ -414,7 +416,6 @@ impl RuntimeDeps { delivery_sink, lineage, awakeable_resolver, - authz_outbox_poller, channel_adapters, audit: Arc::new(audit), }) @@ -665,6 +666,59 @@ fn build_provider_registry( } } +#[cfg(all(feature = "provider-overrides", feature = "integration"))] +fn build_external_job_adapter_registry( + provider_override_active: bool, +) -> Result { + let Some(base_url) = std::env::var("MOA_FIXTURE_EXTERNAL_JOB_ADAPTER_URL") + .ok() + .filter(|value| !value.trim().is_empty()) + else { + return Ok(ExecutionExternalJobAdapterRegistry::default()); + }; + if !provider_override_active { + bail!( + "MOA_FIXTURE_EXTERNAL_JOB_ADAPTER_URL requires the provider-override integration lane" + ); + } + let adapter = Arc::new(FixtureHttpExecutionExternalJobAdapter::new(&base_url)?) + as Arc; + ExecutionExternalJobAdapterRegistry::new([adapter]).map_err(Into::into) +} + +#[cfg(not(all(feature = "provider-overrides", feature = "integration")))] +fn build_external_job_adapter_registry( + _provider_override_active: bool, +) -> Result { + if std::env::var_os("MOA_FIXTURE_EXTERNAL_JOB_ADAPTER_URL").is_some() { + bail!( + "MOA_FIXTURE_EXTERNAL_JOB_ADAPTER_URL requires provider-overrides and integration features" + ); + } + Ok(ExecutionExternalJobAdapterRegistry::default()) +} + +#[cfg(all(feature = "provider-overrides", feature = "integration"))] +fn register_fixture_external_job_tool( + tool_router: ToolRouter, + adapters: &ExecutionExternalJobAdapterRegistry, +) -> Result { + if !adapters.is_empty() { + return tool_router + .with_additional_builtin(Arc::new(FixtureExternalJobTool)) + .map_err(Into::into); + } + Ok(tool_router) +} + +#[cfg(not(all(feature = "provider-overrides", feature = "integration")))] +fn register_fixture_external_job_tool( + tool_router: ToolRouter, + _adapters: &ExecutionExternalJobAdapterRegistry, +) -> Result { + Ok(tool_router) +} + /// Attaches LLM DLP to `registry` when `[llm_dlp].tokenize_enabled` /// is set, otherwise returns it unchanged (providers used directly, zero /// overhead). diff --git a/crates/moa-orchestrator/src/runtime/endpoint.rs b/crates/moa-orchestrator/src/runtime/endpoint.rs index 58e7844be..3dc03a009 100644 --- a/crates/moa-orchestrator/src/runtime/endpoint.rs +++ b/crates/moa-orchestrator/src/runtime/endpoint.rs @@ -37,6 +37,7 @@ use crate::workflows::skill_learning::{SkillLearning, SkillLearningImpl}; use crate::{ objects::{ cron_job::{CronJob, CronJobImpl}, + execution_run_controller::{ExecutionRunController, ExecutionRunControllerImpl}, ingestion::{IngestionVO, IngestionVOImpl}, session::{Session, SessionImpl}, tenant::{TenantImpl, TenantObject}, @@ -49,7 +50,15 @@ use crate::{ artifact_release::{ArtifactRelease, ArtifactReleaseImpl}, artifacts::{Artifacts, ArtifactsImpl}, contacts::{Contacts, ContactsImpl}, + durable_timeout::{DurableTimeout, DurableTimeoutImpl}, execution::{Execution, ExecutionImpl}, + execution_dispatcher::{ + ExecutionDispatchDrain, ExecutionDispatchDrainImpl, ExecutionDispatchReconciler, + ExecutionDispatchReconcilerImpl, ExecutionDispatcher, ExecutionDispatcherImpl, + }, + execution_retention::{ExecutionRetention, ExecutionRetentionImpl}, + execution_schedule::{ExecutionSchedule, ExecutionScheduleImpl}, + execution_trigger::{ExecutionTrigger, ExecutionTriggerImpl}, graph_memory_maint::{GraphMemoryMaint, GraphMemoryMaintImpl}, health::{Health, HealthImpl}, learning_review::{LearningReview, LearningReviewImpl}, @@ -57,13 +66,14 @@ use crate::{ memory::{Memory, MemoryImpl}, session_store::{RestateSessionStore, SessionStoreImpl}, skills::{Skills, SkillsImpl}, - tool_executor::{ToolExecutor, ToolExecutorImpl}, + tool_executor::{ToolExecutor, ToolExecutorDependencies, ToolExecutorImpl}, }, workflows::{ consolidate::{Consolidate, ConsolidateImpl}, - execution_compensation::{ExecutionCompensation, ExecutionCompensationImpl}, - execution_run::{ExecutionRun, ExecutionRunImpl}, - execution_task::{ExecutionTask, ExecutionTaskImpl}, + execution_compensation_attempt::{ + ExecutionCompensationAttempt, ExecutionCompensationAttemptImpl, + }, + execution_task_attempt::{ExecutionTaskAttempt, ExecutionTaskAttemptImpl}, session_retention::{SessionRetention, SessionRetentionImpl}, turn_events::TurnEventAppender, turn_execution::{TurnExecution, implementation::TurnExecutionImpl}, @@ -93,6 +103,13 @@ const CORE_BODY_SERVICE_NAMES: &[&str] = &[ "ToolExecutor", "ActionPolicy", "Execution", + "ExecutionSchedule", + "ExecutionRetention", + "ExecutionTrigger", + "ExecutionDispatcher", + "ExecutionDispatchDrain", + "ExecutionDispatchReconciler", + "DurableTimeout", "GraphMemoryMaint", "Knowledge", "LearningReview", @@ -105,9 +122,9 @@ const CORE_BODY_SERVICE_NAMES: &[&str] = &[ "Worker", "Tenants", "Tenant", - "ExecutionRun", - "ExecutionTask", - "ExecutionCompensation", + "ExecutionRunController", + "ExecutionTaskAttempt", + "ExecutionCompensationAttempt", "KnowledgeSyncIngestion", "Consolidate", "SessionRetention", @@ -122,10 +139,18 @@ const INGRESS_PRIVATE_SERVICE_NAMES: &[&str] = &[ "ToolExecutor", "TurnExecution", "WorkerTurnExecution", - "ExecutionRun", - "ExecutionTask", - "ExecutionCompensation", + "ExecutionRunController", + "ExecutionRetention", + "ExecutionTaskAttempt", + "ExecutionCompensationAttempt", + "ExecutionTrigger", + "ExecutionDispatcher", + "ExecutionDispatchDrain", + "ExecutionDispatchReconciler", + "DurableTimeout", ]; +#[cfg(test)] +const INGRESS_PRIVATE_HANDLER_NAMES: &[(&str, &str)] = &[("ExecutionSchedule", "fire_occurrence")]; const EXPERIMENT_WORKFLOW_SERVICE_NAMES: &[&str] = &[ "ExperimentRun", "ExperimentTrialRun", @@ -163,6 +188,13 @@ fn workflow_options() -> ServiceOptions { service_options().handler("run", HandlerOptions::new().workflow_retention(RETENTION)) } +fn execution_schedule_service_options() -> ServiceOptions { + service_options().handler( + "fire_occurrence", + HandlerOptions::new().ingress_private(true), + ) +} + fn high_cost_internal_service_options() -> ServiceOptions { service_options() .inactivity_timeout(HIGH_COST_INACTIVITY_TIMEOUT) @@ -175,6 +207,10 @@ fn high_cost_internal_workflow_options() -> ServiceOptions { .handler("run", HandlerOptions::new().workflow_retention(RETENTION)) } +fn execution_run_controller_options() -> ServiceOptions { + high_cost_internal_service_options() +} + fn high_cost_public_workflow_options() -> ServiceOptions { service_options() .inactivity_timeout(HIGH_COST_INACTIVITY_TIMEOUT) @@ -299,7 +335,7 @@ pub fn build_endpoint(runtime_deps: &RuntimeDeps) -> Endpoint { service_options(), ) .bind_with_options( - ActionReviewDispatcherImpl::new(pool.clone()).serve(), + ActionReviewDispatcherImpl::new(pool.clone(), config.execution.clone()).serve(), service_options(), ) .bind_with_options( @@ -357,15 +393,17 @@ pub fn build_endpoint(runtime_deps: &RuntimeDeps) -> Endpoint { service_options(), ) .bind_with_options( - ToolExecutorImpl::new( - tool_router.clone(), - connector_catalogs.clone(), + ToolExecutorImpl::new(ToolExecutorDependencies { + router: tool_router.clone(), + connector_catalogs: connector_catalogs.clone(), connector_completion, - session_store.clone(), - session_store.clone(), - pool.clone(), - sandbox_workspace_management, - ) + sessions: session_store.clone(), + events: session_store.clone(), + pool: pool.clone(), + workspace_management: sandbox_workspace_management, + external_job_adapters: runtime_deps.external_job_adapters.clone(), + execution_config: config.execution.clone(), + }) .serve(), high_cost_internal_service_options(), ) @@ -390,6 +428,15 @@ pub fn build_endpoint(runtime_deps: &RuntimeDeps) -> Endpoint { .serve(), service_options(), ) + .bind_with_options( + ExecutionScheduleImpl::new(pool.clone(), authz.clone(), config.execution.clone()) + .serve(), + execution_schedule_service_options(), + ) + .bind_with_options( + ExecutionRetentionImpl::new(pool.clone(), &config.execution).serve(), + high_cost_internal_service_options(), + ) .bind_with_options( GraphMemoryMaintImpl::new(pool.clone(), config.clone()).serve(), service_options(), @@ -495,23 +542,33 @@ pub fn build_endpoint(runtime_deps: &RuntimeDeps) -> Endpoint { high_cost_public_workflow_options(), ) .bind_with_options( - ExecutionRunImpl::new( - pool.clone(), - config.execution.clone(), - moa_core::types::identifiers::ModelId::new( - config - .models - .auxiliary - .clone() - .unwrap_or_else(|| config.models.main.clone()), - ), - ) - .serve(), - high_cost_internal_workflow_options(), + ExecutionTriggerImpl::new(pool.clone(), &config.execution).serve(), + high_cost_internal_service_options(), + ) + .bind_with_options( + ExecutionDispatcherImpl::new(pool.clone()).serve(), + high_cost_internal_service_options(), + ) + .bind_with_options( + ExecutionDispatchDrainImpl::new(pool.clone(), &config.execution).serve(), + high_cost_internal_service_options(), + ) + .bind_with_options( + ExecutionDispatchReconcilerImpl::new(pool.clone(), &config.execution).serve(), + high_cost_internal_service_options(), ) .bind_with_options( - ExecutionTaskImpl::new( + DurableTimeoutImpl::new(pool.clone()).serve(), + high_cost_internal_service_options(), + ) + .bind_with_options( + ExecutionRunControllerImpl::new(pool.clone(), config.execution.clone()).serve(), + execution_run_controller_options(), + ) + .bind_with_options( + ExecutionTaskAttemptImpl::new( pool.clone(), + config.execution.clone(), session_store.clone(), session_limits.clone(), channel_adapters.clone(), @@ -520,7 +577,7 @@ pub fn build_endpoint(runtime_deps: &RuntimeDeps) -> Endpoint { high_cost_internal_workflow_options(), ) .bind_with_options( - ExecutionCompensationImpl::new( + ExecutionCompensationAttemptImpl::new( pool.clone(), session_store.clone(), session_limits.clone(), @@ -675,8 +732,9 @@ mod tests { use restate_sdk::prelude::*; use super::{ - INGRESS_PRIVATE_SERVICE_NAMES, RegisteredDeployment, RegisteredService, - bootstrap_entry_service_options, expected_service_names, + INGRESS_PRIVATE_HANDLER_NAMES, INGRESS_PRIVATE_SERVICE_NAMES, RegisteredDeployment, + RegisteredService, bootstrap_entry_service_options, execution_run_controller_options, + execution_schedule_service_options, expected_service_names, high_cost_internal_service_options, high_cost_internal_workflow_options, high_cost_public_workflow_options, sandbox_workspace_service_options, services_registered_for_mode, services_registered_with_expected, @@ -698,6 +756,40 @@ mod tests { } } + #[restate_sdk::service] + #[name = "ExecutionSchedule"] + trait ExecutionSchedulePolicyProbe { + async fn create() -> Result<(), HandlerError>; + + async fn fire_occurrence() -> Result<(), HandlerError>; + } + + struct ExecutionSchedulePolicyProbeImpl; + + impl ExecutionSchedulePolicyProbe for ExecutionSchedulePolicyProbeImpl { + async fn create(&self, _ctx: Context<'_>) -> Result<(), HandlerError> { + Ok(()) + } + + async fn fire_occurrence(&self, _ctx: Context<'_>) -> Result<(), HandlerError> { + Ok(()) + } + } + + #[restate_sdk::object] + #[name = "ExecutionRunController"] + trait ExecutionRunControllerPolicyProbe { + async fn advance() -> Result<(), HandlerError>; + } + + struct ExecutionRunControllerPolicyProbeImpl; + + impl ExecutionRunControllerPolicyProbe for ExecutionRunControllerPolicyProbeImpl { + async fn advance(&self, _ctx: ObjectContext<'_>) -> Result<(), HandlerError> { + Ok(()) + } + } + #[restate_sdk::workflow] #[name = "WorkflowPolicyProbe"] trait WorkflowPolicyProbe { @@ -821,6 +913,63 @@ mod tests { assert_eq!(run["workflowCompletionRetention"], 86_400_000); } + #[tokio::test] + async fn execution_schedule_keeps_crud_public_and_fire_occurrence_ingress_private() { + // Pins: tenant schedule CRUD remains callable through public ingress, while only trusted + // outbox delivery can invoke the trigger-consuming occurrence handler. + let endpoint = Endpoint::builder() + .bind_with_options( + ExecutionSchedulePolicyProbeImpl.serve(), + execution_schedule_service_options(), + ) + .build(); + let manifest = v4_manifest(endpoint).await; + let service = &manifest["services"][0]; + assert_eq!(service["name"], "ExecutionSchedule"); + assert_ne!(service["ingressPrivate"], true); + + let handlers = service["handlers"] + .as_array() + .expect("schedule handlers should be an array"); + let create = handlers + .iter() + .find(|handler| handler["name"] == "create") + .expect("public create handler should exist"); + let fire_occurrence = handlers + .iter() + .find(|handler| handler["name"] == "fire_occurrence") + .expect("private fire-occurrence handler should exist"); + + assert_ne!(create["ingressPrivate"], true); + assert_eq!(fire_occurrence["ingressPrivate"], true); + } + + #[tokio::test] + async fn execution_run_controller_options_match_the_advance_handler() { + // Pins: the bounded run controller is a virtual object with `advance`, so endpoint + // discovery must not apply workflow-only options for a nonexistent `run` handler. + let endpoint = Endpoint::builder() + .bind_with_options( + ExecutionRunControllerPolicyProbeImpl.serve(), + execution_run_controller_options(), + ) + .build(); + let manifest = v4_manifest(endpoint).await; + let service = &manifest["services"][0]; + assert_eq!(service["name"], "ExecutionRunController"); + assert_eq!(service["ingressPrivate"], true); + let advance = service["handlers"] + .as_array() + .expect("controller handlers should be an array") + .iter() + .find(|handler| handler["name"] == "advance") + .expect("controller advance handler should exist"); + assert_eq!( + advance["workflowCompletionRetention"], + serde_json::Value::Null + ); + } + #[tokio::test] async fn sandbox_workspace_service_uses_configured_durable_retention() { // Pins: Restate cannot evict a workspace operation owner on the generic @@ -884,13 +1033,27 @@ mod tests { "ToolExecutor", "TurnExecution", "WorkerTurnExecution", - "ExecutionRun", - "ExecutionTask", - "ExecutionCompensation", + "ExecutionRunController", + "ExecutionRetention", + "ExecutionTaskAttempt", + "ExecutionCompensationAttempt", + "ExecutionTrigger", + "ExecutionDispatcher", + "ExecutionDispatchDrain", + "ExecutionDispatchReconciler", + "DurableTimeout", ] ); } + #[test] + fn ingress_private_handlers_are_exactly_the_mixed_service_internal_set() { + assert_eq!( + INGRESS_PRIVATE_HANDLER_NAMES, + [("ExecutionSchedule", "fire_occurrence")] + ); + } + #[test] fn product_expected_services_include_experiments() { let names = expected_service_names(SandboxWorkspaceMode::Disabled); @@ -918,10 +1081,10 @@ mod tests { 1 ); assert!( - names.contains(&"ExecutionRun") - && names.contains(&"ExecutionTask") - && names.contains(&"ExecutionCompensation"), - "product readiness should include every durable execution workflow" + names.contains(&"ExecutionRunController") + && names.contains(&"ExecutionTaskAttempt") + && names.contains(&"ExecutionCompensationAttempt"), + "product readiness should include every bounded execution owner" ); assert!( names.contains(&"Execution"), @@ -967,10 +1130,10 @@ mod tests { .copied() .filter(|name| *name != "ExperimentRun") .collect::>(); - let deployment_without_compensation = names + let deployment_without_controller = names .iter() .copied() - .filter(|name| *name != "ExecutionCompensation") + .filter(|name| *name != "ExecutionRunController") .collect::>(); assert!( @@ -989,10 +1152,10 @@ mod tests { ); assert!( !services_registered_with_expected( - &[deployment_with_services(&deployment_without_compensation)], + &[deployment_with_services(&deployment_without_controller)], &names ), - "readiness must reject a deployment missing ExecutionCompensation" + "readiness must reject a deployment missing ExecutionRunController" ); } diff --git a/crates/moa-orchestrator/src/runtime/execution_dispatch.rs b/crates/moa-orchestrator/src/runtime/execution_dispatch.rs new file mode 100644 index 000000000..2073bc7e6 --- /dev/null +++ b/crates/moa-orchestrator/src/runtime/execution_dispatch.rs @@ -0,0 +1,468 @@ +//! Strict conversion from durable execution outbox rows to Restate targets. + +use moa_core::types::identifiers::TenantId; +use moa_execution::{ + repository::outbox::{ExecutionDispatchKind, ExecutionDispatchRecord}, + wire::{ + ExecutionCompensationAttemptCancelRequest, ExecutionCompensationAttemptRequest, + ExecutionExternalJobCancelRequest, ExecutionTaskAttemptCancelRequest, + ExecutionTaskAttemptRequest, + }, +}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use uuid::Uuid; + +use crate::objects::execution_run_controller::ExecutionRunAdvanceRequest; + +/// Journal-safe copy of one claimed outbox row. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct JournaledExecutionDispatch { + /// Immutable outbox identity and downstream idempotency key. + pub dispatch_uid: Uuid, + /// Owning tenant. + pub tenant_id: TenantId, + /// Owning run, when applicable. + pub run_uid: Option, + /// Owning task, when applicable. + pub task_id: Option, + /// Owning compensation, when applicable. + pub compensation_id: Option, + /// Immutable trigger target, when applicable. + pub trigger_uid: Option, + /// Exact asynchronous job target, when applicable. + pub external_job_uid: Option, + /// Closed delivery target. + pub kind: ExecutionDispatchKind, + /// Controller generation fence. + pub controller_generation: Option, + /// Exact scheduling wake. + pub wake_epoch: Option, + /// Task-attempt generation fence. + pub attempt_generation: Option, + /// Compensation logical generation fence. + pub compensation_generation: Option, + /// Compensation-attempt generation fence. + pub compensation_attempt_generation: Option, + /// Immutable target payload. + pub payload: Value, + /// Claim attempt count after the current claim. + pub delivery_attempts: u32, +} + +/// Fully validated downstream request selected from one immutable dispatch. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[serde(tag = "kind", content = "request", rename_all = "snake_case")] +pub enum ExecutionDispatchTarget { + /// Run-controller activation. + RunActivation(ExecutionRunAdvanceRequest), + /// Task-attempt workflow. + TaskAttempt(ExecutionTaskAttemptRequest), + /// Cancellation signal for one exact active task-attempt workflow. + TaskAttemptCancel(ExecutionTaskAttemptCancelRequest), + /// Compensation-attempt workflow. + CompensationAttempt(ExecutionCompensationAttemptRequest), + /// Cancellation signal for one exact active compensation-attempt workflow. + CompensationAttemptCancel(ExecutionCompensationAttemptCancelRequest), + /// Temporal-trigger service. + TriggerDelivery(ExecutionTriggerDeliveryRequest), + /// Tool-executor external cancellation. + ExternalCancel { + /// Immutable outbox identity and downstream idempotency key. + dispatch_uid: Uuid, + /// Exact provider cancellation request. + request: ExecutionExternalJobCancelRequest, + }, +} + +/// Immutable request delivered to `ExecutionTrigger/fire`. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ExecutionTriggerDeliveryRequest { + /// Outbox identity used as the Restate idempotency key. + pub dispatch_uid: Uuid, + /// Owning tenant used to install repository RLS scope. + pub tenant_id: TenantId, + /// Immutable trigger identity reloaded by the handler. + pub trigger_uid: Uuid, +} + +impl From for JournaledExecutionDispatch { + fn from(record: ExecutionDispatchRecord) -> Self { + Self { + dispatch_uid: record.dispatch_uid, + tenant_id: record.tenant_id, + run_uid: record.run_uid, + task_id: record.task_id, + compensation_id: record.compensation_id, + trigger_uid: record.trigger_uid, + external_job_uid: record.external_job_uid, + kind: record.kind, + controller_generation: record.controller_generation, + wake_epoch: record.wake_epoch, + attempt_generation: record.attempt_generation, + compensation_generation: record.compensation_generation, + compensation_attempt_generation: record.compensation_attempt_generation, + payload: record.payload, + delivery_attempts: record.delivery_attempts, + } + } +} + +impl JournaledExecutionDispatch { + /// Strictly decodes and cross-checks the downstream request. + pub fn target(&self) -> Result { + match self.kind { + ExecutionDispatchKind::RunActivation => { + let request = ExecutionRunAdvanceRequest { + dispatch_uid: self.dispatch_uid, + tenant_id: self.tenant_id, + run_uid: required(self.run_uid, "run activation run_uid")?, + controller_generation: required( + self.controller_generation, + "run activation controller_generation", + )?, + wake_epoch: required(self.wake_epoch, "run activation wake_epoch")?, + }; + Ok(ExecutionDispatchTarget::RunActivation(request)) + } + ExecutionDispatchKind::TaskAttempt => { + let request: ExecutionTaskAttemptRequest = decode(&self.payload, "task attempt")?; + require( + request.dispatch_uid == self.dispatch_uid, + "task dispatch UID", + )?; + require(request.tenant_id == self.tenant_id, "task tenant")?; + require( + request.run_uid == required(self.run_uid, "task run_uid")?, + "task run", + )?; + require( + request.task_id.as_uuid() == required(self.task_id, "task task_id")?, + "task identity", + )?; + require( + request.controller_generation + == required(self.controller_generation, "task controller_generation")?, + "task controller generation", + )?; + require( + request.attempt_generation + == required(self.attempt_generation, "task attempt_generation")?, + "task attempt generation", + )?; + Ok(ExecutionDispatchTarget::TaskAttempt(request)) + } + ExecutionDispatchKind::TaskAttemptCancel => { + let request: ExecutionTaskAttemptCancelRequest = + decode(&self.payload, "task attempt cancellation")?; + require( + request.cancellation_dispatch_uid == self.dispatch_uid, + "task cancellation dispatch UID", + )?; + require( + request.tenant_id == self.tenant_id, + "task cancellation tenant", + )?; + require( + request.run_uid == required(self.run_uid, "task cancellation run_uid")?, + "task cancellation run", + )?; + require( + request.task_id.as_uuid() + == required(self.task_id, "task cancellation task_id")?, + "task cancellation identity", + )?; + require( + request.controller_generation + == required( + self.controller_generation, + "task cancellation controller_generation", + )?, + "task cancellation controller generation", + )?; + require( + request.attempt_controller_generation > 0, + "task cancellation attempt controller generation", + )?; + require( + request.attempt_generation + == required( + self.attempt_generation, + "task cancellation attempt_generation", + )?, + "task cancellation attempt generation", + )?; + Ok(ExecutionDispatchTarget::TaskAttemptCancel(request)) + } + ExecutionDispatchKind::CompensationAttempt => { + let request: ExecutionCompensationAttemptRequest = + decode(&self.payload, "compensation attempt")?; + require( + request.dispatch_uid == self.dispatch_uid, + "compensation dispatch UID", + )?; + require(request.tenant_id == self.tenant_id, "compensation tenant")?; + require( + request.run_uid == required(self.run_uid, "compensation run_uid")?, + "compensation run", + )?; + require( + request.compensation_id.as_uuid() + == required(self.compensation_id, "compensation compensation_id")?, + "compensation identity", + )?; + require( + request.controller_generation + == required( + self.controller_generation, + "compensation controller_generation", + )?, + "compensation controller generation", + )?; + require( + request.compensation_generation + == required(self.compensation_generation, "compensation generation")?, + "compensation generation", + )?; + require( + request.compensation_attempt_generation + == required( + self.compensation_attempt_generation, + "compensation attempt_generation", + )?, + "compensation attempt generation", + )?; + Ok(ExecutionDispatchTarget::CompensationAttempt(request)) + } + ExecutionDispatchKind::CompensationAttemptCancel => { + let request: ExecutionCompensationAttemptCancelRequest = + decode(&self.payload, "compensation attempt cancellation")?; + require( + request.cancellation_dispatch_uid == self.dispatch_uid, + "compensation cancellation dispatch UID", + )?; + require( + request.tenant_id == self.tenant_id, + "compensation cancellation tenant", + )?; + require( + request.run_uid == required(self.run_uid, "compensation cancellation run_uid")?, + "compensation cancellation run", + )?; + require( + request.compensation_id.as_uuid() + == required( + self.compensation_id, + "compensation cancellation compensation_id", + )?, + "compensation cancellation identity", + )?; + require( + request.controller_generation + == required( + self.controller_generation, + "compensation cancellation controller_generation", + )?, + "compensation cancellation controller generation", + )?; + require( + request.attempt_controller_generation > 0, + "compensation cancellation attempt controller generation", + )?; + require( + request.compensation_generation + == required( + self.compensation_generation, + "compensation cancellation generation", + )?, + "compensation cancellation generation", + )?; + require( + request.compensation_attempt_generation + == required( + self.compensation_attempt_generation, + "compensation cancellation attempt_generation", + )?, + "compensation cancellation attempt generation", + )?; + Ok(ExecutionDispatchTarget::CompensationAttemptCancel(request)) + } + ExecutionDispatchKind::TriggerDelivery => Ok(ExecutionDispatchTarget::TriggerDelivery( + ExecutionTriggerDeliveryRequest { + dispatch_uid: self.dispatch_uid, + tenant_id: self.tenant_id, + trigger_uid: required(self.trigger_uid, "trigger trigger_uid")?, + }, + )), + ExecutionDispatchKind::ExternalCancel => { + let request: ExecutionExternalJobCancelRequest = + decode(&self.payload, "external cancellation")?; + require( + request.tenant_id == self.tenant_id, + "external cancellation tenant", + )?; + require( + request.external_job_uid + == required(self.external_job_uid, "external external_job_uid")?, + "external cancellation job", + )?; + Ok(ExecutionDispatchTarget::ExternalCancel { + dispatch_uid: self.dispatch_uid, + request, + }) + } + } + } +} + +fn required(value: Option, field: &str) -> Result { + value.ok_or_else(|| invalid(format!("execution dispatch is missing {field}"))) +} + +fn require(condition: bool, field: &str) -> Result<(), moa_execution::Error> { + if condition { + Ok(()) + } else { + Err(invalid(format!( + "execution dispatch payload disagrees with persisted {field}" + ))) + } +} + +fn decode( + payload: &Value, + target: &str, +) -> Result { + serde_json::from_value(payload.clone()).map_err(|error| { + invalid(format!( + "execution {target} dispatch payload is invalid: {error}" + )) + }) +} + +fn invalid(message: String) -> moa_execution::Error { + moa_execution::Error::InvalidRepositoryData { message } +} + +#[cfg(test)] +mod tests { + use moa_execution::{ + state::ExecutionTaskId, + wire::{ExecutionAttemptCancelReason, ExecutionTaskAttemptCancelRequest}, + }; + + use super::*; + + #[test] + fn run_target_uses_only_persisted_fences() { + // Pins: an untrusted payload cannot redirect a run activation. + let dispatch_uid = Uuid::from_u128(1); + let run_uid = Uuid::from_u128(2); + let dispatch = JournaledExecutionDispatch { + dispatch_uid, + tenant_id: TenantId::from(Uuid::from_u128(3)), + run_uid: Some(run_uid), + task_id: None, + compensation_id: None, + trigger_uid: None, + external_job_uid: None, + kind: ExecutionDispatchKind::RunActivation, + controller_generation: Some(4), + wake_epoch: Some(5), + attempt_generation: None, + compensation_generation: None, + compensation_attempt_generation: None, + payload: serde_json::json!({ "run_uid": Uuid::from_u128(99) }), + delivery_attempts: 1, + }; + + assert_eq!( + dispatch.target().expect("run target must decode"), + ExecutionDispatchTarget::RunActivation(ExecutionRunAdvanceRequest { + dispatch_uid, + tenant_id: dispatch.tenant_id, + run_uid, + controller_generation: 4, + wake_epoch: 5, + }) + ); + } + + #[test] + fn task_payload_must_match_outbox_coordinates() { + // Pins: payload corruption cannot move a capacity-owning attempt to another dispatch. + let dispatch = JournaledExecutionDispatch { + dispatch_uid: Uuid::from_u128(1), + tenant_id: TenantId::from(Uuid::from_u128(2)), + run_uid: Some(Uuid::from_u128(3)), + task_id: Some(Uuid::from_u128(4)), + compensation_id: None, + trigger_uid: None, + external_job_uid: None, + kind: ExecutionDispatchKind::TaskAttempt, + controller_generation: Some(1), + wake_epoch: None, + attempt_generation: Some(1), + compensation_generation: None, + compensation_attempt_generation: None, + payload: serde_json::json!({}), + delivery_attempts: 1, + }; + + let error = dispatch + .target() + .expect_err("empty task payload must fail closed"); + assert!(matches!( + error, + moa_execution::Error::InvalidRepositoryData { .. } + )); + } + + #[test] + fn task_cancel_payload_cannot_redirect_persisted_attempt_fences() { + // Pins: dispatcher routing validates the persisted cancellation target + // before the keyed workflow performs its canonical row-lock checks. + let dispatch_uid = Uuid::from_u128(1); + let tenant_id = TenantId::from(Uuid::from_u128(2)); + let run_uid = Uuid::from_u128(3); + let task_id = ExecutionTaskId::from_uuid(Uuid::from_u128(4)); + let payload = ExecutionTaskAttemptCancelRequest { + cancellation_dispatch_uid: dispatch_uid, + tenant_id, + run_uid, + task_id, + controller_generation: 5, + attempt_controller_generation: 5, + task_generation: 6, + attempt_generation: 7, + active_dispatch_uid: Uuid::from_u128(8), + capacity_reservation_uid: Uuid::from_u128(9), + watchdog_trigger_uid: Uuid::from_u128(10), + reason: ExecutionAttemptCancelReason::RunTerminal, + }; + let dispatch = JournaledExecutionDispatch { + dispatch_uid, + tenant_id, + run_uid: Some(run_uid), + task_id: Some(task_id.as_uuid()), + compensation_id: None, + trigger_uid: None, + external_job_uid: None, + kind: ExecutionDispatchKind::TaskAttemptCancel, + controller_generation: Some(5), + wake_epoch: None, + attempt_generation: Some(99), + compensation_generation: None, + compensation_attempt_generation: None, + payload: serde_json::to_value(payload).expect("serialize task cancellation"), + delivery_attempts: 1, + }; + + assert!(matches!( + dispatch.target(), + Err(moa_execution::Error::InvalidRepositoryData { .. }) + )); + } +} diff --git a/crates/moa-orchestrator/src/runtime/jobs.rs b/crates/moa-orchestrator/src/runtime/jobs.rs index e23df7dad..b0dba6b3e 100644 --- a/crates/moa-orchestrator/src/runtime/jobs.rs +++ b/crates/moa-orchestrator/src/runtime/jobs.rs @@ -6,7 +6,10 @@ use anyhow::{Context as AnyhowContext, Result, bail}; use moa_authz::{AwakeableResolver, FgaClient}; use moa_config::AsyncAuthzKind; use moa_config::MoaConfig; -use moa_hands::{HandLeaseReaper, HandLeaseReaperConfig, PostgresExpiredHandLeaseClaims}; +use moa_hands::{ + HandLeaseReaper, HandLeaseReaperConfig, PostgresExpiredHandLeaseClaims, + SandboxProviderInventory, +}; use reqwest::Client; use sqlx::PgPool; use tokio::task::JoinHandle; @@ -15,6 +18,7 @@ use tokio_util::sync::CancellationToken; use crate::services::authz_challenges_reaper::{AuthzChallengeReaper, AuthzChallengeReaperHandle}; use crate::services::action_reviews_reaper::{ActionReviewReaper, ActionReviewReaperHandle}; +use crate::{runtime::kms::KmsRuntime, services::authz_challenges_reaper::HttpAwakeableResolver}; const DEFAULT_RESTATE_INGRESS_PORT: u16 = 8080; /// Initial delay before retrying default cron jobs that failed a reconcile pass. @@ -24,6 +28,147 @@ const CRON_RECONCILE_MAX_BACKOFF: Duration = Duration::from_secs(300); /// How often configured MCP connectors are re-discovered. const MCP_CATALOG_REFRESH_INTERVAL: Duration = Duration::from_secs(300); +/// Minimal dependency graph owned by the standalone maintenance process. +pub struct MaintenanceDependencies { + /// KMS handle used by health checks and durable workspace checkpoints. + pub kms: KmsRuntime, + /// OpenFGA client used only by the authorization outbox owner. + pub fga_client: Option, + /// Restate awakeable resolver used only by builtin approval reconciliation. + pub awakeable_resolver: Option>, + /// Authenticated checkpoint-bucket observation, when workspaces are enabled. + pub checkpoint_versioning_observer: Option< + moa_hands::core::sandbox_workspace::checkpoint::versioning::CheckpointBucketVersioningObserver, + >, + /// Workspace reconciliation, retention, and provider-inventory owner. + pub workspace_maintenance: Option< + Arc, + >, + /// Exact hand providers whose expired compute the maintenance process destroys. + pub hand_providers: Vec>, +} + +/// Builds only the dependencies needed by the standalone maintenance process. +pub async fn build_maintenance_dependencies( + config: &MoaConfig, + runtime_pool: PgPool, + maintenance_pool: Option, + restate_ingress_url: &str, + skip_fga: bool, +) -> Result { + let workspace_enabled = config.sandbox_workspaces.mode.maintenance_enabled(); + let kms = KmsRuntime::build_serving(config, runtime_pool.clone()) + .await + .context("build maintenance runtime KMS")?; + let fga_client = if skip_fga { + tracing::warn!("MOA_SKIP_FGA set; maintenance authz outbox polling disabled"); + None + } else { + let openfga = config + .authz + .openfga + .as_ref() + .context("authz.openfga config missing")?; + let client = FgaClient::new(moa_authz::FgaConfig { + url: openfga.url.clone(), + preshared_key: openfga.preshared_key.clone(), + store_id: openfga.store_id.clone(), + model_id: openfga.model_id.clone(), + timeout_ms: openfga.timeout_ms, + }) + .context("build maintenance OpenFGA client")?; + if workspace_enabled { + let expected_model: serde_json::Value = + serde_json::from_str(moa_authz_schema::SCHEMA_V1_JSON) + .context("decode compiled OpenFGA model")?; + client + .verify_authorization_model(&expected_model) + .await + .context("verify authorization model before workspace maintenance")?; + } + Some(client) + }; + let awakeable_resolver = if config.async_authz.provider == AsyncAuthzKind::Builtin { + Some(Arc::new( + HttpAwakeableResolver::new(restate_ingress_base_url(restate_ingress_url)) + .context("build maintenance Restate awakeable resolver")?, + ) as Arc) + } else { + None + }; + + if !workspace_enabled { + return Ok(MaintenanceDependencies { + kms, + fga_client, + awakeable_resolver, + checkpoint_versioning_observer: None, + workspace_maintenance: None, + hand_providers: Vec::new(), + }); + } + + let maintenance_pool = maintenance_pool + .context("sandbox workspace maintenance requires its dedicated database pool")?; + moa_hands::core::sandbox_workspace::maintenance::WorkspaceMaintenanceCoordinator::verify_maintenance_pool( + &maintenance_pool, + ) + .await + .context("verify dedicated sandbox workspace maintenance database role")?; + crate::runtime::sandbox_workspace_rollout::bootstrap_accounts_and_quotas( + config, + &maintenance_pool, + ) + .await + .context("bootstrap sandbox provider accounts and quota routes")?; + + kms.require_durable("sandbox workspace checkpoints")?; + let (checkpoint_store, checkpoint_versioning_observer) = + moa_hands::core::sandbox_workspace::checkpoint::store::CheckpointObjectStore::from_config_with_versioning_observer( + config, + kms.provider(), + )?; + checkpoint_versioning_observer + .observe_unversioned() + .await + .context("observe checkpoint bucket versioning before maintenance startup")?; + let checkpoint_store = Arc::new(checkpoint_store); + checkpoint_store + .preflight_create_only_namespace() + .await + .context("preflight checkpoint bucket create-only namespace")?; + + let provider_inventory = SandboxProviderInventory::for_maintenance( + config, + &maintenance_pool, + Arc::clone(&checkpoint_store), + kms.provider(), + ) + .await + .context("build maintenance sandbox providers")?; + let hand_providers = provider_inventory.hand_providers(); + let workspace_maintenance = Arc::new( + moa_hands::core::sandbox_workspace::maintenance::WorkspaceMaintenanceCoordinator::new( + maintenance_pool, + checkpoint_store, + provider_inventory.storage_providers(), + hand_providers.clone(), + config.sandbox_checkpoints.retention.clone(), + Duration::from_secs(config.sandbox_workspaces.reconciliation_claim_ttl_seconds), + ) + .context("build sandbox workspace maintenance coordinator")?, + ); + + Ok(MaintenanceDependencies { + kms, + fga_client, + awakeable_resolver, + checkpoint_versioning_observer: Some(checkpoint_versioning_observer), + workspace_maintenance: Some(workspace_maintenance), + hand_providers, + }) +} + /// Starts the OpenFGA outbox poller that drains queued authorization tuple changes. pub fn start_authz_outbox_poller(pool: &PgPool, fga_client: FgaClient) -> moa_authz::PollerHandle { let outbox_poller = @@ -65,11 +210,12 @@ pub fn start_action_review_reaper( /// traffic, because the sandboxes that most need destroying belong to sessions /// that will never send another request. Startup fails when no hand provider is /// registered: a deployment that provisions sandboxes with no way to destroy -/// them is not a deployment MOA should serve. +/// them is not a deployment MOA should serve. Readiness stays closed until the +/// first complete sweep, and any failed pass terminates the supervised owner. pub fn start_hand_lease_reaper( pool: &PgPool, providers: Vec>, -) -> Result> { +) -> Result { if providers.is_empty() { bail!( "durable hand-lease reaper requires at least one registered hand provider; \ @@ -81,13 +227,20 @@ pub fn start_hand_lease_reaper( .map(|provider| provider.provider_name().to_string()) .collect::>() .join(","); + let config = HandLeaseReaperConfig::default(); + let heartbeat_maximum_age_seconds = config.heartbeat_maximum_age.as_secs(); let handle = HandLeaseReaper::new( Arc::new(PostgresExpiredHandLeaseClaims::new(pool.clone())), providers, - HandLeaseReaperConfig::default(), + config, ) - .spawn(); - tracing::info!(providers = %provider_names, "durable hand lease reaper started"); + .spawn() + .context("start durable hand lease reaper")?; + tracing::info!( + providers = %provider_names, + heartbeat_maximum_age_seconds, + "durable hand lease reaper started" + ); Ok(handle) } @@ -108,6 +261,14 @@ pub fn start_workspace_reaper( .reaper_heartbeat_maximum_age_seconds, ); let interval = Duration::from_secs((maximum_age.as_secs() / 3).clamp(1, 10)); + let cadences = moa_hands::core::sandbox_workspace::reaper::WorkspaceReaperCadenceConfig { + safety_interval: interval, + retention_initial_interval: Duration::from_secs(60), + retention_maximum_interval: Duration::from_secs(60 * 60), + inventory_initial_interval: Duration::from_secs(5 * 60), + inventory_maximum_interval: Duration::from_secs(60 * 60), + fleet_metrics_interval: Duration::from_secs(60), + }; let batch_size = i64::from(config.sandbox_checkpoints.retention.gc_batch_size); let reaper = coordinator .workspace_reaper(8) @@ -115,13 +276,18 @@ pub fn start_workspace_reaper( let handle = moa_hands::core::sandbox_workspace::reaper::WorkspaceReaperHandle::spawn( coordinator, reaper, - interval, + cadences, batch_size, maximum_age, ) .context("start durable workspace reaper")?; tracing::info!( - interval_seconds = interval.as_secs(), + safety_interval_seconds = cadences.safety_interval.as_secs(), + retention_initial_interval_seconds = cadences.retention_initial_interval.as_secs(), + retention_maximum_interval_seconds = cadences.retention_maximum_interval.as_secs(), + inventory_initial_interval_seconds = cadences.inventory_initial_interval.as_secs(), + inventory_maximum_interval_seconds = cadences.inventory_maximum_interval.as_secs(), + fleet_metrics_interval_seconds = cadences.fleet_metrics_interval.as_secs(), heartbeat_maximum_age_seconds = maximum_age.as_secs(), batch_size, "durable workspace reaper started" @@ -270,6 +436,26 @@ pub async fn install_default_cron_jobs(ingress_url: &str) -> Result<()> { Ok(()) } +/// Idempotently installs the coarse execution-maintenance repair CronJobs. +/// +/// Exact trigger and outbox work uses persist-then-send on its owning Restate +/// path. This low-frequency job is only the durable repair fence for missed +/// sends and abandoned delivery claims; it never polls from this process. +pub async fn ensure_execution_maintenance_cron_jobs( + ingress_url: &str, + cadence_seconds: u64, +) -> Result<()> { + let client = cron_bootstrap_client()?; + let ingress_url = ingress_url.trim_end_matches('/'); + let dispatch = execution_dispatch_reconciliation_cron_job(cadence_seconds)?; + configure_cron_job(&client, ingress_url, &dispatch) + .await + .context("configure execution dispatch reconciliation CronJob")?; + configure_cron_job(&client, ingress_url, &execution_retention_repair_cron_job()) + .await + .context("configure execution retention repair CronJob") +} + /// Reconciles every default cron job independently, retrying failures forever. /// /// Each pass attempts all still-pending jobs; a per-job failure is collected and @@ -377,12 +563,68 @@ async fn configure_cron_job( Ok(()) } +#[derive(Debug)] struct DefaultCronJob { key: &'static str, body: serde_json::Value, version: &'static str, } +fn execution_dispatch_reconciliation_cron_job(cadence_seconds: u64) -> Result { + let schedule = exact_reconciliation_cron_schedule(cadence_seconds)?; + Ok(DefaultCronJob { + key: "execution_dispatch_reconcile", + body: serde_json::json!({ + "schedule": schedule, + "timezone": "UTC", + "target_service": "ExecutionDispatchReconciler", + "target_handler": "reconcile", + "payload": {} + }), + version: "v1", + }) +} + +fn execution_retention_repair_cron_job() -> DefaultCronJob { + DefaultCronJob { + key: "execution_retention_repair", + body: serde_json::json!({ + "schedule": "0 0 * * * *", + "timezone": "UTC", + "target_service": "ExecutionRetention", + "target_handler": "run", + "payload": {} + }), + version: "v1", + } +} + +fn exact_reconciliation_cron_schedule(cadence_seconds: u64) -> Result { + if cadence_seconds == 0 { + bail!("execution trigger reconciliation cadence must be positive"); + } + if cadence_seconds <= 60 && 60 % cadence_seconds == 0 { + return Ok(if cadence_seconds == 60 { + "0 * * * * *".to_string() + } else { + format!("*/{cadence_seconds} * * * * *") + }); + } + if cadence_seconds.is_multiple_of(60) { + let minutes = cadence_seconds / 60; + if minutes <= 60 && 60 % minutes == 0 { + return Ok(if minutes == 60 { + "0 0 * * * *".to_string() + } else { + format!("0 */{minutes} * * * *") + }); + } + } + bail!( + "execution.trigger_reconciliation_cadence_seconds={cadence_seconds} cannot be represented exactly by the durable wall-clock CronJob" + ) +} + fn default_cron_jobs() -> Vec { vec![ DefaultCronJob { @@ -550,6 +792,43 @@ mod tests { .unwrap_or(0) } + #[test] + fn execution_dispatch_reconciliation_cron_uses_configured_sixty_second_cadence() { + // Pins: repair is a durable CronJob, not a process-local polling loop, + // and targets the bounded repair-plus-dispatch wrapper. + let job = execution_dispatch_reconciliation_cron_job(60) + .expect("default cadence must be exactly representable"); + + assert_eq!(job.key, "execution_dispatch_reconcile"); + assert_eq!(job.body["schedule"], "0 * * * * *"); + assert_eq!(job.body["target_service"], "ExecutionDispatchReconciler"); + assert_eq!(job.body["target_handler"], "reconcile"); + assert_eq!(job.body["payload"], serde_json::json!({})); + } + + #[test] + fn execution_dispatch_reconciliation_rejects_inexact_wall_clock_cadence() { + // Pins: an operator cadence is never silently rounded into a different + // repair SLO by the wall-clock CronJob adapter. + let error = execution_dispatch_reconciliation_cron_job(90) + .expect_err("ninety seconds is not exactly representable by this cron surface"); + + assert!(error.to_string().contains("cannot be represented exactly")); + } + + #[test] + fn execution_retention_cron_is_only_the_coarse_self_schedule_repair() { + // Pins: normal retention cadence is a persisted delayed self-call; this + // hourly Cron only repairs a missing generation after a crash. + let job = execution_retention_repair_cron_job(); + + assert_eq!(job.key, "execution_retention_repair"); + assert_eq!(job.body["schedule"], "0 0 * * * *"); + assert_eq!(job.body["target_service"], "ExecutionRetention"); + assert_eq!(job.body["target_handler"], "run"); + assert_eq!(job.body["payload"], serde_json::json!({})); + } + async fn spawn_s3_versioning_sequence(bodies: [&'static str; 2]) -> (String, JoinHandle<()>) { let listener = tokio::net::TcpListener::bind("127.0.0.1:0") .await diff --git a/crates/moa-orchestrator/src/runtime/mod.rs b/crates/moa-orchestrator/src/runtime/mod.rs index c50586cae..418e355dd 100644 --- a/crates/moa-orchestrator/src/runtime/mod.rs +++ b/crates/moa-orchestrator/src/runtime/mod.rs @@ -5,6 +5,7 @@ pub mod channel_ingress; pub mod database; pub mod deps; pub mod endpoint; +pub mod execution_dispatch; pub mod jobs; pub mod kms; pub mod sandbox_workspace_rollout; diff --git a/crates/moa-orchestrator/src/services/action_policy.rs b/crates/moa-orchestrator/src/services/action_policy.rs index a14eb2e2d..0faa65a54 100644 --- a/crates/moa-orchestrator/src/services/action_policy.rs +++ b/crates/moa-orchestrator/src/services/action_policy.rs @@ -574,6 +574,7 @@ mod tests { run_uid: Uuid::from_u128(72), task_uid: Uuid::from_u128(73), generation: 1, + attempt_generation: 1, }), ); request.capability_policy_context = Some(CapabilityPolicyContext::artifact( @@ -635,6 +636,7 @@ mod tests { run_uid: Uuid::from_u128(78), task_uid: Uuid::from_u128(79), generation: 1, + attempt_generation: 1, }), ); request.capability_policy_context = Some(CapabilityPolicyContext::artifact( @@ -759,6 +761,7 @@ mod tests { run_uid: Uuid::from_u128(10), task_uid: Uuid::from_u128(20), generation: 3, + attempt_generation: 3, }), ); diff --git a/crates/moa-orchestrator/src/services/action_review_dispatcher.rs b/crates/moa-orchestrator/src/services/action_review_dispatcher.rs index f37af6484..aef69b1ba 100644 --- a/crates/moa-orchestrator/src/services/action_review_dispatcher.rs +++ b/crates/moa-orchestrator/src/services/action_review_dispatcher.rs @@ -1,23 +1,24 @@ //! Restate-owned delivery for execution action-review resolutions. -use moa_execution::wire::{ - ExecutionActionReviewAcknowledgement, ExecutionActionReviewResolutionRequest, - ExecutionCompensationReviewAcknowledgement, ExecutionCompensationReviewResolutionRequest, +use chrono::Utc; +use moa_config::ExecutionConfig; +use moa_execution::repository::compensation::CompensationReviewResolutionOutcome; +use moa_execution::repository::task::{ + ResolveTaskAttemptReviewRequest, TaskAttemptReviewResolutionOutcome, }; -use moa_observability::propagation::{ - TRACE_LINK_TRACEPARENT_HEADER, TRACE_LINK_TRACESTATE_HEADER, ValidatedTraceContext, - with_validated_trace_headers, +use moa_execution::repository::{ExecutionRepository, ExecutionScope}; +use moa_execution::wire::{ + ExecutionActionReviewResolutionRequest, ExecutionCompensationReviewResolutionRequest, }; +use moa_observability::propagation::{ValidatedTraceContext, link_validated_context}; use restate_sdk::prelude::*; use serde::{Deserialize, Serialize}; use sqlx::PgPool; use uuid::Uuid; use crate::action_reviews::store as action_review_store; -use crate::workflows::{ - errors::sqlx_error_to_handler_error, execution_compensation::ExecutionCompensationClient, - execution_task::ExecutionTaskClient, -}; +use crate::services::execution_dispatcher::{DispatchExecutionsRequest, ExecutionDispatcherClient}; +use crate::workflows::errors::sqlx_error_to_handler_error; const EXECUTION_REVIEW_DISPATCH_BATCH_SIZE: i64 = 32; @@ -55,6 +56,14 @@ enum JournaledExecutionReviewRequest { Compensation(ExecutionCompensationReviewResolutionRequest), } +#[derive(Debug, Deserialize, Serialize)] +#[serde(tag = "result", rename_all = "snake_case", deny_unknown_fields)] +enum StorageResolutionDisposition { + Delivered, + NotReady, + Failed { message: String }, +} + /// Restate service that owns private workflow delivery for action-review outbox rows. #[restate_sdk::service] #[name = "ActionReviewDispatcher"] @@ -69,13 +78,14 @@ pub trait ActionReviewDispatcher { #[derive(Clone)] pub struct ActionReviewDispatcherImpl { pool: PgPool, + config: ExecutionConfig, } impl ActionReviewDispatcherImpl { /// Creates a dispatcher over the shared control-plane pool. #[must_use] - pub fn new(pool: PgPool) -> Self { - Self { pool } + pub fn new(pool: PgPool, config: ExecutionConfig) -> Self { + Self { pool, config } } } @@ -139,7 +149,7 @@ impl ActionReviewDispatcher for ActionReviewDispatcherImpl { let claimed = deliveries.len(); for delivery in deliveries { - deliver_one(&ctx, &self.pool, delivery).await?; + deliver_one(&ctx, &self.pool, &self.config, delivery).await?; } Ok(Json(DispatchActionReviewsResponse { claimed })) @@ -149,6 +159,7 @@ impl ActionReviewDispatcher for ActionReviewDispatcherImpl { async fn deliver_one( ctx: &Context<'_>, pool: &PgPool, + config: &ExecutionConfig, delivery: JournaledExecutionReviewDelivery, ) -> Result<(), HandlerError> { let resolution_context = ValidatedTraceContext::new( @@ -159,66 +170,116 @@ async fn deliver_one( delivery.task_traceparent.as_deref(), delivery.task_tracestate.as_deref(), ); - macro_rules! with_delivery_trace_headers { - ($request:expr) => {{ - let request = with_validated_trace_headers( - $request, - resolution_context.as_ref(), - |request, name, value| request.header(name, value), - ); - match task_context.as_ref() { - Some(context) => { - let request = request.header( - TRACE_LINK_TRACEPARENT_HEADER.to_string(), - context.traceparent().to_string(), - ); - match context.tracestate() { - Some(tracestate) => request.header( - TRACE_LINK_TRACESTATE_HEADER.to_string(), - tracestate.to_string(), - ), - None => request, - } - } - None => request, - } - }}; + if let Some(context) = resolution_context.as_ref() { + let _ = link_validated_context(&tracing::Span::current(), context); + } + if let Some(context) = task_context.as_ref() { + let _ = link_validated_context(&tracing::Span::current(), context); } let acknowledged = match delivery.request { JournaledExecutionReviewRequest::Task(request) => { - let request = ctx - .workflow_client::(request.task_id.to_string()) - .resolve_action_review(Json(request)) - .idempotency_key(delivery.review_uid.to_string()); - let request = with_delivery_trace_headers!(request); - crate::restate_identity::replay_safe_request(request) - .call() - .await - .map(|Json(acknowledgement)| match acknowledgement { - ExecutionActionReviewAcknowledgement::Applied - | ExecutionActionReviewAcknowledgement::Replayed - | ExecutionActionReviewAcknowledgement::AuditedStale => {} + let repository = ExecutionRepository::new(pool.clone()); + let disposition = ctx + .run(|| async move { + let disposition = match repository + .resolve_task_attempt_review( + config, + ResolveTaskAttemptReviewRequest { + scope: ExecutionScope::ControlPlane, + run_uid: request.run_uid, + task_id: request.task_id, + expected_task_generation: request.generation, + review_uid: request.review_uid, + resolution: request.resolution, + resolved_at: Utc::now(), + }, + ) + .await + { + Ok( + TaskAttemptReviewResolutionOutcome::Applied { .. } + | TaskAttemptReviewResolutionOutcome::Replayed { .. } + | TaskAttemptReviewResolutionOutcome::NotFound + | TaskAttemptReviewResolutionOutcome::Stale, + ) => StorageResolutionDisposition::Delivered, + Ok(TaskAttemptReviewResolutionOutcome::NotReady) => { + StorageResolutionDisposition::NotReady + } + Err(error) => StorageResolutionDisposition::Failed { + message: error.to_string(), + }, + }; + Ok::<_, HandlerError>(Json::from(disposition)) }) + .name(format!( + "resolve_task_attempt_review_{}", + delivery.review_uid + )) + .await? + .into_inner(); + match disposition { + StorageResolutionDisposition::Delivered => Ok(()), + StorageResolutionDisposition::NotReady => { + Err("execution task review owner is not durably parked".to_string()) + } + StorageResolutionDisposition::Failed { message } => Err(message), + } } JournaledExecutionReviewRequest::Compensation(request) => { - let request = ctx - .workflow_client::(request.compensation_id.to_string()) - .resolve_action_review(Json(request)) - .idempotency_key(delivery.review_uid.to_string()); - let request = with_delivery_trace_headers!(request); - crate::restate_identity::replay_safe_request(request) - .call() - .await - .map(|Json(acknowledgement)| match acknowledgement { - ExecutionCompensationReviewAcknowledgement::Applied - | ExecutionCompensationReviewAcknowledgement::Replayed - | ExecutionCompensationReviewAcknowledgement::AuditedStale => {} + let repository = ExecutionRepository::new(pool.clone()); + let disposition = ctx + .run(|| async move { + let disposition = match repository + .resolve_current_compensation_review( + ExecutionScope::ControlPlane, + request.run_uid, + request.compensation_id, + request.generation, + request.review_uid, + &request.resolution, + Utc::now(), + ) + .await + { + Ok( + CompensationReviewResolutionOutcome::Applied { .. } + | CompensationReviewResolutionOutcome::Replayed { .. } + | CompensationReviewResolutionOutcome::NotFound + | CompensationReviewResolutionOutcome::Stale, + ) => StorageResolutionDisposition::Delivered, + Ok(CompensationReviewResolutionOutcome::NotReady) => { + StorageResolutionDisposition::NotReady + } + Err(error) => StorageResolutionDisposition::Failed { + message: error.to_string(), + }, + }; + Ok::<_, HandlerError>(Json::from(disposition)) }) + .name(format!( + "resolve_compensation_attempt_review_{}", + delivery.review_uid + )) + .await? + .into_inner(); + match disposition { + StorageResolutionDisposition::Delivered => Ok(()), + StorageResolutionDisposition::NotReady => { + Err("execution compensation review owner is not durably parked".to_string()) + } + StorageResolutionDisposition::Failed { message } => Err(message), + } } }; match acknowledged { Ok(()) => { + crate::restate_identity::replay_safe_request( + ctx.service_client::() + .dispatch(Json::from(DispatchExecutionsRequest::default())) + .idempotency_key(format!("execution-action-review:{}", delivery.review_uid)), + ) + .send(); let pool = pool.clone(); let review_uid = delivery.review_uid; let attempt_count = delivery.attempt_count; @@ -245,7 +306,6 @@ async fn deliver_one( } } Err(error) => { - let error = error.to_string(); let pool = pool.clone(); let review_uid = delivery.review_uid; let attempt_count = delivery.attempt_count; diff --git a/crates/moa-orchestrator/src/services/action_reviews.rs b/crates/moa-orchestrator/src/services/action_reviews.rs index 438e116ed..d9a5e7b91 100644 --- a/crates/moa-orchestrator/src/services/action_reviews.rs +++ b/crates/moa-orchestrator/src/services/action_reviews.rs @@ -20,7 +20,7 @@ use moa_observability::{ use moa_wire::session_store::AppendEventRequest; use restate_sdk::prelude::*; use serde::{Deserialize, Serialize}; -use std::sync::Arc; +use std::{sync::Arc, time::Duration}; use uuid::Uuid; use crate::action_reviews::app as action_review_app; @@ -28,9 +28,14 @@ use crate::ctx::RequestHeaders; use crate::handlers::authz_shim::AuthzEnforcer; use crate::objects::session::SessionClient; use crate::objects::worker::WorkerClient; +use crate::services::action_review_dispatcher::{ + ActionReviewDispatcherClient, DispatchActionReviewsRequest, +}; +use crate::services::durable_timeout::{DurableTimeoutRequest, schedule_durable_timeout}; use crate::services::session_store::RestateSessionStoreClient; use crate::services::tool_executor::{ - ExecutionToolCallOrigin, ExecutionToolCallOutcome, ExecutionToolCallRequest, ToolExecutorClient, + ExecutionToolCallOrigin, ExecutionToolCallOutcome, ExecutionToolCallPhase, + ExecutionToolCallRequest, ToolExecutorClient, }; use crate::workflows::errors::moa_error_to_handler_error; use moa_core::traits::SessionEventLookupStore; @@ -67,6 +72,8 @@ pub struct ActionReviewSummary { pub deny_reason: Option, /// Creation timestamp. pub created_at: DateTime, + /// Exact durable timeout persisted with the review. + pub expires_at: DateTime, /// Decision timestamp, when present. pub decided_at: Option>, } @@ -115,6 +122,18 @@ pub struct SettleExecutionActionReviewRequest { pub owner: ActionReviewOwner, } +/// Exact execution-owned review whose bounded owner durably parked before acknowledgement. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct AcknowledgeExecutionActionReviewRequest { + /// Tenant that owns the review row. + pub tenant_id: TenantId, + /// Stable review identifier returned by durable review admission. + pub review_id: Uuid, + /// Exact task or compensation owner parked under its generation fence. + pub owner: ActionReviewOwner, +} + /// Durable settlement chosen while holding the action-review row lock. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case", deny_unknown_fields)] @@ -156,6 +175,11 @@ pub trait ActionReviews { async fn settle_execution_owner_review( request: Json, ) -> Result, HandlerError>; + + /// Makes an execution review decision-ready only after its owner park CAS committed. + async fn acknowledge_execution_owner_review( + request: Json, + ) -> Result<(), HandlerError>; } /// Settles an execution-owned action review against the durable row state. @@ -236,8 +260,9 @@ impl ActionReviews for ActionReviewsImpl { .name("action_reviews_request") .await? .into_inner(); - let owner_needs_registration = - !stored.owner_registered && stored.summary.status == ActionReviewStatus::Pending; + let owner_needs_registration = !stored.owner_registered + && stored.summary.status == ActionReviewStatus::Pending + && requires_conversational_registration(&owner); if owner_needs_registration { // The database row is deliberately not decision-ready until the typed // owner has durably acknowledged registration. A crash between these @@ -263,13 +288,32 @@ impl ActionReviews for ActionReviewsImpl { let storage_partition_id = storage_partition_id(stored.summary.tenant_id); let review_id = stored.summary.id; ctx.run(|| async move { - action_review_app::mark_owner_registered(pool, storage_partition_id, review_id) - .await - .map(Json::from) + action_review_app::mark_owner_registered( + pool, + storage_partition_id, + review_id, + None, + ) + .await + .map(Json::from) }) .name("action_reviews_mark_owner_registered") .await?; } + if stored.summary.status == ActionReviewStatus::Pending { + let timeout_secs = u64::try_from(review_timeout_secs).map_err(|error| { + TerminalError::new(format!("action review timeout is invalid: {error}")) + })?; + schedule_durable_timeout( + &ctx, + DurableTimeoutRequest::action_review( + stored.summary.tenant_id, + stored.summary.id, + owner, + ), + Duration::from_secs(timeout_secs), + ); + } if stored.newly_inserted { record_action_review_requested( moa_core::types::action_policy::ActionPolicyEffect::AdminReview, @@ -336,10 +380,13 @@ impl ActionReviews for ActionReviewsImpl { .name("action_reviews_decide") .await? .into_inner(); + let owns_execution_attempt = !decided.owner.is_conversational(); - if let Some(execution_request) = - execution_review_request(&decided.owner, decided.tool_request.as_ref()) - { + if let Some(execution_request) = execution_review_request( + &decided.owner, + decided.review_id, + decided.tool_request.as_ref(), + ) { let execution = crate::restate_identity::replay_safe_request( ctx.service_client::() .execute_execution(Json::from(execution_request)), @@ -401,7 +448,7 @@ impl ActionReviews for ActionReviewsImpl { let mut executed_output: Option = None; if let Some(tool_request) = decided.tool_request.as_ref() { if let Some(execution_request) = - execution_review_request(&decided.owner, Some(tool_request)) + execution_review_request(&decided.owner, decided.review_id, Some(tool_request)) { crate::restate_identity::replay_safe_request( ctx.service_client::() @@ -477,6 +524,15 @@ impl ActionReviews for ActionReviewsImpl { deliver_conversational_resolution(&ctx, receipt).await?; } } + if owns_execution_attempt { + let handle = crate::restate_identity::replay_safe_request( + ctx.service_client::() + .dispatch(Json::from(DispatchActionReviewsRequest::default())) + .idempotency_key(format!("execution-action-review-decision:{review_id}")), + ) + .send(); + let _invocation_id = handle.invocation_id().await?; + } Ok(()) } @@ -499,6 +555,47 @@ impl ActionReviews for ActionReviewsImpl { .name("action_reviews_settle_execution_owner_review") .await?) } + + #[tracing::instrument(skip(self, ctx, request))] + // SAFETY: invoked only after an exact execution attempt generation durably parks its review. + async fn acknowledge_execution_owner_review( + &self, + ctx: Context<'_>, + request: Json, + ) -> Result<(), HandlerError> { + crate::ctx::adopt_incoming_trace_parent(&ctx); + annotate_restate_handler_span("ActionReviews", "acknowledge_execution_owner_review"); + let request = request.into_inner(); + if request.owner.execution_origin().is_none() + && request.owner.compensation_origin().is_none() + { + return Err(TerminalError::new( + "execution review acknowledgement requires a task or compensation owner", + ) + .into()); + } + let pool = self.pool.clone(); + let storage_partition_id = storage_partition_id(request.tenant_id); + ctx.run(|| async move { + action_review_app::mark_owner_registered( + pool, + storage_partition_id, + request.review_id, + Some(&request.owner), + ) + .await + }) + .name("action_reviews_acknowledge_execution_owner") + .await + .map_err(HandlerError::from) + } +} + +fn requires_conversational_registration(owner: &ActionReviewOwner) -> bool { + matches!( + owner, + ActionReviewOwner::Coordinator { .. } | ActionReviewOwner::Worker { .. } + ) } /// Registers one pending conversational review on its typed owner. @@ -621,6 +718,37 @@ async fn release_conversational_review( Ok(()) } +/// Releases a timed-out conversational review without scheduling continuation. +pub(crate) async fn release_timed_out_conversational_review( + ctx: &Context<'_>, + release: moa_core::types::action_policy::ActionReviewRelease, +) -> Result<(), HandlerError> { + let idempotency_key = release.review_id.to_string(); + match release.owner.clone() { + ActionReviewOwner::Coordinator { session_id, .. } => { + crate::restate_identity::replay_safe_request( + ctx.object_client::(session_id.to_string()) + .release_action_review(Json::from(release)) + .idempotency_key(idempotency_key), + ) + .call() + .await?; + } + ActionReviewOwner::Worker { worker_id, .. } => { + crate::restate_identity::replay_safe_request( + ctx.object_client::(worker_id) + .release_action_review(Json::from(release)) + .idempotency_key(idempotency_key), + ) + .call() + .await?; + } + ActionReviewOwner::ExecutionTask { .. } + | ActionReviewOwner::ExecutionCompensation { .. } => {} + } + Ok(()) +} + /// Builds the typed receipt for a conversational owner, or `None` when the /// terminal facts a callback depends on are not durable yet. /// @@ -694,6 +822,13 @@ fn execution_review_resolution( })?, }) } + ExecutionToolCallOutcome::ExternalJob { + external_job_uid, + job, + } => Ok(ExecutionActionReviewResolution::ExternalJob { + external_job_uid, + job, + }), ExecutionToolCallOutcome::UnknownOutcome { message } => { Ok(ExecutionActionReviewResolution::UnknownOutcome { message }) } @@ -705,6 +840,7 @@ fn execution_review_resolution( fn execution_review_request( owner: &ActionReviewOwner, + review_uid: Uuid, tool_request: Option<&moa_core::types::tools::ToolCallRequest>, ) -> Option { let call = tool_request?.clone(); @@ -715,7 +851,11 @@ fn execution_review_request( } ActionReviewOwner::Coordinator { .. } | ActionReviewOwner::Worker { .. } => return None, }; - Some(ExecutionToolCallRequest { call, origin }) + Some(ExecutionToolCallRequest { + call, + origin, + phase: ExecutionToolCallPhase::Reviewed { review_uid }, + }) } /// Loads the terminal fact already durable for one reviewed call. @@ -766,8 +906,13 @@ mod tests { use serde_json::json; use uuid::Uuid; - use super::{executed_terminal_fact, execution_review_request, execution_review_resolution}; - use crate::services::tool_executor::{ExecutionToolCallOrigin, ExecutionToolCallOutcome}; + use super::{ + executed_terminal_fact, execution_review_request, execution_review_resolution, + requires_conversational_registration, + }; + use crate::services::tool_executor::{ + ExecutionToolCallOrigin, ExecutionToolCallOutcome, ExecutionToolCallPhase, + }; use moa_core::types::action_policy::{ToolResultSecurityMetadata, ToolTerminalFact}; use moa_execution::wire::ExecutionActionReviewResolution; @@ -806,6 +951,7 @@ mod tests { run_uid: Uuid::from_u128(10), task_uid: Uuid::from_u128(20), generation: 3, + attempt_generation: 4, }; let call = ToolCallRequest { tool_call_id: ToolCallId::new(), @@ -831,11 +977,16 @@ mod tests { session_id: call.session_id, origin, }; - let request = execution_review_request(&owner, Some(&call)) + let review_uid = Uuid::from_u128(21); + let request = execution_review_request(&owner, review_uid, Some(&call)) .expect("execution provenance should select execution-task dispatch"); assert_eq!(request.call, call); assert_eq!(request.origin, ExecutionToolCallOrigin::Task(origin)); + assert_eq!( + request.phase, + ExecutionToolCallPhase::Reviewed { review_uid } + ); assert!( execution_review_request( &ActionReviewOwner::Coordinator { @@ -843,6 +994,7 @@ mod tests { turn_id: "turn-1".to_string(), generation: 1, }, + Uuid::from_u128(22), Some(&call), ) .is_none() @@ -876,13 +1028,15 @@ mod tests { run_uid: Uuid::from_u128(10), compensation_id: Uuid::from_u128(30), generation: 7, + attempt_generation: 8, }; let owner = ActionReviewOwner::ExecutionCompensation { session_id: call.session_id, origin, }; - let request = execution_review_request(&owner, Some(&call)) + let review_uid = Uuid::from_u128(31); + let request = execution_review_request(&owner, review_uid, Some(&call)) .expect("compensation provenance should select execution dispatch"); assert_eq!(request.call, call); @@ -890,6 +1044,54 @@ mod tests { request.origin, ExecutionToolCallOrigin::Compensation(origin) ); + assert_eq!( + request.phase, + ExecutionToolCallPhase::Reviewed { review_uid } + ); + } + + #[test] + fn execution_review_registration_waits_for_the_attempt_park_offline() { + // Pins: execution-owned reviews cannot become decision-ready in the + // ActionReviews request handler before the attempt's Postgres park CAS. + let session_id = SessionId::new(); + assert!(requires_conversational_registration( + &ActionReviewOwner::Coordinator { + session_id, + turn_id: "turn-1".to_string(), + generation: 1, + } + )); + assert!(requires_conversational_registration( + &ActionReviewOwner::Worker { + session_id, + worker_id: "worker-1".to_string(), + turn_id: "turn-1".to_string(), + generation: 1, + } + )); + assert!(!requires_conversational_registration( + &ActionReviewOwner::ExecutionTask { + session_id, + origin: ExecutionTaskOrigin { + run_uid: Uuid::from_u128(10), + task_uid: Uuid::from_u128(20), + generation: 3, + attempt_generation: 4, + }, + } + )); + assert!(!requires_conversational_registration( + &ActionReviewOwner::ExecutionCompensation { + session_id, + origin: ExecutionCompensationOrigin { + run_uid: Uuid::from_u128(10), + compensation_id: Uuid::from_u128(30), + generation: 7, + attempt_generation: 8, + }, + } + )); } #[test] @@ -932,4 +1134,34 @@ mod tests { ExecutionActionReviewResolution::NotDispatched { reason } ); } + + #[test] + fn execution_review_preserves_pre_admitted_external_job_identity_offline() { + // Pins: review settlement must route the exact MOA-owned job reserved before + // provider dispatch; reconstructing a new UID would orphan capacity and callbacks. + let external_job_uid = Uuid::from_u128(41); + let job = moa_core::types::tools::AsyncToolJob { + provider: "fixture".to_string(), + provider_job_id: "provider-job-41".to_string(), + idempotency_key: "review-job-41".to_string(), + callback_auth_reference: "callback-41".to_string(), + progress_phase: "queued".to_string(), + cancel_supported: true, + next_reconcile_at: chrono::DateTime::UNIX_EPOCH, + }; + + let resolution = execution_review_resolution(ExecutionToolCallOutcome::ExternalJob { + external_job_uid, + job: job.clone(), + }) + .expect("bound execution external job should remain a valid review resolution"); + + assert_eq!( + resolution, + ExecutionActionReviewResolution::ExternalJob { + external_job_uid, + job, + } + ); + } } diff --git a/crates/moa-orchestrator/src/services/action_reviews_reaper.rs b/crates/moa-orchestrator/src/services/action_reviews_reaper.rs index 297f56fbf..116be637c 100644 --- a/crates/moa-orchestrator/src/services/action_reviews_reaper.rs +++ b/crates/moa-orchestrator/src/services/action_reviews_reaper.rs @@ -6,12 +6,19 @@ //! pending-queue depth and oldest-pending-age gauges, mirroring the builtin //! authz challenge reaper's poll-and-emit shape. -use std::time::Duration; +use std::{ + sync::{ + Arc, RwLock, + atomic::{AtomicBool, Ordering}, + }, + time::{Duration, Instant}, +}; use moa_core::{ events::Event, types::action_policy::{ - ActionReviewOwner, ActionReviewStatus, action_review_timed_out_dedupe_key, + ActionReviewOwner, ActionReviewRelease, ActionReviewStatus, + action_review_timed_out_dedupe_key, }, }; use moa_observability::propagation::{ValidatedTraceContext, with_reqwest_validated_trace_headers}; @@ -19,19 +26,42 @@ use moa_observability::{ record_action_review_decision, record_action_review_oldest_pending_age, record_action_review_pending_depth, record_approval_wait, }; +use serde::{Deserialize, Serialize}; use sqlx::PgPool; use thiserror::Error; -use tokio::sync::oneshot; +use tokio::sync::watch; use tokio::time::interval; use crate::action_reviews::store as action_review_store; use crate::services::action_review_dispatcher::{ DispatchActionReviewsRequest, DispatchActionReviewsResponse, }; +use crate::services::durable_timeout::{ + ActionReviewTimeout, DURABLE_TIMEOUT_RECONCILIATION_INTERVAL, +}; use moa_wire::session_store::AppendEventRequest; const OWNER_RELEASE_DISPATCH_BATCH_SIZE: i64 = 32; +/// Durable delivery selected after applying one exact action-review timeout. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "delivery", rename_all = "snake_case", deny_unknown_fields)] +pub enum ActionReviewTimeoutDelivery { + /// The row no longer matches the delayed owner generation/incarnation. + Stale, + /// The exact timeout was already delivered to its conversational owner. + AlreadyDelivered, + /// A conversational owner must have its lifecycle hold released. + Conversational { + /// Timestamp persisted when the review failed closed. + timed_out_at: chrono::DateTime, + /// Exact owner-generation release payload. + release: ActionReviewRelease, + }, + /// An execution outbox row is ready for the Restate-owned dispatcher. + Execution, +} + /// Action-review reaper failures. #[derive(Debug, Error)] pub enum ActionReviewReaperError { @@ -41,6 +71,9 @@ pub enum ActionReviewReaperError { /// The Restate-owned execution-review dispatcher could not be awakened. #[error("Restate action-review dispatcher error: {0}")] Dispatcher(String), + /// The supervised reaper task could not be joined. + #[error("action-review reaper task join failed: {0}")] + Join(String), } /// Background worker that times out expired reviews and samples queue gauges. @@ -57,7 +90,7 @@ impl ActionReviewReaper { pub fn new(pool: PgPool) -> Self { Self { pool, - sweep_interval: Duration::from_secs(30), + sweep_interval: DURABLE_TIMEOUT_RECONCILIATION_INTERVAL, restate_ingress_url: None, client: reqwest::Client::new(), } @@ -68,7 +101,7 @@ impl ActionReviewReaper { pub fn with_restate_ingress(pool: PgPool, restate_ingress_url: String) -> Self { Self { pool, - sweep_interval: Duration::from_secs(30), + sweep_interval: DURABLE_TIMEOUT_RECONCILIATION_INTERVAL, restate_ingress_url: Some(restate_ingress_url.trim_end_matches('/').to_string()), client: reqwest::Client::new(), } @@ -76,28 +109,42 @@ impl ActionReviewReaper { /// Spawn the reaper as a Tokio task. pub fn spawn(self) -> ActionReviewReaperHandle { - let (shutdown, mut shutdown_rx) = oneshot::channel::<()>(); + let health = Arc::new(ActionReviewReaperHealth { + started_at: Instant::now(), + last_success: RwLock::new(None), + exited: AtomicBool::new(false), + }); + let heartbeat_maximum_age = self.sweep_interval.saturating_mul(3); + let (shutdown, mut shutdown_rx) = watch::channel(false); + let task_health = Arc::clone(&health); let task = tokio::spawn(async move { - let mut tick = interval(self.sweep_interval); - loop { - tokio::select! { - biased; - _ = &mut shutdown_rx => { - tracing::info!("action review reaper received shutdown"); - break; + let result = async { + let mut tick = interval(self.sweep_interval); + loop { + tokio::select! { + biased; + _ = shutdown_rx.changed() => { + tracing::info!("action review reaper received shutdown"); + return Ok(()); + } + _ = tick.tick() => {} } - _ = tick.tick() => {} + self.sweep().await?; + // Queue sampling is observational and cannot invalidate a + // completed correctness pass. + let _ = self.sample_gauges().await; + set_action_review_reaper_heartbeat(&task_health); } - if let Err(error) = self.sweep().await { - tracing::error!(error = %error, "action review reaper sweep failed"); - } - // Best-effort in the tick loop; the error is logged inside. - let _ = self.sample_gauges().await; } + .await; + task_health.exited.store(true, Ordering::Release); + result }); ActionReviewReaperHandle { - shutdown: Some(shutdown), + health, + shutdown, task, + heartbeat_maximum_age, } } @@ -109,19 +156,7 @@ impl ActionReviewReaper { resolution_trace_context.as_ref(), ) .await?; - for review in &timed_out { - record_action_review_decision(ActionReviewStatus::Timeout, review.action_class); - let wait = (review.decided_at - review.created_at) - .to_std() - .unwrap_or_default(); - record_approval_wait(review.action_class, wait); - } - if !timed_out.is_empty() { - tracing::warn!( - count = timed_out.len(), - "tenant action reviews timed out and failed closed" - ); - } + record_timed_out_reviews(&timed_out); if self.restate_ingress_url.is_some() { let released = self.dispatch_action_review_releases().await?; if released > 0 { @@ -138,6 +173,31 @@ impl ActionReviewReaper { Ok(timed_out.len()) } + /// Applies one exact delayed timeout and selects its durable delivery. + /// + /// The full persisted owner is compared before any expiry sweep runs. A + /// timer from an older turn, task attempt, or compensation generation is a + /// successful no-op and cannot use a newer row incarnation as its target. + pub async fn apply_timeout( + &self, + timeout: &ActionReviewTimeout, + ) -> Result { + match load_action_review_timeout_state(&self.pool, timeout).await? { + ActionReviewTimeoutState::Stale => return Ok(ActionReviewTimeoutDelivery::Stale), + ActionReviewTimeoutState::Pending => { + let resolution_trace_context = current_trace_context(); + let timed_out = action_review_store::timeout_expired_reviews( + &self.pool, + resolution_trace_context.as_ref(), + ) + .await?; + record_timed_out_reviews(&timed_out); + } + ActionReviewTimeoutState::TimedOut => {} + } + load_action_review_timeout_delivery(&self.pool, timeout).await + } + /// Attempts one bounded batch of persisted owner releases. pub async fn dispatch_action_review_releases(&self) -> Result { let Some(ingress_url) = self.restate_ingress_url.as_deref() else { @@ -297,6 +357,97 @@ impl ActionReviewReaper { } } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ActionReviewTimeoutState { + Stale, + Pending, + TimedOut, +} + +async fn load_action_review_timeout_state( + pool: &PgPool, + timeout: &ActionReviewTimeout, +) -> Result { + let Some(row) = action_review_store::load_action_review_timeout_snapshot( + pool, + action_review_store::ActionReviewTimeoutLookup { + tenant_id: timeout.tenant_id, + review_id: timeout.review_id, + }, + ) + .await? + else { + return Ok(ActionReviewTimeoutState::Stale); + }; + if row.owner != timeout.owner { + return Ok(ActionReviewTimeoutState::Stale); + } + match row.status { + ActionReviewStatus::Pending + if row.is_due && row.owner_registered && row.execution_requested_at.is_none() => + { + Ok(ActionReviewTimeoutState::Pending) + } + ActionReviewStatus::Timeout => Ok(ActionReviewTimeoutState::TimedOut), + _ => Ok(ActionReviewTimeoutState::Stale), + } +} + +async fn load_action_review_timeout_delivery( + pool: &PgPool, + timeout: &ActionReviewTimeout, +) -> Result { + let Some(row) = action_review_store::load_action_review_timeout_snapshot( + pool, + action_review_store::ActionReviewTimeoutLookup { + tenant_id: timeout.tenant_id, + review_id: timeout.review_id, + }, + ) + .await? + else { + return Ok(ActionReviewTimeoutDelivery::Stale); + }; + if row.owner != timeout.owner || row.status != ActionReviewStatus::Timeout { + return Ok(ActionReviewTimeoutDelivery::Stale); + } + if row.owner.is_conversational() { + if row.owner_release_delivered_at.is_some() { + return Ok(ActionReviewTimeoutDelivery::AlreadyDelivered); + } + let Some(timed_out_at) = row.decided_at else { + return Err(sqlx::Error::Protocol( + "timed-out action review has no decision timestamp".to_string(), + )); + }; + return Ok(ActionReviewTimeoutDelivery::Conversational { + timed_out_at, + release: ActionReviewRelease { + review_id: timeout.review_id, + owner: row.owner, + resume_queued: false, + }, + }); + } + Ok(ActionReviewTimeoutDelivery::Execution) +} + +fn record_timed_out_reviews(reviews: &[action_review_store::TimedOutReview]) { + for review in reviews { + record_action_review_decision(ActionReviewStatus::Timeout, review.action_class); + let wait = (review.decided_at - review.created_at) + .to_std() + .unwrap_or_default(); + record_approval_wait(review.action_class, wait); + } + if !reviews.is_empty() { + tracing::warn!( + count = reviews.len(), + "tenant action reviews timed out and failed closed" + ); + } +} + fn current_trace_context() -> Option { let headers = moa_observability::current_trace_headers(); ValidatedTraceContext::from_headers(|name| headers.get(name).cloned()) @@ -304,16 +455,137 @@ fn current_trace_context() -> Option { /// Handle used to stop the action-review reaper. pub struct ActionReviewReaperHandle { - shutdown: Option>, - task: tokio::task::JoinHandle<()>, + health: Arc, + shutdown: watch::Sender, + task: tokio::task::JoinHandle>, + heartbeat_maximum_age: Duration, } impl ActionReviewReaperHandle { - /// Signal shutdown and wait for the task to exit. - pub async fn shutdown(mut self) { - if let Some(shutdown) = self.shutdown.take() { - let _ = shutdown.send(()); + /// Returns a cloneable readiness projection for the supervised reaper. + #[must_use] + pub fn readiness(&self) -> ActionReviewReaperReadiness { + ActionReviewReaperReadiness { + health: Arc::clone(&self.health), + heartbeat_maximum_age: self.heartbeat_maximum_age, + } + } + + /// Waits for the reaper task so unexpected failure can terminate its owner process. + pub async fn task_result(&mut self) -> Result<(), ActionReviewReaperError> { + match (&mut self.task).await { + Ok(result) => result, + Err(error) => Err(ActionReviewReaperError::Join(error.to_string())), + } + } + + /// Signals shutdown and waits for the task to exit. + pub async fn shutdown(mut self) -> Result<(), ActionReviewReaperError> { + let _ = self.shutdown.send(true); + self.task_result().await + } +} + +impl Drop for ActionReviewReaperHandle { + fn drop(&mut self) { + self.health.exited.store(true, Ordering::Release); + let _ = self.shutdown.send(true); + self.task.abort(); + } +} + +/// Cloneable readiness projection for action-review timeout reconciliation. +#[derive(Clone)] +pub struct ActionReviewReaperReadiness { + health: Arc, + heartbeat_maximum_age: Duration, +} + +impl ActionReviewReaperReadiness { + /// Returns the age of the most recent complete successful reconciliation pass. + #[must_use] + pub fn heartbeat_age(&self) -> Duration { + let heartbeat = self + .health + .last_success + .read() + .ok() + .and_then(|value| *value); + heartbeat.map_or_else(|| self.health.started_at.elapsed(), |value| value.elapsed()) + } + + /// Returns why the reaper must not be considered ready. + #[must_use] + pub fn unready_reason(&self) -> Option { + if self.health.exited.load(Ordering::Acquire) { + return Some("action-review reaper exited".to_string()); } - let _ = self.task.await; + reaper_heartbeat_reason( + "action-review reaper", + self.health.started_at, + &self.health.last_success, + self.heartbeat_maximum_age, + ) + } +} + +#[derive(Debug)] +struct ActionReviewReaperHealth { + started_at: Instant, + last_success: RwLock>, + exited: AtomicBool, +} + +fn set_action_review_reaper_heartbeat(health: &ActionReviewReaperHealth) { + if let Ok(mut heartbeat) = health.last_success.write() { + *heartbeat = Some(Instant::now()); + } +} + +/// Returns a bounded readiness reason for a periodic reconciliation owner. +pub(super) fn reaper_heartbeat_reason( + name: &str, + started_at: Instant, + last_success: &RwLock>, + maximum_age: Duration, +) -> Option { + let heartbeat = last_success.read().ok().and_then(|value| *value); + let age = heartbeat.map_or_else(|| started_at.elapsed(), |value| value.elapsed()); + if heartbeat.is_none() { + return Some(format!("{name} has not completed its first pass")); + } + (age > maximum_age).then(|| format!("{name} heartbeat is stale by {:.3}s", age.as_secs_f64())) +} + +#[cfg(test)] +mod readiness_tests { + use super::*; + + #[test] + fn readiness_requires_a_complete_pass_and_rejects_exit() { + // Pins: the maintenance role cannot report ready before this sole + // reconciliation owner succeeds or after it exits. + let health = Arc::new(ActionReviewReaperHealth { + started_at: Instant::now(), + last_success: RwLock::new(None), + exited: AtomicBool::new(false), + }); + let readiness = ActionReviewReaperReadiness { + health: Arc::clone(&health), + heartbeat_maximum_age: Duration::from_secs(60), + }; + assert_eq!( + readiness.unready_reason().as_deref(), + Some("action-review reaper has not completed its first pass") + ); + + set_action_review_reaper_heartbeat(&health); + assert_eq!(readiness.unready_reason(), None); + + health.exited.store(true, Ordering::Release); + assert_eq!( + readiness.unready_reason().as_deref(), + Some("action-review reaper exited") + ); } } diff --git a/crates/moa-orchestrator/src/services/authz_challenges.rs b/crates/moa-orchestrator/src/services/authz_challenges.rs index b9d439f75..b616e9420 100644 --- a/crates/moa-orchestrator/src/services/authz_challenges.rs +++ b/crates/moa-orchestrator/src/services/authz_challenges.rs @@ -11,6 +11,8 @@ use uuid::Uuid; use crate::authz_challenges::app as authz_challenge_app; use crate::authz_challenges::store as authz_challenge_store; use crate::handlers::authz_shim::require_identity; +use crate::services::durable_timeout::{DurableTimeoutRequest, schedule_durable_timeout}; +use crate::workflows::errors::sqlx_error_to_handler_error; /// Async authorization challenge summary returned to users. #[derive(Debug, Clone, Serialize, Deserialize)] @@ -40,6 +42,16 @@ pub struct AuthzChallengeDecisionRequest { pub reason: Option, } +/// Internal request to schedule one builtin challenge's durable timeout. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ScheduleAuthzChallengeTimeoutRequest { + /// Stable challenge row identifier returned by the builtin provider. + pub challenge_id: Uuid, + /// Exact awakeable returned for this challenge incarnation. + pub awakeable_id: String, +} + /// Restate service surface for builtin async authorization challenges. #[restate_sdk::service] #[name = "AuthzChallenges"] @@ -49,6 +61,11 @@ pub trait AuthzChallenges { /// Resolve one async authorization challenge with an approve or deny decision. async fn decide(request: Json) -> Result<(), HandlerError>; + + /// Schedule fail-closed expiry for one newly persisted builtin challenge. + async fn schedule_timeout( + request: Json, + ) -> Result<(), HandlerError>; } /// Concrete async authorization challenge service implementation. @@ -139,6 +156,43 @@ impl AuthzChallenges for AuthzChallengesImpl { .await?; Ok(()) } + + #[tracing::instrument(skip(self, ctx, request))] + // SAFETY: ingress-private creation hook only schedules fail-closed delivery for an exact persisted id/awakeable pair; it cannot approve a challenge. + async fn schedule_timeout( + &self, + ctx: Context<'_>, + request: Json, + ) -> Result<(), HandlerError> { + annotate_restate_handler_span("AuthzChallenges", "schedule_timeout"); + let request = request.into_inner(); + let challenge_id = request.challenge_id; + let awakeable_id = request.awakeable_id; + let lookup = authz_challenge_store::BuiltinChallengeTimeoutLookup { + challenge_id, + awakeable_id: awakeable_id.clone(), + }; + let pool = self.pool.clone(); + let delay = ctx + .run(|| async move { + authz_challenge_store::load_builtin_challenge_timeout_delay(&pool, &lookup) + .await + .map(Json::from) + .map_err(sqlx_error_to_handler_error) + }) + .name("authz_challenges_timeout_delay") + .await? + .into_inner(); + let Some(delay) = delay else { + return Ok(()); + }; + schedule_durable_timeout( + &ctx, + DurableTimeoutRequest::authz_challenge(challenge_id, awakeable_id), + std::time::Duration::from_millis(delay.delay_millis), + ); + Ok(()) + } } fn summary_from_builtin_row(row: BuiltinApprovalRow) -> AuthzChallengeSummary { diff --git a/crates/moa-orchestrator/src/services/authz_challenges_reaper.rs b/crates/moa-orchestrator/src/services/authz_challenges_reaper.rs index fd1f2425e..a2e43c9fa 100644 --- a/crates/moa-orchestrator/src/services/authz_challenges_reaper.rs +++ b/crates/moa-orchestrator/src/services/authz_challenges_reaper.rs @@ -1,7 +1,12 @@ //! Background timeout reaper for builtin async authorization challenges. -use std::sync::Arc; -use std::time::Duration; +use std::{ + sync::{ + Arc, RwLock, + atomic::{AtomicBool, Ordering}, + }, + time::{Duration, Instant}, +}; use moa_authz::{AwakeableResolveError, AwakeableResolver}; use moa_core::traits::ApprovalDecision; @@ -9,12 +14,37 @@ use moa_observability::{ record_builtin_approval_decision, record_builtin_approval_oldest_pending_age, record_builtin_approval_pending_depth, }; +use serde::{Deserialize, Serialize}; use sqlx::PgPool; use thiserror::Error; -use tokio::sync::oneshot; +use tokio::sync::watch; use tokio::time::interval; use crate::authz_challenges::store as authz_challenge_store; +use crate::services::durable_timeout::{ + AuthzChallengeTimeout, DURABLE_TIMEOUT_RECONCILIATION_INTERVAL, +}; + +/// Durable delivery selected after applying one exact authz-challenge timeout. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "delivery", rename_all = "snake_case", deny_unknown_fields)] +pub enum AuthzChallengeTimeoutDelivery { + /// The row no longer matches the delayed challenge incarnation. + Stale, + /// Resolution is complete or is already owned by another exact claim. + AlreadyDelivered, + /// The exact timeout claim must resolve its original awakeable. + Resolve { + /// Stable challenge row identifier. + challenge_id: uuid::Uuid, + /// Exact awakeable fenced by this delayed delivery. + awakeable_id: String, + /// Claim token that must fence the delivery acknowledgement. + resolve_claim_token: uuid::Uuid, + /// Whether this delivery changed the row from pending to timeout. + newly_timed_out: bool, + }, +} /// Authz challenge reaper failures. #[derive(Debug, Error)] @@ -45,6 +75,9 @@ pub enum ReaperError { /// Awakeable resolution through the shared resolver trait failed. #[error("resolve awakeable: {0}")] Resolve(#[from] AwakeableResolveError), + /// The supervised reaper task could not be joined. + #[error("authz-challenge reaper task join failed: {0}")] + Join(String), } /// Background worker that marks expired authz challenges as timed out. @@ -59,34 +92,48 @@ impl AuthzChallengeReaper { pub fn new(pool: PgPool) -> Self { Self { pool, - sweep_interval: Duration::from_secs(30), + sweep_interval: DURABLE_TIMEOUT_RECONCILIATION_INTERVAL, } } /// Spawn the reaper as a Tokio task. pub fn spawn(self, resolver: Arc) -> AuthzChallengeReaperHandle { - let (shutdown, mut shutdown_rx) = oneshot::channel::<()>(); + let health = Arc::new(AuthzChallengeReaperHealth { + started_at: Instant::now(), + last_success: RwLock::new(None), + exited: AtomicBool::new(false), + }); + let heartbeat_maximum_age = self.sweep_interval.saturating_mul(3); + let (shutdown, mut shutdown_rx) = watch::channel(false); + let task_health = Arc::clone(&health); let task = tokio::spawn(async move { - let mut tick = interval(self.sweep_interval); - loop { - tokio::select! { - biased; - _ = &mut shutdown_rx => { - tracing::info!("authz challenge reaper received shutdown"); - break; + let result = async { + let mut tick = interval(self.sweep_interval); + loop { + tokio::select! { + biased; + _ = shutdown_rx.changed() => { + tracing::info!("authz challenge reaper received shutdown"); + return Ok(()); + } + _ = tick.tick() => {} } - _ = tick.tick() => {} + self.sweep(resolver.as_ref()).await?; + // Queue sampling is observational and cannot invalidate a + // completed correctness pass. + let _ = self.sample_gauges().await; + set_authz_challenge_reaper_heartbeat(&task_health); } - if let Err(error) = self.sweep(resolver.as_ref()).await { - tracing::error!(error = %error, "authz challenge reaper sweep failed"); - } - // Best-effort in the tick loop; the error is logged inside. - let _ = self.sample_gauges().await; } + .await; + task_health.exited.store(true, Ordering::Release); + result }); AuthzChallengeReaperHandle { - shutdown: Some(shutdown), + health, + shutdown, task, + heartbeat_maximum_age, } } @@ -173,6 +220,41 @@ impl AuthzChallengeReaper { Ok(resolved_count) } + + /// Applies and claims one exact delayed authz-challenge timeout. + /// + /// Both the row id and original awakeable are compared. A late delivery + /// from an older challenge incarnation therefore returns + /// [`AuthzChallengeTimeoutDelivery::Stale`] without changing or resolving + /// the current row. + pub async fn apply_timeout( + &self, + timeout: &AuthzChallengeTimeout, + ) -> Result { + let request = authz_challenge_store::BuiltinChallengeTimeoutLookup { + challenge_id: timeout.challenge_id, + awakeable_id: timeout.awakeable_id.clone(), + }; + match authz_challenge_store::apply_builtin_challenge_timeout(&self.pool, &request).await? { + authz_challenge_store::BuiltinChallengeTimeoutClaim::Resolve { + challenge_id, + awakeable_id, + resolve_claim_token, + newly_timed_out, + } => Ok(AuthzChallengeTimeoutDelivery::Resolve { + challenge_id, + awakeable_id, + resolve_claim_token, + newly_timed_out, + }), + authz_challenge_store::BuiltinChallengeTimeoutClaim::AlreadyDelivered => { + Ok(AuthzChallengeTimeoutDelivery::AlreadyDelivered) + } + authz_challenge_store::BuiltinChallengeTimeoutClaim::Stale => { + Ok(AuthzChallengeTimeoutDelivery::Stale) + } + } + } } fn missing_awakeable_error(error: &AwakeableResolveError) -> bool { @@ -200,17 +282,90 @@ fn decision_from_unresolved_challenge( /// Handle used to stop the authz challenge reaper. pub struct AuthzChallengeReaperHandle { - shutdown: Option>, - task: tokio::task::JoinHandle<()>, + health: Arc, + shutdown: watch::Sender, + task: tokio::task::JoinHandle>, + heartbeat_maximum_age: Duration, } impl AuthzChallengeReaperHandle { - /// Signal shutdown and wait for the task to exit. - pub async fn shutdown(mut self) { - if let Some(shutdown) = self.shutdown.take() { - let _ = shutdown.send(()); + /// Returns a cloneable readiness projection for the supervised reaper. + #[must_use] + pub fn readiness(&self) -> AuthzChallengeReaperReadiness { + AuthzChallengeReaperReadiness { + health: Arc::clone(&self.health), + heartbeat_maximum_age: self.heartbeat_maximum_age, + } + } + + /// Waits for the reaper task so unexpected failure can terminate its owner process. + pub async fn task_result(&mut self) -> Result<(), ReaperError> { + match (&mut self.task).await { + Ok(result) => result, + Err(error) => Err(ReaperError::Join(error.to_string())), + } + } + + /// Signals shutdown and waits for the task to exit. + pub async fn shutdown(mut self) -> Result<(), ReaperError> { + let _ = self.shutdown.send(true); + self.task_result().await + } +} + +impl Drop for AuthzChallengeReaperHandle { + fn drop(&mut self) { + self.health.exited.store(true, Ordering::Release); + let _ = self.shutdown.send(true); + self.task.abort(); + } +} + +/// Cloneable readiness projection for builtin-authz timeout reconciliation. +#[derive(Clone)] +pub struct AuthzChallengeReaperReadiness { + health: Arc, + heartbeat_maximum_age: Duration, +} + +impl AuthzChallengeReaperReadiness { + /// Returns the age of the most recent complete successful reconciliation pass. + #[must_use] + pub fn heartbeat_age(&self) -> Duration { + let heartbeat = self + .health + .last_success + .read() + .ok() + .and_then(|value| *value); + heartbeat.map_or_else(|| self.health.started_at.elapsed(), |value| value.elapsed()) + } + + /// Returns why the reaper must not be considered ready. + #[must_use] + pub fn unready_reason(&self) -> Option { + if self.health.exited.load(Ordering::Acquire) { + return Some("authz-challenge reaper exited".to_string()); } - let _ = self.task.await; + super::action_reviews_reaper::reaper_heartbeat_reason( + "authz-challenge reaper", + self.health.started_at, + &self.health.last_success, + self.heartbeat_maximum_age, + ) + } +} + +#[derive(Debug)] +struct AuthzChallengeReaperHealth { + started_at: Instant, + last_success: RwLock>, + exited: AtomicBool, +} + +fn set_authz_challenge_reaper_heartbeat(health: &AuthzChallengeReaperHealth) { + if let Ok(mut heartbeat) = health.last_success.write() { + *heartbeat = Some(Instant::now()); } } @@ -301,4 +456,32 @@ mod tests { ApprovalDecision::Timeout ); } + + #[test] + fn readiness_requires_a_complete_pass_and_rejects_exit() { + // Pins: the maintenance role cannot report ready before this sole + // reconciliation owner succeeds or after it exits. + let health = Arc::new(AuthzChallengeReaperHealth { + started_at: Instant::now(), + last_success: RwLock::new(None), + exited: AtomicBool::new(false), + }); + let readiness = AuthzChallengeReaperReadiness { + health: Arc::clone(&health), + heartbeat_maximum_age: Duration::from_secs(60), + }; + assert_eq!( + readiness.unready_reason().as_deref(), + Some("authz-challenge reaper has not completed its first pass") + ); + + set_authz_challenge_reaper_heartbeat(&health); + assert_eq!(readiness.unready_reason(), None); + + health.exited.store(true, Ordering::Release); + assert_eq!( + readiness.unready_reason().as_deref(), + Some("authz-challenge reaper exited") + ); + } } diff --git a/crates/moa-orchestrator/src/services/durable_timeout.rs b/crates/moa-orchestrator/src/services/durable_timeout.rs new file mode 100644 index 000000000..a715b03da --- /dev/null +++ b/crates/moa-orchestrator/src/services/durable_timeout.rs @@ -0,0 +1,329 @@ +//! Shared Restate-delayed timeout delivery for durable approval waits. +//! +//! Normal expiry delivery is one delayed Restate call per persisted wait. The +//! payload carries the immutable persisted owner incarnation, so a late call +//! can only fail closed the wait that originally scheduled it. Process reapers +//! remain a lower-frequency repair path for scheduling or delivery gaps. + +use std::time::Duration; + +use moa_core::types::{ + action_policy::{ActionReviewOwner, ActionReviewRelease}, + identifiers::TenantId, +}; +use moa_observability::restate_observability::annotate_restate_handler_span; +use restate_sdk::prelude::*; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use crate::services::{ + action_review_dispatcher::{ActionReviewDispatcherClient, DispatchActionReviewsRequest}, + action_reviews_reaper::{ActionReviewReaper, ActionReviewTimeoutDelivery}, + authz_challenges_reaper::{AuthzChallengeReaper, AuthzChallengeTimeoutDelivery}, + session_store::RestateSessionStoreClient, +}; +use crate::workflows::errors::sqlx_error_to_handler_error; +use moa_core::{events::Event, traits::ApprovalDecision}; +use moa_wire::session_store::AppendEventRequest; + +/// Low-frequency scanner cadence used only to reconcile missed durable timers. +pub(crate) const DURABLE_TIMEOUT_RECONCILIATION_INTERVAL: Duration = Duration::from_secs(300); + +/// Exact action-review incarnation carried by its delayed timeout. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ActionReviewTimeout { + /// Tenant that owns the persisted review. + pub tenant_id: TenantId, + /// Stable review identifier. + pub review_id: Uuid, + /// Full owner fence, including the originating turn or execution generation. + pub owner: ActionReviewOwner, +} + +/// Exact builtin-authz challenge incarnation carried by its delayed timeout. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct AuthzChallengeTimeout { + /// Stable challenge row identifier. + pub challenge_id: Uuid, + /// Exact Restate awakeable created for this challenge incarnation. + pub awakeable_id: String, +} + +/// One supported durable timeout target. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum DurableTimeoutTarget { + /// Fail one tenant action review closed. + ActionReview(ActionReviewTimeout), + /// Fail one builtin async-authz challenge closed. + AuthzChallenge(AuthzChallengeTimeout), +} + +/// Delayed request accepted by `DurableTimeout/expire`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DurableTimeoutRequest { + /// Immutable idempotency identity for this delayed trigger. + pub trigger_id: Uuid, + /// Persisted wait incarnation that may be expired. + pub target: DurableTimeoutTarget, +} + +impl DurableTimeoutRequest { + /// Builds the timeout trigger for one action-review incarnation. + #[must_use] + pub fn action_review(tenant_id: TenantId, review_id: Uuid, owner: ActionReviewOwner) -> Self { + Self { + trigger_id: review_id, + target: DurableTimeoutTarget::ActionReview(ActionReviewTimeout { + tenant_id, + review_id, + owner, + }), + } + } + + /// Builds the timeout trigger for one builtin-authz challenge incarnation. + #[must_use] + pub fn authz_challenge(challenge_id: Uuid, awakeable_id: String) -> Self { + Self { + trigger_id: challenge_id, + target: DurableTimeoutTarget::AuthzChallenge(AuthzChallengeTimeout { + challenge_id, + awakeable_id, + }), + } + } + + fn has_matching_trigger_id(&self) -> bool { + match &self.target { + DurableTimeoutTarget::ActionReview(timeout) => self.trigger_id == timeout.review_id, + DurableTimeoutTarget::AuthzChallenge(timeout) => { + self.trigger_id == timeout.challenge_id + } + } + } +} + +/// Schedules one replay-safe delayed timeout call. +pub(crate) fn schedule_durable_timeout( + ctx: &Context<'_>, + request: DurableTimeoutRequest, + delay: Duration, +) { + let idempotency_key = format!("durable-timeout:{}", request.trigger_id); + crate::restate_identity::replay_safe_request( + ctx.service_client::() + .expire(Json::from(request)) + .idempotency_key(idempotency_key), + ) + .send_after(delay); +} + +/// Restate service that owns delayed approval timeout delivery. +#[restate_sdk::service] +#[name = "DurableTimeout"] +pub trait DurableTimeout { + /// Delivers one generation/incarnation-fenced timeout. + async fn expire(request: Json) -> Result<(), HandlerError>; +} + +/// PostgreSQL-backed durable timeout implementation. +#[derive(Clone)] +pub struct DurableTimeoutImpl { + pool: sqlx::PgPool, +} + +impl DurableTimeoutImpl { + /// Creates the timeout service over the shared product database. + #[must_use] + pub fn new(pool: sqlx::PgPool) -> Self { + Self { pool } + } +} + +impl DurableTimeout for DurableTimeoutImpl { + #[tracing::instrument(skip(self, ctx, request), fields(trigger_id = %request.0.trigger_id))] + // SAFETY: ingress-private delayed delivery can only fail closed an exact persisted incarnation; mismatches are successful no-ops. + async fn expire( + &self, + ctx: Context<'_>, + request: Json, + ) -> Result<(), HandlerError> { + crate::ctx::adopt_incoming_trace_parent(&ctx); + annotate_restate_handler_span("DurableTimeout", "expire"); + let request = request.into_inner(); + if !request.has_matching_trigger_id() { + tracing::debug!( + trigger_id = %request.trigger_id, + "ignored durable timeout with mismatched trigger identity" + ); + return Ok(()); + } + + match request.target { + DurableTimeoutTarget::ActionReview(timeout) => { + deliver_action_review_timeout(&ctx, self.pool.clone(), timeout).await + } + DurableTimeoutTarget::AuthzChallenge(timeout) => { + deliver_authz_challenge_timeout(&ctx, self.pool.clone(), timeout).await + } + } + } +} + +async fn deliver_action_review_timeout( + ctx: &Context<'_>, + pool: sqlx::PgPool, + timeout: ActionReviewTimeout, +) -> Result<(), HandlerError> { + let release_pool = pool.clone(); + let delivery = ctx + .run(|| async move { + ActionReviewReaper::new(pool) + .apply_timeout(&timeout) + .await + .map(Json::from) + .map_err(sqlx_error_to_handler_error) + }) + .name("durable_timeout_action_review") + .await? + .into_inner(); + + match delivery { + ActionReviewTimeoutDelivery::Stale | ActionReviewTimeoutDelivery::AlreadyDelivered => { + Ok(()) + } + ActionReviewTimeoutDelivery::Execution => { + crate::restate_identity::replay_safe_request( + ctx.service_client::() + .dispatch(Json::from(DispatchActionReviewsRequest::default())), + ) + .call() + .await?; + Ok(()) + } + ActionReviewTimeoutDelivery::Conversational { + timed_out_at, + release, + } => deliver_action_review_release(ctx, release_pool, release, timed_out_at).await, + } +} + +async fn deliver_action_review_release( + ctx: &Context<'_>, + pool: sqlx::PgPool, + release: ActionReviewRelease, + timed_out_at: chrono::DateTime, +) -> Result<(), HandlerError> { + let review_id = release.review_id; + crate::restate_identity::replay_safe_request( + ctx.service_client::() + .append_event(Json(AppendEventRequest { + session_id: release.owner.session_id(), + event: Event::ActionReviewTimedOut { + review_id, + timed_out_at, + }, + dedupe_key: Some( + moa_core::types::action_policy::action_review_timed_out_dedupe_key(review_id), + ), + })), + ) + .call() + .await?; + + crate::services::action_reviews::release_timed_out_conversational_review(ctx, release).await?; + ctx.run(|| async move { + crate::action_reviews::store::mark_action_review_release_delivered(&pool, review_id) + .await + .map_err(sqlx_error_to_handler_error) + }) + .name("durable_timeout_action_review_mark_released") + .await?; + Ok(()) +} + +async fn deliver_authz_challenge_timeout( + ctx: &Context<'_>, + pool: sqlx::PgPool, + timeout: AuthzChallengeTimeout, +) -> Result<(), HandlerError> { + let mark_resolved_pool = pool.clone(); + let delivery = ctx + .run(|| async move { + AuthzChallengeReaper::new(pool) + .apply_timeout(&timeout) + .await + .map(Json::from) + .map_err(sqlx_error_to_handler_error) + }) + .name("durable_timeout_authz_challenge") + .await? + .into_inner(); + + match delivery { + AuthzChallengeTimeoutDelivery::Stale | AuthzChallengeTimeoutDelivery::AlreadyDelivered => { + Ok(()) + } + AuthzChallengeTimeoutDelivery::Resolve { + challenge_id, + awakeable_id, + resolve_claim_token, + newly_timed_out, + } => { + if newly_timed_out { + moa_observability::record_builtin_approval_decision("timeout"); + } + ctx.resolve_awakeable(&awakeable_id, Json::from(ApprovalDecision::Timeout)); + ctx.run(|| async move { + let marked = + crate::authz_challenges::store::mark_claimed_builtin_challenge_resolved( + &mark_resolved_pool, + challenge_id, + resolve_claim_token, + ) + .await + .map_err(sqlx_error_to_handler_error)?; + if !marked { + tracing::debug!( + authz_challenge_id = %challenge_id, + "durable authz timeout acknowledgement lost its exact claim" + ); + } + Ok::<_, HandlerError>(()) + }) + .name("durable_timeout_authz_challenge_mark_resolved") + .await?; + Ok(()) + } + } +} + +#[cfg(test)] +mod tests { + use moa_core::types::identifiers::SessionId; + + use super::*; + + #[test] + fn trigger_identity_is_bound_to_the_persisted_target() { + // Pins: a malformed or replay-substituted trigger id cannot target a different wait. + let request = DurableTimeoutRequest::action_review( + TenantId::new(), + Uuid::from_u128(10), + ActionReviewOwner::Coordinator { + session_id: SessionId::new(), + turn_id: "turn-1".to_string(), + generation: 4, + }, + ); + assert!(request.has_matching_trigger_id()); + + let mut mismatched = request; + mismatched.trigger_id = Uuid::from_u128(11); + assert!(!mismatched.has_matching_trigger_id()); + } +} diff --git a/crates/moa-orchestrator/src/services/execution.rs b/crates/moa-orchestrator/src/services/execution.rs index 2001cdc03..89f3f3166 100644 --- a/crates/moa-orchestrator/src/services/execution.rs +++ b/crates/moa-orchestrator/src/services/execution.rs @@ -7,11 +7,9 @@ use chrono::Utc; use moa_artifacts::document::{ArtifactDefinition, ArtifactKind, ArtifactStatus}; use moa_artifacts::execution_plan::{ CapabilityReference, CompensationInputBinding, CompensationInputMapping, - CompensationValueSource, ExecutionBudgetLimit, ExecutionPlanDefinition, -}; -use moa_artifacts::execution_plan::{ - ExecutionFailureClass, ExecutionTaskOutcome, ExecutionTaskResult, ExecutionUsage, - PlanAmendment, PlanAmendmentOperation, + CompensationValueSource, ExecutionBudgetLimit, ExecutionFailureClass, ExecutionPlanDefinition, + ExecutionTaskOutcome, ExecutionTaskResult, ExecutionUsage, PlanAmendment, + PlanAmendmentOperation, }; use moa_artifacts::reference::ArtifactRef; use moa_artifacts::registry::{ArtifactRegistry, StoredArtifactRevision}; @@ -46,27 +44,27 @@ use moa_execution::capability::{ use moa_execution::{ budget::{BudgetLedger, estimate_fits_limit}, compiler::{CompileExecutionRequest, ValidateAmendmentRequest, compile, validate_amendment}, - completion::{ - CompletionEvaluationRequest, cancellation_terminal_evidence, evaluate_completion, - execution_terminal_reason, terminal_evidence_from_evaluation, - terminal_projection_from_evaluation, - }, + completion::cancellation_terminal_evidence_from_completed_nodes, replan::{ ReplanDecision, ReplanEvaluationRequest, ReplanLoopEvaluationRequest, evaluate_replan_loop_stop, evaluate_replan_resource_stop, evaluate_replan_stop, - failure_fingerprint, replan_stop_gaps, replan_stop_status, + }, + repository::amendment::{ + AmendmentProjectionOutcome, AmendmentProjectionRequest, ExecutionAmendmentSnapshot, }, repository::{ AmendmentReplayOutcome, AmendmentWrite, ConfirmationConflict, ConfirmationOutcome, ExecutionRepository, ExecutionRunPageRequest, ExecutionRunRecord, ExecutionScope, - ExecutionTaskPageRequest, ExecutionTaskRecord, NewExecutionPlanningContext, - NewExecutionRun, PlanningContextWriteOutcome, ReplanStopReceipt, TaskOutcomeWrite, - TerminalFenceOutcome, TransitionOutcome, TransitionRejection, ValidatedAmendment, + ExecutionTaskPageRequest, ExecutionTaskRecord, NewExecutionRun, TaskOutcomeWrite, + TransitionOutcome, TransitionRejection, ValidatedAmendment, + audit::{NewExecutionPlanningContext, PlanningContextWriteOutcome}, + replan_stop::{NewExecutionReplanStopIntent, ReplanStopIntentWriteOutcome}, + terminal::PendingTerminalAdvanceOutcome, }, schema::validate_instance, state::{ - ExecutionRunStatus, ExecutionTaskProjection, ExecutionTaskStatus, ExecutionTerminalCause, - ExecutionTerminalReason, FailureFingerprintInput, PendingExecutionTerminal, + ExecutionRunStatus, ExecutionTaskProjection, ExecutionTaskStatus, ExecutionTerminalReason, + FailureFingerprintInput, PendingExecutionTerminal, }, wire::{ ExecutionAmendmentRequest, ExecutionCancelRequest, ExecutionConfirmRequest, @@ -74,8 +72,7 @@ use moa_execution::{ ExecutionPlanningContextRequest, ExecutionPlanningContextResponse, ExecutionPlanningContextSnapshot, ExecutionReviewDecision, ExecutionReviewDecisionRequest, ExecutionRunCursor, ExecutionRunListRequest, ExecutionRunListResponse, ExecutionRunRequest, - ExecutionRunSummary, ExecutionRunWakeReason, ExecutionRunWakeRequest, - ExecutionSignalRequest, ExecutionStartRequest, ExecutionStartResponse, + ExecutionRunSummary, ExecutionSignalRequest, ExecutionStartRequest, ExecutionStartResponse, ExecutionStatusResponse, ExecutionSynthesisEvidence, ExecutionSynthesisEvidenceRequest, ExecutionTaskCursor, ExecutionTaskListRequest, ExecutionTaskListResponse, PinnedExecutionTemplate, PinnedInstructionSkill, decode_cursor, encode_cursor, @@ -98,10 +95,48 @@ use crate::connector_catalog::ScopedConnectorCatalogProvider; use crate::handlers::authz_shim::AuthzEnforcer; use crate::objects::session::{ExecutionRunStartedDelivery, SessionClient}; use crate::restate_identity::with_identity_headers; -use crate::services::llm_gateway::{LLMCompletionOwner, cancel_completion_owner_from_service}; use crate::workflows::errors::moa_error_to_status_handler_error; -use crate::workflows::execution_run::ExecutionRunClient; -use crate::workflows::execution_task::ExecutionTaskClient; + +/// Authorized compare-and-set request for pausing or resuming one durable run. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ExecutionRunControlRequest { + /// Exact caller-owned run scope. + pub run: ExecutionRunRequest, + /// Current controller generation displayed to the caller. + pub expected_controller_generation: u64, +} + +/// Public pause/resume result including the exact durable controller fence. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[serde(tag = "result", rename_all = "snake_case", deny_unknown_fields)] +pub enum ExecutionRunControlResponse { + /// The control mutation changed durable state. + Applied { + /// Current run projection. + run: ExecutionRunSummary, + /// Exact current controller generation. + controller_generation: u64, + /// Exact current wake epoch. + wake_epoch: u64, + }, + /// The exact generation-fenced mutation was already committed. + Replayed { + /// Current run projection. + run: ExecutionRunSummary, + /// Exact current controller generation. + controller_generation: u64, + /// Exact current wake epoch. + wake_epoch: u64, + }, + /// A stable compare-and-set or lifecycle conflict changed nothing. + Conflict { + /// Stable conflict reason. + reason: ExecutionConflictReason, + }, + /// No scoped run exists. + NotFound, +} /// Restate service surface for durable execution-run operations. #[restate_sdk::service] @@ -147,6 +182,16 @@ pub trait Execution { request: Json, ) -> Result, HandlerError>; + /// Fences new reservations and drains active bounded attempts before parking the run. + async fn pause( + request: Json, + ) -> Result, HandlerError>; + + /// Generation-bumps one fully drained paused run and enqueues exactly one activation. + async fn resume( + request: Json, + ) -> Result, HandlerError>; + /// Delivers audience-authorized input to one waiting task generation. async fn deliver_input( request: Json, @@ -167,11 +212,6 @@ pub trait Execution { request: Json, ) -> Result, HandlerError>; - /// Applies a workflow-generated amendment using only its persisted run scope. - async fn apply_planned_amendment( - request: Json, - ) -> Result, HandlerError>; - /// Lists the tenant's currently invocable compiler capabilities. async fn list_capabilities( request: Json, diff --git a/crates/moa-orchestrator/src/services/execution/capability_catalog.rs b/crates/moa-orchestrator/src/services/execution/capability_catalog.rs index 357a70df2..04a442968 100644 --- a/crates/moa-orchestrator/src/services/execution/capability_catalog.rs +++ b/crates/moa-orchestrator/src/services/execution/capability_catalog.rs @@ -408,7 +408,9 @@ pub(super) fn registered_tool_capability( risk_level: definition.policy.risk_level, default_effect: definition.policy.default_effect, idempotency_class: definition.idempotency_class, + async_mode: definition.async_mode.clone(), execution_class, + requires_sandbox: matches!(execution, ToolExecution::Hand { .. }), source, policy_context, estimate: single_tool_estimate(definition.max_output_tokens), @@ -535,7 +537,9 @@ pub(super) fn action_capability( risk_level: definition.policy.risk_level, default_effect: artifact_effect(admin_review_required, &definition.policy), idempotency_class: definition.idempotency_class, + async_mode: definition.async_mode.clone(), execution_class: execution_class(execution, definition), + requires_sandbox: matches!(execution, ToolExecution::Hand { .. }), source, policy_context, estimate: single_tool_estimate(definition.max_output_tokens), @@ -635,7 +639,9 @@ pub(super) fn append_skill_action(context: SkillActionContext<'_>) -> moa_execut risk_level: definition.policy.risk_level, default_effect: definition.policy.default_effect, idempotency_class: definition.idempotency_class, + async_mode: definition.async_mode.clone(), execution_class: execution_class(execution, definition), + requires_sandbox: matches!(execution, ToolExecution::Hand { .. }), source, policy_context, estimate: single_tool_estimate(definition.max_output_tokens), diff --git a/crates/moa-orchestrator/src/services/execution/handlers.rs b/crates/moa-orchestrator/src/services/execution/handlers.rs index d7d659535..ae1b8eb43 100644 --- a/crates/moa-orchestrator/src/services/execution/handlers.rs +++ b/crates/moa-orchestrator/src/services/execution/handlers.rs @@ -1,7 +1,7 @@ //! Restate execution service handlers and durable mutation operations. use super::capability_catalog::list_capabilities_inner; -use super::planning_context::planning_context_inner; +use super::planning_context::{PlanningContextInput, planning_context_inner}; use super::start::start_inner; use super::support::*; use super::*; @@ -81,21 +81,23 @@ impl Execution for ExecutionImpl { let connector_catalog = self.connector_catalog.clone(); let catalog_identity = identity.clone(); let config = self.config.clone(); + let planning_admitted_at = event_record.timestamp; Ok(ctx .run(|| async move { let scoped_catalog = connector_catalog .for_session(&catalog_identity, &parent) .await .map_err(scoped_catalog_error)?; - planning_context_inner( + planning_context_inner(PlanningContextInput { pool, - scoped_catalog.snapshot().capability_registrations(), + registrations: scoped_catalog.snapshot().capability_registrations(), config, parent, owner_user_id, - event_record.event, + originating_event: event_record.event, + planning_admitted_at, request, - ) + }) .await .map(Json::from) }) @@ -163,9 +165,10 @@ impl Execution for ExecutionImpl { }; let pool = self.pool.clone(); let config = self.config.clone(); + let admitted_identity = identity.clone(); let response = ctx .run(|| async move { - start_inner(pool, config, request, objective) + start_inner(pool, config, request, objective, admitted_identity) .await .map(Json::from) }) @@ -180,6 +183,11 @@ impl Execution for ExecutionImpl { &identity, ) .send(); + if !response.confirmation_required { + // A newly admitted run always starts at controller generation zero. The durable + // RunActivation outbox remains authoritative; this stable kick only reduces latency. + kick_execution_dispatcher(&ctx, response.run.run_uid, 0, "start").await?; + } Ok(Json::from(response)) } @@ -203,16 +211,8 @@ impl Execution for ExecutionImpl { .await? .into_inner(); if let Some(wake_epoch) = accepted.wake_epoch() { - let handoff_started = std::time::Instant::now(); pause_execution_mutation_handoff_for_test().await; - call_run_wake( - &ctx, - run_request.run_uid, - wake_epoch, - ExecutionRunWakeReason::Confirmed, - handoff_started, - ) - .await?; + kick_execution_dispatcher(&ctx, run_request.run_uid, wake_epoch, "confirm").await?; } Ok(Json::from(accepted.into_response())) } @@ -311,31 +311,67 @@ impl Execution for ExecutionImpl { .await?; let run_request = request.run.clone(); let pool = self.pool.clone(); + let config = self.config.clone(); let accepted = ctx - .run(|| async move { cancel_inner(pool, request).await.map(Json::from) }) + .run(|| async move { cancel_inner(pool, config, request).await.map(Json::from) }) .name("execution_cancel") .await? .into_inner(); if let Some(wake_epoch) = accepted.wake_epoch() { - let handoff_started = std::time::Instant::now(); pause_execution_mutation_handoff_for_test().await; - cancel_completion_owner_from_service( - &ctx, - LLMCompletionOwner::execution_run(run_request.run_uid.to_string()), - ) - .await?; - call_run_wake( - &ctx, - run_request.run_uid, - wake_epoch, - ExecutionRunWakeReason::Cancelled, - handoff_started, - ) - .await?; + kick_execution_dispatcher(&ctx, run_request.run_uid, wake_epoch, "cancel").await?; } Ok(Json::from(accepted.into_response())) } + #[tracing::instrument(skip(self, ctx, request))] + async fn pause( + &self, + ctx: Context<'_>, + request: Json, + ) -> Result, HandlerError> { + crate::ctx::adopt_incoming_trace_parent(&ctx); + annotate_restate_handler_span("Execution", "pause"); + let request = request.into_inner(); + self.authz + .authorize_session_participant(&ctx, request.run.session_id) + .await?; + let run_uid = request.run.run_uid; + let pool = self.pool.clone(); + let config = self.config.clone(); + let response = ctx + .run(|| async move { pause_inner(pool, config, request).await.map(Json::from) }) + .name("execution_pause") + .await? + .into_inner(); + kick_control_dispatcher(&ctx, run_uid, "pause", &response).await?; + Ok(Json::from(response)) + } + + #[tracing::instrument(skip(self, ctx, request))] + async fn resume( + &self, + ctx: Context<'_>, + request: Json, + ) -> Result, HandlerError> { + crate::ctx::adopt_incoming_trace_parent(&ctx); + annotate_restate_handler_span("Execution", "resume"); + let request = request.into_inner(); + self.authz + .authorize_session_participant(&ctx, request.run.session_id) + .await?; + let run_uid = request.run.run_uid; + let pool = self.pool.clone(); + let config = self.config.clone(); + let response = ctx + .run(|| async move { resume_inner(pool, config, request).await.map(Json::from) }) + .name("execution_resume") + .await? + .into_inner(); + kick_control_dispatcher(&ctx, run_uid, "resume", &response).await?; + Ok(Json::from(response)) + } + #[tracing::instrument(skip(self, ctx, request))] async fn deliver_input( &self, @@ -363,45 +399,19 @@ impl Execution for ExecutionImpl { } let task_request = request.clone(); let pool = self.pool.clone(); + let config = self.config.clone(); let accepted = ctx - .run(|| async move { deliver_input_inner(pool, request).await.map(Json::from) }) + .run(|| async move { + deliver_input_inner(pool, config, request) + .await + .map(Json::from) + }) .name("execution_deliver_input") .await? .into_inner(); if let Some(wake_epoch) = accepted.wake_epoch() { - let handoff_started = std::time::Instant::now(); pause_execution_mutation_handoff_for_test().await; - if accepted - .task_ids_to_release() - .contains(&task_request.task_id) - { - // The terminal task fence is already committed, and this detached - // cancellation releases a parked promise without claiming completion. - crate::restate_identity::replay_safe_request( - ctx.workflow_client::(task_request.task_id.to_string()) - .cancel(Json::from( - "execution input redispatch reached a terminal admission outcome" - .to_string(), - )), - ) - .send(); - } else { - crate::restate_identity::replay_safe_request( - ctx.workflow_client::(task_request.task_id.to_string()) - .input_delivered(Json::from(task_request.clone())), - ) - .call() - .await - .map_err(HandlerError::from)?; - } - call_run_wake( - &ctx, - task_request.run_uid, - wake_epoch, - ExecutionRunWakeReason::InputDelivered, - handoff_started, - ) - .await?; + kick_execution_dispatcher(&ctx, task_request.run_uid, wake_epoch, "input").await?; } Ok(Json::from(accepted.into_response())) } @@ -420,29 +430,19 @@ impl Execution for ExecutionImpl { .await?; let task_request = request.clone(); let pool = self.pool.clone(); + let config = self.config.clone(); let accepted = ctx - .run(|| async move { decide_review_inner(pool, request).await.map(Json::from) }) + .run(|| async move { + decide_review_inner(pool, config, request) + .await + .map(Json::from) + }) .name("execution_decide_review") .await? .into_inner(); if let Some(wake_epoch) = accepted.wake_epoch() { - let handoff_started = std::time::Instant::now(); pause_execution_mutation_handoff_for_test().await; - crate::restate_identity::replay_safe_request( - ctx.workflow_client::(task_request.task_id.to_string()) - .review_decided(Json::from(task_request.clone())), - ) - .call() - .await - .map_err(HandlerError::from)?; - call_run_wake( - &ctx, - task_request.run_uid, - wake_epoch, - ExecutionRunWakeReason::ReviewDecided, - handoff_started, - ) - .await?; + kick_execution_dispatcher(&ctx, task_request.run_uid, wake_epoch, "review").await?; } Ok(Json::from(accepted.into_response())) } @@ -461,29 +461,19 @@ impl Execution for ExecutionImpl { .await?; let task_request = request.clone(); let pool = self.pool.clone(); + let config = self.config.clone(); let accepted = ctx - .run(|| async move { deliver_signal_inner(pool, request).await.map(Json::from) }) + .run(|| async move { + deliver_signal_inner(pool, config, request) + .await + .map(Json::from) + }) .name("execution_deliver_signal") .await? .into_inner(); if let Some(wake_epoch) = accepted.wake_epoch() { - let handoff_started = std::time::Instant::now(); pause_execution_mutation_handoff_for_test().await; - crate::restate_identity::replay_safe_request( - ctx.workflow_client::(task_request.task_id.to_string()) - .signal_delivered(Json::from(task_request.clone())), - ) - .call() - .await - .map_err(HandlerError::from)?; - call_run_wake( - &ctx, - task_request.run_uid, - wake_epoch, - ExecutionRunWakeReason::SignalDelivered, - handoff_started, - ) - .await?; + kick_execution_dispatcher(&ctx, task_request.run_uid, wake_epoch, "signal").await?; } Ok(Json::from(accepted.into_response())) } @@ -513,64 +503,8 @@ impl Execution for ExecutionImpl { .await? .into_inner(); if let Some(wake_epoch) = accepted.wake_epoch() { - let handoff_started = std::time::Instant::now(); pause_execution_mutation_handoff_for_test().await; - call_run_wake( - &ctx, - run_uid, - wake_epoch, - ExecutionRunWakeReason::AmendmentAccepted, - handoff_started, - ) - .await?; - } - // The amendment transaction already fenced these tasks. Cancellation only - // releases their parked promises and does not claim task completion. - for task_id in accepted.task_ids_to_release() { - crate::restate_identity::replay_safe_request( - ctx.workflow_client::(task_id.to_string()) - .cancel(Json::from( - "execution task superseded or stopped by amendment".to_string(), - )), - ) - .send(); - } - Ok(Json::from(accepted.into_response())) - } - - #[tracing::instrument(skip(self, ctx, request))] - // SAFETY: called only by the keyed ExecutionRun workflow; the request carries no authority and apply_amendment_inner reloads and revision-fences all persisted scope. - async fn apply_planned_amendment( - &self, - ctx: Context<'_>, - request: Json, - ) -> Result, HandlerError> { - crate::ctx::adopt_incoming_trace_parent(&ctx); - annotate_restate_handler_span("Execution", "apply_planned_amendment"); - let request = request.into_inner(); - let pool = self.pool.clone(); - let config = self.config.clone(); - let accepted = ctx - .run(|| async move { - apply_amendment_inner(pool, config, request) - .await - .map(Json::from) - }) - .name("execution_apply_planned_amendment") - .await? - .into_inner(); - // The owning ExecutionRun awaits this mutation and immediately continues - // its drive loop, so a self-wake would be redundant. - // The committed amendment also fences released tasks; these detached - // cancellations only release parked promises and claim no completion. - for task_id in accepted.task_ids_to_release() { - crate::restate_identity::replay_safe_request( - ctx.workflow_client::(task_id.to_string()) - .cancel(Json::from( - "execution task superseded or stopped by amendment".to_string(), - )), - ) - .send(); + kick_execution_dispatcher(&ctx, run_uid, wake_epoch, "amendment").await?; } Ok(Json::from(accepted.into_response())) } @@ -616,7 +550,7 @@ pub(super) async fn confirm_inner( let repository = ExecutionRepository::new(pool); let scope = execution_scope(request.run.tenant_id, request.run.contact_id); let Some(run) = repository - .load_run(scope, request.run.run_uid) + .load_run_for_session(scope, request.run.run_uid, request.run.session_id) .await .map_err(execution_error)? else { @@ -651,7 +585,7 @@ pub(super) async fn status_inner( let repository = ExecutionRepository::new(pool); let scope = execution_scope(request.tenant_id, request.contact_id); let run = repository - .load_run(scope, request.run_uid) + .load_run_for_session(scope, request.run_uid, request.session_id) .await .map_err(execution_error)? .ok_or_else(|| TerminalError::new_with_code(404, "execution run not found"))?; @@ -671,7 +605,7 @@ pub(super) async fn synthesis_evidence_inner( let repository = ExecutionRepository::new(pool); let scope = execution_scope(request.run.tenant_id, request.run.contact_id); let run = repository - .load_run(scope, request.run.run_uid) + .load_run_for_session(scope, request.run.run_uid, request.run.session_id) .await .map_err(execution_error)? .ok_or_else(|| TerminalError::new_with_code(404, "execution run not found"))?; @@ -753,7 +687,7 @@ pub(super) async fn list_tasks_inner( let repository = ExecutionRepository::new(pool); let scope = execution_scope(request.run.tenant_id, request.run.contact_id); let run = repository - .load_run(scope, request.run.run_uid) + .load_run_for_session(scope, request.run.run_uid, request.run.session_id) .await .map_err(execution_error)? .ok_or_else(|| TerminalError::new_with_code(404, "execution run not found"))?; @@ -810,32 +744,41 @@ pub(super) async fn list_tasks_inner( pub(super) async fn cancel_inner( pool: sqlx::PgPool, + config: moa_config::ExecutionConfig, request: ExecutionCancelRequest, ) -> Result { let repository = ExecutionRepository::new(pool); let scope = execution_scope(request.run.tenant_id, request.run.contact_id); - let Some(snapshot) = repository - .load_scheduling_snapshot(scope, request.run.run_uid) + let Some(cancellation) = repository + .load_cancellation_projection_for_session( + scope, + request.run.run_uid, + request.run.session_id, + ) .await .map_err(execution_error)? else { return Ok(not_found_mutation()); }; - verify_run_request(&snapshot.run, &request.run)?; - if snapshot.run.status == ExecutionRunStatus::Cancelled { - return Ok(replayed_mutation(&snapshot.run)); + verify_run_request(&cancellation.run, &request.run)?; + if cancellation.run.status == ExecutionRunStatus::Cancelled { + return Ok(replayed_mutation(&cancellation.run)); } - if let Some(pending) = &snapshot.run.pending_terminal { + if let Some(pending) = &cancellation.run.pending_terminal { return Ok(if pending.status == ExecutionRunStatus::Cancelled { - replayed_mutation(&snapshot.run) + replayed_mutation(&cancellation.run) } else { conflict_mutation(ExecutionConflictReason::AlreadyTerminal) }); } - let terminal_evidence = cancellation_terminal_evidence( - &snapshot.run.goal, - &snapshot.run.active_plan, - &snapshot.projection, + let completed_node_ids = cancellation + .completed_node_ids + .into_iter() + .collect::>(); + let terminal_evidence = cancellation_terminal_evidence_from_completed_nodes( + &cancellation.run.goal, + &cancellation.run.active_plan, + &completed_node_ids, ) .map_err(execution_error)?; let pending_terminal = PendingExecutionTerminal { @@ -849,37 +792,178 @@ pub(super) async fn cancel_inner( }; Ok( match repository - .fence_run_for_terminal( + .fence_completion_terminal_and_enqueue_settlement( + &config, scope, - snapshot.run.run_uid, - snapshot.run.plan_revision, - snapshot.run.wake_epoch, + cancellation.run.run_uid, + cancellation.run.controller_generation, + cancellation.run.wake_epoch, pending_terminal, + chrono::Utc::now(), + u32::try_from( + config + .maximum_activation_steps + .min(config.max_in_flight_tasks) + .min(1_000), + ) + .map_err(|_| invalid_execution_request("terminal page limit exceeds u32"))?, ) .await .map_err(execution_error)? { - TerminalFenceOutcome::Applied(commit) => applied_mutation(&commit.run), - TerminalFenceOutcome::Replayed(commit) => replayed_mutation(&commit.run), - TerminalFenceOutcome::NotFound => not_found_mutation(), - TerminalFenceOutcome::Conflict => { + PendingTerminalAdvanceOutcome::Applied(commit) => applied_mutation(&commit.run), + PendingTerminalAdvanceOutcome::Replayed(commit) => replayed_mutation(&commit.run), + PendingTerminalAdvanceOutcome::NotFound => not_found_mutation(), + PendingTerminalAdvanceOutcome::Conflict => { conflict_mutation(ExecutionConflictReason::AlreadyTerminal) } }, ) } -pub(super) async fn deliver_input_inner( +pub(super) async fn pause_inner( pool: sqlx::PgPool, - request: ExecutionInputRequest, -) -> Result { + config: moa_config::ExecutionConfig, + request: ExecutionRunControlRequest, +) -> Result { + run_control_inner(pool, config, request, true).await +} + +pub(super) async fn resume_inner( + pool: sqlx::PgPool, + config: moa_config::ExecutionConfig, + request: ExecutionRunControlRequest, +) -> Result { + run_control_inner(pool, config, request, false).await +} + +async fn run_control_inner( + pool: sqlx::PgPool, + config: moa_config::ExecutionConfig, + request: ExecutionRunControlRequest, + pause: bool, +) -> Result { + if request.expected_controller_generation == 0 { + return Err(invalid_execution_request( + "expected_controller_generation must be positive", + )); + } let repository = ExecutionRepository::new(pool); - let scope = execution_scope(request.tenant_id, request.contact_id); + let scope = execution_scope(request.run.tenant_id, request.run.contact_id); let Some(run) = repository - .load_run(scope, request.run_uid) + .load_run_for_session(scope, request.run.run_uid, request.run.session_id) .await .map_err(execution_error)? else { + return Ok(ExecutionRunControlResponse::NotFound); + }; + verify_run_request(&run, &request.run)?; + let outcome = if pause { + repository + .pause_run( + scope, + &config, + request.run.run_uid, + request.expected_controller_generation, + ) + .await + } else { + repository + .resume_run( + scope, + &config, + request.run.run_uid, + request.expected_controller_generation, + ) + .await + } + .map_err(execution_error)?; + Ok(match outcome { + TransitionOutcome::RunApplied(run) => ExecutionRunControlResponse::Applied { + run: run_summary(&run), + controller_generation: run.controller_generation, + wake_epoch: run.wake_epoch, + }, + TransitionOutcome::RunAlreadyApplied(run) => ExecutionRunControlResponse::Replayed { + run: run_summary(&run), + controller_generation: run.controller_generation, + wake_epoch: run.wake_epoch, + }, + TransitionOutcome::NotFound => ExecutionRunControlResponse::NotFound, + TransitionOutcome::Rejected(TransitionRejection::GenerationMismatch) => { + ExecutionRunControlResponse::Conflict { + reason: ExecutionConflictReason::GenerationMismatch, + } + } + TransitionOutcome::Rejected(_) => ExecutionRunControlResponse::Conflict { + reason: ExecutionConflictReason::InvalidStatus, + }, + TransitionOutcome::Applied(_) | TransitionOutcome::AlreadyApplied(_) => { + ExecutionRunControlResponse::Conflict { + reason: ExecutionConflictReason::InvalidStatus, + } + } + }) +} + +async fn kick_control_dispatcher( + ctx: &Context<'_>, + run_uid: uuid::Uuid, + action: &str, + response: &ExecutionRunControlResponse, +) -> Result<(), HandlerError> { + let generation = match response { + ExecutionRunControlResponse::Applied { + controller_generation, + .. + } + | ExecutionRunControlResponse::Replayed { + controller_generation, + .. + } => *controller_generation, + ExecutionRunControlResponse::Conflict { .. } | ExecutionRunControlResponse::NotFound => { + return Ok(()); + } + }; + kick_execution_dispatcher(ctx, run_uid, generation, action).await +} + +async fn kick_execution_dispatcher( + ctx: &Context<'_>, + run_uid: uuid::Uuid, + durable_fence: u64, + action: &str, +) -> Result<(), HandlerError> { + use crate::services::execution_dispatcher::{ + DispatchExecutionsRequest, ExecutionDispatcherClient, + }; + let handle = crate::restate_identity::replay_safe_request( + ctx.service_client::() + .dispatch(Json::from(DispatchExecutionsRequest::default())) + .idempotency_key(format!("{run_uid}:{durable_fence}:{action}")), + ) + .send(); + let _invocation_id = handle.invocation_id().await?; + Ok(()) +} + +pub(super) async fn deliver_input_inner( + pool: sqlx::PgPool, + config: moa_config::ExecutionConfig, + request: ExecutionInputRequest, +) -> Result { + let repository = ExecutionRepository::new(pool); + let scope = execution_scope(request.tenant_id, request.contact_id); + let run = match request.session_id { + Some(session_id) => { + repository + .load_run_for_session(scope, request.run_uid, session_id) + .await + } + None => repository.load_run(scope, request.run_uid).await, + } + .map_err(execution_error)?; + let Some(run) = run else { return Ok(not_found_mutation()); }; if run.tenant_id != request.tenant_id @@ -909,6 +993,7 @@ pub(super) async fn deliver_input_inner( let transition = repository .resume_task_with_input( scope, + &config, run.run_uid, task.task_id, request.expected_generation, @@ -921,6 +1006,7 @@ pub(super) async fn deliver_input_inner( pub(super) async fn decide_review_inner( pool: sqlx::PgPool, + config: moa_config::ExecutionConfig, request: ExecutionReviewDecisionRequest, ) -> Result { let repository = ExecutionRepository::new(pool); @@ -942,12 +1028,13 @@ pub(super) async fn decide_review_inner( else { return Ok(not_found_mutation()); }; - if !matches!( - task.kind, - moa_execution::state::LogicalTaskKind::Review { .. } - ) { - return Ok(conflict_mutation(ExecutionConflictReason::InvalidStatus)); - } + let _wait_policy = match &task.kind { + moa_execution::state::LogicalTaskKind::Review { + prompt: _, + wait_policy, + } => wait_policy, + _ => return Ok(conflict_mutation(ExecutionConflictReason::InvalidStatus)), + }; let result = match request.decision { ExecutionReviewDecision::Approved { payload } => ExecutionTaskResult::Completed { output: payload, @@ -960,6 +1047,7 @@ pub(super) async fn decide_review_inner( }; external_wait_mutation( &repository, + &config, scope, &run, &task, @@ -971,6 +1059,7 @@ pub(super) async fn decide_review_inner( pub(super) async fn deliver_signal_inner( pool: sqlx::PgPool, + config: moa_config::ExecutionConfig, request: ExecutionSignalRequest, ) -> Result { let repository = ExecutionRepository::new(pool); @@ -992,16 +1081,19 @@ pub(super) async fn deliver_signal_inner( else { return Ok(not_found_mutation()); }; - let signal_matches = matches!( - &task.kind, - moa_execution::state::LogicalTaskKind::WaitSignal { signal_name } - if signal_name == &request.signal_name - ); + let signal_matches = match &task.kind { + moa_execution::state::LogicalTaskKind::WaitSignal { + signal_name, + wait_policy: _wait_policy, + } => signal_name == &request.signal_name, + _ => false, + }; if !signal_matches { return Ok(conflict_mutation(ExecutionConflictReason::SignalMismatch)); } external_wait_mutation( &repository, + &config, scope, &run, &task, @@ -1016,6 +1108,7 @@ pub(super) async fn deliver_signal_inner( pub(super) async fn external_wait_mutation( repository: &ExecutionRepository, + config: &moa_config::ExecutionConfig, scope: ExecutionScope, run: &ExecutionRunRecord, task: &ExecutionTaskRecord, @@ -1023,7 +1116,12 @@ pub(super) async fn external_wait_mutation( result: ExecutionTaskResult, ) -> Result { if task.generation == generation - && task.status == ExecutionTaskStatus::Running + && matches!( + task.status, + ExecutionTaskStatus::Running + | ExecutionTaskStatus::WaitingReview + | ExecutionTaskStatus::WaitingSignal + ) && let ExecutionTaskResult::Completed { output, .. } = &result { validate_external_wait_payload(&run.active_plan.definition, &task.node_id, output)?; @@ -1031,6 +1129,7 @@ pub(super) async fn external_wait_mutation( let write = repository .complete_external_wait( scope, + config, run.run_uid, task.task_id, generation, @@ -1052,19 +1151,12 @@ pub(super) async fn apply_amendment_inner( ) -> Result { let repository = ExecutionRepository::new(pool); let scope = execution_scope(request.run.tenant_id, request.run.contact_id); - let Some(snapshot) = repository - .load_scheduling_snapshot(scope, request.run.run_uid) - .await - .map_err(execution_error)? - else { - return Ok(not_found_mutation()); - }; - verify_run_request(&snapshot.run, &request.run)?; let amendment_digest = amendment_hash(&request.amendment).map_err(execution_error)?; match repository .recover_amendment_handoff( scope, - snapshot.run.run_uid, + request.run.run_uid, + request.run.session_id, request.expected_plan_revision, &amendment_digest, ) @@ -1084,6 +1176,28 @@ pub(super) async fn apply_amendment_inner( } AmendmentReplayOutcome::NotApplied => {} } + let snapshot = match repository + .load_amendment_projection_for_session( + scope, + &config, + AmendmentProjectionRequest { + run_uid: request.run.run_uid, + session_id: request.run.session_id, + expected_plan_revision: request.expected_plan_revision, + }, + ) + .await + .map_err(execution_error)? + { + AmendmentProjectionOutcome::Ready(snapshot) => *snapshot, + AmendmentProjectionOutcome::NotFound => return Ok(not_found_mutation()), + AmendmentProjectionOutcome::Conflict => { + return Ok(conflict_mutation( + ExecutionConflictReason::PlanRevisionMismatch, + )); + } + }; + verify_run_request(&snapshot.run, &request.run)?; if snapshot.run.plan_revision != request.expected_plan_revision { return Ok(conflict_mutation( ExecutionConflictReason::PlanRevisionMismatch, @@ -1093,12 +1207,7 @@ pub(super) async fn apply_amendment_inner( .budget_ledger .remaining_limit() .map_err(execution_error)?; - let waiting_tasks = snapshot - .projection - .tasks - .iter() - .filter(|task| task.status == ExecutionTaskStatus::WaitingReplan) - .collect::>(); + let waiting_tasks = &snapshot.projection.replan_tasks; let [waiting_task] = waiting_tasks.as_slice() else { return Ok(conflict_mutation(ExecutionConflictReason::InvalidStatus)); }; @@ -1121,8 +1230,8 @@ pub(super) async fn apply_amendment_inner( active_plan: snapshot.run.active_plan.clone(), amendment: request.amendment.clone(), projection: snapshot.projection.clone(), - catalog: snapshot.catalog.clone(), - authorization: snapshot.authorization.clone(), + catalog: snapshot.run.catalog.clone(), + authorization: snapshot.run.authorization.clone(), remaining_budget: remaining_budget.clone(), config: config.clone(), now, @@ -1134,11 +1243,14 @@ pub(super) async fn apply_amendment_inner( return finalize_service_replan_stop( &repository, scope, - &snapshot, - waiting_task, - amendment_digest, - reason, - Some(&request.amendment.reason), + &config, + ServiceReplanStopRequest { + snapshot: &snapshot, + waiting_task, + amendment_digest, + reason, + detail: Some(&request.amendment.reason), + }, ) .await; } @@ -1147,11 +1259,14 @@ pub(super) async fn apply_amendment_inner( return finalize_service_replan_stop( &repository, scope, - &snapshot, - waiting_task, - amendment_digest, - reason, - Some(&request.amendment.reason), + &config, + ServiceReplanStopRequest { + snapshot: &snapshot, + waiting_task, + amendment_digest, + reason, + detail: Some(&request.amendment.reason), + }, ) .await; } @@ -1186,17 +1301,21 @@ pub(super) async fn apply_amendment_inner( return finalize_service_replan_stop( &repository, scope, - &snapshot, - waiting_task, - amendment_digest, - reason, - Some(&request.amendment.reason), + &config, + ServiceReplanStopRequest { + snapshot: &snapshot, + waiting_task, + amendment_digest, + reason, + detail: Some(&request.amendment.reason), + }, ) .await; } let write = repository .append_amendment( scope, + &config, snapshot.run.run_uid, request.expected_plan_revision, ValidatedAmendment { @@ -1223,104 +1342,43 @@ pub(super) async fn apply_amendment_inner( }) } -pub(super) async fn finalize_service_replan_stop( - repository: &ExecutionRepository, - scope: ExecutionScope, - snapshot: &moa_execution::repository::ExecutionSchedulingSnapshot, - waiting_task: &ExecutionTaskProjection, +struct ServiceReplanStopRequest<'a> { + snapshot: &'a ExecutionAmendmentSnapshot, + waiting_task: &'a ExecutionTaskProjection, amendment_digest: ExecutionHash, reason: moa_execution::ReplanStopReason, - detail: Option<&str>, -) -> Result { - let mut evaluation = evaluate_completion(CompletionEvaluationRequest { - goal: snapshot.run.goal.clone(), - plan: snapshot.run.active_plan.clone(), - run_input: snapshot.run.input.clone(), - projection: snapshot.projection.clone(), - terminal_output: snapshot.run.output.clone(), - budget_ledger: snapshot.budget_ledger.clone(), - now: chrono::Utc::now(), - }) - .map_err(execution_error)?; - evaluation.status = replan_stop_status( - snapshot.run.output.is_some(), - evaluation.satisfied_requirement_ids.len(), - ); - let stop_gaps = replan_stop_gaps(reason, detail); - evaluation.gaps.extend(stop_gaps.iter().cloned()); - evaluation.gaps.sort(); - evaluation.gaps.dedup(); - let terminal = terminal_projection_from_evaluation( - &evaluation, - snapshot.run.output.clone(), - None, - None, - None, - ) - .map_err(execution_error)?; - let terminal_status = moa_execution::state::run_status_from_terminal_projection(&terminal); - let terminal_evidence = terminal_evidence_from_evaluation( - ExecutionTerminalCause::ReplanStop { reason }, - &evaluation, - ) - .map_err(execution_error)?; - let terminal_reason = - execution_terminal_reason(&terminal_evidence.cause, &terminal, &evaluation) - .map_err(execution_error)?; - let pending_terminal = PendingExecutionTerminal { - status: terminal_status, - reason: terminal_reason, - terminal_evidence, - output: snapshot.run.output.clone(), - completion_check_results: evaluation - .checks - .iter() - .map(serde_json::to_value) - .collect::, _>>() - .map_err(|error| { - invalid_execution_request(format!( - "serialize replan-stop completion checks: {error}" - )) - })?, - terminal_gaps: evaluation.gaps, - cancellation_reason: None, - }; - Ok( - match repository - .fence_replan_stop( - scope, - snapshot.run.run_uid, - snapshot.run.plan_revision, - snapshot.run.wake_epoch, - pending_terminal, - ReplanStopReceipt { - task_id: waiting_task.task_id, - task_generation: waiting_task.generation, - base_plan_revision: snapshot.run.plan_revision, - amendment_hash: amendment_digest, - }, - ) - .await - .map_err(execution_error)? - { - TerminalFenceOutcome::Applied(commit) => applied_mutation(&commit.run), - TerminalFenceOutcome::Replayed(commit) => replayed_mutation(&commit.run), - TerminalFenceOutcome::Conflict => { - conflict_mutation(ExecutionConflictReason::PlanRevisionMismatch) - } - TerminalFenceOutcome::NotFound => not_found_mutation(), - }, - ) + detail: Option<&'a str>, } -#[cfg(test)] -/// Applies an amendment through the production inner boundary for library regressions. -pub(crate) async fn apply_amendment_for_test( - pool: sqlx::PgPool, - config: ExecutionConfig, - request: ExecutionAmendmentRequest, -) -> Result { - apply_amendment_inner(pool, config, request) +async fn finalize_service_replan_stop( + repository: &ExecutionRepository, + scope: ExecutionScope, + config: &ExecutionConfig, + request: ServiceReplanStopRequest<'_>, +) -> Result { + let write = repository + .request_replan_stop( + scope, + config, + NewExecutionReplanStopIntent { + run_uid: request.snapshot.run.run_uid, + session_id: request.snapshot.run.session_id, + base_plan_revision: request.snapshot.run.plan_revision, + origin_task_id: request.waiting_task.task_id, + task_generation: request.waiting_task.generation, + amendment_hash: request.amendment_digest, + stop_reason: request.reason, + detail: request.detail.map(str::to_string), + }, + ) .await - .map(ExecutionMutationAccepted::into_response) + .map_err(execution_error)?; + Ok(match write { + ReplanStopIntentWriteOutcome::Applied(run) => applied_mutation(&run), + ReplanStopIntentWriteOutcome::Replayed(run) => replayed_mutation(&run), + ReplanStopIntentWriteOutcome::NotFound => not_found_mutation(), + ReplanStopIntentWriteOutcome::Conflict => { + conflict_mutation(ExecutionConflictReason::PlanRevisionMismatch) + } + }) } diff --git a/crates/moa-orchestrator/src/services/execution/planning_context.rs b/crates/moa-orchestrator/src/services/execution/planning_context.rs index d9acd102e..5980ffd39 100644 --- a/crates/moa-orchestrator/src/services/execution/planning_context.rs +++ b/crates/moa-orchestrator/src/services/execution/planning_context.rs @@ -7,15 +7,40 @@ use super::capability_catalog::{ use super::support::{execution_error, execution_scope, invalid_execution_request}; use super::*; +/// Complete bounded input for one immutable planning-context assembly. +pub(super) struct PlanningContextInput { + /// Shared runtime database pool. + pub(super) pool: sqlx::PgPool, + /// Session-scoped capability registrations. + pub(super) registrations: Vec<(ToolDefinition, ToolExecution)>, + /// Validated execution policy. + pub(super) config: ExecutionConfig, + /// Authoritative parent session metadata. + pub(super) parent: moa_core::types::session::SessionMeta, + /// Effective user that owns the planning request. + pub(super) owner_user_id: moa_core::types::identifiers::UserId, + /// Exact persisted user event that originated planning. + pub(super) originating_event: Event, + /// Durable admission timestamp from the originating event. + pub(super) planning_admitted_at: chrono::DateTime, + /// Caller request already authorized by the service boundary. + pub(super) request: ExecutionPlanningContextRequest, +} + +/// Builds and persists one caller-authorized immutable planning context. pub(super) async fn planning_context_inner( - pool: sqlx::PgPool, - registrations: Vec<(ToolDefinition, ToolExecution)>, - config: ExecutionConfig, - parent: moa_core::types::session::SessionMeta, - owner_user_id: moa_core::types::identifiers::UserId, - originating_event: Event, - request: ExecutionPlanningContextRequest, + input: PlanningContextInput, ) -> Result { + let PlanningContextInput { + pool, + registrations, + config, + parent, + owner_user_id, + originating_event, + planning_admitted_at, + request, + } = input; let registrations = registrations .into_iter() .filter_map(|registration| { @@ -104,6 +129,12 @@ pub(super) async fn planning_context_inner( &originating_event, ) .map_err(execution_error)?; + let deadline_at = capped_planning_deadline( + planning_admitted_at, + request.deadline_at, + config.maximum_horizon_seconds, + ) + .map_err(invalid_execution_request)?; let snapshot = ExecutionPlanningContextSnapshot { schema_version: 1, tenant_id: request.tenant_id, @@ -122,7 +153,7 @@ pub(super) async fn planning_context_inner( max_tasks: Some(config.max_tasks), max_tool_calls: Some(config.max_tool_calls), max_retrieved_bytes: Some(config.max_retrieved_bytes), - deadline_at: None, + deadline_at: Some(deadline_at), }, }; let hash = planning_context_hash(&snapshot).map_err(execution_error)?; @@ -159,6 +190,26 @@ pub(super) async fn planning_context_inner( } } +/// Returns the admitted absolute deadline bounded by caller authority and configured horizon. +pub(super) fn capped_planning_deadline( + planning_admitted_at: chrono::DateTime, + authorized_deadline_at: chrono::DateTime, + maximum_horizon_seconds: u64, +) -> Result, &'static str> { + let horizon_seconds = i64::try_from(maximum_horizon_seconds) + .map_err(|_| "execution maximum horizon does not fit the timestamp range")?; + let horizon = chrono::TimeDelta::try_seconds(horizon_seconds) + .ok_or("execution maximum horizon does not fit the timestamp range")?; + let maximum_deadline = planning_admitted_at + .checked_add_signed(horizon) + .ok_or("execution maximum horizon exceeds the timestamp range")?; + let deadline_at = authorized_deadline_at.min(maximum_deadline); + if deadline_at <= planning_admitted_at { + return Err("execution planning deadline must be later than admission time"); + } + Ok(deadline_at) +} + #[derive(Debug)] pub(super) struct PlanningSkillContext { pub(super) revisions: Vec, diff --git a/crates/moa-orchestrator/src/services/execution/start.rs b/crates/moa-orchestrator/src/services/execution/start.rs index e76f8ae10..ff23c26e4 100644 --- a/crates/moa-orchestrator/src/services/execution/start.rs +++ b/crates/moa-orchestrator/src/services/execution/start.rs @@ -8,11 +8,12 @@ pub(super) async fn start_inner( config: ExecutionConfig, request: ExecutionStartRequest, originating_objective: String, + admitted_identity: moa_core::traits::Identity, ) -> Result { let scope = execution_scope(request.tenant_id, request.contact_id); let repository = ExecutionRepository::new(pool); let planning_context = repository - .load_planning_context(scope, request.planning_context_uid) + .load_planning_context_for_session(scope, request.planning_context_uid, request.session_id) .await .map_err(execution_error)? .ok_or_else(|| { @@ -73,7 +74,13 @@ pub(super) async fn start_inner( .map_err(|error| invalid_execution_request(error.to_string()))?; let existing = if let Some(key) = request.idempotency_key.as_deref() { repository - .load_run_by_idempotency_key(scope, request.tenant_id, request.contact_id, key) + .load_run_by_idempotency_key_for_session( + scope, + request.tenant_id, + request.contact_id, + request.session_id, + key, + ) .await .map_err(execution_error)? } else { @@ -103,9 +110,10 @@ pub(super) async fn start_inner( } else { ExecutionRunStatus::Queued }; - let run = repository + let admission = repository .create_run( scope, + &config, NewExecutionRun { tenant_id: request.tenant_id, contact_id: request.contact_id, @@ -114,25 +122,49 @@ pub(super) async fn start_inner( planning_context_uid: request.planning_context_uid, planning_context_hash: expected_context_hash, owner_user_id: snapshot.owner_user_id.clone(), - goal: request.compiled.goal, - plan: request.compiled.plan, + admitted_identity, + goal: request.compiled.goal.clone(), + plan: request.compiled.plan.clone(), catalog: snapshot.catalog.clone(), authorization: snapshot.authorization.clone(), pinned_instruction_skills: snapshot.pinned_instruction_skills.clone(), - source_provenance: request.source_provenance, - input: request.run_input, + source_provenance: request.source_provenance.clone(), + input: request.run_input.clone(), status, approved_budget: snapshot.budget.clone(), - idempotency_key: request.idempotency_key, + idempotency_key: request.idempotency_key.clone(), }, ) .await .map_err(execution_error)?; + let (run, created) = match admission { + moa_execution::repository::run::RunAdmissionOutcome::Admitted(run) => (*run, true), + moa_execution::repository::run::RunAdmissionOutcome::Replayed(run) => { + verify_run_scope( + &run, + request.tenant_id, + request.contact_id, + request.session_id, + )?; + verify_start_replay(&run, &request, snapshot)?; + (*run, false) + } + moa_execution::repository::run::RunAdmissionOutcome::CapacitySaturated { dimension } => { + return Err(TerminalError::new_with_code( + 429, + format!( + "execution {} capacity is exhausted; retry admission later", + dimension.as_str() + ), + ) + .into()); + } + }; Ok(ExecutionStartResponse { active_plan_hash: run.active_plan_hash, estimate: run.active_plan.estimate, run: run_summary(&run), - created: true, + created, confirmation_required, }) } diff --git a/crates/moa-orchestrator/src/services/execution/support.rs b/crates/moa-orchestrator/src/services/execution/support.rs index 1709fa3ca..5a6b2613a 100644 --- a/crates/moa-orchestrator/src/services/execution/support.rs +++ b/crates/moa-orchestrator/src/services/execution/support.rs @@ -1,7 +1,6 @@ //! Shared execution-service mutation handoff types and conversion helpers. use super::*; -use moa_observability::runtime_metrics::record_execution_mutation_run_wake_ack; pub(super) fn scoped_catalog_error( error: crate::connector_catalog::ScopedConnectorCatalogError, @@ -38,13 +37,6 @@ impl ExecutionMutationAccepted { } } - pub(super) fn task_ids_to_release(&self) -> &[moa_execution::state::ExecutionTaskId] { - match self { - Self::Accepted { handoff, .. } => &handoff.task_ids_to_release, - Self::Rejected { .. } => &[], - } - } - pub(super) fn with_task_ids_to_release( mut self, task_ids_to_release: Vec, @@ -62,7 +54,7 @@ impl ExecutionMutationAccepted { } } pub(super) fn replan_evaluation_request( - snapshot: &moa_execution::repository::ExecutionSchedulingSnapshot, + snapshot: &moa_execution::repository::amendment::ExecutionAmendmentSnapshot, proposed_plan: &moa_execution::compiler::CanonicalExecutionPlan, proposed_estimate: ExecutionEstimate, remaining_budget: moa_artifacts::execution_plan::ExecutionBudgetLimit, @@ -97,7 +89,7 @@ pub(super) fn replan_evaluation_request( } pub(super) fn replan_loop_evaluation_request( - snapshot: &moa_execution::repository::ExecutionSchedulingSnapshot, + snapshot: &moa_execution::repository::amendment::ExecutionAmendmentSnapshot, proposed_amendment_fingerprint: ExecutionHash, amendment: PlanAmendment, config: ExecutionConfig, @@ -105,21 +97,7 @@ pub(super) fn replan_loop_evaluation_request( ) -> moa_execution::Result { let seen_amendment_fingerprints = durable_amendment_operation_fingerprints(&snapshot.run.plan_history)?; - let failures = snapshot - .projection - .tasks - .iter() - .filter(|task| task.task_id != waiting_task.task_id) - .filter_map(task_failure_fingerprint) - .collect::>(); let current_failure = task_failure_fingerprint(waiting_task); - let mut failure_fingerprint_counts = - durable_failure_fingerprint_counts(&snapshot.run.plan_history); - for failure in failures { - if let Ok(fingerprint) = failure_fingerprint(&failure) { - *failure_fingerprint_counts.entry(fingerprint).or_insert(0) += 1; - } - } let unresolved_requirement_ids = snapshot .run .goal @@ -143,7 +121,7 @@ pub(super) fn replan_loop_evaluation_request( Ok(ReplanLoopEvaluationRequest { proposed_amendment_fingerprint, seen_amendment_fingerprints, - failure_fingerprint_counts, + failure_fingerprint_counts: snapshot.prior_failure_fingerprint_counts.clone(), current_failure, unresolved_requirement_ids, amendment, @@ -172,30 +150,6 @@ pub(super) fn durable_amendment_operation_fingerprints( Ok(fingerprints) } -pub(super) fn durable_failure_fingerprint_counts( - plan_history: &[Value], -) -> BTreeMap { - let mut counts: BTreeMap = BTreeMap::new(); - for entry in plan_history { - let Some(fingerprint) = entry - .get("failure_fingerprint") - .and_then(Value::as_str) - .and_then(|value| value.parse::().ok()) - else { - continue; - }; - let count = entry - .get("failure_fingerprint_count") - .and_then(Value::as_u64) - .map_or(1, |count| u32::try_from(count).unwrap_or(u32::MAX)); - counts - .entry(fingerprint) - .and_modify(|persisted| *persisted = (*persisted).max(count)) - .or_insert(count); - } - counts -} - pub(super) fn task_failure_fingerprint( task: &ExecutionTaskProjection, ) -> Option { @@ -538,29 +492,6 @@ pub(super) fn execution_run_started_delivery( } } -/// Joins the run wake handoff before an externally visible mutation returns. -pub(super) async fn call_run_wake( - ctx: &Context<'_>, - run_uid: uuid::Uuid, - wake_epoch: u64, - reason: ExecutionRunWakeReason, - handoff_started: std::time::Instant, -) -> Result<(), HandlerError> { - crate::restate_identity::replay_safe_request( - ctx.workflow_client::(run_uid.to_string()) - .wake(Json::from(ExecutionRunWakeRequest { - run_uid, - wake_epoch, - reason, - })), - ) - .call() - .await - .map_err(HandlerError::from)?; - record_execution_mutation_run_wake_ack(handoff_started.elapsed()); - Ok(()) -} - #[cfg(feature = "integration")] pub(super) async fn pause_execution_mutation_handoff_for_test() { if std::env::var("MOA_EXECUTION_TEST_PAUSE_MUTATION_HANDOFF").as_deref() == Ok("true") { @@ -578,11 +509,5 @@ pub(super) fn invalid_execution_request(message: impl Into) -> HandlerEr } pub(super) fn execution_error(error: moa_execution::Error) -> HandlerError { - match error { - moa_execution::Error::Storage { message } => { - TerminalError::new_with_code(503, format!("execution storage unavailable: {message}")) - .into() - } - other => invalid_execution_request(other.to_string()), - } + crate::workflows::errors::execution_error_to_handler_error(error) } diff --git a/crates/moa-orchestrator/src/services/execution/tests.rs b/crates/moa-orchestrator/src/services/execution/tests.rs index c23010b12..e224aa085 100644 --- a/crates/moa-orchestrator/src/services/execution/tests.rs +++ b/crates/moa-orchestrator/src/services/execution/tests.rs @@ -4,12 +4,13 @@ use super::capability_catalog::{ build_capability_response, build_skill_regression_compile_authority, single_tool_estimate, }; use super::planning_context::{ - PlanningSkillContext, build_planning_skill_context, skill_revision_ref, + PlanningSkillContext, build_planning_skill_context, capped_planning_deadline, + skill_revision_ref, }; use super::start::validate_start_source_provenance; use super::support::{ - durable_amendment_operation_fingerprints, durable_failure_fingerprint_counts, - persisted_input_audience, validate_external_wait_payload, + durable_amendment_operation_fingerprints, persisted_input_audience, + validate_external_wait_payload, }; use std::collections::{BTreeMap, BTreeSet}; @@ -34,9 +35,7 @@ use moa_execution::{ capability::{amendment_hash, amendment_operations_fingerprint}, replan::{ ReplanDecision, ReplanLoopEvaluationRequest, ReplanStopReason, evaluate_replan_loop_stop, - failure_fingerprint, }, - state::FailureFingerprintInput, wire::PinnedExecutionTemplate, }; use moa_hands::{McpDiscoveredTool, ToolExecution, ToolRegistry}; @@ -47,6 +46,38 @@ use moa_test_support::fixture_capability::{ use serde_json::{Value, json}; use uuid::Uuid; +#[test] +fn planning_deadline_preserves_authorized_bound_and_caps_configured_horizon_offline() { + // Pins: admission freezes the shorter of caller authority and the configured maximum horizon, + // and never admits an already-expired deadline. + let admitted_at = chrono::DateTime::parse_from_rfc3339("2026-08-11T12:00:00Z") + .expect("fixture timestamp parses") + .with_timezone(&Utc); + let authorized_shorter = admitted_at + chrono::TimeDelta::hours(2); + assert_eq!( + capped_planning_deadline(admitted_at, authorized_shorter, 86_400), + Ok(authorized_shorter) + ); + + let authorized_longer = admitted_at + chrono::TimeDelta::days(7); + assert_eq!( + capped_planning_deadline(admitted_at, authorized_longer, 86_400), + Ok(admitted_at + chrono::TimeDelta::days(1)) + ); + assert_eq!( + capped_planning_deadline(admitted_at, admitted_at, 86_400), + Err("execution planning deadline must be later than admission time") + ); + assert_eq!( + capped_planning_deadline(admitted_at, authorized_longer, u64::MAX), + Err("execution maximum horizon does not fit the timestamp range") + ); + assert_eq!( + capped_planning_deadline(admitted_at, authorized_longer, i64::MAX as u64), + Err("execution maximum horizon does not fit the timestamp range") + ); +} + #[test] fn tool_estimate_reserves_serialized_output_bytes_from_token_budget() { // Pins: a successful non-empty tool result cannot overrun a zero-byte reservation @@ -229,6 +260,13 @@ fn skill_revision(name: &str, revision_uid: u128) -> StoredArtifactRevision { plan: ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, + input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { + expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { + delay_seconds: 86_400, + }, + on_expiry: + moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + }, input_schema: json!({"type": "object"}), output_schema: json!({"type": "object"}), nodes: Vec::new(), @@ -596,6 +634,12 @@ fn accepted_turn_requires_skill_template_provenance_from_planning_snapshot() { }, plan: ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, + input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { + expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { + delay_seconds: 86_400, + }, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + }, input_schema: json!({"type": "object"}), output_schema: json!({"type": "object"}), nodes: Vec::new(), @@ -671,6 +715,12 @@ fn pinned_execution_template( }, plan: ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, + input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { + expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { + delay_seconds: 86_400, + }, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + }, input_schema: json!({"type": "object"}), output_schema: json!({"type": "object"}), nodes: Vec::new(), @@ -955,7 +1005,8 @@ fn capability_catalog_uses_live_execution_metadata_and_omits_non_invocable_decla // source variant contributing a name from the wrong namespace — which // type-checks, and fails only when a live run dispatches it. for capability in &response.catalog.capabilities { - let Ok(dispatch_name) = crate::workflows::execution_task::capability_tool_name(capability) + let Ok(dispatch_name) = + crate::workflows::execution_task_attempt::capability_tool_name(capability) else { continue; }; @@ -1130,33 +1181,6 @@ fn execution_external_wait_payload_is_validated_against_node_schema() { ); } -#[test] -fn replan_failure_counts_include_append_only_superseded_history() { - // Pins: superseding a NeedsReplan task cannot erase its normalized - // failure occurrence from the next amendment stop evaluation. - let failure = FailureFingerprintInput { - class: moa_artifacts::execution_plan::ExecutionFailureClass::Terminal, - node_id: "collect".to_string(), - capability_ref: None, - message: " Source Unavailable ".to_string(), - }; - let fingerprint = failure_fingerprint(&failure).expect("failure should hash"); - let history = vec![ - json!({ - "failure_fingerprint": fingerprint, - "failure_fingerprint_count": 1 - }), - json!({ - "failure_fingerprint": fingerprint, - "failure_fingerprint_count": 2 - }), - ]; - assert_eq!( - durable_failure_fingerprint_counts(&history), - [(fingerprint, 2)].into_iter().collect() - ); -} - #[test] fn replan_history_detects_duplicate_operations_without_exact_replay() { // Pins: the service derives semantic loop identity from persisted amendment values, so a diff --git a/crates/moa-orchestrator/src/services/execution_dispatcher.rs b/crates/moa-orchestrator/src/services/execution_dispatcher.rs new file mode 100644 index 000000000..6c56dfe7f --- /dev/null +++ b/crates/moa-orchestrator/src/services/execution_dispatcher.rs @@ -0,0 +1,1076 @@ +//! Bounded Restate dispatcher and indexed reconciliation for execution outbox rows. + +use std::time::Duration; + +use chrono::{DateTime, Utc}; +use moa_execution::repository::{ + ExecutionRepository, ExecutionScope, + outbox::{ + ExecutionDispatchFailureOutcome, ExecutionDispatchRetryPolicy, ExecutionMaintenanceJobKind, + ExecutionMaintenanceSettlementOutcome, ExecutionQueueBacklogSample, + ExecutionQueueHealthSnapshot, + }, +}; +use restate_sdk::prelude::*; +use serde::{Deserialize, Serialize}; + +use crate::{ + objects::execution_run_controller::ExecutionRunControllerClient, + runtime::execution_dispatch::{ExecutionDispatchTarget, JournaledExecutionDispatch}, + services::{execution_trigger::ExecutionTriggerClient, tool_executor::ToolExecutorClient}, + workflows::{ + errors::execution_error_to_handler_error, + execution_compensation_attempt::ExecutionCompensationAttemptClient, + execution_task_attempt::ExecutionTaskAttemptClient, + }, +}; + +const DISPATCH_CLAIM_TTL: Duration = Duration::from_secs(120); +const MAX_REPOSITORY_BATCH_SIZE: usize = 1_000; +/// Singleton drain-object key matching the fleet-global execution-capacity lock. +pub const EXECUTION_DISPATCH_DRAIN_FLEET_KEY: &str = "fleet"; +const DISPATCH_RETRY_POLICY: ExecutionDispatchRetryPolicy = ExecutionDispatchRetryPolicy { + max_attempts: 8, + base_delay: Duration::from_secs(5), + maximum_delay: Duration::from_secs(300), +}; + +/// Operational request for one bounded fleet outbox pass. +#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct DispatchExecutionsRequest {} + +/// Confirmation that the current indexed outbox head was routed to the fleet drain. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct DispatchExecutionsResponse { + /// Whether a pending outbox head existed and its drain invocation was accepted. + pub scheduled: bool, +} + +/// Summary of one bounded outbox drain pass. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct DrainExecutionDispatchesResponse { + /// Rows claimed with `SKIP LOCKED`. + pub claimed: usize, + /// Downstream invocations durably accepted and acknowledged. + pub acknowledged: usize, + /// Claims released behind bounded retry backoff. + pub retry_scheduled: usize, + /// Claims that exhausted their delivery budget. + pub dead_lettered: usize, + /// Claims changed ownership before settlement. + pub stale_claims: usize, +} + +/// Operational request for one bounded due-trigger repair pass. +#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ReconcileExecutionDispatchesRequest {} + +/// Summary of one bounded indexed reconciliation pass. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ReconcileExecutionDispatchesResponse { + /// Due trigger deliveries and queued run activations requeued transactionally. + pub repaired_dispatches: usize, + /// Bounded fleet drain completed after repair. + pub delivery: DrainExecutionDispatchesResponse, + /// Count-capped fleet queue health sampled after delivery. + pub health: ExecutionQueueHealthReport, +} + +/// Serializable, bounded execution trigger/outbox health report. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ExecutionQueueHealthReport { + /// Canonical database observation time. + pub observed_at: DateTime, + /// Due trigger rows observed up to the sample cap. + pub due_triggers: u32, + /// Whether due trigger depth exceeded the sample cap. + pub due_triggers_saturated: bool, + /// Age of the oldest observed due trigger. + pub trigger_lag_seconds: f64, + /// Claimable outbox rows observed up to the sample cap. + pub claimable_dispatches: u32, + /// Whether claimable dispatch depth exceeded the sample cap. + pub claimable_dispatches_saturated: bool, + /// Age of the oldest observed claimable dispatch. + pub outbox_lag_seconds: f64, + /// Trigger dead letters observed up to the sample cap. + pub dead_letter_triggers: u32, + /// Whether trigger dead letters exceeded the sample cap. + pub dead_letter_triggers_saturated: bool, + /// Dispatch dead letters observed up to the sample cap. + pub dead_letter_dispatches: u32, + /// Whether dispatch dead letters exceeded the sample cap. + pub dead_letter_dispatches_saturated: bool, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +struct JournaledDispatchAckBatch { + delivered_dispatch_uids: Vec, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +enum JournaledDispatchFailure { + RetryScheduled, + DeadLettered, + StaleClaim, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +struct JournaledAdmissionBatch { + admitted_count: usize, + oldest_ready_age_millis: Option, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +enum JournaledMaintenanceSettlement { + Applied { + last_success_age_millis: Option, + }, + StaleOrMissing, +} + +/// Stateless producer-facing router that coalesces kicks by the indexed outbox head. +#[restate_sdk::service] +#[name = "ExecutionDispatcher"] +pub trait ExecutionDispatcher { + /// Routes the current outbox head to the fleet-serialized drain. + async fn dispatch( + request: Json, + ) -> Result, HandlerError>; +} + +/// Fleet-keyed virtual object that owns bounded outbox draining and exact delayed wakes. +#[restate_sdk::object] +#[name = "ExecutionDispatchDrain"] +pub trait ExecutionDispatchDrain { + /// Claims, delivers, and settles one fleet-serialized bounded outbox batch. + async fn drain( + request: Json, + ) -> Result, HandlerError>; +} + +/// Restate target for a low-frequency infrastructure CronJob repair pass. +#[restate_sdk::service] +#[name = "ExecutionDispatchReconciler"] +pub trait ExecutionDispatchReconciler { + /// Repairs only one indexed due-trigger window, then invokes one bounded drain. + async fn reconcile( + request: Json, + ) -> Result, HandlerError>; +} + +/// PostgreSQL-backed producer-facing execution-dispatch router. +#[derive(Clone)] +pub struct ExecutionDispatcherImpl { + repository: ExecutionRepository, +} + +impl ExecutionDispatcherImpl { + /// Creates a router over the canonical execution outbox. + #[must_use] + pub fn new(pool: sqlx::PgPool) -> Self { + Self { + repository: ExecutionRepository::new(pool), + } + } +} + +/// PostgreSQL-backed fleet-serialized bounded execution drain. +#[derive(Clone)] +pub struct ExecutionDispatchDrainImpl { + repository: ExecutionRepository, + config: moa_config::ExecutionConfig, + batch_size: u32, +} + +impl ExecutionDispatchDrainImpl { + /// Creates a drain using the validated execution batch bound. + #[must_use] + pub fn new(pool: sqlx::PgPool, config: &moa_config::ExecutionConfig) -> Self { + Self { + repository: ExecutionRepository::new(pool), + config: config.clone(), + batch_size: config.dispatch_batch_size.min(MAX_REPOSITORY_BATCH_SIZE) as u32, + } + } +} + +impl ExecutionDispatcher for ExecutionDispatcherImpl { + #[tracing::instrument(skip(self, ctx, _request))] + // SAFETY: ingress-private operational target; it accepts no caller-owned resource and routes already-authorized, tenant-fenced rows. + async fn dispatch( + &self, + ctx: Context<'_>, + _request: Json, + ) -> Result, HandlerError> { + crate::ctx::adopt_incoming_trace_parent(&ctx); + moa_observability::restate_observability::annotate_restate_handler_span( + "ExecutionDispatcher", + "dispatch", + ); + let repository = self.repository.clone(); + let wake = ctx + .run(|| async move { + repository + .next_pending_dispatch_wake(ExecutionScope::ControlPlane) + .await + .map(Json::from) + .map_err(execution_error_to_handler_error) + }) + .name("execution_dispatch_route_head") + .await? + .into_inner(); + let (Some(dispatch_uid), Some(next_due_at), Some(head_updated_at)) = + (wake.dispatch_uid, wake.next_due_at, wake.head_updated_at) + else { + return Ok(Json::from(DispatchExecutionsResponse { scheduled: false })); + }; + let handle = crate::restate_identity::replay_safe_request( + ctx.object_client::( + EXECUTION_DISPATCH_DRAIN_FLEET_KEY.to_string(), + ) + .drain(Json::from(DispatchExecutionsRequest::default())) + .idempotency_key(dispatch_head_idempotency_key( + dispatch_uid, + next_due_at, + head_updated_at, + )), + ) + .send_after(next_dispatch_delay(wake.observed_at, next_due_at)); + handle.invocation_id().await?; + Ok(Json::from(DispatchExecutionsResponse { scheduled: true })) + } +} + +impl ExecutionDispatchDrain for ExecutionDispatchDrainImpl { + #[tracing::instrument(skip(self, ctx, _request))] + // SAFETY: ingress-private fleet drain; it accepts no caller-owned resource and drains already-authorized, tenant-fenced rows. + async fn drain( + &self, + ctx: ObjectContext<'_>, + _request: Json, + ) -> Result, HandlerError> { + crate::ctx::adopt_incoming_trace_parent(&ctx); + moa_observability::restate_observability::annotate_restate_handler_span( + "ExecutionDispatchDrain", + "drain", + ); + let claim_owner = format!("execution-dispatcher:{}", ctx.invocation_id()); + let repository = self.repository.clone(); + let batch_size = self.batch_size; + let journaled = ctx + .run(|| { + let claim_owner = claim_owner.clone(); + async move { + repository + .claim_due_dispatches( + ExecutionScope::ControlPlane, + &claim_owner, + batch_size, + DISPATCH_CLAIM_TTL, + ) + .await + .map(|records| { + Json::from( + records + .into_iter() + .map(JournaledExecutionDispatch::from) + .collect::>(), + ) + }) + .map_err(execution_error_to_handler_error) + } + }) + .name("execution_dispatch_claim") + .await? + .into_inner(); + + let mut response = DrainExecutionDispatchesResponse { + claimed: journaled.len(), + acknowledged: 0, + retry_scheduled: 0, + dead_lettered: 0, + stale_claims: 0, + }; + let delivery_results = accept_batch(&ctx, &journaled).await?; + let mut delivered_dispatch_uids = Vec::with_capacity(journaled.len()); + let mut failed_dispatches = Vec::new(); + for (dispatch, accepted) in journaled.into_iter().zip(delivery_results) { + match accepted { + Ok(()) => delivered_dispatch_uids.push(dispatch.dispatch_uid), + Err(error) => failed_dispatches.push((dispatch, error)), + } + } + settle_delivered_batch( + &ctx, + &self.repository, + &claim_owner, + delivered_dispatch_uids, + &mut response, + ) + .await?; + for (dispatch, error) in failed_dispatches { + settle_failure( + &ctx, + &self.repository, + &claim_owner, + dispatch, + error, + &mut response, + ) + .await?; + } + // Synchronous trigger/controller deliveries can materialize Ready tasks without another + // outbox continuation. Admit once more inside this bounded drain episode so the resulting + // TaskAttempt row becomes the indexed head before producers are allowed to coalesce away. + let repository = self.repository.clone(); + let config = self.config.clone(); + let batch_size = self.batch_size; + let admission = ctx + .run(|| async move { + let observed_at = Utc::now(); + repository + .admit_ready_attempts(&config, batch_size, observed_at) + .await + .map(|batch| { + let oldest_ready_age_millis = batch + .oldest_ready_at + .and_then(|oldest| { + observed_at.signed_duration_since(oldest).to_std().ok() + }) + .map(|age| u64::try_from(age.as_millis()).unwrap_or(u64::MAX)); + Json::from(JournaledAdmissionBatch { + admitted_count: batch.admitted.len(), + oldest_ready_age_millis, + }) + }) + .map_err(execution_error_to_handler_error) + }) + .name("execution_dispatch_admit_ready") + .await? + .into_inner(); + moa_observability::runtime_metrics::record_execution_dispatch_batch_size( + admission.admitted_count, + ); + if let Some(age) = admission.oldest_ready_age_millis { + moa_observability::runtime_metrics::record_execution_oldest_ready_age( + Duration::from_millis(age), + ); + } + let repository = self.repository.clone(); + let wake = ctx + .run(|| async move { + repository + .next_pending_dispatch_wake(ExecutionScope::ControlPlane) + .await + .map(Json::from) + .map_err(execution_error_to_handler_error) + }) + .name("execution_dispatch_next_wake") + .await? + .into_inner(); + if let (Some(dispatch_uid), Some(next_due_at), Some(head_updated_at)) = + (wake.dispatch_uid, wake.next_due_at, wake.head_updated_at) + { + let (idempotency_key, delay) = next_dispatch_successor( + dispatch_uid, + next_due_at, + head_updated_at, + wake.observed_at, + response.claimed, + ); + let handle = crate::restate_identity::replay_safe_request( + ctx.object_client::( + EXECUTION_DISPATCH_DRAIN_FLEET_KEY.to_string(), + ) + .drain(Json::from(DispatchExecutionsRequest::default())) + .idempotency_key(idempotency_key), + ) + .send_after(delay); + handle.invocation_id().await?; + } + Ok(Json::from(response)) + } +} + +fn dispatch_head_idempotency_key( + dispatch_uid: uuid::Uuid, + next_due_at: DateTime, + head_updated_at: DateTime, +) -> String { + format!( + "execution-dispatch-head:{dispatch_uid}:{}:{}", + next_due_at.timestamp_micros(), + head_updated_at.timestamp_micros() + ) +} + +fn reconciliation_drain_idempotency_key(generation: u64) -> String { + format!("execution-reconcile-drain:{generation}") +} + +fn next_dispatch_delay(observed_at: DateTime, next_due_at: DateTime) -> Duration { + next_due_at + .signed_duration_since(observed_at) + .to_std() + .unwrap_or(Duration::ZERO) +} + +fn next_dispatch_successor( + dispatch_uid: uuid::Uuid, + next_due_at: DateTime, + head_updated_at: DateTime, + observed_at: DateTime, + claimed: usize, +) -> (String, Duration) { + let base_key = dispatch_head_idempotency_key(dispatch_uid, next_due_at, head_updated_at); + let delay = next_dispatch_delay(observed_at, next_due_at); + if claimed == 0 && next_due_at > observed_at { + return ( + format!("{base_key}:early-empty:{}", observed_at.timestamp_micros()), + delay.max(Duration::from_millis(1)), + ); + } + (base_key, delay) +} + +/// PostgreSQL-backed bounded reconciliation target. +#[derive(Clone)] +pub struct ExecutionDispatchReconcilerImpl { + repository: ExecutionRepository, + batch_size: u32, +} + +impl ExecutionDispatchReconcilerImpl { + /// Creates the low-frequency repair target using the validated batch bound. + #[must_use] + pub fn new(pool: sqlx::PgPool, config: &moa_config::ExecutionConfig) -> Self { + Self { + repository: ExecutionRepository::new(pool), + batch_size: config.dispatch_batch_size.min(MAX_REPOSITORY_BATCH_SIZE) as u32, + } + } +} + +impl ExecutionDispatchReconciler for ExecutionDispatchReconcilerImpl { + #[tracing::instrument(skip(self, ctx, _request))] + // SAFETY: ingress-private infrastructure CronJob target; it scans one bounded indexed due window and accepts no caller-owned identifiers. + async fn reconcile( + &self, + ctx: Context<'_>, + _request: Json, + ) -> Result, HandlerError> { + crate::ctx::adopt_incoming_trace_parent(&ctx); + moa_observability::restate_observability::annotate_restate_handler_span( + "ExecutionDispatchReconciler", + "reconcile", + ); + let repository = self.repository.clone(); + let generation = ctx + .run(|| async move { + repository + .begin_execution_maintenance( + ExecutionScope::ControlPlane, + ExecutionMaintenanceJobKind::DispatchReconciliation, + ) + .await + .map(|checkpoint| Json::from(checkpoint.generation)) + .map_err(execution_error_to_handler_error) + }) + .name("execution_dispatch_reconciliation_begin") + .await? + .into_inner(); + + let work = self.reconcile_bounded(&ctx, generation).await; + let response = match work { + Ok(response) => response, + Err(error) => { + let last_success_age = record_reconciliation_failure( + &ctx, + &self.repository, + generation, + &crate::workflows::errors::handler_error_message(&error), + ) + .await; + moa_observability::runtime_metrics::record_execution_maintenance( + false, + last_success_age, + ); + return Err(error); + } + }; + + let repository = self.repository.clone(); + let settlement = match ctx + .run(|| async move { + repository + .complete_execution_maintenance( + ExecutionScope::ControlPlane, + ExecutionMaintenanceJobKind::DispatchReconciliation, + generation, + ) + .await + .map(|outcome| { + Json::from(match outcome { + ExecutionMaintenanceSettlementOutcome::Applied(checkpoint) => { + JournaledMaintenanceSettlement::Applied { + last_success_age_millis: checkpoint_success_age_millis( + &checkpoint, + ), + } + } + ExecutionMaintenanceSettlementOutcome::StaleOrMissing => { + JournaledMaintenanceSettlement::StaleOrMissing + } + }) + }) + .map_err(execution_error_to_handler_error) + }) + .name("execution_dispatch_reconciliation_complete") + .await + { + Ok(settlement) => settlement.into_inner(), + Err(error) => { + let last_success_age = record_reconciliation_failure( + &ctx, + &self.repository, + generation, + &error.to_string(), + ) + .await; + moa_observability::runtime_metrics::record_execution_maintenance( + false, + last_success_age, + ); + return Err(error.into()); + } + }; + let JournaledMaintenanceSettlement::Applied { + last_success_age_millis, + } = settlement + else { + moa_observability::runtime_metrics::record_execution_maintenance(false, None); + return Err(TerminalError::new_with_code( + 409, + "execution dispatch reconciliation was superseded before completion", + ) + .into()); + }; + moa_observability::runtime_metrics::record_execution_maintenance( + true, + last_success_age_millis.map(Duration::from_millis), + ); + Ok(Json::from(response)) + } +} + +impl ExecutionDispatchReconcilerImpl { + async fn reconcile_bounded( + &self, + ctx: &Context<'_>, + generation: u64, + ) -> Result { + let repository = self.repository.clone(); + let batch_size = self.batch_size; + let repaired_dispatches = ctx + .run(|| async move { + repository + .reconcile_due_trigger_dispatches(ExecutionScope::ControlPlane, batch_size) + .await + .map(|dispatches| Json::from(dispatches.len())) + .map_err(execution_error_to_handler_error) + }) + .name("execution_trigger_reconcile_due_window") + .await? + .into_inner(); + + // Recovery requeues preserve the original dispatch identity and due time. Address the + // drain with this maintenance generation so Restate cannot memoize the original completed + // head invocation when redriving a downstream-lost accepted delivery. + let delivery = crate::restate_identity::replay_safe_request( + ctx.object_client::( + EXECUTION_DISPATCH_DRAIN_FLEET_KEY.to_string(), + ) + .drain(Json::from(DispatchExecutionsRequest::default())) + .idempotency_key(reconciliation_drain_idempotency_key(generation)), + ) + .call() + .await? + .into_inner(); + let repository = self.repository.clone(); + let sample_limit = self.batch_size; + let health = ctx + .run(|| async move { + repository + .sample_execution_queue_health(ExecutionScope::ControlPlane, sample_limit) + .await + .map(queue_health_report) + .map(Json::from) + .map_err(execution_error_to_handler_error) + }) + .name("execution_queue_health_sample") + .await? + .into_inner(); + record_queue_health(&health); + Ok(ReconcileExecutionDispatchesResponse { + repaired_dispatches, + delivery, + health, + }) + } +} + +async fn record_reconciliation_failure( + ctx: &Context<'_>, + repository: &ExecutionRepository, + generation: u64, + error: &str, +) -> Option { + let repository = repository.clone(); + let error = error.to_string(); + let settlement = ctx + .run(|| async move { + repository + .fail_execution_maintenance( + ExecutionScope::ControlPlane, + ExecutionMaintenanceJobKind::DispatchReconciliation, + generation, + &error, + ) + .await + .map(|outcome| { + Json::from(match outcome { + ExecutionMaintenanceSettlementOutcome::Applied(checkpoint) => { + JournaledMaintenanceSettlement::Applied { + last_success_age_millis: checkpoint_success_age_millis(&checkpoint), + } + } + ExecutionMaintenanceSettlementOutcome::StaleOrMissing => { + JournaledMaintenanceSettlement::StaleOrMissing + } + }) + }) + .map_err(execution_error_to_handler_error) + }) + .name("execution_dispatch_reconciliation_fail") + .await; + match settlement { + Ok(result) => match result.into_inner() { + JournaledMaintenanceSettlement::Applied { + last_success_age_millis, + } => last_success_age_millis.map(Duration::from_millis), + JournaledMaintenanceSettlement::StaleOrMissing => { + tracing::warn!( + checkpoint_generation = generation, + "execution dispatch reconciliation failure checkpoint was superseded" + ); + None + } + }, + Err(settlement_error) => { + tracing::warn!( + checkpoint_generation = generation, + error = %settlement_error, + "failed to persist execution dispatch reconciliation failure checkpoint" + ); + None + } + } +} + +fn checkpoint_success_age_millis( + checkpoint: &moa_execution::repository::outbox::ExecutionMaintenanceCheckpoint, +) -> Option { + checkpoint + .last_succeeded_at + .and_then(|succeeded_at| { + checkpoint + .updated_at + .signed_duration_since(succeeded_at) + .to_std() + .ok() + }) + .map(|age| u64::try_from(age.as_millis()).unwrap_or(u64::MAX)) +} + +fn queue_health_report(snapshot: ExecutionQueueHealthSnapshot) -> ExecutionQueueHealthReport { + ExecutionQueueHealthReport { + observed_at: snapshot.observed_at, + due_triggers: snapshot.due_triggers.observed_count, + due_triggers_saturated: snapshot.due_triggers.saturated, + trigger_lag_seconds: backlog_age(snapshot.observed_at, &snapshot.due_triggers) + .as_secs_f64(), + claimable_dispatches: snapshot.claimable_dispatches.observed_count, + claimable_dispatches_saturated: snapshot.claimable_dispatches.saturated, + outbox_lag_seconds: backlog_age(snapshot.observed_at, &snapshot.claimable_dispatches) + .as_secs_f64(), + dead_letter_triggers: snapshot.dead_letter_triggers.observed_count, + dead_letter_triggers_saturated: snapshot.dead_letter_triggers.saturated, + dead_letter_dispatches: snapshot.dead_letter_dispatches.observed_count, + dead_letter_dispatches_saturated: snapshot.dead_letter_dispatches.saturated, + } +} + +fn backlog_age(observed_at: DateTime, sample: &ExecutionQueueBacklogSample) -> Duration { + sample + .oldest_at + .map(|oldest_at| observed_at.signed_duration_since(oldest_at)) + .and_then(|age| age.to_std().ok()) + .unwrap_or(Duration::ZERO) +} + +fn record_queue_health(health: &ExecutionQueueHealthReport) { + moa_observability::runtime_metrics::record_execution_trigger_queue( + Duration::from_secs_f64(health.trigger_lag_seconds), + u64::from(health.due_triggers), + health.due_triggers_saturated, + u64::from(health.dead_letter_triggers), + health.dead_letter_triggers_saturated, + ); + moa_observability::runtime_metrics::record_execution_outbox_queue( + Duration::from_secs_f64(health.outbox_lag_seconds), + u64::from(health.claimable_dispatches), + health.claimable_dispatches_saturated, + u64::from(health.dead_letter_dispatches), + health.dead_letter_dispatches_saturated, + ); +} + +async fn settle_delivered_batch( + ctx: &ObjectContext<'_>, + repository: &ExecutionRepository, + claim_owner: &str, + dispatch_uids: Vec, + response: &mut DrainExecutionDispatchesResponse, +) -> Result<(), HandlerError> { + if dispatch_uids.is_empty() { + return Ok(()); + } + let requested = dispatch_uids.len(); + let repository = repository.clone(); + let claim_owner = claim_owner.to_string(); + let outcome = ctx + .run(|| async move { + repository + .mark_dispatches_delivered( + ExecutionScope::ControlPlane, + &dispatch_uids, + &claim_owner, + ) + .await + .map(|delivered_dispatch_uids| { + Json::from(JournaledDispatchAckBatch { + delivered_dispatch_uids, + }) + }) + .map_err(execution_error_to_handler_error) + }) + .name("execution_dispatch_ack_batch") + .await? + .into_inner(); + let acknowledged = outcome.delivered_dispatch_uids.len(); + if acknowledged > requested { + return Err(TerminalError::new( + "execution dispatch batch acknowledgement exceeded its request", + ) + .into()); + } + response.acknowledged += acknowledged; + response.stale_claims += requested - acknowledged; + Ok(()) +} + +async fn settle_failure( + ctx: &ObjectContext<'_>, + repository: &ExecutionRepository, + claim_owner: &str, + dispatch: JournaledExecutionDispatch, + error: String, + response: &mut DrainExecutionDispatchesResponse, +) -> Result<(), HandlerError> { + let repository = repository.clone(); + let claim_owner = claim_owner.to_string(); + let dispatch_uid = dispatch.dispatch_uid; + let outcome = ctx + .run(|| async move { + repository + .record_dispatch_failure( + ExecutionScope::ControlPlane, + dispatch_uid, + &claim_owner, + &error, + DISPATCH_RETRY_POLICY, + ) + .await + .map(|outcome| { + Json::from(match outcome { + ExecutionDispatchFailureOutcome::RetryScheduled { .. } => { + JournaledDispatchFailure::RetryScheduled + } + ExecutionDispatchFailureOutcome::DeadLettered => { + JournaledDispatchFailure::DeadLettered + } + ExecutionDispatchFailureOutcome::StaleClaim => { + JournaledDispatchFailure::StaleClaim + } + }) + }) + .map_err(execution_error_to_handler_error) + }) + .name(format!("execution_dispatch_fail_{dispatch_uid}")) + .await? + .into_inner(); + match outcome { + JournaledDispatchFailure::RetryScheduled => response.retry_scheduled += 1, + JournaledDispatchFailure::DeadLettered => response.dead_lettered += 1, + JournaledDispatchFailure::StaleClaim => response.stale_claims += 1, + } + Ok(()) +} + +async fn accept_batch( + ctx: &ObjectContext<'_>, + dispatches: &[JournaledExecutionDispatch], +) -> Result>, HandlerError> { + let targets = dispatches + .iter() + .map(JournaledExecutionDispatch::target) + .collect::>(); + let mut results = (0..targets.len()).map(|_| None).collect::>(); + let mut run_slots = Vec::new(); + let mut run_calls = DurableFuturesUnordered::new(); + let mut trigger_slots = Vec::new(); + let mut trigger_calls = DurableFuturesUnordered::new(); + + // Restate command creation follows stable claimed-row order. Async send targets only await + // durable acceptance at their exact slot; the expensive synchronous controller/trigger calls + // are retained in homogeneous durable fan-ins and reassembled by stable slot below. + for (slot, target) in targets.into_iter().enumerate() { + match target { + Ok(ExecutionDispatchTarget::RunActivation(request)) => { + let dispatch_uid = request.dispatch_uid; + run_slots.push(slot); + run_calls.push( + crate::restate_identity::replay_safe_request( + ctx.object_client::( + request.run_uid.to_string(), + ) + .advance(Json::from(request)) + .idempotency_key(dispatch_uid.to_string()), + ) + .call(), + ); + } + Ok(ExecutionDispatchTarget::TriggerDelivery(request)) => { + let dispatch_uid = request.dispatch_uid; + trigger_slots.push(slot); + trigger_calls.push( + crate::restate_identity::replay_safe_request( + ctx.service_client::() + .fire(Json::from(request)) + .idempotency_key(dispatch_uid.to_string()), + ) + .call(), + ); + } + Ok(target) => results[slot] = Some(accept_target(ctx, target).await.map(|_| ())), + Err(error) => results[slot] = Some(Err(error.to_string())), + } + } + + while let Some((fanout_slot, result)) = run_calls.next().await? { + results[run_slots[fanout_slot]] = + Some(result.map(|_| ()).map_err(|error| error.to_string())); + } + while let Some((fanout_slot, result)) = trigger_calls.next().await? { + results[trigger_slots[fanout_slot]] = + Some(result.map(|_| ()).map_err(|error| error.to_string())); + } + results + .into_iter() + .map(|result| { + result.ok_or_else(|| { + HandlerError::from(TerminalError::new( + "execution dispatch fan-in dropped a result before settlement", + )) + }) + }) + .collect() +} + +async fn accept_target( + ctx: &ObjectContext<'_>, + target: ExecutionDispatchTarget, +) -> Result { + match target { + ExecutionDispatchTarget::RunActivation(_) | ExecutionDispatchTarget::TriggerDelivery(_) => { + Err("synchronous dispatch target bypassed bounded fan-out".to_string()) + } + ExecutionDispatchTarget::TaskAttempt(request) => { + let dispatch_uid = request.dispatch_uid; + let handle = crate::restate_identity::replay_safe_request( + ctx.workflow_client::(dispatch_uid.to_string()) + .run(Json::from(request)) + .idempotency_key(dispatch_uid.to_string()), + ) + .send(); + handle + .invocation_id() + .await + .map_err(|error| error.to_string()) + } + ExecutionDispatchTarget::TaskAttemptCancel(request) => { + let dispatch_uid = request.cancellation_dispatch_uid; + let workflow_key = request.active_dispatch_uid.to_string(); + let handle = crate::restate_identity::replay_safe_request( + ctx.workflow_client::(workflow_key) + .cancel(Json::from(request)) + .idempotency_key(dispatch_uid.to_string()), + ) + .send(); + handle + .invocation_id() + .await + .map_err(|error| error.to_string()) + } + ExecutionDispatchTarget::CompensationAttempt(request) => { + let dispatch_uid = request.dispatch_uid; + let handle = crate::restate_identity::replay_safe_request( + ctx.workflow_client::(dispatch_uid.to_string()) + .run(Json::from(request)) + .idempotency_key(dispatch_uid.to_string()), + ) + .send(); + handle + .invocation_id() + .await + .map_err(|error| error.to_string()) + } + ExecutionDispatchTarget::CompensationAttemptCancel(request) => { + let dispatch_uid = request.cancellation_dispatch_uid; + let workflow_key = request.active_dispatch_uid.to_string(); + let handle = crate::restate_identity::replay_safe_request( + ctx.workflow_client::(workflow_key) + .cancel(Json::from(request)) + .idempotency_key(dispatch_uid.to_string()), + ) + .send(); + handle + .invocation_id() + .await + .map_err(|error| error.to_string()) + } + ExecutionDispatchTarget::ExternalCancel { + dispatch_uid, + request, + } => { + let handle = crate::restate_identity::replay_safe_request( + ctx.service_client::() + .cancel_external_job(Json::from(request)) + .idempotency_key(dispatch_uid.to_string()), + ) + .send(); + handle + .invocation_id() + .await + .map_err(|error| error.to_string()) + } + } +} + +#[cfg(test)] +mod tests { + use super::{ + dispatch_head_idempotency_key, next_dispatch_delay, next_dispatch_successor, + reconciliation_drain_idempotency_key, + }; + use chrono::{TimeDelta, Utc}; + use std::time::Duration; + + #[test] + fn next_dispatch_delay_preserves_future_deadline_and_clamps_due_work() { + // Pins: the dispatcher sleeps until a future database deadline but immediately drains + // work already due, including small clock differences between the query and scheduler. + let observed_at = Utc::now(); + assert_eq!( + next_dispatch_delay(observed_at, observed_at + TimeDelta::seconds(5)), + Duration::from_secs(5) + ); + assert_eq!( + next_dispatch_delay(observed_at, observed_at - TimeDelta::milliseconds(1)), + Duration::ZERO + ); + } + + #[test] + fn dispatch_head_identity_coalesces_same_head_and_advances_with_changed_head() { + // Pins: concurrent producer kicks that observe one indexed head address one Restate + // invocation, while either a new head row or a rearmed due time addresses new work. + let due_at = Utc::now() + TimeDelta::seconds(5); + let first_uid = uuid::Uuid::from_u128(1); + let updated_at = Utc::now(); + let same_head = dispatch_head_idempotency_key(first_uid, due_at, updated_at); + assert_eq!( + same_head, + dispatch_head_idempotency_key(first_uid, due_at, updated_at) + ); + assert_ne!( + same_head, + dispatch_head_idempotency_key(uuid::Uuid::from_u128(2), due_at, updated_at) + ); + assert_ne!( + same_head, + dispatch_head_idempotency_key(first_uid, due_at + TimeDelta::seconds(1), updated_at) + ); + assert_ne!( + same_head, + dispatch_head_idempotency_key( + first_uid, + due_at, + updated_at + TimeDelta::milliseconds(1) + ) + ); + } + + #[test] + fn early_empty_drain_retries_same_head_with_distinct_identity_after_one_millisecond() { + // Pins: Restate stores delayed-send deadlines at millisecond precision. If that truncates + // a sub-millisecond future head into an early empty drain, its successor must not reuse + // the completing invocation's identity and be swallowed by idempotency memoization. + let observed_at = Utc::now(); + let due_at = observed_at + TimeDelta::microseconds(500); + let dispatch_uid = uuid::Uuid::from_u128(1); + let updated_at = observed_at - TimeDelta::seconds(1); + let base_key = dispatch_head_idempotency_key(dispatch_uid, due_at, updated_at); + + let (retry_key, retry_delay) = + next_dispatch_successor(dispatch_uid, due_at, updated_at, observed_at, 0); + assert_ne!(retry_key, base_key); + assert_eq!(retry_delay, Duration::from_millis(1)); + + let (normal_key, normal_delay) = + next_dispatch_successor(dispatch_uid, due_at, updated_at, observed_at, 1); + assert_eq!(normal_key, base_key); + assert_eq!(normal_delay, Duration::from_micros(500)); + } + + #[test] + fn reconciliation_redrive_does_not_reuse_a_completed_head_identity() { + // Pins: repair can requeue the same dispatch UID and due time after downstream loss; each + // persisted maintenance generation must therefore bypass the normal completed head key. + let head_key = + dispatch_head_idempotency_key(uuid::Uuid::from_u128(1), Utc::now(), Utc::now()); + let first_repair = reconciliation_drain_idempotency_key(7); + assert_ne!(head_key, first_repair); + assert_eq!(first_repair, reconciliation_drain_idempotency_key(7)); + assert_ne!(first_repair, reconciliation_drain_idempotency_key(8)); + } +} diff --git a/crates/moa-orchestrator/src/services/execution_retention.rs b/crates/moa-orchestrator/src/services/execution_retention.rs new file mode 100644 index 000000000..116276884 --- /dev/null +++ b/crates/moa-orchestrator/src/services/execution_retention.rs @@ -0,0 +1,283 @@ +//! Durable, self-scheduling terminal execution-detail retention. + +use std::time::Duration; + +use moa_execution::repository::{ + ExecutionRepository, ExecutionScope, + retention::{ + ExecutionRetentionClaimOutcome, ExecutionRetentionPageOutcome, + ExecutionRetentionScheduleReceipt, + }, +}; +use restate_sdk::prelude::*; +use serde::{Deserialize, Serialize}; + +use crate::workflows::errors::execution_error_to_handler_error; + +const RETENTION_PAGE_SIZE: u32 = 64; +const BACKLOG_DELAY: Duration = Duration::from_secs(5); +const INITIAL_IDLE_DELAY: Duration = Duration::from_secs(60); +const MAXIMUM_IDLE_DELAY: Duration = Duration::from_secs(60 * 60); +const FAILURE_RETRY_DELAY: Duration = Duration::from_secs(30); + +/// Generation-fenced invocation accepted from bootstrap repair or the prior pass. +#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ExecutionRetentionRequest { + /// Exact scheduled generation; absent only for the coarse repair Cron. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub expected_generation: Option, +} + +/// Bounded pass response exposed to operational callers. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ExecutionRetentionResponse { + /// Work performed by this pass, or `None` when another schedule owns it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub page: Option, + /// Durable generation accepted by the next delayed invocation. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub scheduled_generation: Option, + /// Delay until the next normal pass. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub next_delay_seconds: Option, +} + +/// Ingress-private bounded retention target. +#[restate_sdk::service] +#[name = "ExecutionRetention"] +pub trait ExecutionRetention { + /// Archives or deletes at most one bounded page, then schedules the next generation. + async fn run( + request: Json, + ) -> Result, HandlerError>; +} + +/// PostgreSQL-backed execution retention implementation. +#[derive(Clone)] +pub struct ExecutionRetentionImpl { + repository: ExecutionRepository, + retention_days: u64, +} + +impl ExecutionRetentionImpl { + /// Creates the retention target from the validated fleet policy. + #[must_use] + pub fn new(pool: sqlx::PgPool, config: &moa_config::ExecutionConfig) -> Self { + Self { + repository: ExecutionRepository::new(pool), + retention_days: config.terminal_detail_retention_days, + } + } +} + +impl ExecutionRetention for ExecutionRetentionImpl { + #[tracing::instrument(skip(self, ctx, request))] + // SAFETY: ingress-private maintenance uses control-plane RLS and rechecks terminal and legal-hold fences. + async fn run( + &self, + ctx: Context<'_>, + request: Json, + ) -> Result, HandlerError> { + crate::ctx::adopt_incoming_trace_parent(&ctx); + moa_observability::restate_observability::annotate_restate_handler_span( + "ExecutionRetention", + "run", + ); + let repository = self.repository.clone(); + let expected_generation = request.into_inner().expected_generation; + let claim = ctx + .run(|| async move { + repository + .claim_execution_retention(ExecutionScope::ControlPlane, expected_generation) + .await + .map(Json::from) + .map_err(execution_error_to_handler_error) + }) + .name("execution_retention_claim") + .await? + .into_inner(); + let ExecutionRetentionClaimOutcome::Claimed { + generation, + previous_delay_seconds, + } = claim + else { + return Ok(Json::from(ExecutionRetentionResponse { + page: None, + scheduled_generation: None, + next_delay_seconds: None, + })); + }; + + let repository = self.repository.clone(); + let retention_days = self.retention_days; + let page = match ctx + .run(|| async move { + repository + .advance_execution_retention_page( + ExecutionScope::ControlPlane, + retention_days, + RETENTION_PAGE_SIZE, + ) + .await + .map(Json::from) + .map_err(execution_error_to_handler_error) + }) + .name(format!("execution_retention_page_{generation}")) + .await + { + Ok(page) => page.into_inner(), + Err(error) => { + schedule_after_failure(&ctx, &self.repository, generation, &error.to_string()) + .await; + return Err(error.into()); + } + }; + let delay = retention_delay(&page, previous_delay_seconds); + let receipt = + persist_next_schedule(&ctx, &self.repository, generation, delay, None).await?; + send_next(&ctx, &receipt, delay).await?; + + Ok(Json::from(ExecutionRetentionResponse { + page: Some(page), + scheduled_generation: Some(receipt.scheduled_generation), + next_delay_seconds: Some(delay.as_secs()), + })) + } +} + +async fn persist_next_schedule( + ctx: &Context<'_>, + repository: &ExecutionRepository, + generation: u64, + delay: Duration, + failure: Option, +) -> Result { + let repository = repository.clone(); + ctx.run(|| async move { + repository + .schedule_execution_retention( + ExecutionScope::ControlPlane, + generation, + delay.as_secs(), + failure.as_deref(), + ) + .await + .map(Json::from) + .map_err(execution_error_to_handler_error) + }) + .name(format!("execution_retention_schedule_{generation}")) + .await + .map(Json::into_inner) + .map_err(HandlerError::from) +} + +async fn send_next( + ctx: &Context<'_>, + receipt: &ExecutionRetentionScheduleReceipt, + delay: Duration, +) -> Result<(), HandlerError> { + let request = ExecutionRetentionRequest { + expected_generation: Some(receipt.scheduled_generation), + }; + let handle = crate::restate_identity::replay_safe_request( + ctx.service_client::() + .run(Json::from(request)) + .idempotency_key(format!( + "execution-retention-generation-{}", + receipt.scheduled_generation + )), + ) + .send_after(delay); + let _invocation_id = handle.invocation_id().await?; + Ok(()) +} + +async fn schedule_after_failure( + ctx: &Context<'_>, + repository: &ExecutionRepository, + generation: u64, + error: &str, +) { + match persist_next_schedule( + ctx, + repository, + generation, + FAILURE_RETRY_DELAY, + Some(error.to_string()), + ) + .await + { + Ok(receipt) => { + if let Err(schedule_error) = send_next(ctx, &receipt, FAILURE_RETRY_DELAY).await { + tracing::warn!( + ?schedule_error, + "failed to accept delayed execution retention retry" + ); + } + } + Err(schedule_error) => { + tracing::warn!( + ?schedule_error, + "failed to persist execution retention retry" + ); + } + } +} + +fn retention_delay( + page: &ExecutionRetentionPageOutcome, + previous_delay_seconds: Option, +) -> Duration { + if !matches!(page, ExecutionRetentionPageOutcome::Idle) { + return BACKLOG_DELAY; + } + let previous = Duration::from_secs(previous_delay_seconds.unwrap_or(0)); + if previous < INITIAL_IDLE_DELAY { + INITIAL_IDLE_DELAY + } else { + previous.saturating_mul(2).min(MAXIMUM_IDLE_DELAY) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use uuid::Uuid; + + #[test] + fn idle_retention_backs_off_but_backlog_resets_to_short_delay() { + // Pins: empty fleets stop waking frequently while any archive/delete work + // restores the short bounded cadence. + assert_eq!( + retention_delay(&ExecutionRetentionPageOutcome::Idle, None), + INITIAL_IDLE_DELAY + ); + assert_eq!( + retention_delay( + &ExecutionRetentionPageOutcome::Idle, + Some(INITIAL_IDLE_DELAY.as_secs()) + ), + INITIAL_IDLE_DELAY.saturating_mul(2) + ); + assert_eq!( + retention_delay( + &ExecutionRetentionPageOutcome::Idle, + Some(MAXIMUM_IDLE_DELAY.as_secs()) + ), + MAXIMUM_IDLE_DELAY + ); + assert_eq!( + retention_delay( + &ExecutionRetentionPageOutcome::SegmentArchived { + run_uid: Uuid::nil(), + segment_kind: "task".to_string(), + records: 1, + }, + Some(MAXIMUM_IDLE_DELAY.as_secs()) + ), + BACKLOG_DELAY + ); + } +} diff --git a/crates/moa-orchestrator/src/services/execution_schedule.rs b/crates/moa-orchestrator/src/services/execution_schedule.rs new file mode 100644 index 000000000..009ca73a7 --- /dev/null +++ b/crates/moa-orchestrator/src/services/execution_schedule.rs @@ -0,0 +1,742 @@ +//! Authenticated tenant control surface for recurring durable executions. + +use chrono::{DateTime, Duration, LocalResult, NaiveDateTime, TimeZone, Utc}; +use chrono_tz::Tz; +use croner::Cron; +use moa_authz_schema::Relation; +use moa_config::ExecutionConfig; +use moa_core::types::execution_planning::{ + ExecutionScheduleCreateRequest, ExecutionScheduleDstPolicy, ExecutionScheduleListRequest, + ExecutionScheduleMissedFirePolicy, ExecutionScheduleOriginSource, ExecutionSchedulePage, + ExecutionSchedulePolicy, ExecutionScheduleRecord, ExecutionScheduleRequest, + ExecutionScheduleUpdateRequest, +}; +use moa_execution::repository::{ + ExecutionRepository, ExecutionScope, + schedule::{ + ExecutionScheduleCreateOutcome, ExecutionScheduleMutationOutcome, + ExecutionScheduleOccurrence, ExecutionScheduleRunAdmission, + ExecutionScheduleRunAdmissionOutcome, execution_schedule_run_blueprint, + }, +}; +use moa_observability::restate_observability::annotate_restate_handler_span; +use restate_sdk::prelude::*; +use sqlx::PgPool; +use uuid::Uuid; + +use crate::{ + handlers::authz_shim::AuthzEnforcer, + services::execution_dispatcher::{DispatchExecutionsRequest, ExecutionDispatcherClient}, + workflows::errors::execution_error_to_handler_error, +}; + +/// Restate service surface for tenant recurring execution schedules. +#[restate_sdk::service] +#[name = "ExecutionSchedule"] +pub trait ExecutionSchedule { + /// Creates one immutable pinned schedule and arms its first occurrence. + async fn create( + request: Json, + ) -> Result, HandlerError>; + + /// Loads one schedule status after tenant authorization. + async fn status( + request: Json, + ) -> Result>, HandlerError>; + + /// Lists one bounded stable page of tenant schedules. + async fn list( + request: Json, + ) -> Result, HandlerError>; + + /// Replaces mutable timing/resource policy behind an incarnation fence. + async fn update( + request: Json, + ) -> Result, HandlerError>; + + /// Pauses future occurrences without changing immutable inputs. + async fn pause( + request: Json, + ) -> Result, HandlerError>; + + /// Resumes a paused schedule using its persisted missed-fire policy. + async fn resume( + request: Json, + ) -> Result, HandlerError>; + + /// Permanently fences future occurrences while retaining audit state. + async fn cancel( + request: Json, + ) -> Result, HandlerError>; + + /// Consumes one exact persisted schedule-occurrence trigger. + async fn fire_occurrence( + request: Json, + ) -> Result, HandlerError>; +} + +/// Durable disposition of one trusted schedule-occurrence delivery. +#[derive(Clone, Debug, serde::Deserialize, Eq, PartialEq, serde::Serialize)] +#[serde(tag = "outcome", rename_all = "snake_case")] +pub enum ExecutionScheduleFireResponse { + /// A fresh run and its initial controller activation committed. + Admitted { + /// Deterministic fresh occurrence run. + run_uid: Uuid, + /// Initial controller activation outbox identity. + activation_dispatch_uid: Uuid, + }, + /// Overlap policy consumed the occurrence without creating a run. + Skipped, + /// The same deterministic occurrence already committed. + Replayed { + /// Existing run when the occurrence was admitted rather than skipped. + run_uid: Option, + /// Existing activation outbox when a run was admitted. + activation_dispatch_uid: Option, + }, + /// A pause, cancellation, or newer incarnation fenced the trigger. + Stale, +} + +#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] +struct JournaledScheduleWrite { + record: ExecutionScheduleRecord, + dispatcher_kick_uid: Option, +} + +/// Concrete authenticated recurring-execution schedule service. +#[derive(Clone)] +pub struct ExecutionScheduleImpl { + repository: ExecutionRepository, + authz: AuthzEnforcer, + config: ExecutionConfig, +} + +impl ExecutionScheduleImpl { + /// Creates the service over the shared runtime Postgres pool. + #[must_use] + pub(crate) fn new(pool: PgPool, authz: AuthzEnforcer, config: ExecutionConfig) -> Self { + Self { + repository: ExecutionRepository::new(pool), + authz, + config, + } + } +} + +impl ExecutionSchedule for ExecutionScheduleImpl { + #[tracing::instrument(skip(self, ctx, request))] + async fn create( + &self, + ctx: Context<'_>, + request: Json, + ) -> Result, HandlerError> { + prepare_handler(&ctx, "create"); + let request = request.into_inner(); + let identity = self + .authz + .authorize_tenant(&ctx, request.tenant_id, Relation::Operator) + .await?; + if request.origin.created_by != identity || request.run_as_identity != identity { + return Err(TerminalError::new_with_code( + 403, + "schedule creator and run_as_identity must equal the authorized operator", + ) + .into()); + } + if request.origin.source != ExecutionScheduleOriginSource::TenantApi { + return Err(TerminalError::new_with_code( + 400, + "ExecutionSchedule create accepts only tenant_api origin; session origins require the session-owned admission path", + ) + .into()); + } + let repository = self.repository.clone(); + let config = self.config.clone(); + let write = ctx + .run(|| async move { + let first = next_occurrence(&request.policy, Utc::now(), true)?; + let tenant_id = request.tenant_id; + match repository + .create_schedule( + ExecutionScope::Tenant { tenant_id }, + &config, + request, + first, + ) + .await + .map_err(execution_error_to_handler_error)? + { + ExecutionScheduleCreateOutcome::Created { schedule, trigger } => { + Ok(Json::from(JournaledScheduleWrite { + record: *schedule, + dispatcher_kick_uid: trigger.map(|write| write.dispatch.dispatch_uid), + })) + } + ExecutionScheduleCreateOutcome::Replayed(schedule) => { + Ok(Json::from(JournaledScheduleWrite { + record: *schedule, + dispatcher_kick_uid: None, + })) + } + ExecutionScheduleCreateOutcome::Conflict => Err(TerminalError::new_with_code( + 409, + "schedule_uid is already bound to different immutable inputs", + ) + .into()), + } + }) + .name("execution_schedule_create") + .await? + .into_inner(); + kick_new_schedule_trigger(&ctx, write.dispatcher_kick_uid).await?; + Ok(Json::from(write.record)) + } + + #[tracing::instrument(skip(self, ctx, request))] + async fn status( + &self, + ctx: Context<'_>, + request: Json, + ) -> Result>, HandlerError> { + prepare_handler(&ctx, "status"); + let request = request.into_inner(); + self.authz + .authorize_tenant(&ctx, request.tenant_id, Relation::Operator) + .await?; + let repository = self.repository.clone(); + Ok(ctx + .run(|| async move { + repository + .load_schedule( + ExecutionScope::Tenant { + tenant_id: request.tenant_id, + }, + request.tenant_id, + request.schedule_uid, + ) + .await + .map(Json::from) + .map_err(execution_error_to_handler_error) + }) + .name("execution_schedule_status") + .await?) + } + + #[tracing::instrument(skip(self, ctx, request))] + async fn list( + &self, + ctx: Context<'_>, + request: Json, + ) -> Result, HandlerError> { + prepare_handler(&ctx, "list"); + let request = request.into_inner(); + self.authz + .authorize_tenant(&ctx, request.tenant_id, Relation::Operator) + .await?; + let repository = self.repository.clone(); + Ok(ctx + .run(|| async move { + repository + .list_schedules( + ExecutionScope::Tenant { + tenant_id: request.tenant_id, + }, + request.tenant_id, + request.limit, + request.cursor, + ) + .await + .map(Json::from) + .map_err(execution_error_to_handler_error) + }) + .name("execution_schedule_list") + .await?) + } + + #[tracing::instrument(skip(self, ctx, request))] + async fn update( + &self, + ctx: Context<'_>, + request: Json, + ) -> Result, HandlerError> { + prepare_handler(&ctx, "update"); + let request = request.into_inner(); + self.authz + .authorize_tenant(&ctx, request.tenant_id, Relation::Operator) + .await?; + let repository = self.repository.clone(); + let config = self.config.clone(); + let write = ctx + .run(|| async move { + let next = next_occurrence(&request.policy, Utc::now(), true)?; + let tenant_id = request.tenant_id; + mutation_write( + repository + .update_schedule( + ExecutionScope::Tenant { tenant_id }, + &config, + request, + next, + ) + .await + .map_err(execution_error_to_handler_error)?, + ) + }) + .name("execution_schedule_update") + .await? + .into_inner(); + kick_new_schedule_trigger(&ctx, write.dispatcher_kick_uid).await?; + Ok(Json::from(write.record)) + } + + #[tracing::instrument(skip(self, ctx, request))] + async fn pause( + &self, + ctx: Context<'_>, + request: Json, + ) -> Result, HandlerError> { + prepare_handler(&ctx, "pause"); + let request = request.into_inner(); + self.authz + .authorize_tenant(&ctx, request.tenant_id, Relation::Operator) + .await?; + let repository = self.repository.clone(); + let config = self.config.clone(); + Ok(ctx + .run(|| async move { + mutation_record( + repository + .pause_schedule( + ExecutionScope::Tenant { + tenant_id: request.tenant_id, + }, + &config, + request.tenant_id, + request.schedule_uid, + ) + .await + .map_err(execution_error_to_handler_error)?, + ) + }) + .name("execution_schedule_pause") + .await?) + } + + #[tracing::instrument(skip(self, ctx, request))] + async fn resume( + &self, + ctx: Context<'_>, + request: Json, + ) -> Result, HandlerError> { + prepare_handler(&ctx, "resume"); + let request = request.into_inner(); + self.authz + .authorize_tenant(&ctx, request.tenant_id, Relation::Operator) + .await?; + let repository = self.repository.clone(); + let config = self.config.clone(); + let write = ctx + .run(|| async move { + let scope = ExecutionScope::Tenant { + tenant_id: request.tenant_id, + }; + let schedule = repository + .load_schedule(scope, request.tenant_id, request.schedule_uid) + .await + .map_err(execution_error_to_handler_error)? + .ok_or_else(not_found)?; + let now = Utc::now(); + let next = resume_occurrence(&schedule, now)?; + mutation_write( + repository + .resume_schedule( + scope, + &config, + request.tenant_id, + request.schedule_uid, + next, + ) + .await + .map_err(execution_error_to_handler_error)?, + ) + }) + .name("execution_schedule_resume") + .await? + .into_inner(); + kick_new_schedule_trigger(&ctx, write.dispatcher_kick_uid).await?; + Ok(Json::from(write.record)) + } + + #[tracing::instrument(skip(self, ctx, request))] + async fn cancel( + &self, + ctx: Context<'_>, + request: Json, + ) -> Result, HandlerError> { + prepare_handler(&ctx, "cancel"); + let request = request.into_inner(); + self.authz + .authorize_tenant(&ctx, request.tenant_id, Relation::Operator) + .await?; + let repository = self.repository.clone(); + let config = self.config.clone(); + Ok(ctx + .run(|| async move { + mutation_record( + repository + .cancel_schedule( + ExecutionScope::Tenant { + tenant_id: request.tenant_id, + }, + &config, + request.tenant_id, + request.schedule_uid, + ) + .await + .map_err(execution_error_to_handler_error)?, + ) + }) + .name("execution_schedule_cancel") + .await?) + } + + #[tracing::instrument(skip(self, ctx, request), fields( + dispatch_uid = %request.0.dispatch_uid, + trigger_uid = %request.0.trigger_uid, + ))] + // SAFETY: ingress-private delivery reloads the exact trigger, schedule, run-as identity, blueprint, and incarnation under tenant RLS. + async fn fire_occurrence( + &self, + ctx: Context<'_>, + request: Json, + ) -> Result, HandlerError> { + prepare_handler(&ctx, "fire_occurrence"); + let request = request.into_inner(); + let trigger_uid = request.trigger_uid; + let repository = self.repository.clone(); + let config = self.config.clone(); + let response = ctx + .run(|| async move { + let scope = ExecutionScope::Tenant { + tenant_id: request.tenant_id, + }; + let trigger = repository + .load_trigger(scope, request.trigger_uid) + .await + .map_err(execution_error_to_handler_error)? + .ok_or_else(|| { + TerminalError::new_with_code(404, "schedule trigger not found") + })?; + let (Some(schedule_uid), Some(schedule_incarnation), Some(occurrence_sequence)) = ( + trigger.schedule_uid, + trigger.schedule_incarnation, + trigger.occurrence_sequence, + ) else { + return Err(TerminalError::new_with_code( + 409, + "trigger is not a schedule occurrence", + ) + .into()); + }; + let Some(schedule) = repository + .load_schedule(scope, request.tenant_id, schedule_uid) + .await + .map_err(execution_error_to_handler_error)? + else { + let _ = repository + .fire_trigger(scope, request.trigger_uid) + .await + .map_err(execution_error_to_handler_error)?; + return Ok(Json::from(ExecutionScheduleFireResponse::Stale)); + }; + // The trigger freezes the occurrence. Mutable schedule cursor fields may + // already be cleared by successful completion, pause, or cancellation + // when this exact delivery is replayed. + let local = serde_json::from_value::( + trigger + .payload + .get("occurrence_local") + .cloned() + .ok_or_else(|| { + TerminalError::new_with_code( + 500, + "schedule trigger is missing its frozen local occurrence", + ) + })?, + ) + .map_err(|_| { + TerminalError::new_with_code( + 500, + "schedule trigger has an invalid frozen local occurrence", + ) + })?; + let occurrence = ExecutionScheduleOccurrence { + at: trigger.due_at, + local, + }; + let blueprint = execution_schedule_run_blueprint(&schedule) + .map_err(execution_error_to_handler_error)?; + let run = blueprint + .instantiate( + &schedule, + occurrence, + occurrence_sequence, + config.maximum_horizon_seconds, + ) + .map_err(execution_error_to_handler_error)?; + let next = + next_occurrence(&schedule.policy, occurrence.at + Duration::seconds(1), true)?; + let outcome = repository + .admit_schedule_occurrence( + scope, + &config, + ExecutionScheduleRunAdmission { + tenant_id: request.tenant_id, + schedule_uid, + schedule_incarnation, + occurrence_sequence, + trigger_uid: request.trigger_uid, + trigger_dispatch_uid: request.dispatch_uid, + occurrence, + run, + next_occurrence: next, + }, + ) + .await + .map_err(execution_error_to_handler_error)?; + Ok(Json::from(match outcome { + ExecutionScheduleRunAdmissionOutcome::Admitted { + run, activation, .. + } => ExecutionScheduleFireResponse::Admitted { + run_uid: run.run_uid, + activation_dispatch_uid: activation.dispatch_uid, + }, + ExecutionScheduleRunAdmissionOutcome::Skipped { .. } => { + ExecutionScheduleFireResponse::Skipped + } + ExecutionScheduleRunAdmissionOutcome::Replayed { + run_uid, + activation_dispatch_uid, + } => ExecutionScheduleFireResponse::Replayed { + run_uid, + activation_dispatch_uid, + }, + ExecutionScheduleRunAdmissionOutcome::Stale => { + ExecutionScheduleFireResponse::Stale + } + })) + }) + .name(format!("execution_schedule_fire_{trigger_uid}")) + .await? + .into_inner(); + Ok(Json::from(response)) + } +} + +fn prepare_handler(ctx: &Context<'_>, handler: &'static str) { + crate::ctx::adopt_incoming_trace_parent(ctx); + annotate_restate_handler_span("ExecutionSchedule", handler); +} + +fn mutation_record( + outcome: ExecutionScheduleMutationOutcome, +) -> Result, HandlerError> { + match outcome { + ExecutionScheduleMutationOutcome::Updated { schedule, .. } => Ok(Json::from(*schedule)), + ExecutionScheduleMutationOutcome::NotFound => Err(not_found()), + ExecutionScheduleMutationOutcome::Stale => Err(TerminalError::new_with_code( + 409, + "schedule lifecycle state or incarnation is stale", + ) + .into()), + } +} + +fn mutation_write( + outcome: ExecutionScheduleMutationOutcome, +) -> Result, HandlerError> { + match outcome { + ExecutionScheduleMutationOutcome::Updated { schedule, trigger } => { + Ok(Json::from(JournaledScheduleWrite { + record: *schedule, + dispatcher_kick_uid: trigger.map(|write| write.dispatch.dispatch_uid), + })) + } + ExecutionScheduleMutationOutcome::NotFound => Err(not_found()), + ExecutionScheduleMutationOutcome::Stale => Err(TerminalError::new_with_code( + 409, + "schedule lifecycle state or incarnation is stale", + ) + .into()), + } +} + +async fn kick_new_schedule_trigger( + ctx: &Context<'_>, + dispatch_uid: Option, +) -> Result<(), HandlerError> { + let Some(dispatch_uid) = dispatch_uid else { + return Ok(()); + }; + // Schedule mutations occur outside a dispatcher chain. Accept exactly one dispatcher kick + // only when the transaction armed a new outbox-backed trigger; replay and empty schedules do + // not create another invocation. + let handle = crate::restate_identity::replay_safe_request( + ctx.service_client::() + .dispatch(Json::from(DispatchExecutionsRequest::default())) + .idempotency_key(format!("execution-schedule-trigger:{dispatch_uid}")), + ) + .send(); + handle.invocation_id().await?; + Ok(()) +} + +fn not_found() -> HandlerError { + TerminalError::new_with_code(404, "execution schedule not found").into() +} + +fn resume_occurrence( + schedule: &ExecutionScheduleRecord, + now: DateTime, +) -> Result, HandlerError> { + let anchor = match schedule.policy.missed_fire_policy { + ExecutionScheduleMissedFirePolicy::Skip => now, + ExecutionScheduleMissedFirePolicy::FireOnce => schedule.paused_at.unwrap_or(now), + }; + let occurrence = next_occurrence(&schedule.policy, anchor, true)?; + Ok(apply_missed_fire_policy( + schedule.policy.missed_fire_policy, + occurrence, + now, + )) +} + +fn apply_missed_fire_policy( + policy: ExecutionScheduleMissedFirePolicy, + occurrence: Option, + now: DateTime, +) -> Option { + match (policy, occurrence) { + (ExecutionScheduleMissedFirePolicy::FireOnce, Some(missed)) if missed.at <= now => { + Some(ExecutionScheduleOccurrence { + at: now, + local: missed.local, + }) + } + (_, occurrence) => occurrence, + } +} + +fn next_occurrence( + policy: &ExecutionSchedulePolicy, + anchor: DateTime, + inclusive: bool, +) -> Result, HandlerError> { + let cron = Cron::new(&policy.calendar_expression) + .with_seconds_optional() + .parse() + .map_err(|error| TerminalError::new(format!("invalid calendar expression: {error}")))?; + let timezone: Tz = policy + .timezone + .parse() + .map_err(|_| TerminalError::new(format!("invalid IANA timezone: {}", policy.timezone)))?; + let mut cursor = anchor.max(policy.start_at); + for _ in 0..8 { + let candidate = cron + .find_next_occurrence(&cursor.with_timezone(&timezone), inclusive) + .map_err(|error| TerminalError::new(format!("no next schedule occurrence: {error}")))?; + let local = candidate.naive_local(); + let resolved = resolve_local(timezone, local, policy.dst_policy); + let Some(at) = resolved else { + cursor = candidate.with_timezone(&Utc) + Duration::seconds(1); + continue; + }; + if policy.end_at.is_some_and(|end_at| at >= end_at) { + return Ok(None); + } + return Ok(Some(ExecutionScheduleOccurrence { at, local })); + } + Err(TerminalError::new("schedule DST policy skipped too many consecutive candidates").into()) +} + +fn resolve_local( + timezone: Tz, + local: NaiveDateTime, + policy: ExecutionScheduleDstPolicy, +) -> Option> { + match timezone.from_local_datetime(&local) { + LocalResult::Single(value) => Some(value.with_timezone(&Utc)), + LocalResult::Ambiguous(earlier, later) => match policy { + ExecutionScheduleDstPolicy::Earliest => Some(earlier.with_timezone(&Utc)), + ExecutionScheduleDstPolicy::Latest => Some(later.with_timezone(&Utc)), + ExecutionScheduleDstPolicy::Skip => None, + }, + LocalResult::None => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn dst_fall_back_policy_selects_exact_utc_instant() { + // Pins: an ambiguous New York 01:30 occurrence does not depend on library default choice. + let local = NaiveDateTime::parse_from_str("2026-11-01 01:30:00", "%Y-%m-%d %H:%M:%S") + .expect("valid local timestamp"); + let earlier = resolve_local( + chrono_tz::America::New_York, + local, + ExecutionScheduleDstPolicy::Earliest, + ) + .expect("earlier ambiguous instant"); + let later = resolve_local( + chrono_tz::America::New_York, + local, + ExecutionScheduleDstPolicy::Latest, + ) + .expect("later ambiguous instant"); + + assert_eq!(later - earlier, Duration::hours(1)); + assert_eq!( + resolve_local( + chrono_tz::America::New_York, + local, + ExecutionScheduleDstPolicy::Skip + ), + None + ); + } + + #[test] + fn missed_fire_policy_coalesces_once_without_rewriting_local_identity() { + // Pins: resume never replays every missed occurrence; FireOnce emits one immediate + // occurrence with the frozen missed local time, while Skip retains the next future fire. + let now = DateTime::parse_from_rfc3339("2026-08-11T12:00:00Z") + .expect("valid UTC test time") + .with_timezone(&Utc); + let missed = ExecutionScheduleOccurrence { + at: now - Duration::hours(3), + local: (now - Duration::hours(3)).naive_utc(), + }; + let fired = apply_missed_fire_policy( + ExecutionScheduleMissedFirePolicy::FireOnce, + Some(missed), + now, + ) + .expect("fire-once keeps one occurrence"); + assert_eq!(fired.at, now); + assert_eq!(fired.local, missed.local); + + let future = ExecutionScheduleOccurrence { + at: now + Duration::hours(1), + local: (now + Duration::hours(1)).naive_utc(), + }; + assert_eq!( + apply_missed_fire_policy(ExecutionScheduleMissedFirePolicy::Skip, Some(future), now,), + Some(future) + ); + } +} diff --git a/crates/moa-orchestrator/src/services/execution_trigger.rs b/crates/moa-orchestrator/src/services/execution_trigger.rs new file mode 100644 index 000000000..41d6f49b4 --- /dev/null +++ b/crates/moa-orchestrator/src/services/execution_trigger.rs @@ -0,0 +1,819 @@ +//! Restate delivery for exact, generation-fenced execution triggers. + +use std::time::Duration; + +use moa_execution::repository::{ + ExecutionRepository, ExecutionScope, TransitionOutcome, + terminal::PendingTerminalAdvanceOutcome, + trigger::{ + ExecutionExternalReconcileTriggerOutcome, ExecutionExternalStartRecoveryTriggerOutcome, + ExecutionRunDeadlineTriggerOutcome, ExecutionTriggerFireOutcome, ExecutionTriggerKind, + ExecutionTriggerNoOp, ExecutionWatchdogTriggerOutcome, + }, +}; +use moa_execution::wire::{ + ExecutionAttemptWatchdogResponseOutcome, ExecutionCompensationAttemptWatchdogRequest, + ExecutionExternalJobReconcileRequest, ExecutionExternalJobReconcileResponseOutcome, + ExecutionExternalJobStartRecoveryRequest, ExecutionExternalJobStartRecoveryResponseOutcome, + ExecutionTaskAttemptWatchdogRequest, +}; +use restate_sdk::prelude::*; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use crate::{ + runtime::execution_dispatch::ExecutionTriggerDeliveryRequest, + services::execution_schedule::{ExecutionScheduleClient, ExecutionScheduleFireResponse}, + services::tool_executor::ToolExecutorClient, + workflows::{ + errors::execution_error_to_handler_error, + execution_compensation_attempt::ExecutionCompensationAttemptClient, + execution_task_attempt::ExecutionTaskAttemptClient, + }, +}; + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[serde(tag = "disposition", rename_all = "snake_case")] +enum ExternalReconcileRoute { + Ready { + request: ExecutionExternalJobReconcileRequest, + }, + NoOp { + response: ExecutionTriggerFireResponse, + }, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[serde(tag = "disposition", rename_all = "snake_case")] +enum ExternalStartRecoveryRoute { + Ready { + request: Box, + }, + NoOp { + response: ExecutionTriggerFireResponse, + }, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[serde(tag = "disposition", rename_all = "snake_case")] +enum RunDeadlineRoute { + Fenced { + response: ExecutionTriggerFireResponse, + }, + NoOp { + response: ExecutionTriggerFireResponse, + }, +} + +#[derive(Debug, thiserror::Error)] +#[error("run deadline fence changed while its trigger delivery was in flight")] +struct RunDeadlineFenceRace; + +#[derive(Debug, thiserror::Error)] +#[error("run deadline delivery arrived before its canonical database due time")] +struct RunDeadlineNotDue; + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[serde(tag = "disposition", rename_all = "snake_case")] +enum WatchdogRoute { + Task { + request: ExecutionTaskAttemptWatchdogRequest, + }, + Compensation { + request: ExecutionCompensationAttemptWatchdogRequest, + }, + NoOp { + response: ExecutionTriggerFireResponse, + }, +} + +#[derive(Debug, thiserror::Error)] +#[error("watchdog receiver remains current after requesting redelivery")] +struct WatchdogReceiverStillCurrent; + +/// Successful disposition of one immutable trigger delivery. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(tag = "outcome", rename_all = "snake_case")] +pub enum ExecutionTriggerFireResponse { + /// Canonical trigger state advanced and may have enqueued a run activation. + Delivered { + /// Same-transaction run-activation dispatch, when the trigger owns a run. + activation_dispatch_uid: Option, + }, + /// No visible trigger has the supplied immutable identity. + NotFound, + /// The same trigger was already delivered. + Duplicate, + /// Cancellation or supersession fenced the trigger. + Inactive, + /// A newer run, task, compensation, or schedule incarnation won. + StaleGeneration, + /// Delivery arrived before the canonical absolute due time. + NotDue, +} + +/// Restate service that owns temporal-trigger delivery. +#[restate_sdk::service] +#[name = "ExecutionTrigger"] +pub trait ExecutionTrigger { + /// Fires one exact trigger after reloading its canonical database state. + async fn fire( + request: Json, + ) -> Result, HandlerError>; +} + +/// PostgreSQL-backed exact trigger delivery. +#[derive(Clone)] +pub struct ExecutionTriggerImpl { + repository: ExecutionRepository, + config: moa_config::ExecutionConfig, +} + +impl ExecutionTriggerImpl { + /// Creates a trigger service over the shared execution repository. + #[must_use] + pub fn new(pool: sqlx::PgPool, config: &moa_config::ExecutionConfig) -> Self { + Self { + repository: ExecutionRepository::new(pool), + config: config.clone(), + } + } +} + +impl ExecutionTrigger for ExecutionTriggerImpl { + #[tracing::instrument(skip(self, ctx, request), fields( + dispatch_uid = %request.0.dispatch_uid, + trigger_uid = %request.0.trigger_uid, + ))] + // SAFETY: ingress-private outbox delivery; the handler reloads the exact tenant-scoped trigger and every persisted generation fence. + async fn fire( + &self, + ctx: Context<'_>, + request: Json, + ) -> Result, HandlerError> { + crate::ctx::adopt_incoming_trace_parent(&ctx); + moa_observability::restate_observability::annotate_restate_handler_span( + "ExecutionTrigger", + "fire", + ); + let request = request.into_inner(); + let delivery_dispatch_uid = request.dispatch_uid; + let trigger_uid = request.trigger_uid; + let tenant_id = request.tenant_id; + let repository = self.repository.clone(); + let trigger_kind = ctx + .run(|| async move { + repository + .load_trigger(ExecutionScope::Tenant { tenant_id }, trigger_uid) + .await + .map(|trigger| { + Json::from(trigger.map(|trigger| trigger.kind.as_str().to_string())) + }) + .map_err(execution_error_to_handler_error) + }) + .name(format!("execution_trigger_route_{trigger_uid}")) + .await? + .into_inner(); + if trigger_kind.is_none() { + return Ok(Json::from(ExecutionTriggerFireResponse::NotFound)); + } + if trigger_kind.as_deref() == Some(ExecutionTriggerKind::ScheduleOccurrence.as_str()) { + let response = crate::restate_identity::replay_safe_request( + ctx.service_client::() + .fire_occurrence(Json::from(request.clone())) + .idempotency_key(request.dispatch_uid.to_string()), + ) + .call() + .await? + .into_inner(); + let response = schedule_trigger_response(response); + return Ok(Json::from(response)); + } + if trigger_kind.as_deref() == Some(ExecutionTriggerKind::RunDeadline.as_str()) { + let repository = self.repository.clone(); + let config = self.config.clone(); + let page_limit = + u32::try_from(self.config.maximum_activation_steps).unwrap_or(u32::MAX); + let route = ctx + .run(move || async move { + let prepared = repository + .prepare_run_deadline_trigger( + ExecutionScope::Tenant { tenant_id }, + trigger_uid, + ) + .await + .map_err(execution_error_to_handler_error)?; + let route = match prepared { + ExecutionRunDeadlineTriggerOutcome::Ready { + run_uid, + controller_generation, + wake_epoch, + observed_at, + } => { + let outcome = repository + .fence_deadline_and_enqueue_settlement( + &config, + ExecutionScope::Tenant { tenant_id }, + run_uid, + controller_generation, + wake_epoch, + observed_at, + page_limit, + ) + .await + .map_err(execution_error_to_handler_error)?; + run_deadline_fence_route(outcome).map_err(HandlerError::from)? + } + ExecutionRunDeadlineTriggerOutcome::NoOp(reason) => { + run_deadline_noop_route(reason).map_err(HandlerError::from)? + } + }; + Ok::<_, HandlerError>(Json::from(route)) + }) + .name(format!("execution_run_deadline_fence_{trigger_uid}")) + .retry_policy(run_deadline_retry_policy()) + .await? + .into_inner(); + let response = match route { + RunDeadlineRoute::Fenced { response } => response, + RunDeadlineRoute::NoOp { response } => return Ok(Json::from(response)), + }; + let repository = self.repository.clone(); + ctx.run(|| async move { + repository + .settle_run_deadline_trigger(ExecutionScope::Tenant { tenant_id }, trigger_uid) + .await + .map(|_| Json::from(())) + .map_err(execution_error_to_handler_error) + }) + .name(format!("execution_run_deadline_settle_{trigger_uid}")) + .await?; + return Ok(Json::from(response)); + } + if matches!( + trigger_kind.as_deref(), + Some(kind) + if kind == ExecutionTriggerKind::TaskTimer.as_str() + || kind == ExecutionTriggerKind::WaitExpiry.as_str() + ) { + // These deliveries are accepted synchronously by ExecutionDispatcher. Their commit is + // visible when that owning dispatcher selects its next outbox head; another kick here + // would only create duplicate, unkeyed dispatcher invocations. + let repository = self.repository.clone(); + let config = self.config.clone(); + let response = ctx + .run(|| async move { + repository + .fire_wait_trigger( + ExecutionScope::Tenant { + tenant_id: request.tenant_id, + }, + &config, + request.trigger_uid, + ) + .await + .map(wait_trigger_response) + .map(Json::from) + .map_err(execution_error_to_handler_error) + }) + .name(format!("execution_wait_trigger_fire_{trigger_uid}")) + .await?; + return Ok(response); + } + if matches!( + trigger_kind.as_deref(), + Some(kind) + if kind == ExecutionTriggerKind::TaskWatchdog.as_str() + || kind == ExecutionTriggerKind::CompensationWatchdog.as_str() + ) { + let repository = self.repository.clone(); + let prepared = ctx + .run(|| async move { + let outcome = repository + .prepare_watchdog_trigger(ExecutionScope::Tenant { tenant_id }, trigger_uid) + .await + .map_err(execution_error_to_handler_error)?; + let route = watchdog_route(outcome); + Ok::<_, HandlerError>(Json::from(route)) + }) + .name(format!("execution_watchdog_prepare_{trigger_uid}")) + .await? + .into_inner(); + let watchdog_outcome = match prepared { + WatchdogRoute::Task { request } => { + crate::restate_identity::replay_safe_request( + ctx.workflow_client::( + request.dispatch_uid.to_string(), + ) + .watchdog(Json::from(request)) + .idempotency_key(delivery_dispatch_uid.to_string()), + ) + .call() + .await? + .into_inner() + .outcome + } + WatchdogRoute::Compensation { request } => { + crate::restate_identity::replay_safe_request( + ctx.workflow_client::( + request.dispatch_uid.to_string(), + ) + .watchdog(Json::from(request)) + .idempotency_key(delivery_dispatch_uid.to_string()), + ) + .call() + .await? + .into_inner() + .outcome + } + WatchdogRoute::NoOp { response } => return Ok(Json::from(response)), + }; + if watchdog_outcome == ExecutionAttemptWatchdogResponseOutcome::RetryDelivery { + // The receiver can race a different durable owner transition. Revalidate after + // its response so a watchdog superseded by that transition completes as a stale + // delivery instead of blocking the fleet-serialized drain behind endless retry. + let repository = self.repository.clone(); + let response = ctx + .run(|| async move { + let refreshed = repository + .prepare_watchdog_trigger( + ExecutionScope::Tenant { tenant_id }, + trigger_uid, + ) + .await + .map(watchdog_route) + .map_err(execution_error_to_handler_error)?; + watchdog_retry_response(refreshed).map(Json::from) + }) + .name(format!("execution_watchdog_revalidate_{trigger_uid}")) + .retry_policy(watchdog_revalidation_retry_policy()) + .await? + .into_inner(); + return Ok(Json::from(response)); + } + let repository = self.repository.clone(); + ctx.run(|| async move { + repository + .settle_watchdog_trigger(ExecutionScope::Tenant { tenant_id }, trigger_uid) + .await + .map(|_| Json::from(())) + .map_err(execution_error_to_handler_error) + }) + .name(format!("execution_watchdog_settle_{trigger_uid}")) + .await?; + return Ok(Json::from(ExecutionTriggerFireResponse::Delivered { + activation_dispatch_uid: None, + })); + } + if trigger_kind.as_deref() == Some(ExecutionTriggerKind::ExternalStartRecovery.as_str()) { + let repository = self.repository.clone(); + let prepared = ctx + .run(|| async move { + let outcome = repository + .prepare_external_start_recovery_trigger( + ExecutionScope::Tenant { tenant_id }, + trigger_uid, + ) + .await + .map_err(execution_error_to_handler_error)?; + let route = match outcome { + ExecutionExternalStartRecoveryTriggerOutcome::Ready(request) => { + ExternalStartRecoveryRoute::Ready { + request: Box::new(request), + } + } + ExecutionExternalStartRecoveryTriggerOutcome::NoOp(reason) => { + ExternalStartRecoveryRoute::NoOp { + response: trigger_response(ExecutionTriggerFireOutcome::NoOp( + reason, + )), + } + } + }; + Ok::<_, HandlerError>(Json::from(route)) + }) + .name(format!( + "execution_external_start_recovery_prepare_{trigger_uid}" + )) + .await? + .into_inner(); + let recovery_request = match prepared { + ExternalStartRecoveryRoute::Ready { request } => *request, + ExternalStartRecoveryRoute::NoOp { response } => { + return Ok(Json::from(response)); + } + }; + let recovery_response = crate::restate_identity::replay_safe_request( + ctx.service_client::() + .recover_external_job_start(Json::from(recovery_request)) + .idempotency_key(delivery_dispatch_uid.to_string()), + ) + .call() + .await? + .into_inner(); + if recovery_response.outcome + == ExecutionExternalJobStartRecoveryResponseOutcome::UnknownPreserved + { + // ToolExecutor atomically rearmed the same durable outbox row. The owning + // dispatcher observes its new head deadline after this synchronous call returns. + return Ok(Json::from(ExecutionTriggerFireResponse::NotDue)); + } + let repository = self.repository.clone(); + ctx.run(|| async move { + repository + .settle_external_start_recovery_trigger( + ExecutionScope::Tenant { tenant_id }, + trigger_uid, + ) + .await + .map(|_| Json::from(())) + .map_err(execution_error_to_handler_error) + }) + .name(format!( + "execution_external_start_recovery_settle_{trigger_uid}" + )) + .await?; + return Ok(Json::from(match recovery_response.outcome { + ExecutionExternalJobStartRecoveryResponseOutcome::NotStartedReleased + | ExecutionExternalJobStartRecoveryResponseOutcome::StartedBound => { + ExecutionTriggerFireResponse::Delivered { + activation_dispatch_uid: None, + } + } + ExecutionExternalJobStartRecoveryResponseOutcome::StaleDelivery => { + ExecutionTriggerFireResponse::StaleGeneration + } + ExecutionExternalJobStartRecoveryResponseOutcome::AlreadySettled => { + ExecutionTriggerFireResponse::Inactive + } + ExecutionExternalJobStartRecoveryResponseOutcome::UnknownPreserved => { + return Err(crate::workflows::errors::moa_error_to_handler_error( + moa_core::error::MoaError::ProviderTransport( + "external start recovery remained ambiguous without durable rearm" + .to_string(), + ), + )); + } + })); + } + if trigger_kind.as_deref() == Some(ExecutionTriggerKind::ExternalReconcile.as_str()) { + let repository = self.repository.clone(); + let prepared = ctx + .run(|| async move { + let outcome = repository + .prepare_external_reconcile_trigger( + ExecutionScope::Tenant { tenant_id }, + trigger_uid, + ) + .await + .map_err(execution_error_to_handler_error)?; + let route = match outcome { + ExecutionExternalReconcileTriggerOutcome::Ready(request) => { + ExternalReconcileRoute::Ready { request } + } + ExecutionExternalReconcileTriggerOutcome::NoOp(reason) => { + ExternalReconcileRoute::NoOp { + response: trigger_response(ExecutionTriggerFireOutcome::NoOp( + reason, + )), + } + } + }; + Ok::<_, HandlerError>(Json::from(route)) + }) + .name(format!( + "execution_external_reconcile_prepare_{trigger_uid}" + )) + .await? + .into_inner(); + let reconcile_request = match prepared { + ExternalReconcileRoute::Ready { request } => request, + ExternalReconcileRoute::NoOp { response } => { + return Ok(Json::from(response)); + } + }; + let reconcile_response = crate::restate_identity::replay_safe_request( + ctx.service_client::() + .reconcile_external_job(Json::from(reconcile_request)) + .idempotency_key(delivery_dispatch_uid.to_string()), + ) + .call() + .await? + .into_inner(); + let repository = self.repository.clone(); + ctx.run(|| async move { + repository + .settle_external_reconcile_trigger( + ExecutionScope::Tenant { tenant_id }, + trigger_uid, + ) + .await + .map(|_| Json::from(())) + .map_err(execution_error_to_handler_error) + }) + .name(format!("execution_external_reconcile_settle_{trigger_uid}")) + .await?; + let response = external_reconcile_response(reconcile_response.outcome); + return Ok(Json::from(response)); + } + let repository = self.repository.clone(); + let response = ctx + .run(|| async move { + repository + .fire_trigger( + ExecutionScope::Tenant { + tenant_id: request.tenant_id, + }, + request.trigger_uid, + ) + .await + .map(trigger_response) + .map(Json::from) + .map_err(execution_error_to_handler_error) + }) + .name(format!("execution_trigger_fire_{trigger_uid}")) + .await?; + Ok(Json::from(response.into_inner())) + } +} + +fn watchdog_route(outcome: ExecutionWatchdogTriggerOutcome) -> WatchdogRoute { + match outcome { + ExecutionWatchdogTriggerOutcome::Task(request) => WatchdogRoute::Task { request }, + ExecutionWatchdogTriggerOutcome::Compensation(request) => { + WatchdogRoute::Compensation { request } + } + ExecutionWatchdogTriggerOutcome::NoOp(reason) => WatchdogRoute::NoOp { + response: trigger_response(ExecutionTriggerFireOutcome::NoOp(reason)), + }, + } +} + +fn watchdog_retry_response( + route: WatchdogRoute, +) -> Result { + match route { + WatchdogRoute::Task { .. } | WatchdogRoute::Compensation { .. } => { + Err(WatchdogReceiverStillCurrent.into()) + } + WatchdogRoute::NoOp { response } => Ok(response), + } +} + +fn watchdog_revalidation_retry_policy() -> RunRetryPolicy { + RunRetryPolicy::new() + .initial_delay(Duration::from_millis(10)) + .exponentiation_factor(2.0) + .max_delay(Duration::from_secs(1)) +} + +fn run_deadline_fence_route( + outcome: PendingTerminalAdvanceOutcome, +) -> Result { + match outcome { + PendingTerminalAdvanceOutcome::Applied(commit) => Ok(RunDeadlineRoute::Fenced { + response: ExecutionTriggerFireResponse::Delivered { + activation_dispatch_uid: commit.continuation.map(|dispatch| dispatch.dispatch_uid), + }, + }), + PendingTerminalAdvanceOutcome::Replayed(_) => Ok(RunDeadlineRoute::Fenced { + response: ExecutionTriggerFireResponse::Duplicate, + }), + PendingTerminalAdvanceOutcome::NotFound => Ok(RunDeadlineRoute::NoOp { + response: ExecutionTriggerFireResponse::NotFound, + }), + PendingTerminalAdvanceOutcome::Conflict => Err(RunDeadlineFenceRace), + } +} + +fn run_deadline_noop_route( + reason: ExecutionTriggerNoOp, +) -> Result { + if reason == ExecutionTriggerNoOp::NotDue { + return Err(RunDeadlineNotDue); + } + Ok(RunDeadlineRoute::NoOp { + response: trigger_response(ExecutionTriggerFireOutcome::NoOp(reason)), + }) +} + +fn run_deadline_retry_policy() -> RunRetryPolicy { + RunRetryPolicy::new() + .initial_delay(Duration::from_millis(10)) + .exponentiation_factor(2.0) + .max_delay(Duration::from_secs(1)) +} + +fn wait_trigger_response( + outcome: ( + TransitionOutcome, + Option, + ), +) -> ExecutionTriggerFireResponse { + match outcome { + (TransitionOutcome::Applied(_), activation) => ExecutionTriggerFireResponse::Delivered { + activation_dispatch_uid: activation.map(|dispatch| dispatch.dispatch_uid), + }, + (TransitionOutcome::AlreadyApplied(_), _) => ExecutionTriggerFireResponse::Duplicate, + (TransitionOutcome::NotFound, _) => ExecutionTriggerFireResponse::NotFound, + (TransitionOutcome::Rejected(_), _) => ExecutionTriggerFireResponse::StaleGeneration, + (TransitionOutcome::RunApplied(_), _) | (TransitionOutcome::RunAlreadyApplied(_), _) => { + ExecutionTriggerFireResponse::StaleGeneration + } + } +} + +fn external_reconcile_response( + outcome: ExecutionExternalJobReconcileResponseOutcome, +) -> ExecutionTriggerFireResponse { + match outcome { + ExecutionExternalJobReconcileResponseOutcome::Applied { .. } => { + ExecutionTriggerFireResponse::Delivered { + activation_dispatch_uid: None, + } + } + ExecutionExternalJobReconcileResponseOutcome::StaleDelivery => { + ExecutionTriggerFireResponse::StaleGeneration + } + ExecutionExternalJobReconcileResponseOutcome::AlreadyTerminal => { + ExecutionTriggerFireResponse::Inactive + } + ExecutionExternalJobReconcileResponseOutcome::NotFound => { + ExecutionTriggerFireResponse::NotFound + } + } +} + +fn schedule_trigger_response( + outcome: ExecutionScheduleFireResponse, +) -> ExecutionTriggerFireResponse { + match outcome { + ExecutionScheduleFireResponse::Admitted { + activation_dispatch_uid, + .. + } => ExecutionTriggerFireResponse::Delivered { + activation_dispatch_uid: Some(activation_dispatch_uid), + }, + ExecutionScheduleFireResponse::Skipped => ExecutionTriggerFireResponse::Delivered { + activation_dispatch_uid: None, + }, + ExecutionScheduleFireResponse::Replayed { .. } => ExecutionTriggerFireResponse::Duplicate, + ExecutionScheduleFireResponse::Stale => ExecutionTriggerFireResponse::StaleGeneration, + } +} + +fn trigger_response(outcome: ExecutionTriggerFireOutcome) -> ExecutionTriggerFireResponse { + match outcome { + ExecutionTriggerFireOutcome::Delivered { activation } => { + ExecutionTriggerFireResponse::Delivered { + activation_dispatch_uid: activation.map(|dispatch| dispatch.dispatch_uid), + } + } + ExecutionTriggerFireOutcome::NoOp(reason) => match reason { + ExecutionTriggerNoOp::NotFound => ExecutionTriggerFireResponse::NotFound, + ExecutionTriggerNoOp::Duplicate => ExecutionTriggerFireResponse::Duplicate, + ExecutionTriggerNoOp::Inactive => ExecutionTriggerFireResponse::Inactive, + ExecutionTriggerNoOp::StaleGeneration => ExecutionTriggerFireResponse::StaleGeneration, + ExecutionTriggerNoOp::NotDue => ExecutionTriggerFireResponse::NotDue, + }, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn every_stale_delivery_is_a_successful_no_op_response() { + // Pins: Restate retries cannot turn stale trigger delivery into product failure. + let cases = [ + ( + ExecutionTriggerNoOp::NotFound, + ExecutionTriggerFireResponse::NotFound, + ), + ( + ExecutionTriggerNoOp::Duplicate, + ExecutionTriggerFireResponse::Duplicate, + ), + ( + ExecutionTriggerNoOp::Inactive, + ExecutionTriggerFireResponse::Inactive, + ), + ( + ExecutionTriggerNoOp::StaleGeneration, + ExecutionTriggerFireResponse::StaleGeneration, + ), + ( + ExecutionTriggerNoOp::NotDue, + ExecutionTriggerFireResponse::NotDue, + ), + ]; + for (reason, expected) in cases { + assert_eq!( + trigger_response(ExecutionTriggerFireOutcome::NoOp(reason)), + expected + ); + } + } + + #[test] + fn watchdog_retry_revalidation_completes_a_superseded_delivery_offline() + -> Result<(), HandlerError> { + // Pins: a receiver RetryDelivery is retried only while repository revalidation still + // resolves an active receiver; a concurrently superseded watchdog is a successful no-op. + assert_eq!( + watchdog_retry_response(watchdog_route(ExecutionWatchdogTriggerOutcome::NoOp( + ExecutionTriggerNoOp::Inactive, + )))?, + ExecutionTriggerFireResponse::Inactive + ); + assert_eq!( + watchdog_retry_response(watchdog_route(ExecutionWatchdogTriggerOutcome::NoOp( + ExecutionTriggerNoOp::StaleGeneration, + )))?, + ExecutionTriggerFireResponse::StaleGeneration + ); + assert!( + watchdog_retry_response(WatchdogRoute::Task { + request: ExecutionTaskAttemptWatchdogRequest { + dispatch_uid: Uuid::from_u128(1), + capacity_reservation_uid: Uuid::from_u128(2), + watchdog_trigger_uid: Uuid::from_u128(3), + run_uid: Uuid::from_u128(4), + task_id: moa_execution::state::ExecutionTaskId::from_uuid(Uuid::from_u128(5)), + controller_generation: 1, + attempt_generation: 1, + tenant_id: moa_core::types::identifiers::TenantId::new(), + }, + }) + .is_err(), + "an exact current receiver must keep the ctx.run operation retrying" + ); + Ok::<_, HandlerError>(()) + } + + #[test] + fn schedule_occurrence_dispositions_preserve_delivery_and_no_op_semantics() { + // Pins: schedule occurrence routing never turns overlap, replay, or an + // obsolete schedule incarnation into a failed trigger delivery. + let activation_dispatch_uid = Uuid::from_u128(1); + assert_eq!( + schedule_trigger_response(ExecutionScheduleFireResponse::Admitted { + run_uid: Uuid::from_u128(2), + activation_dispatch_uid, + }), + ExecutionTriggerFireResponse::Delivered { + activation_dispatch_uid: Some(activation_dispatch_uid), + } + ); + assert_eq!( + schedule_trigger_response(ExecutionScheduleFireResponse::Skipped), + ExecutionTriggerFireResponse::Delivered { + activation_dispatch_uid: None, + } + ); + assert_eq!( + schedule_trigger_response(ExecutionScheduleFireResponse::Replayed { + run_uid: None, + activation_dispatch_uid: None, + }), + ExecutionTriggerFireResponse::Duplicate + ); + assert_eq!( + schedule_trigger_response(ExecutionScheduleFireResponse::Stale), + ExecutionTriggerFireResponse::StaleGeneration + ); + } + + #[test] + fn run_deadline_conflict_retries_without_exposing_a_settlement_route() { + // Pins: a pause or wake transition between deadline preparation and fencing is a + // retryable race, never a stale-success response that allows the trigger to settle. + assert!(matches!( + run_deadline_fence_route(PendingTerminalAdvanceOutcome::Conflict), + Err(RunDeadlineFenceRace) + )); + assert!(matches!( + run_deadline_fence_route(PendingTerminalAdvanceOutcome::NotFound), + Ok(RunDeadlineRoute::NoOp { + response: ExecutionTriggerFireResponse::NotFound, + }) + )); + } + + #[test] + fn early_run_deadline_delivery_retries_until_database_due_time() { + // Pins: an application/Restate clock that reaches a delayed invocation slightly before + // PostgreSQL must not memoize NotDue under the immutable trigger-delivery identity. + assert!(matches!( + run_deadline_noop_route(ExecutionTriggerNoOp::NotDue), + Err(RunDeadlineNotDue) + )); + assert!(matches!( + run_deadline_noop_route(ExecutionTriggerNoOp::StaleGeneration), + Ok(RunDeadlineRoute::NoOp { + response: ExecutionTriggerFireResponse::StaleGeneration, + }) + )); + } +} diff --git a/crates/moa-orchestrator/src/services/llm_gateway.rs b/crates/moa-orchestrator/src/services/llm_gateway.rs index cca50a02c..17752bc06 100644 --- a/crates/moa-orchestrator/src/services/llm_gateway.rs +++ b/crates/moa-orchestrator/src/services/llm_gateway.rs @@ -58,8 +58,8 @@ pub(crate) enum LLMCompletionOwnerKind { RootTurn, /// A worker turn workflow. WorkerTurn, - /// A durable execution run and every task it owns. - ExecutionRun, + /// One immutable bounded execution-task attempt. + ExecutionTaskAttempt, } impl LLMCompletionOwnerKind { @@ -67,7 +67,7 @@ impl LLMCompletionOwnerKind { match self { Self::RootTurn => "root_turn", Self::WorkerTurn => "worker_turn", - Self::ExecutionRun => "execution_run", + Self::ExecutionTaskAttempt => "execution_task_attempt", } } } @@ -96,11 +96,11 @@ impl LLMCompletionOwner { } } - /// Creates the owner for one durable execution run and all of its task workflows. - pub(crate) fn execution_run(workflow_key: impl Into) -> Self { + /// Creates the owner for one immutable bounded execution-task attempt. + pub(crate) fn execution_task_attempt(dispatch_uid: uuid::Uuid) -> Self { Self { - kind: LLMCompletionOwnerKind::ExecutionRun, - workflow_key: workflow_key.into(), + kind: LLMCompletionOwnerKind::ExecutionTaskAttempt, + workflow_key: dispatch_uid.to_string(), } } @@ -339,12 +339,6 @@ pub(crate) enum LLMCompletionAction { WorkerModel { turn: usize }, /// One execution-task generation and model-loop turn. ExecutionTaskModel { generation: u64, turn: u32 }, - /// One execution amendment generation or repair attempt. - ExecutionAmendment { - run_uid: Uuid, - plan_revision: u64, - attempt: usize, - }, /// One behavior-lab simulator turn. ExperimentSimulator { trial_uid: Uuid, turn: u32 }, } @@ -361,11 +355,6 @@ impl LLMCompletionAction { Self::ExecutionTaskModel { generation, turn } => { format!("execution-task-model:{generation}:{turn}") } - Self::ExecutionAmendment { - run_uid, - plan_revision, - attempt, - } => format!("execution-amendment:{run_uid}:{plan_revision}:{attempt}"), Self::ExperimentSimulator { trial_uid, turn } => { format!("experiment-simulator:{trial_uid}:{turn}") } @@ -418,34 +407,6 @@ pub(crate) async fn cancel_completion_owner( Ok(()) } -/// Durably fences an owner from an ordinary Restate service handler. -pub(crate) async fn cancel_completion_owner_from_service( - ctx: &Context<'_>, - owner: LLMCompletionOwner, -) -> Result<(), HandlerError> { - crate::restate_identity::replay_safe_request( - ctx.service_client::() - .cancel_owner(Json::from(owner)), - ) - .call() - .await?; - Ok(()) -} - -/// Durably fences an owner from its keyed execution-run workflow. -pub(crate) async fn cancel_completion_owner_from_workflow( - ctx: &WorkflowContext<'_>, - owner: LLMCompletionOwner, -) -> Result<(), HandlerError> { - crate::restate_identity::replay_safe_request( - ctx.service_client::() - .cancel_owner(Json::from(owner)), - ) - .call() - .await?; - Ok(()) -} - async fn wait_for_completion_owner_cancellation( runtime_cache: Arc, owner: LLMCompletionOwner, @@ -978,7 +939,7 @@ mod tests { fn completion_owner_is_typed_stripped_and_zero_usage_offline() { // Pins: the internal workflow owner never leaks into a provider request, // and a fenced completion cannot contribute content or billable usage. - let owner = LLMCompletionOwner::execution_run(uuid::Uuid::from_u128(41).to_string()); + let owner = LLMCompletionOwner::execution_task_attempt(uuid::Uuid::from_u128(41)); let mut request = CompletionRequest::new("cancel me"); attach_completion_owner(&mut request, &owner); @@ -1011,16 +972,20 @@ mod tests { let worker = LLMCompletionOwner::worker_turn(raw_key) .cancellation_key() .expect("worker owner should produce a cache key"); - let run = LLMCompletionOwner::execution_run(raw_key) + let attempt_one = LLMCompletionOwner::execution_task_attempt(uuid::Uuid::from_u128(41)) + .cancellation_key() + .expect("first task-attempt owner should produce a cache key"); + let attempt_two = LLMCompletionOwner::execution_task_attempt(uuid::Uuid::from_u128(42)) .cancellation_key() - .expect("execution run owner should produce a cache key"); + .expect("second task-attempt owner should produce a cache key"); assert_ne!(root, worker); - assert_ne!(worker, run); - assert_ne!(root, run); + assert_ne!(attempt_one, attempt_two); + assert_ne!(attempt_one, root); + assert_ne!(attempt_one, worker); assert!(!root.contains(raw_key)); assert!(!worker.contains(raw_key)); - assert!(!run.contains(raw_key)); + assert!(!attempt_one.contains(&uuid::Uuid::from_u128(41).to_string())); } #[test] diff --git a/crates/moa-orchestrator/src/services/mod.rs b/crates/moa-orchestrator/src/services/mod.rs index 117b61fb8..db4d754e5 100644 --- a/crates/moa-orchestrator/src/services/mod.rs +++ b/crates/moa-orchestrator/src/services/mod.rs @@ -16,7 +16,12 @@ pub mod authz_challenges_reaper; pub mod connectors; pub mod contacts; pub mod dual_control; +pub mod durable_timeout; pub mod execution; +pub mod execution_dispatcher; +pub mod execution_retention; +pub mod execution_schedule; +pub mod execution_trigger; pub mod experiments; pub mod graph_memory_maint; pub mod health; diff --git a/crates/moa-orchestrator/src/services/skill_regression/gate.rs b/crates/moa-orchestrator/src/services/skill_regression/gate.rs index a4be393e5..f1531f9c3 100644 --- a/crates/moa-orchestrator/src/services/skill_regression/gate.rs +++ b/crates/moa-orchestrator/src/services/skill_regression/gate.rs @@ -9,7 +9,8 @@ use moa_core::{ types::{action_policy::ActionRuleScope, experience::LearningCandidate, provider::ModelTask}, }; use moa_eval_core::TestSuite; -use moa_execution::repository::{CompileAuditWriteOutcome, ExecutionRepository, ExecutionScope}; +use moa_execution::repository::audit::CompileAuditWriteOutcome; +use moa_execution::repository::{ExecutionRepository, ExecutionScope}; use moa_providers::ProviderRegistry; use moa_session::PostgresSessionStore; use moa_skills::{ diff --git a/crates/moa-orchestrator/src/services/tool_executor.rs b/crates/moa-orchestrator/src/services/tool_executor.rs index 97c2b31ce..58ae59875 100644 --- a/crates/moa-orchestrator/src/services/tool_executor.rs +++ b/crates/moa-orchestrator/src/services/tool_executor.rs @@ -1,8 +1,10 @@ //! Durable Restate facade over the configured tool router. +use std::collections::{BTreeMap, HashMap}; use std::sync::Arc; use std::time::Duration; +use moa_config::ExecutionConfig; use moa_connectors::executor::{ ConnectorInvocationCompletionService, ConnectorInvocationCompletionTicket, SecuredConnectorOutputMetadata, @@ -20,13 +22,21 @@ use moa_core::{ types::events_stream::EventRecord, types::hands::SandboxFile, types::identifiers::{ - ExecutionRunScopeId, ExecutionTaskScopeId, SessionId, TenantId, ToolCallId, + ExecutionCompensationScopeId, ExecutionRunScopeId, ExecutionTaskScopeId, SessionId, + TenantId, ToolCallId, }, + types::sandbox_workspace::ExecutionHandReleaseOwner, + types::sandbox_workspace::ExecutionHandReleaseReceipt, types::sandbox_workspace::SandboxWorkspaceScope, types::security::ToolCapabilityId, types::session::SessionMeta, + types::tools::AsyncToolJob, + types::tools::AsyncToolJobCallbackOutcome, + types::tools::AsyncToolJobCancelOutcome, + types::tools::ExternalJobStartContext, types::tools::IdempotencyClass, types::tools::SecuredToolOutput, + types::tools::ToolAsyncMode, types::tools::ToolCallRequest, types::tools::ToolDefinition, types::tools::ToolOutput, @@ -35,12 +45,31 @@ use moa_core::{ types::tools::TrustedSandboxFileManifestRef, }; use moa_execution::repository::{ - ExecutionEffectAdmissionOutcome, ExecutionEffectOwner, ExecutionRepository, ExecutionScope, + ExecutionEffectAdmissionOutcome, ExecutionEffectOwner, ExecutionEffectPhase, + ExecutionRepository, ExecutionScope, + compensation::{CompensationAttemptExternalOutcome, CompensationAttemptWriteOutcome}, + external_job::{ + ExecutionExternalJobBinding, ExecutionExternalJobCallback, + ExecutionExternalJobCallbackUpdate, ExecutionExternalJobCancellation, + ExecutionExternalJobCancellationOutcome, ExecutionExternalJobIntentReleaseOutcome, + ExecutionExternalJobOwner, ExecutionExternalJobStartRecoveryAdoptionOutcome, + ExecutionExternalJobState, NewExecutionExternalJobIntent, + }, + trigger::ExecutionExternalStartRecoveryRearmOutcome, +}; +use moa_execution::wire::{ + ExecutionCompensationAttemptCancelRequest, ExecutionCompensationReleaseIntent, + ExecutionExternalJobCancelRequest, ExecutionExternalJobCancelResponse, + ExecutionExternalJobCancelResponseOutcome, ExecutionExternalJobReconcileRequest, + ExecutionExternalJobReconcileResponse, ExecutionExternalJobReconcileResponseOutcome, + ExecutionExternalJobStartRecoveryOwner, ExecutionExternalJobStartRecoveryRequest, + ExecutionExternalJobStartRecoveryResponse, ExecutionExternalJobStartRecoveryResponseOutcome, + ExecutionToolDispatchRejection, }; -use moa_execution::wire::ExecutionToolDispatchRejection; use moa_hands::{ - DeferredWorkspaceToolOutput, JournaledWorkspaceCommit, PendingConnectorToolOutput, - ToolCallScope, ToolCatalogPin, ToolCatalogSnapshot, ToolExecution, ToolRouter, + DeferredWorkspaceToolOutput, ExecutionHandReleaseRequest, JournaledWorkspaceCommit, + PendingConnectorToolOutput, SessionHandReleasePageOutcome, ToolCallScope, ToolCatalogPin, + ToolCatalogSnapshot, ToolExecution, ToolRouter, }; use moa_security::{ OutputClassification, ToolInputCanaryScreening, classify_tool_output, @@ -52,17 +81,421 @@ use restate_sdk::prelude::*; use serde_json::Value; use sha2::{Digest, Sha256}; +use crate::services::execution_dispatcher::{DispatchExecutionsRequest, ExecutionDispatcherClient}; use crate::services::sandbox_workspaces::SandboxWorkspaceManagement; use crate::services::session_store::RestateSessionStoreClient; use crate::turn::util::{blocked_canary_message, blocked_canary_tool_output}; use crate::workflows::errors::{ - authz_error_to_handler_error, moa_error_to_handler_error, sqlx_error_to_handler_error, + authz_error_to_handler_error, execution_error_to_handler_error, moa_error_to_handler_error, + sqlx_error_to_handler_error, }; use moa_observability::restate_observability::annotate_restate_handler_span; use tracing_opentelemetry::OpenTelemetrySpanExt; use crate::connector_catalog::ScopedConnectorCatalogProvider; +/// Transient authentication material presented by a provider callback. +/// +/// This value must be consumed before entering a durable Restate handler so raw +/// signatures and headers are never journaled. Adapters compare it with the +/// persisted callback-authentication reference without exposing the referenced +/// secret to execution workflows. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ExecutionExternalJobCallbackAuthentication { + /// Canonical lower-case provider headers selected by the ingress boundary. + pub headers: BTreeMap, + /// SHA-256 digest of the exact callback body. + pub body_sha256: [u8; 32], +} + +/// Bounded provider callback fields parsed only after transient authentication succeeds. +#[derive(Clone, Debug, PartialEq)] +pub struct ExecutionExternalJobAdapterCallback { + /// Provider-issued job identity asserted by the callback. + pub provider_job_id: String, + /// Stable provider event identity used for durable deduplication. + pub provider_event_id: String, + /// Typed progress or terminal observation. + pub outcome: AsyncToolJobCallbackOutcome, +} + +/// Exact admitted call passed to an asynchronous provider after durable intent reservation. +#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] +#[serde(deny_unknown_fields)] +pub struct ExecutionExternalJobStartRequest { + /// Reserved identity and provider idempotency key that must be used on the network call. + pub context: ExternalJobStartContext, + /// Fully governed tool request admitted against the pinned catalog. + pub call: ToolCallRequest, +} + +/// Bounded result of one declared asynchronous-capable provider start. +#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)] +#[serde(tag = "outcome", rename_all = "snake_case", deny_unknown_fields)] +pub enum ExecutionExternalJobStartOutcome { + /// The provider completed synchronously and owns no durable job. + Completed(Box), + /// The provider committed asynchronous work under the reserved idempotency key. + ExternalJob(AsyncToolJob), +} + +/// Bounded recovery observation for an unbound start intent after runtime loss. +#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)] +#[serde(tag = "outcome", rename_all = "snake_case", deny_unknown_fields)] +pub enum ExecutionExternalJobStartRecovery { + /// Provider evidence proves that no start was committed. + NotStarted, + /// The reserved idempotency key resolves to one committed provider job. + Started(AsyncToolJob), + /// Provider evidence cannot prove whether the start committed. + Unknown { + /// Stable operator-visible reconciliation evidence. + error: serde_json::Value, + }, +} + +/// Provider-specific bounded operations for durable asynchronous tool jobs. +#[async_trait::async_trait] +pub trait ExecutionExternalJobAdapter: Send + Sync { + /// Stable registry key persisted in every external-job row. + fn provider_key(&self) -> &'static str; + + /// Starts one governed asynchronous-capable call using the reserved provider identity. + async fn start( + &self, + request: &ExecutionExternalJobStartRequest, + ) -> moa_core::error::Result; + + /// Recovers one reserved start by provider idempotency key without replaying the task. + async fn recover_start( + &self, + context: &ExternalJobStartContext, + ) -> moa_core::error::Result; + + /// Authenticates transient callback evidence against a persisted reference. + async fn authenticate_callback( + &self, + callback_auth_reference: &str, + authentication: &ExecutionExternalJobCallbackAuthentication, + body: &[u8], + ) -> moa_core::error::Result; + + /// Parses one size-bounded raw callback after authentication, before durable persistence. + async fn parse_callback( + &self, + authentication: &ExecutionExternalJobCallbackAuthentication, + body: &[u8], + ) -> moa_core::error::Result; + + /// Requests cancellation for one exact provider-job generation. + async fn cancel( + &self, + request: &ExecutionExternalJobCancelRequest, + ) -> moa_core::error::Result; + + /// Performs one bounded sparse reconciliation observation. + async fn reconcile( + &self, + request: &ExecutionExternalJobReconcileRequest, + ) -> moa_core::error::Result; +} + +/// Immutable fail-closed registry of asynchronous provider-job adapters. +#[derive(Clone, Default)] +pub struct ExecutionExternalJobAdapterRegistry { + adapters: Arc>>, +} + +impl ExecutionExternalJobAdapterRegistry { + /// Builds a registry and rejects blank or duplicate provider keys. + pub fn new( + adapters: impl IntoIterator>, + ) -> moa_core::error::Result { + let mut keyed = HashMap::new(); + for adapter in adapters { + let provider = adapter.provider_key().trim(); + if provider.is_empty() { + return Err(MoaError::ValidationError( + "external-job adapter provider key must not be blank".to_string(), + )); + } + if keyed.insert(provider.to_string(), adapter).is_some() { + return Err(MoaError::ValidationError(format!( + "duplicate external-job adapter provider key `{provider}`" + ))); + } + } + Ok(Self { + adapters: Arc::new(keyed), + }) + } + + /// Returns the exact registered adapter or fails closed for an unknown provider. + pub fn require( + &self, + provider: &str, + ) -> moa_core::error::Result> { + self.adapters.get(provider).cloned().ok_or_else(|| { + MoaError::ValidationError(format!( + "external-job provider `{provider}` is not registered" + )) + }) + } + + /// Returns whether no asynchronous provider adapter is registered. + #[must_use] + pub fn is_empty(&self) -> bool { + self.adapters.is_empty() + } +} + +/// Provider key reserved for the deterministic integration-only external-job adapter. +#[cfg(all(feature = "provider-overrides", feature = "integration"))] +pub const FIXTURE_EXTERNAL_JOB_PROVIDER: &str = "fixture-external-job"; + +/// Catalog tool exposed only beside the deterministic integration adapter. +#[cfg(all(feature = "provider-overrides", feature = "integration"))] +pub struct FixtureExternalJobTool; + +#[cfg(all(feature = "provider-overrides", feature = "integration"))] +#[async_trait::async_trait] +impl moa_core::traits::BuiltInTool for FixtureExternalJobTool { + fn name(&self) -> &'static str { + "fixture_external_job" + } + + fn description(&self) -> &'static str { + "Starts one deterministic asynchronous fixture job." + } + + fn input_schema(&self) -> serde_json::Value { + serde_json::json!({ + "type": "object", + "properties": { "value": { "type": "string" } }, + "required": ["value"], + "additionalProperties": false + }) + } + + fn policy_spec(&self) -> moa_core::types::tools::ToolPolicySpec { + moa_core::types::tools::ToolPolicySpec { + risk_level: moa_core::types::action_policy::RiskLevel::High, + default_effect: moa_core::types::action_policy::ActionPolicyEffect::Allow, + action_class: moa_core::types::action_policy::ActionClass::ExternalWrite, + input_shape: moa_core::types::tools::ToolInputShape::Json, + diff_strategy: moa_core::types::tools::ToolDiffStrategy::None, + } + } + + fn idempotency_class(&self) -> IdempotencyClass { + IdempotencyClass::NonIdempotent + } + + fn definition(&self) -> ToolDefinition { + ToolDefinition { + name: self.name().to_string(), + description: self.description().to_string(), + schema: self.input_schema(), + policy: self.policy_spec(), + idempotency_class: self.idempotency_class(), + async_mode: ToolAsyncMode::MayReturnExternalJob { + provider: FIXTURE_EXTERNAL_JOB_PROVIDER.to_string(), + }, + rollback: None, + max_output_tokens: 256, + } + } + + async fn execute( + &self, + _input: &serde_json::Value, + _ctx: &moa_core::traits::ToolContext<'_>, + ) -> moa_core::error::Result { + Err(MoaError::ValidationError( + "fixture external-job tool bypassed its declared asynchronous adapter".to_string(), + )) + } +} + +/// Loopback HTTP adapter used by the normal provider-override integration lane. +/// +/// Production composition never constructs this type. It exists so spawned orchestrator E2E +/// processes exercise the real reserve, provider-start, recovery, callback, cancellation, and +/// reconciliation boundaries against a restart-stable parent-process fixture. +#[cfg(all(feature = "provider-overrides", feature = "integration"))] +#[derive(Clone)] +pub struct FixtureHttpExecutionExternalJobAdapter { + client: reqwest::Client, + base_url: reqwest::Url, +} + +#[cfg(all(feature = "provider-overrides", feature = "integration"))] +impl FixtureHttpExecutionExternalJobAdapter { + /// Builds the adapter for one loopback fixture endpoint. + pub fn new(base_url: &str) -> moa_core::error::Result { + let mut base_url = reqwest::Url::parse(base_url).map_err(|error| { + MoaError::ConfigError(format!("parse external-job fixture URL: {error}")) + })?; + if !base_url.host_str().is_some_and(|host| { + host == "localhost" + || host + .parse::() + .is_ok_and(|ip| ip.is_loopback()) + }) { + return Err(MoaError::ConfigError( + "external-job fixture adapter requires a loopback URL".to_string(), + )); + } + if !base_url.path().ends_with('/') { + base_url.set_path(&format!("{}/", base_url.path())); + } + Ok(Self { + client: reqwest::Client::new(), + base_url, + }) + } + + async fn post_json( + &self, + route: &str, + request: &Request, + ) -> moa_core::error::Result + where + Request: serde::Serialize + Sync, + Response: serde::de::DeserializeOwned, + { + let url = self.base_url.join(route).map_err(|error| { + MoaError::ConfigError(format!("join external-job fixture route: {error}")) + })?; + self.client + .post(url) + .json(request) + .send() + .await + .map_err(|error| MoaError::ProviderTransport(error.to_string()))? + .error_for_status() + .map_err(|error| MoaError::ProviderError(error.to_string()))? + .json() + .await + .map_err(|error| MoaError::SerializationError(error.to_string())) + } +} + +#[cfg(all(feature = "provider-overrides", feature = "integration"))] +async fn fixture_external_job_after_bind_barrier( + ctx: &Context<'_>, + provider: &str, + context: &ExternalJobStartContext, +) -> Result<(), HandlerError> { + if provider != FIXTURE_EXTERNAL_JOB_PROVIDER { + return Ok(()); + } + let base_url = std::env::var("MOA_FIXTURE_EXTERNAL_JOB_ADAPTER_URL").map_err(|_| { + TerminalError::new("fixture external-job adapter URL disappeared after runtime startup") + })?; + let adapter = FixtureHttpExecutionExternalJobAdapter::new(&base_url) + .map_err(moa_error_to_handler_error)?; + let context = context.clone(); + let external_job_uid = context.external_job_uid; + ctx.run(|| async move { + adapter + .post_json::<_, ()>("after_bind", &context) + .await + .map(Json::from) + .map_err(moa_error_to_handler_error) + }) + .name(format!( + "fixture_external_job_after_bind:{external_job_uid}" + )) + .retry_policy(RunRetryPolicy::new().max_attempts(1)) + .await?; + Ok(()) +} + +#[cfg(not(all(feature = "provider-overrides", feature = "integration")))] +async fn fixture_external_job_after_bind_barrier( + _ctx: &Context<'_>, + _provider: &str, + _context: &ExternalJobStartContext, +) -> Result<(), HandlerError> { + Ok(()) +} + +#[cfg(all(feature = "provider-overrides", feature = "integration"))] +#[derive(serde::Deserialize)] +#[serde(deny_unknown_fields)] +struct FixtureExternalJobCallbackEnvelope { + provider_job_id: String, + provider_event_id: String, + outcome: AsyncToolJobCallbackOutcome, +} + +#[cfg(all(feature = "provider-overrides", feature = "integration"))] +#[async_trait::async_trait] +impl ExecutionExternalJobAdapter for FixtureHttpExecutionExternalJobAdapter { + fn provider_key(&self) -> &'static str { + FIXTURE_EXTERNAL_JOB_PROVIDER + } + + async fn start( + &self, + request: &ExecutionExternalJobStartRequest, + ) -> moa_core::error::Result { + self.post_json("start", request).await + } + + async fn recover_start( + &self, + context: &ExternalJobStartContext, + ) -> moa_core::error::Result { + self.post_json("recover_start", context).await + } + + async fn authenticate_callback( + &self, + callback_auth_reference: &str, + authentication: &ExecutionExternalJobCallbackAuthentication, + body: &[u8], + ) -> moa_core::error::Result { + let presented = authentication + .headers + .get("authorization") + .map(String::as_str); + let digest: [u8; 32] = Sha256::digest(body).into(); + Ok(callback_auth_reference == "fixture-callback-token" + && presented == Some("Bearer fixture-callback-token") + && authentication.body_sha256 == digest) + } + + async fn parse_callback( + &self, + _authentication: &ExecutionExternalJobCallbackAuthentication, + body: &[u8], + ) -> moa_core::error::Result { + let envelope: FixtureExternalJobCallbackEnvelope = serde_json::from_slice(body) + .map_err(|error| MoaError::SerializationError(error.to_string()))?; + Ok(ExecutionExternalJobAdapterCallback { + provider_job_id: envelope.provider_job_id, + provider_event_id: envelope.provider_event_id, + outcome: envelope.outcome, + }) + } + + async fn cancel( + &self, + request: &ExecutionExternalJobCancelRequest, + ) -> moa_core::error::Result { + self.post_json("cancel", request).await + } + + async fn reconcile( + &self, + request: &ExecutionExternalJobReconcileRequest, + ) -> moa_core::error::Result { + self.post_json("reconcile", request).await + } +} + /// Restate service surface for durable tool execution. #[restate_sdk::service] pub trait ToolExecutor { @@ -76,6 +509,21 @@ pub trait ToolExecutor { request: Json, ) -> Result, HandlerError>; + /// Cancels one exact asynchronous provider-job generation. + async fn cancel_external_job( + request: Json, + ) -> Result, HandlerError>; + + /// Reconciles one exact asynchronous provider-job generation. + async fn reconcile_external_job( + request: Json, + ) -> Result, HandlerError>; + + /// Recovers one expired pre-provider start intent without replaying its task attempt. + async fn recover_external_job_start( + request: Json, + ) -> Result, HandlerError>; + /// Lists tools in one authenticated session and agent catalog scope. async fn list_tools( request: Json, @@ -96,6 +544,11 @@ pub trait ToolExecutor { request: Json, ) -> Result<(), HandlerError>; + /// Releases one exact bounded execution-attempt sandbox before a durable yield. + async fn checkpoint_and_release_execution_hands( + request: Json, + ) -> Result, HandlerError>; + /// Releases the generation-independent hand scope owned by one compensation. async fn release_execution_compensation_hands( request: Json, @@ -132,6 +585,19 @@ pub enum ExecutionToolCallOrigin { Compensation(ExecutionCompensationOrigin), } +/// Exact persisted phase authorized to begin one execution-scoped provider effect. +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(tag = "phase", rename_all = "snake_case", deny_unknown_fields)] +pub enum ExecutionToolCallPhase { + /// A currently running bounded attempt is dispatching its own effect. + Direct, + /// A storage-owned action review is dispatching the one effect it approved. + Reviewed { + /// Exact action-review identity persisted in the attempt checkpoint. + review_uid: uuid::Uuid, + }, +} + /// Tool request owned by one persisted execution operation. #[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] #[serde(deny_unknown_fields)] @@ -140,6 +606,8 @@ pub struct ExecutionToolCallRequest { pub call: ToolCallRequest, /// Required typed execution provenance. pub origin: ExecutionToolCallOrigin, + /// Exact running or reviewed phase admitted by the row-locked repository fence. + pub phase: ExecutionToolCallPhase, } /// Typed execution-only result that keeps ambiguous external effects out of errors. @@ -151,6 +619,13 @@ pub enum ExecutionToolCallOutcome { /// Classified tool output journaled by ToolExecutor. output: Box, }, + /// The provider committed asynchronous work and returned its durable recovery contract. + ExternalJob { + /// MOA-owned job identity reserved before provider dispatch. + external_job_uid: uuid::Uuid, + /// Immutable provider job identity, callback reference, and reconciliation schedule. + job: AsyncToolJob, + }, /// A non-idempotent external effect may have committed and cannot be resent safely. UnknownOutcome { /// Stable diagnostic requiring operator reconciliation. @@ -187,6 +662,24 @@ pub struct ReleaseExecutionTaskHandsRequest { pub task_id: moa_execution::state::ExecutionTaskId, } +/// Exact bounded execution sandbox ownership to release before parking. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CheckpointAndReleaseExecutionHandsRequest { + /// Verified tenant that owns the execution run and lease. + pub tenant_id: TenantId, + /// Authoritative parent session loaded from the run. + pub session_id: SessionId, + /// Owning execution run. + pub run_uid: uuid::Uuid, + /// Exact task or compensation owner and logical generation. + pub owner: ExecutionHandReleaseOwner, + /// Exact active-attempt generation relinquishing ownership. + pub attempt_generation: u64, + /// Fresh absolute bound for checkpoint, provider destroy, and receipt verification. + pub release_deadline_at: chrono::DateTime, +} + /// Request to release one settled compensation's scoped hands. #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] #[serde(deny_unknown_fields)] @@ -203,11 +696,30 @@ pub struct ReleaseExecutionCompensationHandsRequest { /// Request to release every hand under a session at terminal teardown. #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] pub struct ReleaseSessionHandsRequest { /// Verified tenant that owns the session leases. pub tenant_id: TenantId, /// Session whose hands and durable leases should be reclaimed. pub session_id: SessionId, + /// Consecutive no-progress continuations used to bound outage retry cadence. + pub continuation_attempt: u32, +} + +fn session_release_continuation( + outcome: SessionHandReleasePageOutcome, + continuation_attempt: u32, +) -> Option<(u32, Duration)> { + match outcome { + SessionHandReleasePageOutcome::Complete => None, + SessionHandReleasePageOutcome::Progressed => Some((0, Duration::from_millis(100))), + SessionHandReleasePageOutcome::Waiting => { + let next_attempt = continuation_attempt.saturating_add(1); + let exponent = next_attempt.min(8); + let delay_ms = 100_u64.saturating_mul(1_u64 << exponent).min(30_000); + Some((next_attempt, Duration::from_millis(delay_ms))) + } + } } #[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] @@ -246,6 +758,90 @@ enum JournaledExecutionEffectAdmission { }, } +#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] +#[serde(deny_unknown_fields)] +struct JournaledExternalJobForCancel { + tenant_id: TenantId, + job_generation: u64, + provider: Option, + provider_job_id: Option, + idempotency_key: String, + cancel_supported: bool, + terminal: bool, +} + +#[derive(Clone, Copy, Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "snake_case")] +enum JournaledExternalJobCallbackOutcome { + Applied, + Duplicate, + StaleGeneration, + AlreadyTerminal, + NotFound, +} + +impl From + for JournaledExternalJobCallbackOutcome +{ + fn from( + outcome: moa_execution::repository::external_job::ExecutionExternalJobCallbackOutcome, + ) -> Self { + match outcome { + moa_execution::repository::external_job::ExecutionExternalJobCallbackOutcome::Applied( + _, + ) => Self::Applied, + moa_execution::repository::external_job::ExecutionExternalJobCallbackOutcome::Duplicate => { + Self::Duplicate + } + moa_execution::repository::external_job::ExecutionExternalJobCallbackOutcome::StaleGeneration => { + Self::StaleGeneration + } + moa_execution::repository::external_job::ExecutionExternalJobCallbackOutcome::AlreadyTerminal => { + Self::AlreadyTerminal + } + moa_execution::repository::external_job::ExecutionExternalJobCallbackOutcome::NotFound => { + Self::NotFound + } + } + } +} + +impl From + for JournaledExternalJobForCancel +{ + fn from(record: moa_execution::repository::external_job::ExecutionExternalJobRecord) -> Self { + Self { + tenant_id: record.tenant_id, + job_generation: record.job_generation, + provider: record.provider, + provider_job_id: record.provider_job_id, + idempotency_key: record.idempotency_key, + cancel_supported: record.cancel_supported, + terminal: record.state.is_terminal(), + } + } +} + +#[derive(Clone, Copy, Debug, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "snake_case")] +enum JournaledExternalJobCancellationOutcome { + Applied, + StaleGeneration, + AlreadyTerminal, + NotFound, +} + +impl From for JournaledExternalJobCancellationOutcome { + fn from(outcome: ExecutionExternalJobCancellationOutcome) -> Self { + match outcome { + ExecutionExternalJobCancellationOutcome::Applied(_) => Self::Applied, + ExecutionExternalJobCancellationOutcome::StaleGeneration => Self::StaleGeneration, + ExecutionExternalJobCancellationOutcome::AlreadyTerminal => Self::AlreadyTerminal, + ExecutionExternalJobCancellationOutcome::NotFound => Self::NotFound, + } + } +} + impl From for JournaledExecutionEffectAdmission { fn from(outcome: ExecutionEffectAdmissionOutcome) -> Self { match outcome { @@ -284,6 +880,28 @@ struct SessionAccess { events: Arc, } +/// Fully validated dependencies for the durable tool-execution boundary. +pub(crate) struct ToolExecutorDependencies { + /// Tenant-scoped tool router. + pub(crate) router: Arc, + /// Connector catalog authority used for every invocation. + pub(crate) connector_catalogs: ScopedConnectorCatalogProvider, + /// Durable connector completion service. + pub(crate) connector_completion: ConnectorInvocationCompletionService, + /// Session metadata store. + pub(crate) sessions: Arc, + /// Session event lookup store. + pub(crate) events: Arc, + /// Shared runtime database pool. + pub(crate) pool: sqlx::PgPool, + /// Sandbox workspace lifecycle service. + pub(crate) workspace_management: SandboxWorkspaceManagement, + /// Registered asynchronous provider-job adapters. + pub(crate) external_job_adapters: ExecutionExternalJobAdapterRegistry, + /// Validated execution capacity and recovery policy. + pub(crate) execution_config: ExecutionConfig, +} + /// Concrete Restate service implementation backed by a shared `ToolRouter`. #[derive(Clone)] pub struct ToolExecutorImpl { @@ -293,20 +911,25 @@ pub struct ToolExecutorImpl { session_access: SessionAccess, execution_repository: ExecutionRepository, workspace_management: SandboxWorkspaceManagement, + external_job_adapters: ExecutionExternalJobAdapterRegistry, + execution_config: ExecutionConfig, } impl ToolExecutorImpl { /// Creates the fully configured durable tool-execution service. #[must_use] - pub(crate) fn new( - router: Arc, - connector_catalogs: ScopedConnectorCatalogProvider, - connector_completion: ConnectorInvocationCompletionService, - sessions: Arc, - events: Arc, - pool: sqlx::PgPool, - workspace_management: SandboxWorkspaceManagement, - ) -> Self { + pub(crate) fn new(dependencies: ToolExecutorDependencies) -> Self { + let ToolExecutorDependencies { + router, + connector_catalogs, + connector_completion, + sessions, + events, + pool, + workspace_management, + external_job_adapters, + execution_config, + } = dependencies; Self { router, connector_catalogs, @@ -314,7 +937,137 @@ impl ToolExecutorImpl { session_access: SessionAccess { sessions, events }, execution_repository: ExecutionRepository::new(pool.clone()), workspace_management, + external_job_adapters, + execution_config, + } + } + + async fn finalize_recovered_compensation_external_start( + &self, + ctx: &Context<'_>, + request: ExecutionCompensationAttemptCancelRequest, + external_job_uid: Option, + recovered_at: chrono::DateTime, + ) -> Result<(), HandlerError> { + let repository = self.execution_repository.clone(); + let scope = ExecutionScope::Tenant { + tenant_id: request.tenant_id, + }; + let run_uid = request.run_uid; + let run = ctx + .run(|| async move { + repository + .load_run(scope, run_uid) + .await + .map(Json::from) + .map_err(execution_error_to_handler_error) + }) + .name(format!( + "load_recovered_compensation_release_run:{}", + request.cancellation_dispatch_uid + )) + .await? + .into_inner() + .ok_or_else(|| { + TerminalError::new("recovered compensation release lost its authoritative run") + })?; + let receipt = crate::restate_identity::replay_safe_request( + ctx.service_client::() + .checkpoint_and_release_execution_hands(Json::from( + CheckpointAndReleaseExecutionHandsRequest { + tenant_id: request.tenant_id, + session_id: run.session_id, + run_uid: request.run_uid, + owner: ExecutionHandReleaseOwner::Compensation { + compensation_id: ExecutionCompensationScopeId( + request.compensation_id.as_uuid(), + ), + logical_generation: request.compensation_generation, + }, + attempt_generation: request.compensation_attempt_generation, + release_deadline_at: recovered_at + chrono::Duration::minutes(5), + }, + )) + .idempotency_key(format!( + "external-start-recovery-hand-release:{}", + request.cancellation_dispatch_uid + )), + ) + .call() + .await? + .into_inner(); + let settled_at = ctx + .run(|| async { Ok::<_, HandlerError>(Json::from(chrono::Utc::now())) }) + .name(format!( + "recovered_compensation_release_clock:{}", + request.cancellation_dispatch_uid + )) + .await? + .into_inner(); + let repository = self.execution_repository.clone(); + let request_for_finalizer = request.clone(); + let outcome = ctx + .run(|| async move { + let applied = match request_for_finalizer.intent { + ExecutionCompensationReleaseIntent::Retry => { + let outcome = repository + .yield_released_compensation_attempt_after_external_not_started( + &request_for_finalizer, + settled_at, + Some(receipt), + ) + .await?; + matches!( + outcome, + CompensationAttemptWriteOutcome::Applied(_) + | CompensationAttemptWriteOutcome::Replayed(_) + ) + } + ExecutionCompensationReleaseIntent::ExternalJob => { + let external_job_uid = external_job_uid.ok_or_else(|| { + moa_execution::Error::InvalidRepositoryInput { + message: "recovered external compensation lost its job identity" + .to_string(), + } + })?; + let outcome = repository + .yield_released_compensation_attempt_to_external_job( + &request_for_finalizer, + external_job_uid, + Some(receipt), + settled_at, + ) + .await?; + matches!( + outcome, + CompensationAttemptExternalOutcome::Applied { .. } + | CompensationAttemptExternalOutcome::Replayed { .. } + ) + } + _ => { + return Err(execution_error_to_handler_error( + moa_execution::Error::InvalidRepositoryInput { + message: "recovered compensation carried an invalid release intent" + .to_string(), + }, + )); + } + }; + Ok::<_, HandlerError>(Json::from(applied)) + }) + .name(format!( + "finalize_recovered_compensation_external_start:{}", + request.cancellation_dispatch_uid + )) + .await? + .into_inner(); + if !outcome { + return Err(anyhow::anyhow!( + "recovered compensation external-start release remains unsettled" + ) + .into()); } + Ok(()) } async fn scoped_catalog_for_session( @@ -887,8 +1640,9 @@ impl ToolExecutor for ToolExecutorImpl { let admission_scope = execution_scope_for_session(&session); let admission_run_uid = execution_run_uid(origin); let admission_session_id = session.id; - let admission_owner = execution_effect_owner(origin); - let admission_name = execution_effect_admission_run_name(origin, request.call.tool_call_id); + let admission_owner = execution_effect_owner(origin, request.phase); + let admission_name = + execution_effect_admission_run_name(origin, request.phase, request.call.tool_call_id); let effect_admission = ctx .run(|| async move { admission_repository @@ -912,6 +1666,17 @@ impl ToolExecutor for ToolExecutorImpl { })); } + if requires_sandbox + && matches!( + &definition.async_mode, + ToolAsyncMode::MayReturnExternalJob { .. } + ) + { + return Err(TerminalError::new( + "asynchronous provider adapters cannot own a live sandbox hand", + ) + .into()); + } if requires_sandbox { let exact_scope = workspace_scope.clone().ok_or_else(|| { TerminalError::new( @@ -933,23 +1698,141 @@ impl ToolExecutor for ToolExecutorImpl { .await?; } - // This journaled admission is the linearization cut point. A terminal fence that wins - // the same run-row lock returns above without calling the router; an admitted replay must - // continue into this stable Restate effect operation and be joined by terminal settlement. - let journaled = ctx - .run(|| async move { - classify_execution_tool_result( - service - .execute_scoped_with_scope( - &session_for_run, - &request_for_run, - Some(hand_scope.as_str()), - workspace_scope_for_run.as_ref(), - ) - .await, + if let ToolAsyncMode::MayReturnExternalJob { provider } = &definition.async_mode { + let adapter = self + .external_job_adapters + .require(provider) + .map_err(moa_error_to_handler_error)?; + let expires_at = request.call.resource_budget.deadline.ok_or_else(|| { + TerminalError::new( + "asynchronous execution tool calls require an absolute attempt deadline", ) - .map(Json::from) - .map_err(moa_error_to_handler_error) + })?; + let intent = execution_external_job_intent( + origin, + session.tenant_id, + request.call.tool_call_id, + provider, + expires_at, + ); + let repository = self.execution_repository.clone(); + let config = self.execution_config.clone(); + let intent_for_reserve = intent.clone(); + ctx.run(|| async move { + repository + .reserve_external_job_intent( + ExecutionScope::ControlPlane, + &config, + intent_for_reserve, + ) + .await + .map(|record| Json::from(record.external_job_uid)) + .map_err(execution_error_to_handler_error) + }) + .name(format!( + "execution_external_job_reserve:{}", + intent.external_job_uid + )) + .await?; + + let start_context = ExternalJobStartContext { + external_job_uid: intent.external_job_uid, + provider: provider.clone(), + idempotency_key: intent.idempotency_key.clone(), + }; + let start_request = ExecutionExternalJobStartRequest { + context: start_context.clone(), + call: request.call.clone(), + }; + let start = ctx + .run(|| async move { + adapter + .start(&start_request) + .await + .map(Json::from) + .map_err(moa_error_to_handler_error) + }) + .name(format!( + "execution_external_job_start:{}", + intent.external_job_uid + )) + .retry_policy(RunRetryPolicy::new().max_attempts(1)) + .await? + .into_inner(); + return match start { + ExecutionExternalJobStartOutcome::Completed(output) => { + let repository = self.execution_repository.clone(); + let intent_for_release = intent.clone(); + ctx.run(|| async move { + repository + .release_external_job_intent( + ExecutionScope::ControlPlane, + intent_for_release, + ) + .await + .and_then(|outcome| match outcome { + ExecutionExternalJobIntentReleaseOutcome::Released + | ExecutionExternalJobIntentReleaseOutcome::AlreadyReleased => { + Ok(()) + } + ExecutionExternalJobIntentReleaseOutcome::Stale + | ExecutionExternalJobIntentReleaseOutcome::AlreadyBound => { + Err(moa_execution::Error::InvalidRepositoryData { + message: "synchronous provider result could not release its unbound external-job intent".to_string(), + }) + } + }) + .map_err(execution_error_to_handler_error) + }) + .name(format!( + "execution_external_job_release:{}", + intent.external_job_uid + )) + .await?; + Ok(Json::from(ExecutionToolCallOutcome::Completed { output })) + } + ExecutionExternalJobStartOutcome::ExternalJob(job) => { + let binding = execution_external_job_binding(&intent, provider, job.clone()); + let repository = self.execution_repository.clone(); + let config = self.execution_config.clone(); + ctx.run(|| async move { + repository + .bind_external_job(ExecutionScope::ControlPlane, &config, binding) + .await + .map(|record| Json::from(record.external_job_uid)) + .map_err(execution_error_to_handler_error) + }) + .name(format!( + "execution_external_job_bind:{}", + intent.external_job_uid + )) + .await?; + fixture_external_job_after_bind_barrier(&ctx, provider, &start_context).await?; + Ok(Json::from(ExecutionToolCallOutcome::ExternalJob { + external_job_uid: intent.external_job_uid, + job, + })) + } + }; + } + + // This journaled admission is the linearization cut point. A terminal fence that wins + // the same run-row lock returns above without calling the router; an admitted replay must + // continue into this stable Restate effect operation and be joined by terminal settlement. + let journaled = ctx + .run(|| async move { + classify_execution_tool_result( + service + .execute_scoped_with_scope( + &session_for_run, + &request_for_run, + Some(hand_scope.as_str()), + workspace_scope_for_run.as_ref(), + ) + .await, + ) + .map(Json::from) + .map_err(moa_error_to_handler_error) }) .name(run_name) .retry_policy(RunRetryPolicy::new().max_attempts(1)) @@ -1018,6 +1901,481 @@ impl ToolExecutor for ToolExecutorImpl { })) } + #[tracing::instrument(skip(self, ctx, request))] + // SAFETY: internal generation-fenced cancellation delivery; the tenant-owned job row is + // loaded before any provider call and no caller-owned payload is returned. + async fn cancel_external_job( + &self, + ctx: Context<'_>, + request: Json, + ) -> Result, HandlerError> { + crate::ctx::adopt_incoming_trace_parent(&ctx); + annotate_restate_handler_span("ToolExecutor", "cancel_external_job"); + let request = request.into_inner(); + let repository = self.execution_repository.clone(); + let scope = ExecutionScope::ControlPlane; + let external_job_uid = request.external_job_uid; + let loaded = ctx + .run(|| async move { + repository + .load_external_job(scope, external_job_uid) + .await + .map(|record| record.map(JournaledExternalJobForCancel::from)) + .map(Json::from) + .map_err(execution_error_to_handler_error) + }) + .name(format!("load_external_job_for_cancel:{external_job_uid}")) + .await? + .into_inner(); + let Some(job) = loaded else { + return Ok(Json::from(external_job_cancel_response( + &request, + ExecutionExternalJobCancelResponseOutcome::NotFound, + ))); + }; + let (Some(provider), Some(provider_job_id)) = + (job.provider.as_deref(), job.provider_job_id.as_deref()) + else { + return Ok(Json::from(external_job_cancel_response( + &request, + ExecutionExternalJobCancelResponseOutcome::StaleDelivery, + ))); + }; + if job.tenant_id != request.tenant_id + || job.job_generation != request.job_generation + || provider != request.provider + || provider_job_id != request.provider_job_id + || job.idempotency_key != request.idempotency_key + { + return Ok(Json::from(external_job_cancel_response( + &request, + ExecutionExternalJobCancelResponseOutcome::StaleDelivery, + ))); + } + if job.terminal { + return Ok(Json::from(external_job_cancel_response( + &request, + ExecutionExternalJobCancelResponseOutcome::AlreadyTerminal, + ))); + } + + let provider_outcome = if job.cancel_supported { + let adapter = self + .external_job_adapters + .require(provider) + .map_err(moa_error_to_handler_error)?; + let request_for_provider = request.clone(); + ctx.run(|| async move { + adapter + .cancel(&request_for_provider) + .await + .map(Json::from) + .map_err(moa_error_to_handler_error) + }) + .name(format!( + "cancel_external_job_provider:{}:{}", + request.external_job_uid, request.job_generation + )) + .retry_policy(RunRetryPolicy::new().max_attempts(1)) + .await? + .into_inner() + } else { + AsyncToolJobCancelOutcome::Unsupported + }; + + let cancellation = external_job_cancellation(&request, &provider_outcome); + let repository = self.execution_repository.clone(); + let config = self.execution_config.clone(); + let settlement = ctx + .run(|| async move { + repository + .settle_external_job_cancellation(scope, &config, cancellation) + .await + .map(JournaledExternalJobCancellationOutcome::from) + .map(Json::from) + .map_err(execution_error_to_handler_error) + }) + .name(format!( + "settle_external_job_cancel:{}:{}", + request.external_job_uid, request.job_generation + )) + .await? + .into_inner(); + let outcome = match settlement { + JournaledExternalJobCancellationOutcome::Applied => { + ExecutionExternalJobCancelResponseOutcome::Applied { provider_outcome } + } + JournaledExternalJobCancellationOutcome::StaleGeneration => { + ExecutionExternalJobCancelResponseOutcome::StaleDelivery + } + JournaledExternalJobCancellationOutcome::AlreadyTerminal => { + ExecutionExternalJobCancelResponseOutcome::AlreadyTerminal + } + JournaledExternalJobCancellationOutcome::NotFound => { + ExecutionExternalJobCancelResponseOutcome::NotFound + } + }; + Ok(Json::from(external_job_cancel_response(&request, outcome))) + } + + #[tracing::instrument(skip(self, ctx, request))] + // SAFETY: internal generation-fenced sparse reconciliation; the handler reloads the exact + // provider job before calling a registered adapter and persists through the canonical callback transaction. + async fn reconcile_external_job( + &self, + ctx: Context<'_>, + request: Json, + ) -> Result, HandlerError> { + crate::ctx::adopt_incoming_trace_parent(&ctx); + annotate_restate_handler_span("ToolExecutor", "reconcile_external_job"); + let request = request.into_inner(); + if request.trigger_uid.is_nil() { + return Err( + TerminalError::new("external reconcile trigger UID must not be nil").into(), + ); + } + let repository = self.execution_repository.clone(); + let external_job_uid = request.external_job_uid; + let loaded = ctx + .run(|| async move { + repository + .load_external_job(ExecutionScope::ControlPlane, external_job_uid) + .await + .map(|record| record.map(JournaledExternalJobForCancel::from)) + .map(Json::from) + .map_err(execution_error_to_handler_error) + }) + .name(format!( + "load_external_job_for_reconcile:{external_job_uid}" + )) + .await? + .into_inner(); + let Some(job) = loaded else { + return Ok(Json::from(external_job_reconcile_response( + &request, + ExecutionExternalJobReconcileResponseOutcome::NotFound, + ))); + }; + let (Some(provider), Some(provider_job_id)) = + (job.provider.as_deref(), job.provider_job_id.as_deref()) + else { + return Ok(Json::from(external_job_reconcile_response( + &request, + ExecutionExternalJobReconcileResponseOutcome::StaleDelivery, + ))); + }; + if job.tenant_id != request.tenant_id + || job.job_generation != request.job_generation + || provider != request.provider + || provider_job_id != request.provider_job_id + || job.idempotency_key != request.idempotency_key + { + return Ok(Json::from(external_job_reconcile_response( + &request, + ExecutionExternalJobReconcileResponseOutcome::StaleDelivery, + ))); + } + if job.terminal { + return Ok(Json::from(external_job_reconcile_response( + &request, + ExecutionExternalJobReconcileResponseOutcome::AlreadyTerminal, + ))); + } + + let adapter = self + .external_job_adapters + .require(provider) + .map_err(moa_error_to_handler_error)?; + let request_for_provider = request.clone(); + let provider_outcome = ctx + .run(|| async move { + adapter + .reconcile(&request_for_provider) + .await + .map(Json::from) + .map_err(moa_error_to_handler_error) + }) + .name(format!( + "reconcile_external_job_provider:{}:{}:{}", + request.external_job_uid, request.job_generation, request.trigger_uid + )) + .retry_policy(RunRetryPolicy::new().max_attempts(1)) + .await? + .into_inner(); + + let callback = ExecutionExternalJobCallback { + external_job_uid: request.external_job_uid, + job_generation: request.job_generation, + provider: request.provider.clone(), + provider_job_id: request.provider_job_id.clone(), + provider_event_id: format!("external-reconcile:{}", request.trigger_uid), + update: ExecutionExternalJobCallbackUpdate::from(provider_outcome.clone()), + }; + let repository = self.execution_repository.clone(); + let config = self.execution_config.clone(); + let settlement = ctx + .run(|| async move { + repository + .apply_external_job_callback_and_activate( + ExecutionScope::ControlPlane, + &config, + callback, + ) + .await + .map(|write| JournaledExternalJobCallbackOutcome::from(write.outcome)) + .map(Json::from) + .map_err(execution_error_to_handler_error) + }) + .name(format!( + "settle_external_job_reconcile:{}:{}:{}", + request.external_job_uid, request.job_generation, request.trigger_uid + )) + .await? + .into_inner(); + let outcome = match settlement { + JournaledExternalJobCallbackOutcome::Applied + | JournaledExternalJobCallbackOutcome::Duplicate => { + ExecutionExternalJobReconcileResponseOutcome::Applied { provider_outcome } + } + JournaledExternalJobCallbackOutcome::StaleGeneration => { + ExecutionExternalJobReconcileResponseOutcome::StaleDelivery + } + JournaledExternalJobCallbackOutcome::AlreadyTerminal => { + ExecutionExternalJobReconcileResponseOutcome::AlreadyTerminal + } + JournaledExternalJobCallbackOutcome::NotFound => { + ExecutionExternalJobReconcileResponseOutcome::NotFound + } + }; + Ok(Json::from(external_job_reconcile_response( + &request, outcome, + ))) + } + + #[tracing::instrument(skip(self, ctx, request))] + // SAFETY: internal generation-fenced delivery prepared from canonical unbound owner storage. + async fn recover_external_job_start( + &self, + ctx: Context<'_>, + request: Json, + ) -> Result, HandlerError> { + crate::ctx::adopt_incoming_trace_parent(&ctx); + annotate_restate_handler_span("ToolExecutor", "recover_external_job_start"); + let request = request.into_inner(); + if request.trigger_uid.is_nil() + || request.external_job_uid.is_nil() + || request.job_generation == 0 + || request.provider.trim().is_empty() + || request.idempotency_key.trim().is_empty() + { + return Err(TerminalError::new( + "external start recovery requires exact non-empty trigger and provider identity", + ) + .into()); + } + let adapter = self + .external_job_adapters + .require(&request.provider) + .map_err(moa_error_to_handler_error)?; + let context = ExternalJobStartContext { + external_job_uid: request.external_job_uid, + provider: request.provider.clone(), + idempotency_key: request.idempotency_key.clone(), + }; + let context_for_provider = context.clone(); + let recovery = ctx + .run(|| async move { + adapter + .recover_start(&context_for_provider) + .await + .map(Json::from) + .map_err(moa_error_to_handler_error) + }) + .name(format!( + "recover_external_job_start_provider:{}:{}", + request.external_job_uid, request.job_generation + )) + .retry_policy(RunRetryPolicy::new().max_attempts(1)) + .await? + .into_inner(); + let recovered_at = ctx + .run(|| async { Ok::<_, HandlerError>(Json::from(chrono::Utc::now())) }) + .name(format!( + "external_start_recovered_at:{}", + request.external_job_uid + )) + .await? + .into_inner(); + let outcome = match recovery { + ExecutionExternalJobStartRecovery::NotStarted => { + let repository = self.execution_repository.clone(); + let request_for_adoption = request.clone(); + let adoption = ctx + .run(|| async move { + repository + .recover_external_job_start_not_started( + &request_for_adoption, + recovered_at, + ) + .await + .map(Json::from) + .map_err(execution_error_to_handler_error) + }) + .name(format!( + "adopt_not_started_external_job_start:{}", + request.external_job_uid + )) + .await? + .into_inner(); + match adoption { + ExecutionExternalJobStartRecoveryAdoptionOutcome::Applied { + compensation_release, + } + | ExecutionExternalJobStartRecoveryAdoptionOutcome::Replayed { + compensation_release, + } => { + if let Some(release) = compensation_release { + self.finalize_recovered_compensation_external_start( + &ctx, + *release, + None, + recovered_at, + ) + .await?; + } + ExecutionExternalJobStartRecoveryResponseOutcome::NotStartedReleased + } + ExecutionExternalJobStartRecoveryAdoptionOutcome::AlreadySettled => { + ExecutionExternalJobStartRecoveryResponseOutcome::AlreadySettled + } + ExecutionExternalJobStartRecoveryAdoptionOutcome::NotFound + | ExecutionExternalJobStartRecoveryAdoptionOutcome::Stale => { + ExecutionExternalJobStartRecoveryResponseOutcome::StaleDelivery + } + ExecutionExternalJobStartRecoveryAdoptionOutcome::InvalidState => { + return Err(anyhow::anyhow!( + "external NotStarted recovery owner is not ready for safe requeue" + ) + .into()); + } + } + } + ExecutionExternalJobStartRecovery::Started(job) => { + let intent = execution_external_job_intent_from_recovery(&request); + let binding = + execution_external_job_binding(&intent, &request.provider, job.clone()); + let repository = self.execution_repository.clone(); + let config = self.execution_config.clone(); + let request_for_adoption = request.clone(); + let adoption = ctx + .run(|| async move { + repository + .recover_external_job_start_started( + &config, + &request_for_adoption, + binding, + recovered_at, + ) + .await + .map(Json::from) + .map_err(execution_error_to_handler_error) + }) + .name(format!( + "adopt_started_external_job_start:{}", + request.external_job_uid + )) + .await? + .into_inner(); + match adoption { + ExecutionExternalJobStartRecoveryAdoptionOutcome::Applied { + compensation_release, + } + | ExecutionExternalJobStartRecoveryAdoptionOutcome::Replayed { + compensation_release, + } => { + if let Some(release) = compensation_release { + self.finalize_recovered_compensation_external_start( + &ctx, + *release, + Some(request.external_job_uid), + recovered_at, + ) + .await?; + } + ExecutionExternalJobStartRecoveryResponseOutcome::StartedBound + } + ExecutionExternalJobStartRecoveryAdoptionOutcome::AlreadySettled => { + ExecutionExternalJobStartRecoveryResponseOutcome::AlreadySettled + } + ExecutionExternalJobStartRecoveryAdoptionOutcome::NotFound + | ExecutionExternalJobStartRecoveryAdoptionOutcome::Stale => { + ExecutionExternalJobStartRecoveryResponseOutcome::StaleDelivery + } + ExecutionExternalJobStartRecoveryAdoptionOutcome::InvalidState => { + return Err(anyhow::anyhow!( + "recovered provider start is contained but its owner still requires repair" + ) + .into()); + } + } + } + ExecutionExternalJobStartRecovery::Unknown { error } => { + let retry_at = recovered_at + + chrono::Duration::seconds( + i64::try_from(self.execution_config.trigger_reconciliation_cadence_seconds) + .unwrap_or(i64::MAX), + ); + let error = bounded_external_start_recovery_error(&error); + let repository = self.execution_repository.clone(); + let request_for_rearm = request.clone(); + let rearm = ctx + .run(|| async move { + repository + .rearm_external_start_recovery( + ExecutionScope::ControlPlane, + &request_for_rearm, + retry_at, + &error, + ) + .await + .map(|outcome| { + Json::from(matches!( + outcome, + ExecutionExternalStartRecoveryRearmOutcome::Rearmed(_) + )) + }) + .map_err(execution_error_to_handler_error) + }) + .name(format!( + "rearm_external_start_recovery:{}", + request.external_job_uid + )) + .await? + .into_inner(); + match rearm { + true => ExecutionExternalJobStartRecoveryResponseOutcome::UnknownPreserved, + false => ExecutionExternalJobStartRecoveryResponseOutcome::StaleDelivery, + } + } + }; + if let Some(idempotency_key) = external_start_recovery_dispatch_key(&request, outcome) { + // Recovery commits its replacement activation, reconciliation, or rearmed delivery + // before this wake. Repair remains a fallback rather than the normal delivery path. + let handle = crate::restate_identity::replay_safe_request( + ctx.service_client::() + .dispatch(Json::from(DispatchExecutionsRequest::default())) + .idempotency_key(idempotency_key), + ) + .send(); + let _invocation_id = handle.invocation_id().await?; + } + Ok(Json::from(ExecutionExternalJobStartRecoveryResponse { + external_job_uid: request.external_job_uid, + job_generation: request.job_generation, + outcome, + })) + } + #[tracing::instrument(skip(self, ctx, request))] // SAFETY: internal authenticated catalog projection; session admission owns the caller identity. async fn list_tools( @@ -1092,6 +2450,62 @@ impl ToolExecutor for ToolExecutorImpl { Ok(()) } + #[tracing::instrument(skip(self, ctx, request))] + // SAFETY: internal bounded-attempt yield; the authoritative Session is loaded and its exact + // tenant/run/owner generation is fenced again by the durable release repository. + async fn checkpoint_and_release_execution_hands( + &self, + ctx: Context<'_>, + request: Json, + ) -> Result, HandlerError> { + crate::ctx::adopt_incoming_trace_parent(&ctx); + annotate_restate_handler_span("ToolExecutor", "checkpoint_and_release_execution_hands"); + let request = request.into_inner(); + let session_store = self.session_access.sessions.clone(); + let session_id = request.session_id; + let session = ctx + .run(|| async move { + session_store + .get_session(session_id) + .await + .map(Json::from) + .map_err(moa_error_to_handler_error) + }) + .name(format!("load_task_yield_session:{session_id}")) + .await? + .into_inner(); + if session.tenant_id != request.tenant_id { + return Err(TerminalError::new("execution yield session tenant mismatch").into()); + } + let router = self.router.clone(); + let run_id = ExecutionRunScopeId(request.run_uid); + let owner = request.owner; + let attempt_generation = request.attempt_generation; + let release_deadline_at = request.release_deadline_at; + Ok(ctx + .run(|| async move { + router + .checkpoint_and_release_execution_hand(ExecutionHandReleaseRequest { + session: &session, + run_id, + owner, + attempt_generation, + scope: ToolCallScope::unbounded().with_budget( + moa_core::types::resource::ResourceBudget::until(release_deadline_at), + ), + }) + .await + .map(Json::from) + .map_err(moa_error_to_handler_error) + }) + .name(format!( + "checkpoint_release_execution_hand:{}:{:?}:{}", + request.run_uid, request.owner, request.attempt_generation + )) + .retry_policy(RunRetryPolicy::new().max_attempts(1)) + .await?) + } + #[tracing::instrument(skip(self, ctx, request))] // SAFETY: internal terminal-task teardown reclaims only the typed run/task hand scope and returns no caller-owned data. async fn release_execution_task_hands( @@ -1106,6 +2520,7 @@ impl ToolExecutor for ToolExecutorImpl { run_uid: request.run_uid, task_uid: request.task_id.as_uuid(), generation: 1, + attempt_generation: 1, }); if !self .router @@ -1131,6 +2546,7 @@ impl ToolExecutor for ToolExecutorImpl { run_uid: request.run_uid, compensation_id: request.compensation_id.as_uuid(), generation: 1, + attempt_generation: 1, }); if !self .router @@ -1156,9 +2572,31 @@ impl ToolExecutor for ToolExecutorImpl { crate::ctx::adopt_incoming_trace_parent(&ctx); annotate_restate_handler_span("ToolExecutor", "release_session_hands"); let request = request.into_inner(); - self.router - .reclaim_hands(request.tenant_id, &request.session_id, None) + let outcome = self + .router + .reclaim_session_hands_page(request.tenant_id, &request.session_id) .await; + if let Some((continuation_attempt, delay)) = + session_release_continuation(outcome, request.continuation_attempt) + { + let invocation_id = ctx.invocation_id(); + let continuation = crate::restate_identity::replay_safe_request( + ctx.service_client::() + .release_session_hands(Json::from(ReleaseSessionHandsRequest { + continuation_attempt, + ..request.clone() + })) + .idempotency_key(format!( + "release-session-hands:{}:{invocation_id}", + request.session_id + )), + ) + .send_after(delay); + continuation + .invocation_id() + .await + .map_err(HandlerError::from)?; + } Ok(()) } } @@ -1285,44 +2723,263 @@ fn execution_run_uid(origin: ExecutionToolCallOrigin) -> uuid::Uuid { } } -fn execution_effect_owner(origin: ExecutionToolCallOrigin) -> ExecutionEffectOwner { +fn execution_effect_owner( + origin: ExecutionToolCallOrigin, + phase: ExecutionToolCallPhase, +) -> ExecutionEffectOwner { + let phase = match phase { + ExecutionToolCallPhase::Direct => ExecutionEffectPhase::Direct, + ExecutionToolCallPhase::Reviewed { review_uid } => { + ExecutionEffectPhase::Reviewed { review_uid } + } + }; match origin { ExecutionToolCallOrigin::Task(origin) => ExecutionEffectOwner::Task { task_id: moa_execution::state::ExecutionTaskId::from_uuid(origin.task_uid), generation: origin.generation, + attempt_generation: origin.attempt_generation, + phase, }, ExecutionToolCallOrigin::Compensation(origin) => ExecutionEffectOwner::Compensation { compensation_id: moa_execution::state::CompensationId::from_uuid( origin.compensation_id, ), generation: origin.generation, + attempt_generation: origin.attempt_generation, + phase, + }, + } +} + +const EXECUTION_EXTERNAL_JOB_NAMESPACE: uuid::Uuid = + uuid::Uuid::from_u128(0x62c0_5ead_8b32_5daa_86bf_b05c_7d27_7441); + +fn execution_external_job_intent( + origin: ExecutionToolCallOrigin, + tenant_id: TenantId, + tool_call_id: ToolCallId, + provider: &str, + expires_at: chrono::DateTime, +) -> NewExecutionExternalJobIntent { + let run_uid = execution_run_uid(origin); + let (owner, owner_identity) = match origin { + ExecutionToolCallOrigin::Task(origin) => ( + ExecutionExternalJobOwner::Task { + task_id: origin.task_uid, + attempt_generation: origin.attempt_generation, + }, + format!("task:{}:{}", origin.task_uid, origin.attempt_generation), + ), + ExecutionToolCallOrigin::Compensation(origin) => ( + ExecutionExternalJobOwner::Compensation { + compensation_id: origin.compensation_id, + compensation_generation: origin.generation, + compensation_attempt_generation: origin.attempt_generation, + }, + format!( + "compensation:{}:{}:{}", + origin.compensation_id, origin.generation, origin.attempt_generation + ), + ), + }; + // The provider key deliberately excludes attempt generation: a provider retry after a + // runtime-loss recovery fence must join the same committed start instead of duplicating it. + // MOA's external-job UID still includes the exact attempt owner, and a successor attempt is + // admitted only after the preceding unbound intent is proven NotStarted and released. + let idempotency_key = format!("execution-external:{provider}:{tool_call_id}"); + let canonical_identity = format!( + "v1|tenant:{tenant_id}|run:{run_uid}|owner:{owner_identity}|call:{tool_call_id}|provider:{provider}" + ); + let external_job_uid = uuid::Uuid::new_v5( + &EXECUTION_EXTERNAL_JOB_NAMESPACE, + canonical_identity.as_bytes(), + ); + NewExecutionExternalJobIntent { + external_job_uid, + tenant_id, + run_uid, + owner, + job_generation: 1, + provider: provider.to_string(), + idempotency_key, + expires_at, + } +} + +fn execution_external_job_intent_from_recovery( + request: &ExecutionExternalJobStartRecoveryRequest, +) -> NewExecutionExternalJobIntent { + let owner = match request.owner { + ExecutionExternalJobStartRecoveryOwner::Task { + task_id, + attempt_generation, + } => ExecutionExternalJobOwner::Task { + task_id, + attempt_generation, + }, + ExecutionExternalJobStartRecoveryOwner::Compensation { + compensation_id, + compensation_generation, + compensation_attempt_generation, + } => ExecutionExternalJobOwner::Compensation { + compensation_id, + compensation_generation, + compensation_attempt_generation, }, + }; + NewExecutionExternalJobIntent { + external_job_uid: request.external_job_uid, + tenant_id: request.tenant_id, + run_uid: request.run_uid, + owner, + job_generation: request.job_generation, + provider: request.provider.clone(), + idempotency_key: request.idempotency_key.clone(), + // Exact intent matching excludes expiry; release is allowed only after provider recovery + // proved NotStarted, so no synthetic wall-clock value is used as an authorization fence. + expires_at: chrono::DateTime::::MAX_UTC, + } +} + +fn external_start_recovery_dispatch_key( + request: &ExecutionExternalJobStartRecoveryRequest, + outcome: ExecutionExternalJobStartRecoveryResponseOutcome, +) -> Option { + matches!( + outcome, + ExecutionExternalJobStartRecoveryResponseOutcome::NotStartedReleased + | ExecutionExternalJobStartRecoveryResponseOutcome::StartedBound + | ExecutionExternalJobStartRecoveryResponseOutcome::UnknownPreserved + ) + .then(|| { + format!( + "external-start-recovery-dispatch:{}:{}:{}", + request.external_job_uid, request.job_generation, request.trigger_uid + ) + }) +} + +fn bounded_external_start_recovery_error(error: &serde_json::Value) -> String { + error.to_string().chars().take(4_096).collect() +} + +fn execution_external_job_binding( + intent: &NewExecutionExternalJobIntent, + provider: &str, + job: AsyncToolJob, +) -> ExecutionExternalJobBinding { + let provider_contract_violation = + (job.provider != provider || job.idempotency_key != intent.idempotency_key).then(|| { + format!( + "declared_provider={provider}; returned_provider={}; idempotency_key_matches={}", + job.provider, + job.idempotency_key == intent.idempotency_key + ) + }); + ExecutionExternalJobBinding { + external_job_uid: intent.external_job_uid, + tenant_id: intent.tenant_id, + run_uid: intent.run_uid, + owner: intent.owner, + job_generation: intent.job_generation, + idempotency_key: intent.idempotency_key.clone(), + provider: provider.to_string(), + provider_job_id: job.provider_job_id, + callback_auth_reference: job.callback_auth_reference, + state: ExecutionExternalJobState::Running, + progress_phase: Some(job.progress_phase), + cancel_supported: job.cancel_supported, + next_reconcile_at: Some(job.next_reconcile_at), + provider_contract_violation, } } fn execution_effect_admission_run_name( origin: ExecutionToolCallOrigin, + phase: ExecutionToolCallPhase, tool_call_id: ToolCallId, ) -> String { + let phase = match phase { + ExecutionToolCallPhase::Direct => "direct".to_string(), + ExecutionToolCallPhase::Reviewed { review_uid } => format!("reviewed:{review_uid}"), + }; match origin { ExecutionToolCallOrigin::Task(origin) => format!( - "execution_effect_admission:task:{}:{}:{}:{tool_call_id}", - origin.run_uid, origin.task_uid, origin.generation + "execution_effect_admission:task:{}:{}:{}:{}:{phase}:{tool_call_id}", + origin.run_uid, origin.task_uid, origin.generation, origin.attempt_generation ), ExecutionToolCallOrigin::Compensation(origin) => format!( - "execution_effect_admission:compensation:{}:{}:{}:{tool_call_id}", - origin.run_uid, origin.compensation_id, origin.generation + "execution_effect_admission:compensation:{}:{}:{}:{}:{phase}:{tool_call_id}", + origin.run_uid, origin.compensation_id, origin.generation, origin.attempt_generation ), } } -fn execution_repository_error(error: moa_execution::Error) -> HandlerError { - match error { - error @ moa_execution::Error::Storage { .. } => HandlerError::from(error), - error => TerminalError::new(format!("execution effect admission failed: {error}")).into(), +fn external_job_cancel_response( + request: &ExecutionExternalJobCancelRequest, + outcome: ExecutionExternalJobCancelResponseOutcome, +) -> ExecutionExternalJobCancelResponse { + ExecutionExternalJobCancelResponse { + external_job_uid: request.external_job_uid, + job_generation: request.job_generation, + outcome, + } +} + +fn external_job_reconcile_response( + request: &ExecutionExternalJobReconcileRequest, + outcome: ExecutionExternalJobReconcileResponseOutcome, +) -> ExecutionExternalJobReconcileResponse { + ExecutionExternalJobReconcileResponse { + external_job_uid: request.external_job_uid, + job_generation: request.job_generation, + outcome, } } +fn external_job_cancellation( + request: &ExecutionExternalJobCancelRequest, + outcome: &AsyncToolJobCancelOutcome, +) -> ExecutionExternalJobCancellation { + let (state, next_reconcile_at, error) = match outcome { + AsyncToolJobCancelOutcome::Cancelled => (ExecutionExternalJobState::Cancelled, None, None), + AsyncToolJobCancelOutcome::Accepted { + next_reconcile_at, .. + } => ( + ExecutionExternalJobState::CancelRequested, + Some(*next_reconcile_at), + None, + ), + AsyncToolJobCancelOutcome::Unsupported => ( + ExecutionExternalJobState::UnknownOutcome, + None, + Some(serde_json::json!({ + "kind": "cancellation_unsupported", + "provider": request.provider, + "provider_job_id": request.provider_job_id, + })), + ), + AsyncToolJobCancelOutcome::UnknownOutcome { error } => ( + ExecutionExternalJobState::UnknownOutcome, + None, + Some(error.clone()), + ), + }; + ExecutionExternalJobCancellation { + external_job_uid: request.external_job_uid, + job_generation: request.job_generation, + provider: request.provider.clone(), + provider_job_id: request.provider_job_id.clone(), + state, + next_reconcile_at, + error, + } +} + +fn execution_repository_error(error: moa_execution::Error) -> HandlerError { + execution_error_to_handler_error(error) +} + /// Builds the Restate run-operation name fenced by execution generation. pub fn execution_task_tool_run_name( definition: &ToolDefinition, @@ -1832,10 +3489,11 @@ mod tests { types::identifiers::ExecutionTaskScopeId, types::identifiers::HandProvisioningOperationId, types::identifiers::SessionId, types::identifiers::TenantId, types::identifiers::ToolCallId, types::sandbox_workspace::SandboxWorkspaceScope, - types::security::SensitivityClass, types::session::SessionMeta, - types::tools::IdempotencyClass, types::tools::ToolCallRequest, - types::tools::ToolDiffStrategy, types::tools::ToolInputShape, types::tools::ToolOutput, - types::tools::ToolPolicySpec, + types::security::SensitivityClass, types::session::SessionMeta, types::tools::AsyncToolJob, + types::tools::AsyncToolJobCallbackOutcome, types::tools::AsyncToolJobCancelOutcome, + types::tools::AsyncToolJobTerminalOutcome, types::tools::IdempotencyClass, + types::tools::ToolCallRequest, types::tools::ToolDiffStrategy, + types::tools::ToolInputShape, types::tools::ToolOutput, types::tools::ToolPolicySpec, }; use moa_hands::{ HandRoute, PinnedToolContract, PinnedToolOwner, ToolCatalogPin, ToolExecution, @@ -1849,18 +3507,369 @@ mod tests { use uuid::Uuid; use super::{ - ExecutionToolCallOrigin, ExecutionToolCallOutcome, ExecutionToolCallRequest, - JournaledExecutionEffectAdmission, ScopedToolCatalogRequest, agent_deployment_tool_denial, - blocked_canary_tool_output, classify_execution_tool_result, - execute_buffered_with_trusted_files, execution_compensation_hand_scope, - execution_hand_scope, execution_task_hand_scope, execution_task_tool_run_name, - execution_tool_run_name, execution_workspace_scope, has_prior_tool_call_event, - is_installed_connector_action, root_trusted_file_read, tool_contract_denial, - worker_workspace_scope, + ExecutionExternalJobAdapter, ExecutionExternalJobAdapterRegistry, + ExecutionExternalJobCallbackAuthentication, ExecutionToolCallOrigin, + ExecutionToolCallOutcome, ExecutionToolCallRequest, JournaledExecutionEffectAdmission, + ScopedToolCatalogRequest, agent_deployment_tool_denial, blocked_canary_tool_output, + classify_execution_tool_result, execute_buffered_with_trusted_files, + execution_compensation_hand_scope, execution_external_job_intent, execution_hand_scope, + execution_task_hand_scope, execution_task_tool_run_name, execution_tool_run_name, + execution_workspace_scope, external_start_recovery_dispatch_key, has_prior_tool_call_event, + is_installed_connector_action, root_trusted_file_read, session_release_continuation, + tool_contract_denial, worker_workspace_scope, }; + use moa_core::types::tools::ExternalJobStartContext; + use moa_execution::wire::{ + ExecutionExternalJobCancelRequest, ExecutionExternalJobReconcileRequest, + ExecutionExternalJobStartRecoveryOwner, ExecutionExternalJobStartRecoveryRequest, + ExecutionExternalJobStartRecoveryResponseOutcome, + }; + use moa_hands::SessionHandReleasePageOutcome; struct ConnectorLookingBuiltIn; + struct FixtureExternalJobAdapter; + + #[test] + fn committed_external_start_recovery_outcomes_wake_dispatch_exactly_by_trigger() { + // Pins: recovery outcomes that commit replacement outbox work wake its dispatcher, while + // stale/already-settled deliveries remain side-effect free. The exact trigger identity + // makes replay coalesce without suppressing another job generation. + let request = ExecutionExternalJobStartRecoveryRequest { + tenant_id: TenantId::from(Uuid::from_u128(1)), + run_uid: Uuid::from_u128(2), + owner: ExecutionExternalJobStartRecoveryOwner::Task { + task_id: Uuid::from_u128(3), + attempt_generation: 4, + }, + external_job_uid: Uuid::from_u128(5), + job_generation: 6, + provider: "fixture".to_string(), + idempotency_key: "provider-start-6".to_string(), + trigger_uid: Uuid::from_u128(7), + }; + let expected = format!( + "external-start-recovery-dispatch:{}:6:{}", + request.external_job_uid, request.trigger_uid + ); + for outcome in [ + ExecutionExternalJobStartRecoveryResponseOutcome::NotStartedReleased, + ExecutionExternalJobStartRecoveryResponseOutcome::StartedBound, + ExecutionExternalJobStartRecoveryResponseOutcome::UnknownPreserved, + ] { + assert_eq!( + external_start_recovery_dispatch_key(&request, outcome), + Some(expected.clone()) + ); + } + for outcome in [ + ExecutionExternalJobStartRecoveryResponseOutcome::StaleDelivery, + ExecutionExternalJobStartRecoveryResponseOutcome::AlreadySettled, + ] { + assert_eq!( + external_start_recovery_dispatch_key(&request, outcome), + None + ); + } + } + + #[test] + fn session_release_backoff_is_bounded_and_resets_after_progress() { + // Pins: a persistent provider/reaper outage cannot create a fixed-rate continuation + // storm, while a page that makes progress resumes the fast drain cadence. + let (_, first_wait) = + session_release_continuation(SessionHandReleasePageOutcome::Waiting, 0) + .expect("waiting cleanup continues"); + let (saturated_attempt, saturated_wait) = + session_release_continuation(SessionHandReleasePageOutcome::Waiting, u32::MAX) + .expect("waiting cleanup remains retryable"); + assert!(first_wait >= Duration::from_millis(200)); + assert_eq!(saturated_attempt, u32::MAX); + assert_eq!(saturated_wait, Duration::from_millis(25_600)); + + let (reset_attempt, progress_wait) = + session_release_continuation(SessionHandReleasePageOutcome::Progressed, 12) + .expect("a partial page schedules its successor"); + assert_eq!(reset_attempt, 0); + assert_eq!(progress_wait, Duration::from_millis(100)); + assert!( + session_release_continuation(SessionHandReleasePageOutcome::Complete, 12,).is_none() + ); + } + + // Pins: the pre-provider UID is a versioned wire identity, not Rust Debug output, and every + // durable owner coordinate changes it while the provider idempotency key remains call-stable. + #[test] + fn external_job_intent_uid_is_canonical_and_exact_offline() { + let tenant_id = TenantId::from( + Uuid::parse_str("11111111-1111-1111-1111-111111111111").expect("fixture tenant UUID"), + ); + let run_uid = + Uuid::parse_str("22222222-2222-2222-2222-222222222222").expect("fixture run UUID"); + let task_uid = + Uuid::parse_str("33333333-3333-3333-3333-333333333333").expect("fixture task UUID"); + let tool_call_id = ToolCallId::from( + Uuid::parse_str("44444444-4444-4444-4444-444444444444") + .expect("fixture tool-call UUID"), + ); + let expires_at = Utc::now() + chrono::Duration::minutes(5); + let task_origin = ExecutionTaskOrigin { + run_uid, + task_uid, + generation: 5, + attempt_generation: 7, + }; + let origin = ExecutionToolCallOrigin::Task(task_origin); + let expected = + execution_external_job_intent(origin, tenant_id, tool_call_id, "fixture", expires_at); + assert_eq!( + expected.external_job_uid, + Uuid::parse_str("913531e0-959c-5a11-9260-4ae8636e92e0") + .expect("pinned external-job UUID") + ); + assert_eq!( + expected.idempotency_key, + format!("execution-external:fixture:{tool_call_id}") + ); + + let mutations = [ + execution_external_job_intent( + origin, + TenantId::new(), + tool_call_id, + "fixture", + expires_at, + ), + execution_external_job_intent( + ExecutionToolCallOrigin::Task(ExecutionTaskOrigin { + run_uid: Uuid::new_v4(), + ..task_origin + }), + tenant_id, + tool_call_id, + "fixture", + expires_at, + ), + execution_external_job_intent( + ExecutionToolCallOrigin::Task(ExecutionTaskOrigin { + task_uid: Uuid::new_v4(), + ..task_origin + }), + tenant_id, + tool_call_id, + "fixture", + expires_at, + ), + execution_external_job_intent( + ExecutionToolCallOrigin::Task(ExecutionTaskOrigin { + attempt_generation: 8, + ..task_origin + }), + tenant_id, + tool_call_id, + "fixture", + expires_at, + ), + execution_external_job_intent( + origin, + tenant_id, + ToolCallId::new(), + "fixture", + expires_at, + ), + execution_external_job_intent( + origin, + tenant_id, + tool_call_id, + "other-provider", + expires_at, + ), + ]; + assert!( + mutations + .iter() + .all(|intent| { intent.external_job_uid != expected.external_job_uid }) + ); + assert_eq!(mutations[3].idempotency_key, expected.idempotency_key); + } + + #[async_trait] + impl ExecutionExternalJobAdapter for FixtureExternalJobAdapter { + fn provider_key(&self) -> &'static str { + "fixture" + } + + async fn start( + &self, + request: &super::ExecutionExternalJobStartRequest, + ) -> moa_core::error::Result { + Ok(super::ExecutionExternalJobStartOutcome::ExternalJob( + AsyncToolJob { + provider: request.context.provider.clone(), + provider_job_id: format!("fixture-job-{}", request.context.external_job_uid), + idempotency_key: request.context.idempotency_key.clone(), + callback_auth_reference: "vault://fixture/callback".to_string(), + progress_phase: "queued".to_string(), + cancel_supported: true, + next_reconcile_at: chrono::Utc::now() + chrono::Duration::minutes(5), + }, + )) + } + + async fn recover_start( + &self, + context: &ExternalJobStartContext, + ) -> moa_core::error::Result { + Ok(super::ExecutionExternalJobStartRecovery::Started( + AsyncToolJob { + provider: context.provider.clone(), + provider_job_id: format!("fixture-job-{}", context.external_job_uid), + idempotency_key: context.idempotency_key.clone(), + callback_auth_reference: "vault://fixture/callback".to_string(), + progress_phase: "queued".to_string(), + cancel_supported: true, + next_reconcile_at: chrono::Utc::now() + chrono::Duration::minutes(5), + }, + )) + } + + async fn authenticate_callback( + &self, + callback_auth_reference: &str, + authentication: &ExecutionExternalJobCallbackAuthentication, + body: &[u8], + ) -> moa_core::error::Result { + Ok(callback_auth_reference == "vault://fixture/callback" + && authentication.body_sha256 == [7; 32] + && body == b"fixture-callback") + } + + async fn parse_callback( + &self, + _authentication: &ExecutionExternalJobCallbackAuthentication, + _body: &[u8], + ) -> moa_core::error::Result { + Ok(super::ExecutionExternalJobAdapterCallback { + provider_job_id: "job-7".to_string(), + provider_event_id: "event-11".to_string(), + outcome: AsyncToolJobCallbackOutcome::Terminal { + outcome: AsyncToolJobTerminalOutcome::Cancelled, + }, + }) + } + + async fn cancel( + &self, + request: &ExecutionExternalJobCancelRequest, + ) -> moa_core::error::Result { + Ok(AsyncToolJobCancelOutcome::UnknownOutcome { + error: serde_json::json!({ + "provider_job_id": request.provider_job_id, + "fixture": true, + }), + }) + } + + async fn reconcile( + &self, + _request: &ExecutionExternalJobReconcileRequest, + ) -> moa_core::error::Result { + Ok(AsyncToolJobCallbackOutcome::Terminal { + outcome: AsyncToolJobTerminalOutcome::Cancelled, + }) + } + } + + // Pins: an asynchronous outcome is usable only through its exact registered provider + // adapter, whose callback authentication and bounded provider operations are deterministic. + #[tokio::test] + async fn external_job_adapter_registry_fails_closed_and_routes_exact_provider_offline() { + let registry = ExecutionExternalJobAdapterRegistry::new([ + Arc::new(FixtureExternalJobAdapter) as Arc, + ]) + .expect("fixture registry should be valid"); + assert!(registry.require("missing").is_err()); + + let adapter = registry + .require("fixture") + .expect("fixture provider should be registered"); + let start_context = ExternalJobStartContext { + external_job_uid: Uuid::from_u128(17), + provider: "fixture".to_string(), + idempotency_key: "fixture-start-17".to_string(), + }; + let started = adapter + .start(&super::ExecutionExternalJobStartRequest { + context: start_context.clone(), + call: tool_request("fixture_external_job"), + }) + .await + .expect("fixture start should complete"); + let super::ExecutionExternalJobStartOutcome::ExternalJob(started) = started else { + panic!("fixture async adapter must return a provider job"); + }; + assert_eq!(started.provider, start_context.provider); + assert_eq!(started.idempotency_key, start_context.idempotency_key); + assert_eq!( + started.provider_job_id, + format!("fixture-job-{}", start_context.external_job_uid) + ); + let recovered = adapter + .recover_start(&start_context) + .await + .expect("fixture start recovery should complete"); + let super::ExecutionExternalJobStartRecovery::Started(recovered) = recovered else { + panic!("fixture recovery must resolve the same started job"); + }; + assert_eq!(recovered.provider_job_id, started.provider_job_id); + assert_eq!(recovered.idempotency_key, started.idempotency_key); + assert!( + adapter + .authenticate_callback( + "vault://fixture/callback", + &ExecutionExternalJobCallbackAuthentication { + headers: std::collections::BTreeMap::new(), + body_sha256: [7; 32], + }, + b"fixture-callback", + ) + .await + .expect("fixture authentication should complete") + ); + let request = ExecutionExternalJobCancelRequest { + tenant_id: TenantId::new(), + external_job_uid: Uuid::new_v4(), + job_generation: 3, + provider: "fixture".to_string(), + provider_job_id: "job-7".to_string(), + idempotency_key: "cancel-job-7".to_string(), + }; + assert_eq!( + adapter + .cancel(&request) + .await + .expect("fixture cancellation should complete"), + AsyncToolJobCancelOutcome::UnknownOutcome { + error: serde_json::json!({ + "provider_job_id": "job-7", + "fixture": true, + }), + } + ); + let parsed = adapter + .parse_callback( + &ExecutionExternalJobCallbackAuthentication { + headers: std::collections::BTreeMap::new(), + body_sha256: [7; 32], + }, + br#"{"state":"cancelled"}"#, + ) + .await + .expect("fixture callback parsing should complete"); + assert_eq!(parsed.provider_event_id, "event-11"); + assert_eq!(parsed.provider_job_id, "job-7"); + } + #[async_trait] impl BuiltInTool for ConnectorLookingBuiltIn { fn name(&self) -> &'static str { @@ -2202,6 +4211,7 @@ mod tests { run_uid: Uuid::from_u128(10), task_uid: Uuid::from_u128(20), generation: 1, + attempt_generation: 1, }; let next_generation = ExecutionTaskOrigin { generation: 2, @@ -2233,6 +4243,7 @@ mod tests { run_uid: first.run_uid, compensation_id: Uuid::from_u128(30), generation: 1, + attempt_generation: 1, }, )), None, @@ -2276,6 +4287,7 @@ mod tests { run_uid: Uuid::from_u128(10), task_uid: Uuid::from_u128(20), generation: 3, + attempt_generation: 3, }; let next = ExecutionTaskOrigin { generation: 4, @@ -2303,6 +4315,7 @@ mod tests { run_uid: Uuid::from_u128(10), compensation_id: Uuid::from_u128(30), generation: 3, + attempt_generation: 3, }; let next = ExecutionCompensationOrigin { generation: 4, diff --git a/crates/moa-orchestrator/src/tool_invocation/governed.rs b/crates/moa-orchestrator/src/tool_invocation/governed.rs index 98ae4ddcc..152c4ffc0 100644 --- a/crates/moa-orchestrator/src/tool_invocation/governed.rs +++ b/crates/moa-orchestrator/src/tool_invocation/governed.rs @@ -4,6 +4,7 @@ use std::collections::{BTreeSet, HashMap}; use std::sync::Arc; use std::time::{Duration, Instant}; +use chrono::{DateTime, Utc}; use moa_config::SessionLimitsConfig; use moa_core::traits::ChannelAdapter; use moa_core::{ @@ -36,8 +37,8 @@ use crate::services::{ action_reviews::{ActionReviewsClient, RequestActionReview}, session_store::RestateSessionStoreClient, tool_executor::{ - ExecutionToolCallOrigin, ExecutionToolCallOutcome, ExecutionToolCallRequest, - ScopedToolCatalogRequest, ToolExecutorClient, + ExecutionToolCallOrigin, ExecutionToolCallOutcome, ExecutionToolCallPhase, + ExecutionToolCallRequest, ScopedToolCatalogRequest, ToolExecutorClient, }, }; use crate::turn::util::{ @@ -96,6 +97,8 @@ pub(crate) enum GovernedInvocationOrigin<'a> { task_uid: uuid::Uuid, /// Task generation fenced by the execution workflow. generation: u64, + /// Exact bounded task-attempt generation that owns the provider call. + attempt_generation: u64, }, /// Tool call belongs to one persisted execution compensation. ExecutionCompensation { @@ -105,6 +108,8 @@ pub(crate) enum GovernedInvocationOrigin<'a> { compensation_id: uuid::Uuid, /// Compensation generation fenced by the execution workflow. generation: u64, + /// Exact bounded compensation-attempt generation that owns the provider call. + attempt_generation: u64, }, } @@ -134,24 +139,28 @@ impl GovernedInvocationOrigin<'_> { run_uid, task_uid, generation, + attempt_generation, } => ActionReviewOwner::ExecutionTask { session_id, origin: ExecutionTaskOrigin { run_uid, task_uid, generation, + attempt_generation, }, }, Self::ExecutionCompensation { run_uid, compensation_id, generation, + attempt_generation, } => ActionReviewOwner::ExecutionCompensation { session_id, origin: ExecutionCompensationOrigin { run_uid, compensation_id, generation, + attempt_generation, }, }, } @@ -200,10 +209,21 @@ pub(crate) struct GovernedInvocationResult { pub(crate) output: SecuredToolOutput, /// Outcome classification for workflow-local recording. pub(crate) disposition: GovernedInvocationDisposition, + /// Exact durable review wait, present only for a storage-backed review boundary. + pub(crate) review: Option, /// Event ownership plan used for the output event. pub(crate) event_plan: GovernedInvocationEventPlan, } +/// Exact storage-backed review reference returned by action-review admission. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct GovernedReviewPending { + /// Stable action-review identity. + pub(crate) review_uid: uuid::Uuid, + /// Exact persisted review expiry owned by the durable timeout service. + pub(crate) expires_at: DateTime, +} + impl GovernedInvocationResult { /// Returns whether the caller should record a successful segment tool use. pub(crate) fn should_record_segment_tool_use(&self) -> bool { @@ -256,6 +276,17 @@ pub(crate) enum GovernedInvocationEventPlan { pub(crate) enum GovernedInvocationOutcome { /// Tool call was fully handled by the governed coordinator. Completed(Box), + /// Provider work was durably started and must resume from callback or reconciliation. + ExternalJob { + /// MOA-owned job identity reserved before provider dispatch. + external_job_uid: uuid::Uuid, + /// Stable tool-call id for durable external-job ownership. + tool_id: ToolCallId, + /// Invocation that committed the provider job. + invocation: ToolInvocation, + /// Immutable provider job identity and recovery contract. + job: moa_core::types::tools::AsyncToolJob, + }, /// Tool call is a delegation tool and must stay on the workflow-owned path. Delegation { /// Stable tool-call id for the delegation path. @@ -285,6 +316,10 @@ pub(crate) enum GovernedInvocationOutcome { enum GovernedDispatchOutcome { Completed(Box), + ExternalJob { + external_job_uid: uuid::Uuid, + job: moa_core::types::tools::AsyncToolJob, + }, UnknownOutcome { message: String, }, @@ -508,7 +543,13 @@ fn bounded_review_refusal( request: &GovernedInvocationRequest<'_>, invocation: &ToolInvocation, ) -> Option { - (!request.resource_budget.is_unbounded()).then(|| { + (!request.resource_budget.is_unbounded() + && !matches!( + request.origin, + GovernedInvocationOrigin::ExecutionTask { .. } + | GovernedInvocationOrigin::ExecutionCompensation { .. } + )) + .then(|| { denied_tool_output(format!( "Tool {} requires admin review, which a resource-bounded turn cannot detach.", invocation.name @@ -539,7 +580,7 @@ async fn request_action_review( ))); } - crate::restate_identity::replay_safe_request( + let review = crate::restate_identity::replay_safe_request( ctx.service_client::() .request(Json::from(RequestActionReview { envelope: prepared_action.envelope, @@ -548,17 +589,27 @@ async fn request_action_review( })), ) .call() - .await?; + .await? + .into_inner(); + if review.id != request.tool_id.0 || review.tool_call_id != request.tool_id { + return Err(TerminalError::new( + "action-review admission returned a mismatched durable review identity", + ) + .into()); + } let output = pending_review_output(&invocation, &prepared_action.input_summary); append_synthetic_tool_result(ctx, &request, &invocation, &output).await?; - Ok(GovernedInvocationOutcome::Completed(Box::new( - completed_result( - request.tool_id, - invocation, - output, - GovernedInvocationDisposition::ReviewPending, - ), - ))) + let mut result = completed_result( + request.tool_id, + invocation, + output, + GovernedInvocationDisposition::ReviewPending, + ); + result.review = Some(GovernedReviewPending { + review_uid: review.id, + expires_at: review.expires_at, + }); + Ok(GovernedInvocationOutcome::Completed(Box::new(result))) } async fn execute_allowed_tool( @@ -593,6 +644,7 @@ async fn execute_allowed_tool( run_uid, task_uid, generation, + attempt_generation, } => span .in_scope(|| { crate::restate_identity::replay_safe_request( @@ -603,7 +655,9 @@ async fn execute_allowed_tool( run_uid, task_uid, generation, + attempt_generation, }), + phase: ExecutionToolCallPhase::Direct, })), ) }) @@ -616,6 +670,7 @@ async fn execute_allowed_tool( run_uid, compensation_id, generation, + attempt_generation, } => span .in_scope(|| { crate::restate_identity::replay_safe_request( @@ -627,8 +682,10 @@ async fn execute_allowed_tool( run_uid, compensation_id, generation, + attempt_generation, }, ), + phase: ExecutionToolCallPhase::Direct, })), ) }) @@ -653,6 +710,17 @@ async fn execute_allowed_tool( record_turn_tool_dispatch_duration(dispatch_started.elapsed(), 1); let output = match dispatch { GovernedDispatchOutcome::Completed(output) => *output, + GovernedDispatchOutcome::ExternalJob { + external_job_uid, + job, + } => { + return Ok(GovernedInvocationOutcome::ExternalJob { + external_job_uid, + tool_id: request.tool_id, + invocation, + job, + }); + } GovernedDispatchOutcome::UnknownOutcome { message } => { return Ok(GovernedInvocationOutcome::UnknownOutcome { tool_id: request.tool_id, @@ -675,6 +743,7 @@ async fn execute_allowed_tool( invocation, output, disposition: GovernedInvocationDisposition::Executed, + review: None, event_plan: GovernedInvocationEventPlan::ToolExecutorResult, }, ))) @@ -684,6 +753,13 @@ impl From for GovernedDispatchOutcome { fn from(outcome: ExecutionToolCallOutcome) -> Self { match outcome { ExecutionToolCallOutcome::Completed { output } => Self::Completed(output), + ExecutionToolCallOutcome::ExternalJob { + external_job_uid, + job, + } => Self::ExternalJob { + external_job_uid, + job, + }, ExecutionToolCallOutcome::UnknownOutcome { message } => { Self::UnknownOutcome { message } } @@ -727,6 +803,7 @@ fn completed_result( invocation, output, disposition, + review: None, event_plan: GovernedInvocationEventPlan::WorkflowSyntheticResult { success: false }, } } @@ -1173,6 +1250,7 @@ mod tests { run_uid: Uuid::from_u128(83), task_uid: Uuid::from_u128(84), generation: 2, + attempt_generation: 3, }, ); request.capability_policy_context = Some(&context); @@ -1216,6 +1294,44 @@ mod tests { assert!(output.to_text().contains("cannot detach")); } + #[test] + fn bounded_execution_origins_detach_only_into_storage_backed_reviews() { + // Pins: only execution task and compensation owners may cross a bounded + // invocation boundary because their exact generation is parked in Postgres. + let session = test_session_meta(); + let tool_call = tool_call(); + let allowed_tools = BTreeSet::from(["file_read".to_string()]); + for origin in [ + GovernedInvocationOrigin::ExecutionTask { + run_uid: Uuid::from_u128(51), + task_uid: Uuid::from_u128(52), + generation: 3, + attempt_generation: 4, + }, + GovernedInvocationOrigin::ExecutionCompensation { + run_uid: Uuid::from_u128(51), + compensation_id: Uuid::from_u128(53), + generation: 4, + attempt_generation: 5, + }, + ] { + let mut request = request(&session, &tool_call, &allowed_tools, origin); + request.resource_budget = moa_core::types::resource::ResourceBudget::new( + None, + Some(moa_core::types::resource::ResourceAmounts { + tool_calls: 1, + ..moa_core::types::resource::ResourceAmounts::ZERO + }), + ); + + assert_eq!( + bounded_review_refusal(&request, &tool_call.invocation), + None, + "execution-owned bounded reviews must park through their durable owner" + ); + } + } + #[test] fn governed_origin_maps_to_exactly_one_typed_action_review_owner() { // Pins: who is resumed after a review is decided at the moment the tool call is @@ -1257,6 +1373,7 @@ mod tests { run_uid: Uuid::from_u128(50), task_uid: Uuid::from_u128(51), generation: 2, + attempt_generation: 3, } .action_review_owner(session_id); assert_eq!( @@ -1267,6 +1384,7 @@ mod tests { run_uid: Uuid::from_u128(50), task_uid: Uuid::from_u128(51), generation: 2, + attempt_generation: 3, }, } ); @@ -1274,6 +1392,7 @@ mod tests { run_uid: Uuid::from_u128(50), compensation_id: Uuid::from_u128(52), generation: 7, + attempt_generation: 8, } .action_review_owner(session_id); assert_eq!( @@ -1284,6 +1403,7 @@ mod tests { run_uid: Uuid::from_u128(50), compensation_id: Uuid::from_u128(52), generation: 7, + attempt_generation: 8, }, } ); @@ -1368,6 +1488,7 @@ mod tests { run_uid: Uuid::from_u128(40), task_uid: Uuid::from_u128(41), generation: 2, + attempt_generation: 3, }, ); request.capability_provenance = Some(&capability); @@ -1385,6 +1506,7 @@ mod tests { run_uid: Uuid::from_u128(40), task_uid: Uuid::from_u128(41), generation: 2, + attempt_generation: 3, }) ); } @@ -1452,6 +1574,7 @@ mod tests { run_uid: Uuid::from_u128(40), task_uid: Uuid::from_u128(41), generation: 2, + attempt_generation: 3, }, ); @@ -1499,6 +1622,7 @@ mod tests { run_uid: Uuid::from_u128(40), task_uid: Uuid::from_u128(41), generation: 2, + attempt_generation: 3, } )); } @@ -1651,6 +1775,7 @@ mod tests { moa_core::types::security::ToolCapabilityId::builtin("noop"), ), disposition: GovernedInvocationDisposition::Executed, + review: None, event_plan: GovernedInvocationEventPlan::ToolExecutorResult, }; diff --git a/crates/moa-orchestrator/src/workflows/errors.rs b/crates/moa-orchestrator/src/workflows/errors.rs index d5552eebb..a1a76a557 100644 --- a/crates/moa-orchestrator/src/workflows/errors.rs +++ b/crates/moa-orchestrator/src/workflows/errors.rs @@ -1,7 +1,5 @@ //! Shared workflow error conversion for Restate handlers. -use std::borrow::Cow; - use moa_authz::{AuthzCheckError, AuthzError}; use moa_core::error::{FailureProvenance, MoaError}; use restate_sdk::prelude::*; @@ -80,20 +78,24 @@ pub(crate) fn classify_authz_error(error: &AuthzError) -> RestateErrorClass { /// Classifies a Postgres error without parsing its display text. #[must_use] pub(crate) fn classify_sqlx_error(error: &sqlx::Error) -> RestateErrorClass { - let transient = match error { - sqlx::Error::PoolTimedOut | sqlx::Error::PoolClosed | sqlx::Error::WorkerCrashed => true, - sqlx::Error::Io(error) => transient_io(error.kind()), - sqlx::Error::Database(error) => transient_sqlstate(error.code()), - _ => false, - }; - - if transient { + if moa_db::is_retryable_sqlx_error(error) { RestateErrorClass::Retryable } else { RestateErrorClass::Terminal { status: None } } } +/// Classifies an execution-domain failure at the Restate boundary. +#[must_use] +pub(crate) fn classify_execution_error(error: &moa_execution::Error) -> RestateErrorClass { + match error { + moa_execution::Error::Database { source } => classify_sqlx_error(source), + moa_execution::Error::StorageUnavailable { .. } + | moa_execution::Error::CapacitySaturated { .. } => RestateErrorClass::Retryable, + _ => RestateErrorClass::Terminal { status: None }, + } +} + /// Converts a [`MoaError`] into a Restate handler error. pub(crate) fn moa_error_to_handler_error(error: MoaError) -> HandlerError { error_to_handler_error(classify_moa_error(&error), error) @@ -119,6 +121,11 @@ pub(crate) fn sqlx_error_to_handler_error(error: sqlx::Error) -> HandlerError { error_to_handler_error(classify_sqlx_error(&error), error) } +/// Converts an execution-domain failure into a Restate handler error. +pub(crate) fn execution_error_to_handler_error(error: moa_execution::Error) -> HandlerError { + error_to_handler_error(classify_execution_error(&error), error) +} + fn error_to_handler_error( class: RestateErrorClass, error: impl std::error::Error + Send + Sync + 'static, @@ -134,34 +141,6 @@ fn error_to_handler_error( } } -fn transient_io(kind: std::io::ErrorKind) -> bool { - matches!( - kind, - std::io::ErrorKind::Interrupted - | std::io::ErrorKind::WouldBlock - | std::io::ErrorKind::TimedOut - | std::io::ErrorKind::ConnectionReset - | std::io::ErrorKind::ConnectionAborted - | std::io::ErrorKind::ConnectionRefused - | std::io::ErrorKind::NotConnected - | std::io::ErrorKind::NetworkDown - | std::io::ErrorKind::NetworkUnreachable - | std::io::ErrorKind::HostUnreachable - ) -} - -fn transient_sqlstate(code: Option>) -> bool { - let Some(code) = code else { - return false; - }; - code.starts_with("08") - || code.starts_with("40") - || matches!( - code.as_ref(), - "53P01" | "53P02" | "53P03" | "55P03" | "57P01" | "57P02" | "57P03" - ) -} - /// Builds a terminal `400` handler error from a message. pub(crate) fn bad_request(message: impl Into) -> HandlerError { TerminalError::new_with_code(400, message.into()).into() @@ -378,4 +357,56 @@ mod tests { terminal(None) ); } + + #[test] + fn execution_errors_preserve_retryability_at_the_restate_boundary() { + // Pins: transient execution storage failures stay replayable while + // deterministic repository corruption terminates the invocation. + let cases = [ + ( + moa_execution::Error::Database { + source: sqlx::Error::PoolClosed, + }, + RestateErrorClass::Retryable, + ), + ( + moa_execution::Error::StorageUnavailable { + message: "database restarting".to_string(), + }, + RestateErrorClass::Retryable, + ), + ( + moa_execution::Error::CapacitySaturated { + dimension: "scheduled_triggers", + }, + RestateErrorClass::Retryable, + ), + ( + moa_execution::Error::Database { + source: sqlx::Error::RowNotFound, + }, + terminal(None), + ), + ( + moa_execution::Error::InvalidRepositoryData { + message: "invalid persisted status".to_string(), + }, + terminal(None), + ), + ( + moa_execution::Error::Storage { + message: "missing required row".to_string(), + }, + terminal(None), + ), + ]; + + for (error, expected) in cases { + assert_eq!( + classify_execution_error(&error), + expected, + "unexpected class for {error}" + ); + } + } } diff --git a/crates/moa-orchestrator/src/workflows/execution_compensation.rs b/crates/moa-orchestrator/src/workflows/execution_compensation.rs deleted file mode 100644 index c313ee57c..000000000 --- a/crates/moa-orchestrator/src/workflows/execution_compensation.rs +++ /dev/null @@ -1,975 +0,0 @@ -//! Durable keyed workflow for one exact execution compensation. - -use std::{collections::BTreeSet, sync::Arc}; - -use moa_artifacts::execution_plan::{ExecutionFailureClass, ExecutionUsage}; -use moa_config::SessionLimitsConfig; -use moa_core::{ - traits::{ChannelAdapter, SessionStore as _}, - types::{ - action_policy::{ActionClass, CapabilityProvenance}, - channel::Channel, - completion::{ToolCallContent, ToolInvocation}, - identifiers::ToolCallId, - session::SessionMeta, - tools::IdempotencyClass, - }, -}; -use moa_execution::{ - capability::{CapabilitySource, ExecutionCapability}, - repository::{ - ActionReviewResolutionWrite, CompensationClaimOutcome, CompensationOutcomeWrite, - ExecutionRepository, ExecutionRunRecord, ExecutionScope, ExecutionTaskRecord, - }, - schema::validate_instance, - state::{ - CompensationRegistrationProjection, CompensationStatus, ExecutionCompensationOutcome, - ExecutionRunStatus, LogicalTaskKind, - }, - wire::{ - ExecutionActionReviewResolution, ExecutionCompensationReviewAcknowledgement, - ExecutionCompensationReviewResolutionRequest, ExecutionCompensationWorkflowRequest, - ExecutionToolDispatchRejection, - }, -}; -use moa_observability::{ - propagation::link_remote_context_from_link_headers, - restate_observability::annotate_restate_handler_span, -}; -use moa_session::PostgresSessionStore; -use restate_sdk::prelude::*; -use serde::{Deserialize, Serialize}; -use serde_json::Value; -use tracing_opentelemetry::OpenTelemetrySpanExt; - -use crate::{ - ctx::RequestHeaders, - services::tool_executor::{ReleaseExecutionCompensationHandsRequest, ToolExecutorClient}, - tool_invocation::governed::{ - GovernedInvocationDisposition, GovernedInvocationOrigin, GovernedInvocationOutcome, - GovernedInvocationRequest, GovernedInvocationResult, invoke_governed_tool, - }, -}; - -/// Durable workflow surface for one stable compensation registration. -#[restate_sdk::workflow] -pub trait ExecutionCompensation { - /// Executes the exact pinned compensator through bounded generation-fenced retries. - async fn run(request: Json) -> Result<(), HandlerError>; - - /// Persists and resolves one compensation action-review delivery. - #[shared] - async fn resolve_action_review( - request: Json, - ) -> Result, HandlerError>; -} - -/// Runtime dependencies for one governed compensation workflow. -#[derive(Clone)] -pub struct ExecutionCompensationImpl { - repository: ExecutionRepository, - session_store: Arc, - session_limits: SessionLimitsConfig, - channel_adapters: Arc>>, -} - -impl ExecutionCompensationImpl { - /// Creates one compensation workflow over the exact execution and tool-runtime stores. - #[must_use] - pub fn new( - pool: sqlx::PgPool, - session_store: Arc, - session_limits: SessionLimitsConfig, - channel_adapters: Arc>>, - ) -> Self { - Self { - repository: ExecutionRepository::new(pool), - session_store, - session_limits, - channel_adapters, - } - } -} - -impl ExecutionCompensation for ExecutionCompensationImpl { - #[tracing::instrument(skip(self, ctx, request))] - // SAFETY: dispatched only by the owning ExecutionRun workflow after reverse-order repository claim. - async fn run( - &self, - ctx: WorkflowContext<'_>, - request: Json, - ) -> Result<(), HandlerError> { - crate::ctx::adopt_incoming_trace_parent(&ctx); - annotate_restate_handler_span("ExecutionCompensation", "run"); - let request = request.into_inner(); - require_compensation_key(ctx.key(), request.compensation_id)?; - if request.identity.tenant_id != request.tenant_id { - return Err(TerminalError::new_with_code( - 409, - "execution compensation identity tenant mismatch", - ) - .into()); - } - annotate_compensation_span(request.run_uid, request.compensation_id); - let scope = execution_scope(&request); - let mut generation = request.generation; - let mut operation_index = 0_u64; - loop { - let repository = self.repository.clone(); - let load_request = request.clone(); - let prepared = ctx - .run(|| async move { - prepare_compensation(repository, scope, &load_request, generation) - .await - .map(Json::from) - }) - .name(format!("execution_compensation_prepare_{operation_index}")) - .await? - .into_inner(); - operation_index = operation_index.saturating_add(1); - let prepared = match prepared { - PreparedCompensation::Ready(prepared) => prepared, - PreparedCompensation::Settled => { - cleanup_compensation_hands(&ctx, &request).await?; - return Ok(()); - } - }; - - let outcome = execute_compensation_attempt(self, &ctx, &request, &prepared).await?; - let repository = self.repository.clone(); - let run_uid = request.run_uid; - let compensation_id = request.compensation_id; - let recorded = ctx - .run(|| async move { - let write = repository - .record_compensation_outcome( - scope, - run_uid, - compensation_id, - generation, - outcome, - ) - .await - .map_err(execution_error)?; - compensation_record_step(write).map(Json::from) - }) - .name(format!("execution_compensation_outcome_{operation_index}")) - .await? - .into_inner(); - operation_index = operation_index.saturating_add(1); - match recorded { - CompensationRecordStep::Settled => { - cleanup_compensation_hands(&ctx, &request).await?; - return Ok(()); - } - CompensationRecordStep::Retry { next_generation } => { - let repository = self.repository.clone(); - let claim = ctx - .run(|| async move { - let outcome = repository - .claim_next_compensation( - scope, - run_uid, - compensation_id, - next_generation, - ) - .await - .map_err(execution_error)?; - claim_retry_step(outcome, next_generation).map(Json::from) - }) - .name(format!("execution_compensation_reclaim_{operation_index}")) - .await? - .into_inner(); - operation_index = operation_index.saturating_add(1); - let CompensationRetryClaim::Claimed { - generation: claimed, - } = claim - else { - cleanup_compensation_hands(&ctx, &request).await?; - return Ok(()); - }; - generation = claimed; - } - CompensationRecordStep::Conflict => { - cleanup_compensation_hands(&ctx, &request).await?; - return Ok(()); - } - } - } - } - - #[tracing::instrument(skip(self, ctx, request))] - // SAFETY: invoked only by the bounded compensation-review outbox dispatcher from a terminal persisted review row. - async fn resolve_action_review( - &self, - ctx: SharedWorkflowContext<'_>, - request: Json, - ) -> Result, HandlerError> { - crate::ctx::adopt_incoming_trace_parent(&ctx); - annotate_restate_handler_span("ExecutionCompensation", "resolve_action_review"); - let headers = ctx.request_headers(); - let _ = link_remote_context_from_link_headers(&tracing::Span::current(), |name| { - headers.get(name).cloned() - }); - let request = request.into_inner(); - require_compensation_key(ctx.key(), request.compensation_id)?; - annotate_compensation_span(request.run_uid, request.compensation_id); - let repository = self.repository.clone(); - let record_request = request.clone(); - let write = ctx - .run(|| async move { - repository - .record_compensation_action_review_resolution( - ExecutionScope::ControlPlane, - record_request.run_uid, - record_request.compensation_id, - record_request.generation, - record_request.review_uid, - &record_request.resolution, - ) - .await - .map(Json::from) - .map_err(execution_error) - }) - .name(format!( - "execution_compensation_review_resolution_{}", - request.review_uid - )) - .await? - .into_inner(); - let acknowledgement = match write { - ActionReviewResolutionWrite::Applied => { - ctx.resolve_promise( - &action_review_promise_key(request.review_uid, request.generation), - Json::from(request.resolution), - ); - ExecutionCompensationReviewAcknowledgement::Applied - } - ActionReviewResolutionWrite::Replayed => { - ctx.resolve_promise( - &action_review_promise_key(request.review_uid, request.generation), - Json::from(request.resolution), - ); - ExecutionCompensationReviewAcknowledgement::Replayed - } - ActionReviewResolutionWrite::AuditedStale | ActionReviewResolutionWrite::NotFound => { - ExecutionCompensationReviewAcknowledgement::AuditedStale - } - }; - Ok(Json::from(acknowledgement)) - } -} - -#[derive(Clone, Debug, Deserialize, Serialize)] -#[serde(tag = "state", rename_all = "snake_case")] -enum PreparedCompensation { - Ready(Box), - Settled, -} - -#[derive(Clone, Debug, Deserialize, Serialize)] -struct PreparedCompensationAttempt { - run: ExecutionRunRecord, - registration: CompensationRegistrationProjection, - forward_task: ExecutionTaskRecord, -} - -async fn prepare_compensation( - repository: ExecutionRepository, - scope: ExecutionScope, - request: &ExecutionCompensationWorkflowRequest, - generation: u64, -) -> Result { - let snapshot = repository - .load_compensation_snapshot(scope, request.run_uid) - .await - .map_err(execution_error)? - .ok_or_else(|| TerminalError::new_with_code(404, "execution run not found"))?; - if snapshot.run.tenant_id != request.tenant_id - || snapshot.run.contact_id != request.contact_id - || snapshot.run.session_id != request.session_id - { - return Err( - TerminalError::new_with_code(409, "execution compensation scope mismatch").into(), - ); - } - if snapshot.run.status != ExecutionRunStatus::Compensating { - return Ok(PreparedCompensation::Settled); - } - let Some(registration) = snapshot - .registrations - .into_iter() - .find(|candidate| candidate.compensation_id == request.compensation_id) - else { - return Err(TerminalError::new_with_code(404, "execution compensation not found").into()); - }; - if registration.status.is_settled() { - return Ok(PreparedCompensation::Settled); - } - if registration.status != CompensationStatus::Running || registration.generation != generation { - return Ok(PreparedCompensation::Settled); - } - let forward_task = repository - .load_task(scope, request.run_uid, registration.forward_task_id) - .await - .map_err(execution_error)? - .ok_or_else(|| TerminalError::new_with_code(404, "compensation forward task not found"))?; - Ok(PreparedCompensation::Ready(Box::new( - PreparedCompensationAttempt { - run: snapshot.run, - registration, - forward_task, - }, - ))) -} - -async fn execute_compensation_attempt( - workflow: &ExecutionCompensationImpl, - ctx: &WorkflowContext<'_>, - request: &ExecutionCompensationWorkflowRequest, - prepared: &PreparedCompensationAttempt, -) -> Result { - let mut usage = prepared - .registration - .outcome - .as_ref() - .map(ExecutionCompensationOutcome::usage) - .cloned() - .unwrap_or(ExecutionUsage { - cost_microusd: 0, - tokens: 0, - tool_calls: 0, - retrieved_bytes: 0, - }); - let result = invoke_exact_compensator(workflow, ctx, request, prepared).await?; - if result.is_ok() { - usage.tool_calls = usage.tool_calls.saturating_add(1); - } - Ok(match result { - Ok(CompensatorResult::Output(output)) => { - usage.retrieved_bytes = usage - .retrieved_bytes - .saturating_add(serialized_len(&output)); - let capability = match find_catalog_capability( - &prepared.run, - &prepared.registration.compensator.compensator, - ) { - Ok(capability) => capability, - Err(message) => return Ok(failed_compensation(message, false, usage)), - }; - if let Err(error) = validate_instance( - &capability.output_schema, - &output, - "execution_compensation.output", - ) { - return Ok(ExecutionCompensationOutcome::UnknownOutcome { - message: format!( - "compensator returned an invalid output after possible commit: {error}" - ), - usage, - }); - } - ExecutionCompensationOutcome::Completed { output, usage } - } - Ok(CompensatorResult::Failed { message, retryable }) => { - failed_compensation(message, retryable, usage) - } - Ok(CompensatorResult::UnknownOutcome { message }) => { - ExecutionCompensationOutcome::UnknownOutcome { message, usage } - } - Err(message) => failed_compensation(message, false, usage), - }) -} - -enum CompensatorResult { - Output(Value), - Failed { message: String, retryable: bool }, - UnknownOutcome { message: String }, -} - -async fn invoke_exact_compensator( - workflow: &ExecutionCompensationImpl, - ctx: &WorkflowContext<'_>, - request: &ExecutionCompensationWorkflowRequest, - prepared: &PreparedCompensationAttempt, -) -> Result, HandlerError> { - let capability = match validate_runtime_contract(prepared) { - Ok(capability) => capability, - Err(message) => return Ok(Err(message)), - }; - if let Err(error) = validate_instance( - &capability.input_schema, - &prepared.registration.mapped_input, - "execution_compensation.input", - ) { - return Ok(Err(format!( - "compensator mapped input failed pinned schema: {error}" - ))); - } - let session = load_session( - workflow, - ctx, - request.session_id, - request.compensation_id, - prepared.registration.generation, - prepared.registration.attempt, - ) - .await?; - let tool_name = match capability_tool_name(capability) { - Ok(tool_name) => tool_name, - Err(message) => return Ok(Err(message)), - }; - let tool_id = ToolCallId(uuid::Uuid::new_v5( - &request.compensation_id.as_uuid(), - format!("generation:{}", prepared.registration.generation).as_bytes(), - )); - let tool_call = ToolCallContent { - invocation: ToolInvocation { - id: Some(tool_id.to_string()), - name: tool_name.clone(), - input: prepared.registration.mapped_input.clone(), - }, - provider_metadata: None, - }; - let allowed_tools = BTreeSet::from([tool_name]); - let provenance = CapabilityProvenance { - kind: Some(capability_source_kind(&capability.source).to_string()), - id: Some(format!( - "{}@{}", - capability.reference.name, capability.reference.version - )), - step_id: Some(format!( - "compensation:{}", - prepared.registration.forward_task_id - )), - }; - let governed = invoke_governed_tool( - ctx, - GovernedInvocationRequest { - session: &session, - identity: &request.identity, - session_id: request.session_id, - tool_id, - tool_call: &tool_call, - allowed_tools: &allowed_tools, - expected_tool_contract_revision: Some(&capability.contract_revision), - active_canary: None, - trusted_sandbox_manifest: None, - origin: GovernedInvocationOrigin::ExecutionCompensation { - run_uid: request.run_uid, - compensation_id: request.compensation_id.as_uuid(), - generation: prepared.registration.generation, - }, - capability_provenance: Some(&provenance), - capability_policy_context: Some(&capability.policy_context), - resource_budget: moa_core::types::resource::ResourceBudget::UNBOUNDED, - }, - &workflow.session_limits, - workflow.session_store.clone(), - workflow.channel_adapters.as_ref(), - ) - .await?; - let result = match classify_governed_compensation_outcome(governed) { - GovernedCompensationOutcome::Completed(result) => result, - GovernedCompensationOutcome::Settled(result) => return Ok(Ok(result)), - }; - if result.disposition == GovernedInvocationDisposition::ReviewPending { - let resolution = ctx - .promise::>(&action_review_promise_key( - result.tool_id.0, - prepared.registration.generation, - )) - .await? - .into_inner(); - return Ok(Ok(compensation_review_result(resolution))); - } - if result.output.is_error() { - return Ok(Ok(CompensatorResult::Failed { - message: result.output.safe_output.to_text(), - retryable: result.disposition == GovernedInvocationDisposition::Executed, - })); - } - Ok(Ok(CompensatorResult::Output( - result - .output - .safe_output - .structured_payload() - .cloned() - .unwrap_or_else(|| Value::String(result.output.safe_output.to_text())), - ))) -} - -enum GovernedCompensationOutcome { - Completed(Box), - Settled(CompensatorResult), -} - -fn classify_governed_compensation_outcome( - outcome: GovernedInvocationOutcome, -) -> GovernedCompensationOutcome { - match outcome { - GovernedInvocationOutcome::Completed(result) => { - GovernedCompensationOutcome::Completed(result) - } - GovernedInvocationOutcome::UnknownOutcome { message, .. } => { - GovernedCompensationOutcome::Settled(CompensatorResult::UnknownOutcome { message }) - } - GovernedInvocationOutcome::NotDispatched { reason, .. } => { - GovernedCompensationOutcome::Settled(CompensatorResult::Failed { - message: execution_dispatch_rejection_message(reason), - retryable: false, - }) - } - GovernedInvocationOutcome::Delegation { .. } => { - GovernedCompensationOutcome::Settled(CompensatorResult::Failed { - message: "compensator attempted an unsupported delegation path".to_string(), - retryable: false, - }) - } - } -} - -fn validate_runtime_contract( - prepared: &PreparedCompensationAttempt, -) -> Result<&ExecutionCapability, String> { - let LogicalTaskKind::Capability { - reference: forward_reference, - } = &prepared.forward_task.kind - else { - return Err("registered compensation forward task is not a direct capability".to_string()); - }; - if prepared.forward_task.compensation_contract.as_ref() - != Some(&prepared.registration.compensator) - { - return Err("registered compensation drifted from the forward task contract".to_string()); - } - let forward = find_catalog_capability(&prepared.run, forward_reference)?; - if !forward - .rollback - .as_ref() - .is_some_and(|rollback| rollback.matches(&prepared.registration.compensator)) - { - return Err("pinned forward capability no longer promises the exact rollback".to_string()); - } - let compensator = find_catalog_capability( - &prepared.run, - &prepared.registration.compensator.compensator, - )?; - if compensator.action_class == ActionClass::Read { - return Err("compensator catalog entry is read-only".to_string()); - } - if compensator.idempotency_class != IdempotencyClass::Idempotent { - return Err("compensator catalog entry is not idempotent".to_string()); - } - Ok(compensator) -} - -fn find_catalog_capability<'a>( - run: &'a ExecutionRunRecord, - reference: &moa_artifacts::execution_plan::CapabilityReference, -) -> Result<&'a ExecutionCapability, String> { - if !run.authorization.capability_refs.contains(reference) { - return Err("compensator is outside the persisted authorization envelope".to_string()); - } - run.catalog - .capabilities - .iter() - .find(|capability| capability.reference == *reference) - .ok_or_else(|| "compensator is absent from the persisted catalog".to_string()) -} - -fn capability_tool_name(capability: &ExecutionCapability) -> Result { - capability - .source - .model_visible_tool_name() - .map(str::to_string) - .ok_or_else(|| "compensator has no governed tool owner".to_string()) -} - -const fn capability_source_kind(source: &CapabilitySource) -> &'static str { - match source { - CapabilitySource::BuiltInTool { .. } => "built_in_tool", - CapabilitySource::HandTool { .. } => "hand_tool", - CapabilitySource::McpTool { .. } => "mcp_tool", - CapabilitySource::ActionArtifact { .. } => "action_artifact", - CapabilitySource::ConnectorAction { .. } => "connector_action", - CapabilitySource::InstalledConnectorAction { .. } => "installed_connector_action", - CapabilitySource::SkillAction { .. } => "skill_action", - CapabilitySource::SkillCode { .. } => "skill_code", - CapabilitySource::Memory { .. } => "memory", - CapabilitySource::Knowledge { .. } => "knowledge", - CapabilitySource::Model => "model", - } -} - -fn compensation_review_result(resolution: ExecutionActionReviewResolution) -> CompensatorResult { - match resolution { - ExecutionActionReviewResolution::Completed { tool_output } => { - let output = match serde_json::from_value::( - tool_output, - ) { - Ok(output) => output, - Err(error) => { - return CompensatorResult::UnknownOutcome { - message: format!("invalid compensation review tool output: {error}"), - }; - } - }; - if output.is_error() { - CompensatorResult::Failed { - message: output.safe_output.to_text(), - retryable: true, - } - } else { - CompensatorResult::Output( - output - .safe_output - .structured_payload() - .cloned() - .unwrap_or_else(|| Value::String(output.safe_output.to_text())), - ) - } - } - ExecutionActionReviewResolution::Failed { class, message } => CompensatorResult::Failed { - message, - retryable: class == ExecutionFailureClass::Retryable, - }, - ExecutionActionReviewResolution::UnknownOutcome { message } => { - CompensatorResult::UnknownOutcome { message } - } - ExecutionActionReviewResolution::NotDispatched { reason } => CompensatorResult::Failed { - message: execution_dispatch_rejection_message(reason), - retryable: false, - }, - ExecutionActionReviewResolution::Denied { reason } - | ExecutionActionReviewResolution::TimedOut { reason } => CompensatorResult::Failed { - message: reason, - retryable: false, - }, - } -} - -fn execution_dispatch_rejection_message(reason: ExecutionToolDispatchRejection) -> String { - let reason = match reason { - ExecutionToolDispatchRejection::OriginNotFound => "origin_not_found", - ExecutionToolDispatchRejection::StaleGeneration => "stale_generation", - ExecutionToolDispatchRejection::OperationNotRunning => "operation_not_running", - ExecutionToolDispatchRejection::RunNotDispatchable => "run_not_dispatchable", - }; - format!("execution effect was not dispatched: {reason}") -} - -fn failed_compensation( - message: String, - retryable: bool, - usage: ExecutionUsage, -) -> ExecutionCompensationOutcome { - ExecutionCompensationOutcome::Failed { - message, - retryable, - usage, - } -} - -#[derive(Clone, Copy, Debug, Deserialize, Serialize)] -#[serde(tag = "state", rename_all = "snake_case")] -enum CompensationRecordStep { - Settled, - Retry { next_generation: u64 }, - Conflict, -} - -fn compensation_record_step( - write: CompensationOutcomeWrite, -) -> Result { - match write { - CompensationOutcomeWrite::Completed(_) - | CompensationOutcomeWrite::Failed(_) - | CompensationOutcomeWrite::UnknownOutcome(_) => Ok(CompensationRecordStep::Settled), - CompensationOutcomeWrite::Requeued(registration) => Ok(CompensationRecordStep::Retry { - next_generation: registration.generation, - }), - CompensationOutcomeWrite::Replayed(registration) => { - if registration.status == CompensationStatus::Pending { - Ok(CompensationRecordStep::Retry { - next_generation: registration.generation, - }) - } else { - Ok(CompensationRecordStep::Settled) - } - } - CompensationOutcomeWrite::NotFound => { - Err(TerminalError::new_with_code(404, "execution compensation not found").into()) - } - CompensationOutcomeWrite::Conflict => Ok(CompensationRecordStep::Conflict), - } -} - -#[derive(Clone, Copy, Debug, Deserialize, Serialize)] -#[serde(tag = "state", rename_all = "snake_case")] -enum CompensationRetryClaim { - Claimed { generation: u64 }, - Settled, -} - -fn claim_retry_step( - outcome: CompensationClaimOutcome, - expected_generation: u64, -) -> Result { - match outcome { - CompensationClaimOutcome::Claimed(registration) - | CompensationClaimOutcome::Replayed(registration) - if registration.generation == expected_generation => - { - Ok(CompensationRetryClaim::Claimed { - generation: registration.generation, - }) - } - CompensationClaimOutcome::Claimed(_) | CompensationClaimOutcome::Replayed(_) => { - Ok(CompensationRetryClaim::Settled) - } - CompensationClaimOutcome::BudgetRejected(_) => Ok(CompensationRetryClaim::Settled), - CompensationClaimOutcome::Conflict => Ok(CompensationRetryClaim::Settled), - CompensationClaimOutcome::NotFound => { - Err(TerminalError::new_with_code(404, "execution compensation not found").into()) - } - } -} - -async fn load_session( - workflow: &ExecutionCompensationImpl, - ctx: &WorkflowContext<'_>, - session_id: moa_core::types::identifiers::SessionId, - compensation_id: moa_execution::state::CompensationId, - generation: u64, - attempt: u64, -) -> Result { - let store = workflow.session_store.clone(); - Ok(ctx - .run(|| async move { - store - .get_session(session_id) - .await - .map(Json::from) - .map_err(crate::workflows::errors::moa_error_to_handler_error) - }) - .name(format!( - "execution_compensation_load_session_{compensation_id}_{generation}_{attempt}" - )) - .await? - .into_inner()) -} - -fn action_review_promise_key(review_uid: uuid::Uuid, generation: u64) -> String { - format!("execution_compensation_action_review:{review_uid}:{generation}") -} - -fn require_compensation_key( - key: &str, - compensation_id: moa_execution::state::CompensationId, -) -> Result<(), HandlerError> { - if key == compensation_id.to_string() { - Ok(()) - } else { - Err(TerminalError::new_with_code(404, "execution compensation id mismatch").into()) - } -} - -fn execution_scope(request: &ExecutionCompensationWorkflowRequest) -> ExecutionScope { - request.contact_id.map_or( - ExecutionScope::Tenant { - tenant_id: request.tenant_id, - }, - |contact_id| ExecutionScope::Contact { - tenant_id: request.tenant_id, - contact_id, - }, - ) -} - -fn annotate_compensation_span( - run_uid: uuid::Uuid, - compensation_id: moa_execution::state::CompensationId, -) { - let span = tracing::Span::current(); - span.set_attribute("moa.execution.run_uid", run_uid.to_string()); - span.set_attribute("moa.execution.compensation_id", compensation_id.to_string()); -} - -fn serialized_len(value: &Value) -> u64 { - serde_json::to_vec(value) - .map(|bytes| bytes.len() as u64) - .unwrap_or_default() -} - -async fn cleanup_compensation_hands( - ctx: &WorkflowContext<'_>, - request: &ExecutionCompensationWorkflowRequest, -) -> Result<(), HandlerError> { - crate::restate_identity::replay_safe_request( - ctx.service_client::() - .release_execution_compensation_hands(Json::from( - ReleaseExecutionCompensationHandsRequest { - tenant_id: request.tenant_id, - session_id: request.session_id, - run_uid: request.run_uid, - compensation_id: request.compensation_id, - }, - )), - ) - .call() - .await?; - Ok(()) -} - -fn execution_error(error: moa_execution::Error) -> HandlerError { - match error { - storage @ moa_execution::Error::Storage { .. } => HandlerError::from(storage), - deterministic => TerminalError::new(format!( - "execution compensation workflow failed: {deterministic}" - )) - .into(), - } -} - -#[cfg(test)] -mod tests { - use super::*; - use moa_core::types::completion::ToolInvocation; - use serde_json::json; - - #[test] - fn stable_compensation_tool_id_changes_only_with_generation() { - // Pins: replay of one compensation generation addresses the same governed - // invocation, while a persisted retry generation receives a fresh identity. - let compensation_id = - moa_execution::state::CompensationId::from_uuid(uuid::Uuid::from_u128(0xc011)); - let first = ToolCallId(uuid::Uuid::new_v5( - &compensation_id.as_uuid(), - b"generation:1", - )); - let replay = ToolCallId(uuid::Uuid::new_v5( - &compensation_id.as_uuid(), - b"generation:1", - )); - let retry = ToolCallId(uuid::Uuid::new_v5( - &compensation_id.as_uuid(), - b"generation:2", - )); - assert_eq!(first, replay); - assert_ne!(first, retry); - } - - #[test] - fn review_timeout_is_terminal_not_retryable() { - // Pins: an expired reviewed undo requires manual repair and never resends - // the external effect under a new compensation generation. - let result = compensation_review_result(ExecutionActionReviewResolution::TimedOut { - reason: "review expired".to_string(), - }); - assert!(matches!( - result, - CompensatorResult::Failed { - retryable: false, - .. - } - )); - } - - #[test] - fn governed_compensation_ambiguity_is_terminal_unknown() { - // Pins: an ambiguous compensator effect is never retried because the undo - // may already have committed and a second dispatch could double-apply it. - let classified = - classify_governed_compensation_outcome(GovernedInvocationOutcome::UnknownOutcome { - tool_id: ToolCallId(uuid::Uuid::from_u128(91)), - invocation: ToolInvocation { - id: Some("tool-91".to_string()), - name: "fixture_undo".to_string(), - input: json!({"id": 91}), - }, - message: "undo result is ambiguous".to_string(), - }); - assert!(matches!( - classified, - GovernedCompensationOutcome::Settled(CompensatorResult::UnknownOutcome { message }) - if message == "undo result is ambiguous" - )); - } - - #[test] - fn governed_compensation_admission_rejection_is_terminal_not_unknown() { - // Pins: compensation admission rejection is definitive zero-effect; the - // reverse driver stops for manual repair without retrying or claiming ambiguity. - let classified = - classify_governed_compensation_outcome(GovernedInvocationOutcome::NotDispatched { - tool_id: ToolCallId(uuid::Uuid::from_u128(92)), - invocation: ToolInvocation { - id: Some("tool-92".to_string()), - name: "fixture_undo".to_string(), - input: json!({"id": 92}), - }, - reason: ExecutionToolDispatchRejection::StaleGeneration, - }); - assert!(matches!( - classified, - GovernedCompensationOutcome::Settled(CompensatorResult::Failed { - retryable: false, - message, - }) if message.ends_with("stale_generation") - )); - } - - #[test] - fn reviewed_compensation_ambiguity_is_terminal_unknown() { - // Pins: the durable review dispatcher preserves ToolExecutor ambiguity all - // the way into the compensation state machine without retry classification. - let result = compensation_review_result(ExecutionActionReviewResolution::UnknownOutcome { - message: "reviewed undo is ambiguous".to_string(), - }); - assert!(matches!( - result, - CompensatorResult::UnknownOutcome { message } - if message == "reviewed undo is ambiguous" - )); - } - - #[test] - fn reviewed_compensator_error_preserves_idempotent_retry() { - // Pins: compiler and runtime admission require an idempotent compensator, - // so a definitive reviewed error has the same bounded retry eligibility as - // a direct compensator error; denial, timeout, and ambiguity remain terminal. - let output = moa_core::types::tools::SecuredToolOutput::assessed_safe( - moa_core::types::tools::ToolOutput::from(moa_core::error::ToolFailureClass::Fatal { - reason: "temporary undo failure".to_string(), - }), - moa_core::types::security::ToolCapabilityId::builtin("fixture_undo"), - ); - let result = compensation_review_result(ExecutionActionReviewResolution::Completed { - tool_output: serde_json::to_value(output).expect("secured output should serialize"), - }); - assert!(matches!( - result, - CompensatorResult::Failed { - retryable: true, - message, - } if message.contains("temporary undo failure") - )); - } - - #[test] - fn reviewed_compensation_admission_rejection_is_terminal_not_unknown() { - // Pins: the atomic owner fence proved that no undo began, so admission - // rejection requires manual repair without retry and never claims ambiguity. - let result = compensation_review_result(ExecutionActionReviewResolution::NotDispatched { - reason: ExecutionToolDispatchRejection::OperationNotRunning, - }); - assert!(matches!( - result, - CompensatorResult::Failed { - retryable: false, - message, - } if message.ends_with("operation_not_running") - )); - } -} diff --git a/crates/moa-orchestrator/src/workflows/execution_compensation_attempt.rs b/crates/moa-orchestrator/src/workflows/execution_compensation_attempt.rs new file mode 100644 index 000000000..cb69c108c --- /dev/null +++ b/crates/moa-orchestrator/src/workflows/execution_compensation_attempt.rs @@ -0,0 +1,354 @@ +//! One immutable, bounded compensation-attempt workflow per durable dispatch identity. + +mod active; +mod external; +mod yielding; + +use std::{collections::HashMap, sync::Arc}; + +use chrono::Utc; +use moa_config::SessionLimitsConfig; +use moa_core::{traits::ChannelAdapter, types::channel::Channel}; +use moa_execution::repository::{ + ExecutionRepository, ExecutionScope, + compensation::{ + CompensationAttemptFence, CompensationAttemptRecord, CompensationAttemptWriteOutcome, + }, +}; +use moa_execution::wire::{ + ExecutionAttemptWatchdogResponse, ExecutionAttemptWatchdogResponseOutcome, + ExecutionCompensationAttemptCancelRequest, ExecutionCompensationAttemptRequest, + ExecutionCompensationAttemptWatchdogRequest, ExecutionCompensationReleaseIntent, +}; +use moa_observability::restate_observability::annotate_restate_handler_span; +use moa_session::PostgresSessionStore; +use restate_sdk::prelude::*; +use uuid::Uuid; + +use crate::workflows::errors::execution_error_to_handler_error; +use crate::{ + restate_identity::replay_safe_request, + services::execution_dispatcher::{DispatchExecutionsRequest, ExecutionDispatcherClient}, +}; + +/// Durable surface for one strict reverse-order compensation slice. +#[restate_sdk::workflow] +pub trait ExecutionCompensationAttempt { + /// Executes at most one admitted compensation generation and then returns. + async fn run(request: Json) -> Result<(), HandlerError>; + + /// Classifies one exact active compensation whose watchdog became due. + #[shared] + async fn watchdog( + request: Json, + ) -> Result, HandlerError>; + + /// Checkpoints and relinquishes one exact active compensation after a durable run fence. + #[shared] + async fn cancel( + request: Json, + ) -> Result<(), HandlerError>; +} + +/// Runtime dependencies for one immutable bounded compensation attempt. +#[derive(Clone)] +pub struct ExecutionCompensationAttemptImpl { + repository: ExecutionRepository, + session_store: Arc, + session_limits: SessionLimitsConfig, + channel_adapters: Arc>>, +} + +impl ExecutionCompensationAttemptImpl { + /// Creates the bounded compensation-attempt workflow over authoritative stores. + #[must_use] + pub fn new( + pool: sqlx::PgPool, + session_store: Arc, + session_limits: SessionLimitsConfig, + channel_adapters: Arc>>, + ) -> Self { + Self { + repository: ExecutionRepository::new(pool), + session_store, + session_limits, + channel_adapters, + } + } +} + +impl ExecutionCompensationAttempt for ExecutionCompensationAttemptImpl { + #[tracing::instrument(skip(self, ctx, request))] + // SAFETY: only the durable dispatch outbox invokes this identity-free workflow; + // the locked run supplies the authoritative principal, session, and catalog. + async fn run( + &self, + ctx: WorkflowContext<'_>, + request: Json, + ) -> Result<(), HandlerError> { + crate::ctx::adopt_incoming_trace_parent(&ctx); + annotate_restate_handler_span("ExecutionCompensationAttempt", "run"); + let request = request.into_inner(); + require_dispatch_key(ctx.key(), request.dispatch_uid)?; + let now = journal_now(&ctx, "compensation_attempt_started_at").await?; + let repository = self.repository.clone(); + let fence = compensation_attempt_fence(&request); + let started = ctx + .run(|| async move { + repository + .start_compensation_attempt(ExecutionScope::ControlPlane, fence, now) + .await + .and_then(started_record) + .map(Json::from) + .map_err(execution_error_to_handler_error) + }) + .name("start_compensation_attempt") + .await? + .into_inner(); + let Some(started) = started else { + return Ok(()); + }; + validate_authoritative_attempt(&request, &started)?; + let exit = active::execute_compensation_attempt(self, &ctx, &request, &started).await?; + let progress_at = journal_now(&ctx, "compensation_attempt_progress_at").await?; + let repository = self.repository.clone(); + let fence = compensation_attempt_fence(&request); + ctx.run(|| async move { + repository + .record_compensation_attempt_progress( + ExecutionScope::ControlPlane, + fence, + progress_at, + ) + .await + .and_then(active::write_applied) + .map_err(execution_error_to_handler_error) + }) + .name("record_compensation_attempt_progress") + .await?; + settle_active_exit(self, &ctx, &request, &started, exit).await?; + kick_dispatcher(&ctx, request.dispatch_uid, "run").await + } + + #[tracing::instrument(skip(self, ctx, request))] + // SAFETY: only exact durable watchdog delivery invokes this shared handler; + // the repository revalidates dispatch, logical, attempt, and trigger fences. + async fn watchdog( + &self, + ctx: SharedWorkflowContext<'_>, + request: Json, + ) -> Result, HandlerError> { + crate::ctx::adopt_incoming_trace_parent(&ctx); + annotate_restate_handler_span("ExecutionCompensationAttempt", "watchdog"); + let request = request.into_inner(); + require_dispatch_key(ctx.key(), request.dispatch_uid)?; + let release_request = ExecutionCompensationAttemptCancelRequest { + cancellation_dispatch_uid: Uuid::new_v5( + &request.dispatch_uid, + b"compensation-watchdog-release-v1", + ), + tenant_id: request.tenant_id, + run_uid: request.run_uid, + compensation_id: request.compensation_id, + controller_generation: request.controller_generation, + attempt_controller_generation: request.controller_generation, + compensation_generation: request.compensation_generation, + compensation_attempt_generation: request.compensation_attempt_generation, + active_dispatch_uid: request.dispatch_uid, + capacity_reservation_uid: request.capacity_reservation_uid, + watchdog_trigger_uid: request.watchdog_trigger_uid, + intent: ExecutionCompensationReleaseIntent::Watchdog, + }; + let outcome = yielding::release_and_settle_compensation_shared( + self, + &ctx, + release_request, + yielding::SharedReleaseSettlement::WatchdogExpired, + ) + .await?; + // TriggerDelivery awaits this handler, so its owning dispatcher observes every outbox row + // committed by watchdog settlement before selecting the next durable timing head. + Ok(Json::from(ExecutionAttemptWatchdogResponse { outcome })) + } + + #[tracing::instrument(skip(self, ctx, request))] + // SAFETY: exact terminal-fence cancellation delivery is validated against the + // immutable dispatch before any sandbox or capacity ownership is released. + async fn cancel( + &self, + ctx: SharedWorkflowContext<'_>, + request: Json, + ) -> Result<(), HandlerError> { + crate::ctx::adopt_incoming_trace_parent(&ctx); + annotate_restate_handler_span("ExecutionCompensationAttempt", "cancel"); + let request = request.into_inner(); + require_dispatch_key(ctx.key(), request.active_dispatch_uid)?; + let outcome = yielding::cancel_compensation_attempt(self, &ctx, request.clone()).await?; + if outcome == ExecutionAttemptWatchdogResponseOutcome::RetryDelivery { + return Err(anyhow::anyhow!( + "compensation cancellation could not prove exact resource release" + ) + .into()); + } + kick_dispatcher_shared(&ctx, request.cancellation_dispatch_uid, "cancel").await + } +} + +async fn settle_active_exit( + workflow: &ExecutionCompensationAttemptImpl, + ctx: &WorkflowContext<'_>, + request: &ExecutionCompensationAttemptRequest, + started: &CompensationAttemptRecord, + exit: active::ActiveCompensationAttemptExit, +) -> Result<(), HandlerError> { + match exit { + active::ActiveCompensationAttemptExit::Outcome(outcome) => { + let intent = if matches!( + outcome, + moa_execution::state::ExecutionCompensationOutcome::Failed { + retryable: true, + .. + } + ) { + ExecutionCompensationReleaseIntent::Retry + } else { + ExecutionCompensationReleaseIntent::Outcome + }; + let Some((release_request, receipt)) = + yielding::release_compensation_hands_workflow(workflow, ctx, request, intent) + .await? + else { + return Ok(()); + }; + let now = journal_now(ctx, "compensation_attempt_settled_at").await?; + let repository = workflow.repository.clone(); + ctx.run(|| async move { + repository + .settle_released_compensation_attempt( + &release_request, + outcome, + now, + Some(receipt), + ) + .await + .and_then(active::write_applied) + .map_err(execution_error_to_handler_error) + }) + .name("settle_compensation_attempt") + .await?; + Ok(()) + } + active::ActiveCompensationAttemptExit::ReviewPending(review) => { + yielding::park_compensation_review(workflow, ctx, request, started, review).await + } + active::ActiveCompensationAttemptExit::ExternalJob(external_job_uid) => { + external::yield_external_job(workflow, ctx, request, external_job_uid).await + } + } +} + +fn compensation_attempt_fence( + request: &ExecutionCompensationAttemptRequest, +) -> CompensationAttemptFence { + CompensationAttemptFence { + run_uid: request.run_uid, + compensation_id: request.compensation_id, + controller_generation: request.controller_generation, + compensation_generation: request.compensation_generation, + attempt_generation: request.compensation_attempt_generation, + dispatch_uid: request.dispatch_uid, + } +} + +fn started_record( + outcome: CompensationAttemptWriteOutcome, +) -> Result, moa_execution::Error> { + match outcome { + CompensationAttemptWriteOutcome::Applied(record) + | CompensationAttemptWriteOutcome::Replayed(record) => Ok(Some(record)), + CompensationAttemptWriteOutcome::NotFound | CompensationAttemptWriteOutcome::Conflict => { + Ok(None) + } + } +} + +fn validate_authoritative_attempt( + request: &ExecutionCompensationAttemptRequest, + started: &CompensationAttemptRecord, +) -> Result<(), HandlerError> { + if started.run.tenant_id != request.tenant_id + || started.registration.compensation_id != request.compensation_id + || started.registration.generation != request.compensation_generation + || started.attempt_generation != request.compensation_attempt_generation + || started.attempt_deadline_at != Some(request.attempt_deadline_at) + { + return Err( + TerminalError::new("compensation dispatch drifted from authoritative state").into(), + ); + } + Ok(()) +} + +async fn journal_now( + ctx: &WorkflowContext<'_>, + name: &'static str, +) -> Result, HandlerError> { + Ok(ctx + .run(|| async { Ok::<_, HandlerError>(Json::from(Utc::now())) }) + .name(name) + .await? + .into_inner()) +} + +async fn journal_now_shared( + ctx: &SharedWorkflowContext<'_>, + name: &'static str, +) -> Result, HandlerError> { + Ok(ctx + .run(|| async { Ok::<_, HandlerError>(Json::from(Utc::now())) }) + .name(name) + .await? + .into_inner()) +} + +fn require_dispatch_key(key: &str, dispatch_uid: Uuid) -> Result<(), HandlerError> { + if key == dispatch_uid.to_string() { + Ok(()) + } else { + Err(TerminalError::new_with_code(404, "compensation attempt dispatch mismatch").into()) + } +} + +async fn kick_dispatcher( + ctx: &WorkflowContext<'_>, + dispatch_uid: Uuid, + boundary: &'static str, +) -> Result<(), HandlerError> { + let handle = replay_safe_request( + ctx.service_client::() + .dispatch(Json::from(DispatchExecutionsRequest::default())) + .idempotency_key(format!( + "compensation-attempt-dispatch:{dispatch_uid}:{boundary}" + )), + ) + .send(); + let _invocation_id = handle.invocation_id().await?; + Ok(()) +} + +async fn kick_dispatcher_shared( + ctx: &SharedWorkflowContext<'_>, + dispatch_uid: Uuid, + boundary: &'static str, +) -> Result<(), HandlerError> { + let handle = replay_safe_request( + ctx.service_client::() + .dispatch(Json::from(DispatchExecutionsRequest::default())) + .idempotency_key(format!( + "compensation-attempt-dispatch:{dispatch_uid}:{boundary}" + )), + ) + .send(); + let _invocation_id = handle.invocation_id().await?; + Ok(()) +} diff --git a/crates/moa-orchestrator/src/workflows/execution_compensation_attempt/active.rs b/crates/moa-orchestrator/src/workflows/execution_compensation_attempt/active.rs new file mode 100644 index 000000000..a75b0c51c --- /dev/null +++ b/crates/moa-orchestrator/src/workflows/execution_compensation_attempt/active.rs @@ -0,0 +1,427 @@ +//! Governed execution of one bounded compensation effect. + +use std::collections::BTreeSet; + +use moa_artifacts::execution_plan::ExecutionUsage; +use moa_core::{ + traits::SessionStore as _, + types::{ + action_policy::{ActionClass, CapabilityProvenance}, + completion::{ToolCallContent, ToolInvocation}, + identifiers::ToolCallId, + resource::ResourceBudget, + tools::IdempotencyClass, + }, +}; +use moa_execution::{ + capability::{CapabilitySource, ExecutionCapability}, + repository::{ + ExecutionScope, + compensation::{CompensationAttemptRecord, CompensationAttemptWriteOutcome}, + }, + schema::validate_instance, + state::{ExecutionCompensationOutcome, LogicalTaskKind}, + wire::{ExecutionCompensationAttemptRequest, ExecutionToolDispatchRejection}, +}; +use restate_sdk::prelude::*; +use serde::Serialize; +use serde_json::Value; +use uuid::Uuid; + +use crate::{ + tool_invocation::governed::{ + GovernedInvocationDisposition, GovernedInvocationOrigin, GovernedInvocationOutcome, + GovernedInvocationRequest, GovernedReviewPending, invoke_governed_tool, + }, + workflows::{ + errors::{execution_error_to_handler_error, moa_error_to_handler_error}, + execution_compensation_attempt::ExecutionCompensationAttemptImpl, + }, +}; + +/// Complete set of boundaries at which a compensation workflow must return. +#[derive(Clone, Debug, PartialEq)] +pub(super) enum ActiveCompensationAttemptExit { + /// The exact compensator produced a completed, failed, or ambiguous outcome. + Outcome(ExecutionCompensationOutcome), + /// Action policy persisted a review; resolution proceeds through storage. + ReviewPending(GovernedReviewPending), + /// The provider accepted durable asynchronous work owned by this attempt. + ExternalJob(Uuid), +} + +/// Executes the compiler-pinned compensator at most once. +pub(super) async fn execute_compensation_attempt( + workflow: &ExecutionCompensationAttemptImpl, + ctx: &WorkflowContext<'_>, + request: &ExecutionCompensationAttemptRequest, + started: &CompensationAttemptRecord, +) -> Result { + let scope = execution_scope(started); + let repository = workflow.repository.clone(); + let run_uid = started.run.run_uid; + let forward_task_id = started.registration.forward_task_id; + let forward_task = ctx + .run(|| async move { + repository + .load_task(scope, run_uid, forward_task_id) + .await + .map(Json::from) + .map_err(execution_error_to_handler_error) + }) + .name("load_compensation_forward_task") + .await? + .into_inner() + .ok_or_else(|| TerminalError::new("compensation forward task was not found"))?; + let capability = match validate_runtime_contract(started, &forward_task) { + Ok(capability) => capability, + Err(message) => { + return Ok(ActiveCompensationAttemptExit::Outcome(failed( + message, + false, + cumulative_usage(started), + ))); + } + }; + if let Err(error) = validate_instance( + &capability.input_schema, + &started.registration.mapped_input, + "execution_compensation.input", + ) { + return Ok(ActiveCompensationAttemptExit::Outcome(failed( + format!("compensator mapped input failed pinned schema: {error}"), + false, + cumulative_usage(started), + ))); + } + let session = load_session(workflow, ctx, started).await?; + if session.tenant_id != started.run.tenant_id { + return Err( + TerminalError::new("authoritative compensation session tenant mismatch").into(), + ); + } + let tool_name = match capability.source.model_visible_tool_name() { + Some(name) => name.to_string(), + None => { + return Ok(ActiveCompensationAttemptExit::Outcome(failed( + "compensator has no governed tool owner".to_string(), + false, + cumulative_usage(started), + ))); + } + }; + let tool_id = + stable_compensator_tool_id(request.compensation_id, request.compensation_generation); + let tool_call = ToolCallContent { + invocation: ToolInvocation { + id: Some(tool_id.to_string()), + name: tool_name.clone(), + input: started.registration.mapped_input.clone(), + }, + provider_metadata: None, + }; + let allowed_tools = BTreeSet::from([tool_name]); + let provenance = CapabilityProvenance { + kind: Some(capability_source_kind(&capability.source).to_string()), + id: Some(format!( + "{}@{}", + capability.reference.name, capability.reference.version + )), + step_id: Some(format!( + "compensation:{}", + started.registration.forward_task_id + )), + }; + let outcome = invoke_governed_tool( + ctx, + GovernedInvocationRequest { + session: &session, + identity: &started.run.admitted_identity, + session_id: started.run.session_id, + tool_id, + tool_call: &tool_call, + allowed_tools: &allowed_tools, + expected_tool_contract_revision: Some(&capability.contract_revision), + active_canary: None, + trusted_sandbox_manifest: None, + origin: GovernedInvocationOrigin::ExecutionCompensation { + run_uid: started.run.run_uid, + compensation_id: started.registration.compensation_id.as_uuid(), + generation: started.registration.generation, + attempt_generation: started.attempt_generation, + }, + capability_provenance: Some(&provenance), + capability_policy_context: Some(&capability.policy_context), + resource_budget: ResourceBudget::until(request.attempt_deadline_at), + }, + &workflow.session_limits, + workflow.session_store.clone(), + workflow.channel_adapters.as_ref(), + ) + .await?; + Ok(classify_outcome(started, capability, outcome)) +} + +fn classify_outcome( + started: &CompensationAttemptRecord, + capability: &ExecutionCapability, + outcome: GovernedInvocationOutcome, +) -> ActiveCompensationAttemptExit { + let mut usage = cumulative_usage(started); + match outcome { + GovernedInvocationOutcome::Completed(result) + if result.disposition == GovernedInvocationDisposition::ReviewPending => + { + match result.review { + Some(review) => ActiveCompensationAttemptExit::ReviewPending(review), + None => ActiveCompensationAttemptExit::Outcome( + ExecutionCompensationOutcome::UnknownOutcome { + message: "governed review admission omitted its persisted review reference" + .to_string(), + usage, + }, + ), + } + } + GovernedInvocationOutcome::Completed(result) => { + usage.tool_calls = usage.tool_calls.saturating_add(1); + usage.retrieved_bytes = usage.retrieved_bytes.saturating_add(serialized_len( + &result.output.safe_output.structured_payload(), + )); + if result.output.is_error() { + return ActiveCompensationAttemptExit::Outcome(failed( + result.output.safe_output.to_text(), + true, + usage, + )); + } + let output = result + .output + .safe_output + .structured_payload() + .cloned() + .unwrap_or_else(|| Value::String(result.output.safe_output.to_text())); + if let Err(error) = validate_instance( + &capability.output_schema, + &output, + "execution_compensation.output", + ) { + ActiveCompensationAttemptExit::Outcome( + ExecutionCompensationOutcome::UnknownOutcome { + message: format!( + "compensator returned invalid output after possible commit: {error}" + ), + usage, + }, + ) + } else { + ActiveCompensationAttemptExit::Outcome(ExecutionCompensationOutcome::Completed { + output, + usage, + }) + } + } + GovernedInvocationOutcome::UnknownOutcome { message, .. } => { + ActiveCompensationAttemptExit::Outcome(ExecutionCompensationOutcome::UnknownOutcome { + message, + usage, + }) + } + GovernedInvocationOutcome::ExternalJob { + external_job_uid, .. + } => ActiveCompensationAttemptExit::ExternalJob(external_job_uid), + GovernedInvocationOutcome::NotDispatched { reason, .. } => { + ActiveCompensationAttemptExit::Outcome(failed( + execution_dispatch_rejection_message(reason), + false, + usage, + )) + } + GovernedInvocationOutcome::Delegation { .. } => { + ActiveCompensationAttemptExit::Outcome(failed( + "compensators cannot invoke delegation capabilities".to_string(), + false, + usage, + )) + } + } +} + +fn validate_runtime_contract<'a>( + started: &'a CompensationAttemptRecord, + forward_task: &moa_execution::repository::ExecutionTaskRecord, +) -> Result<&'a ExecutionCapability, String> { + let LogicalTaskKind::Capability { + reference: forward_reference, + } = &forward_task.kind + else { + return Err("registered compensation forward task is not a direct capability".to_string()); + }; + if forward_task.compensation_contract.as_ref() != Some(&started.registration.compensator) { + return Err("registered compensation drifted from the forward task contract".to_string()); + } + let forward = find_catalog_capability(&started.run, forward_reference)?; + if !forward + .rollback + .as_ref() + .is_some_and(|rollback| rollback.matches(&started.registration.compensator)) + { + return Err("pinned forward capability no longer promises the exact rollback".to_string()); + } + let compensator = + find_catalog_capability(&started.run, &started.registration.compensator.compensator)?; + if compensator.action_class == ActionClass::Read { + return Err("compensator catalog entry is read-only".to_string()); + } + if compensator.idempotency_class != IdempotencyClass::Idempotent { + return Err("compensator catalog entry is not idempotent".to_string()); + } + Ok(compensator) +} + +fn find_catalog_capability<'a>( + run: &'a moa_execution::repository::ExecutionRunRecord, + reference: &moa_artifacts::execution_plan::CapabilityReference, +) -> Result<&'a ExecutionCapability, String> { + if !run.authorization.capability_refs.contains(reference) { + return Err("compensator is outside the persisted authorization envelope".to_string()); + } + run.catalog + .capabilities + .iter() + .find(|capability| capability.reference == *reference) + .ok_or_else(|| "compensator is absent from the persisted catalog".to_string()) +} + +async fn load_session( + workflow: &ExecutionCompensationAttemptImpl, + ctx: &WorkflowContext<'_>, + started: &CompensationAttemptRecord, +) -> Result { + let store = workflow.session_store.clone(); + let session_id = started.run.session_id; + Ok(ctx + .run(|| async move { + store + .get_session(session_id) + .await + .map(Json::from) + .map_err(moa_error_to_handler_error) + }) + .name("compensation_attempt_load_session") + .await? + .into_inner()) +} + +pub(super) fn execution_scope(started: &CompensationAttemptRecord) -> ExecutionScope { + started.run.contact_id.map_or( + ExecutionScope::Tenant { + tenant_id: started.run.tenant_id, + }, + |contact_id| ExecutionScope::Contact { + tenant_id: started.run.tenant_id, + contact_id, + }, + ) +} + +pub(super) fn cumulative_usage(started: &CompensationAttemptRecord) -> ExecutionUsage { + started + .registration + .outcome + .as_ref() + .map(ExecutionCompensationOutcome::usage) + .cloned() + .unwrap_or(ExecutionUsage { + cost_microusd: 0, + tokens: 0, + tool_calls: 0, + retrieved_bytes: 0, + }) +} + +pub(super) fn write_applied( + outcome: CompensationAttemptWriteOutcome, +) -> Result<(), moa_execution::Error> { + match outcome { + CompensationAttemptWriteOutcome::Applied(_) + | CompensationAttemptWriteOutcome::Replayed(_) + | CompensationAttemptWriteOutcome::NotFound => Ok(()), + CompensationAttemptWriteOutcome::Conflict => { + Err(moa_execution::Error::InvalidRepositoryData { + message: "compensation attempt transition lost its exact generation fence" + .to_string(), + }) + } + } +} + +fn failed(message: String, retryable: bool, usage: ExecutionUsage) -> ExecutionCompensationOutcome { + ExecutionCompensationOutcome::Failed { + message, + retryable, + usage, + } +} + +fn stable_compensator_tool_id( + compensation_id: moa_execution::state::CompensationId, + logical_generation: u64, +) -> ToolCallId { + ToolCallId(Uuid::new_v5( + &compensation_id.as_uuid(), + format!("generation:{logical_generation}").as_bytes(), + )) +} + +const fn capability_source_kind(source: &CapabilitySource) -> &'static str { + match source { + CapabilitySource::BuiltInTool { .. } => "built_in_tool", + CapabilitySource::HandTool { .. } => "hand_tool", + CapabilitySource::McpTool { .. } => "mcp_tool", + CapabilitySource::ActionArtifact { .. } => "action_artifact", + CapabilitySource::ConnectorAction { .. } => "connector_action", + CapabilitySource::InstalledConnectorAction { .. } => "installed_connector_action", + CapabilitySource::SkillAction { .. } => "skill_action", + CapabilitySource::SkillCode { .. } => "skill_code", + CapabilitySource::Memory { .. } => "memory", + CapabilitySource::Knowledge { .. } => "knowledge", + CapabilitySource::Model => "model", + } +} + +fn execution_dispatch_rejection_message(reason: ExecutionToolDispatchRejection) -> String { + let label = match reason { + ExecutionToolDispatchRejection::OriginNotFound => "origin_not_found", + ExecutionToolDispatchRejection::StaleGeneration => "stale_generation", + ExecutionToolDispatchRejection::OperationNotRunning => "operation_not_running", + ExecutionToolDispatchRejection::RunNotDispatchable => "run_not_dispatchable", + }; + format!("execution effect was not dispatched: {label}") +} + +fn serialized_len(value: &T) -> u64 { + serde_json::to_vec(value) + .map(|bytes| bytes.len() as u64) + .unwrap_or_default() +} + +#[cfg(test)] +mod tests { + use super::*; + + // Pins: retries of one logical effect reuse the tool identity, while a new + // logical generation cannot alias the prior provider idempotency key. + #[test] + fn compensator_tool_identity_is_fenced_by_logical_generation_offline() { + let compensation_id = moa_execution::state::CompensationId::from_uuid(Uuid::from_u128(7)); + assert_eq!( + stable_compensator_tool_id(compensation_id, 7), + stable_compensator_tool_id(compensation_id, 7) + ); + assert_ne!( + stable_compensator_tool_id(compensation_id, 7), + stable_compensator_tool_id(compensation_id, 8) + ); + } +} diff --git a/crates/moa-orchestrator/src/workflows/execution_compensation_attempt/external.rs b/crates/moa-orchestrator/src/workflows/execution_compensation_attempt/external.rs new file mode 100644 index 000000000..647fbf50c --- /dev/null +++ b/crates/moa-orchestrator/src/workflows/execution_compensation_attempt/external.rs @@ -0,0 +1,136 @@ +//! Durable ownership handoff for asynchronous compensation provider jobs. + +use moa_execution::{ + repository::compensation::CompensationAttemptExternalOutcome, + wire::{ExecutionCompensationAttemptRequest, ExecutionCompensationReleaseIntent}, +}; +use restate_sdk::prelude::*; +use uuid::Uuid; + +use crate::services::tool_executor::ToolExecutorClient; +use crate::workflows::{ + errors::execution_error_to_handler_error, + execution_compensation_attempt::{ + ExecutionCompensationAttemptImpl, journal_now, + yielding::{release_hands_request, release_request}, + }, +}; + +/// Parks one compensation on its already-durable external job after verified compute release. +pub(super) async fn yield_external_job( + workflow: &ExecutionCompensationAttemptImpl, + ctx: &WorkflowContext<'_>, + request: &ExecutionCompensationAttemptRequest, + external_job_uid: Uuid, +) -> Result<(), HandlerError> { + let release_request = release_request(request, ExecutionCompensationReleaseIntent::ExternalJob); + let claimed_at = journal_now(ctx, "compensation_external_release_claimed_at").await?; + let repository = workflow.repository.clone(); + let request_for_claim = release_request.clone(); + let started = ctx + .run(|| async move { + repository + .begin_compensation_external_release( + &request_for_claim, + external_job_uid, + claimed_at, + ) + .await + .and_then(external_release_claimed) + .map(Json::from) + .map_err(execution_error_to_handler_error) + }) + .name("begin_compensation_external_release") + .await? + .into_inner(); + let Some(started) = started else { + return Ok(()); + }; + let receipt = crate::restate_identity::replay_safe_request( + ctx.service_client::() + .checkpoint_and_release_execution_hands(Json::from(release_hands_request( + &started, claimed_at, + ))), + ) + .call() + .await? + .into_inner(); + let yielded_at = journal_now(ctx, "compensation_external_job_yielded_at").await?; + let repository = workflow.repository.clone(); + ctx.run(|| async move { + repository + .yield_released_compensation_attempt_to_external_job( + &release_request, + external_job_uid, + Some(receipt), + yielded_at, + ) + .await + .and_then(external_yielded) + .map_err(execution_error_to_handler_error) + }) + .name("yield_compensation_attempt_to_external_job") + .await?; + Ok(()) +} + +fn external_release_claimed( + outcome: CompensationAttemptExternalOutcome, +) -> Result< + Option, + moa_execution::Error, +> { + match outcome { + CompensationAttemptExternalOutcome::Applied { attempt, .. } + | CompensationAttemptExternalOutcome::Replayed { attempt, .. } => Ok(Some(attempt)), + CompensationAttemptExternalOutcome::NotFound + | CompensationAttemptExternalOutcome::Stale => Ok(None), + CompensationAttemptExternalOutcome::InvalidState => { + Err(moa_execution::Error::InvalidRepositoryData { + message: "active compensation external-job release was rejected".to_string(), + }) + } + } +} + +fn external_yielded( + outcome: CompensationAttemptExternalOutcome, +) -> Result<(), moa_execution::Error> { + match outcome { + CompensationAttemptExternalOutcome::Applied { .. } + | CompensationAttemptExternalOutcome::Replayed { .. } + | CompensationAttemptExternalOutcome::NotFound + | CompensationAttemptExternalOutcome::Stale => Ok(()), + CompensationAttemptExternalOutcome::InvalidState => { + Err(moa_execution::Error::InvalidRepositoryData { + message: "active compensation external-job yield was rejected".to_string(), + }) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // Pins: a live attempt in any state other than exact Cancelling cannot silently + // release capacity or park on provider work. + #[test] + fn compensation_external_job_yield_rejects_invalid_state_offline() { + assert!(external_yielded(CompensationAttemptExternalOutcome::InvalidState).is_err()); + } + + // Pins: stale deliveries are replay-safe no-ops, while an invalid live transition + // cannot silently skip the provider-ownership release boundary. + #[test] + fn compensation_external_job_claim_is_fenced_offline() { + assert!( + external_release_claimed(CompensationAttemptExternalOutcome::Stale) + .expect("stale delivery should be harmless") + .is_none() + ); + assert!( + external_release_claimed(CompensationAttemptExternalOutcome::InvalidState).is_err() + ); + } +} diff --git a/crates/moa-orchestrator/src/workflows/execution_compensation_attempt/yielding.rs b/crates/moa-orchestrator/src/workflows/execution_compensation_attempt/yielding.rs new file mode 100644 index 000000000..7648413bb --- /dev/null +++ b/crates/moa-orchestrator/src/workflows/execution_compensation_attempt/yielding.rs @@ -0,0 +1,441 @@ +//! Storage-only compensation review and verified cancellation boundaries. + +use moa_core::types::action_policy::{ActionReviewOwner, ExecutionCompensationOrigin}; +use moa_core::types::{ + identifiers::ExecutionCompensationScopeId, sandbox_workspace::ExecutionHandReleaseOwner, +}; +use moa_execution::{ + repository::compensation::{ + CompensationAttemptRecord, CompensationAttemptReleaseClaimOutcome, + CompensationAttemptWriteOutcome, + }, + state::ExecutionCompensationOutcome, + wire::{ + ExecutionAttemptWatchdogResponseOutcome, ExecutionCompensationAttemptCancelRequest, + ExecutionCompensationAttemptRequest, ExecutionCompensationReleaseIntent, + }, +}; +use restate_sdk::prelude::*; +use serde::{Deserialize, Serialize}; + +use crate::{ + services::action_reviews::{AcknowledgeExecutionActionReviewRequest, ActionReviewsClient}, + services::tool_executor::{CheckpointAndReleaseExecutionHandsRequest, ToolExecutorClient}, + tool_invocation::governed::GovernedReviewPending, + workflows::{ + errors::execution_error_to_handler_error, + execution_compensation_attempt::{ExecutionCompensationAttemptImpl, journal_now}, + }, +}; + +/// Persists an exact review wait, then makes its decision claimable. +pub(super) async fn park_compensation_review( + workflow: &ExecutionCompensationAttemptImpl, + ctx: &WorkflowContext<'_>, + request: &ExecutionCompensationAttemptRequest, + started: &CompensationAttemptRecord, + review: GovernedReviewPending, +) -> Result<(), HandlerError> { + let Some((release_request, receipt)) = release_compensation_hands_workflow( + workflow, + ctx, + request, + ExecutionCompensationReleaseIntent::Review, + ) + .await? + else { + return Ok(()); + }; + let now = journal_now(ctx, "compensation_review_parked_at").await?; + let repository = workflow.repository.clone(); + let parked = ctx + .run(|| async move { + repository + .park_released_compensation_review( + &release_request, + review.review_uid, + review.expires_at, + now, + Some(receipt), + ) + .await + .and_then(review_parked) + .map(Json::from) + .map_err(execution_error_to_handler_error) + }) + .name("park_compensation_review") + .await? + .into_inner(); + if !parked { + return Ok(()); + } + let owner = ActionReviewOwner::ExecutionCompensation { + session_id: started.run.session_id, + origin: ExecutionCompensationOrigin { + run_uid: started.run.run_uid, + compensation_id: started.registration.compensation_id.as_uuid(), + generation: started.registration.generation, + attempt_generation: started.attempt_generation, + }, + }; + crate::restate_identity::replay_safe_request( + ctx.service_client::() + .acknowledge_execution_owner_review(Json::from( + AcknowledgeExecutionActionReviewRequest { + tenant_id: started.run.tenant_id, + review_id: review.review_uid, + owner, + }, + )), + ) + .call() + .await?; + Ok(()) +} + +/// Claims Cancelling and obtains provider-verified release proof before settlement. +pub(super) async fn release_compensation_hands_workflow( + workflow: &ExecutionCompensationAttemptImpl, + ctx: &WorkflowContext<'_>, + request: &ExecutionCompensationAttemptRequest, + intent: ExecutionCompensationReleaseIntent, +) -> Result< + Option<( + ExecutionCompensationAttemptCancelRequest, + moa_core::types::sandbox_workspace::ExecutionHandReleaseReceipt, + )>, + HandlerError, +> { + let claimed_at = journal_now(ctx, "compensation_release_claimed_at").await?; + let release_request = release_request(request, intent); + let repository = workflow.repository.clone(); + let claim_request = release_request.clone(); + let claim = ctx + .run(|| async move { + repository + .begin_compensation_attempt_release(&claim_request, claimed_at) + .await + .and_then(release_claimed) + .map(Json::from) + .map_err(execution_error_to_handler_error) + }) + .name("begin_compensation_attempt_release") + .await? + .into_inner(); + let started = match claim { + SharedReleaseClaim::Claimed(started) => started, + SharedReleaseClaim::ReplayedOrStale => return Ok(None), + SharedReleaseClaim::RetryDelivery => { + return Err( + anyhow::anyhow!("compensation release claim could not safely settle").into(), + ); + } + }; + let receipt = crate::restate_identity::replay_safe_request( + ctx.service_client::() + .checkpoint_and_release_execution_hands(Json::from(release_hands_request( + &started, claimed_at, + ))), + ) + .call() + .await? + .into_inner(); + Ok(Some((release_request, receipt))) +} + +fn review_parked(outcome: CompensationAttemptWriteOutcome) -> Result { + match outcome { + CompensationAttemptWriteOutcome::Applied(_) + | CompensationAttemptWriteOutcome::Replayed(_) => Ok(true), + CompensationAttemptWriteOutcome::NotFound => Ok(false), + CompensationAttemptWriteOutcome::Conflict => { + Err(moa_execution::Error::InvalidRepositoryData { + message: "compensation review park lost its exact attempt fence".to_string(), + }) + } + } +} + +/// Cancels one exact active compensation without releasing ownership prematurely. +pub(super) async fn cancel_compensation_attempt( + workflow: &ExecutionCompensationAttemptImpl, + ctx: &SharedWorkflowContext<'_>, + request: ExecutionCompensationAttemptCancelRequest, +) -> Result { + let settlement = shared_release_settlement(request.intent).ok_or_else(|| { + TerminalError::new("compensation cancel delivery carried an internal release intent") + })?; + release_and_settle_compensation_shared(workflow, ctx, request, settlement).await +} + +/// Outcome applied after a shared exact release claim obtains teardown proof. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum SharedReleaseSettlement { + /// Terminal-fence cancellation. + Cancelled, + /// Due watchdog retry of an idempotent compensator. + WatchdogExpired, + /// Authorized pause that returns the exact compensation to idle storage state. + Paused, +} + +const fn shared_release_settlement( + intent: ExecutionCompensationReleaseIntent, +) -> Option { + match intent { + ExecutionCompensationReleaseIntent::Pause => Some(SharedReleaseSettlement::Paused), + ExecutionCompensationReleaseIntent::Deadline + | ExecutionCompensationReleaseIntent::RunTerminal => { + Some(SharedReleaseSettlement::Cancelled) + } + ExecutionCompensationReleaseIntent::Watchdog => { + Some(SharedReleaseSettlement::WatchdogExpired) + } + ExecutionCompensationReleaseIntent::Outcome + | ExecutionCompensationReleaseIntent::Retry + | ExecutionCompensationReleaseIntent::Review + | ExecutionCompensationReleaseIntent::ExternalJob => None, + } +} + +/// Releases sandbox ownership before a shared cancellation or watchdog settles. +pub(super) async fn release_and_settle_compensation_shared( + workflow: &ExecutionCompensationAttemptImpl, + ctx: &SharedWorkflowContext<'_>, + request: ExecutionCompensationAttemptCancelRequest, + settlement: SharedReleaseSettlement, +) -> Result { + let claimed_at = super::journal_now_shared(ctx, "compensation_release_claimed_at").await?; + let repository = workflow.repository.clone(); + let claim_request = request.clone(); + let claim = ctx + .run(|| async move { + repository + .begin_compensation_attempt_release(&claim_request, claimed_at) + .await + .and_then(release_claimed) + .map(Json::from) + .map_err(execution_error_to_handler_error) + }) + .name("begin_compensation_attempt_release") + .await? + .into_inner(); + let started = match claim { + SharedReleaseClaim::Claimed(started) => started, + SharedReleaseClaim::ReplayedOrStale => { + return Ok(ExecutionAttemptWatchdogResponseOutcome::ReplayedOrStale); + } + SharedReleaseClaim::RetryDelivery => { + return Ok(ExecutionAttemptWatchdogResponseOutcome::RetryDelivery); + } + }; + let receipt = crate::restate_identity::replay_safe_request( + ctx.service_client::() + .checkpoint_and_release_execution_hands(Json::from(release_hands_request( + &started, claimed_at, + ))), + ) + .call() + .await? + .into_inner(); + let settled_at = super::journal_now_shared(ctx, "compensation_cancel_settled_at").await?; + let outcome = match settlement { + SharedReleaseSettlement::Cancelled => ExecutionCompensationOutcome::Failed { + message: format!( + "bounded compensation attempt cancelled: {:?}", + request.intent + ), + retryable: false, + usage: super::active::cumulative_usage(&started), + }, + SharedReleaseSettlement::WatchdogExpired => ExecutionCompensationOutcome::Failed { + message: "compensation attempt watchdog expired".to_string(), + retryable: true, + usage: super::active::cumulative_usage(&started), + }, + SharedReleaseSettlement::Paused => { + let repository = workflow.repository.clone(); + let outcome = ctx + .run(|| async move { + repository + .yield_released_compensation_attempt(&request, settled_at, Some(receipt)) + .await + .map(shared_write_outcome) + .map(Json::from) + .map_err(execution_error_to_handler_error) + }) + .name("yield_released_compensation_attempt") + .await? + .into_inner(); + return Ok(outcome); + } + }; + let repository = workflow.repository.clone(); + let outcome = ctx + .run(|| async move { + repository + .settle_released_compensation_attempt(&request, outcome, settled_at, Some(receipt)) + .await + .map(shared_write_outcome) + .map(Json::from) + .map_err(execution_error_to_handler_error) + }) + .name("settle_released_compensation_attempt") + .await? + .into_inner(); + Ok(outcome) +} + +pub(super) fn release_request( + request: &ExecutionCompensationAttemptRequest, + intent: ExecutionCompensationReleaseIntent, +) -> ExecutionCompensationAttemptCancelRequest { + ExecutionCompensationAttemptCancelRequest { + cancellation_dispatch_uid: uuid::Uuid::new_v5( + &request.dispatch_uid, + b"compensation-attempt-release-v1", + ), + tenant_id: request.tenant_id, + run_uid: request.run_uid, + compensation_id: request.compensation_id, + controller_generation: request.controller_generation, + attempt_controller_generation: request.controller_generation, + compensation_generation: request.compensation_generation, + compensation_attempt_generation: request.compensation_attempt_generation, + active_dispatch_uid: request.dispatch_uid, + capacity_reservation_uid: request.capacity_reservation_uid, + watchdog_trigger_uid: request.watchdog_trigger_uid, + intent, + } +} + +pub(super) fn release_hands_request( + started: &CompensationAttemptRecord, + claimed_at: chrono::DateTime, +) -> CheckpointAndReleaseExecutionHandsRequest { + CheckpointAndReleaseExecutionHandsRequest { + tenant_id: started.run.tenant_id, + session_id: started.run.session_id, + run_uid: started.run.run_uid, + owner: ExecutionHandReleaseOwner::Compensation { + compensation_id: ExecutionCompensationScopeId( + started.registration.compensation_id.as_uuid(), + ), + logical_generation: started.registration.generation, + }, + attempt_generation: started.attempt_generation, + release_deadline_at: claimed_at + chrono::Duration::minutes(5), + } +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "snake_case", deny_unknown_fields)] +enum SharedReleaseClaim { + Claimed(Box), + ReplayedOrStale, + RetryDelivery, +} + +fn release_claimed( + outcome: CompensationAttemptReleaseClaimOutcome, +) -> Result { + match outcome { + CompensationAttemptReleaseClaimOutcome::Applied(record) + | CompensationAttemptReleaseClaimOutcome::Replayed(record) => { + Ok(SharedReleaseClaim::Claimed(Box::new(record))) + } + CompensationAttemptReleaseClaimOutcome::NotFound + | CompensationAttemptReleaseClaimOutcome::Stale => Ok(SharedReleaseClaim::ReplayedOrStale), + CompensationAttemptReleaseClaimOutcome::InvalidState => { + Ok(SharedReleaseClaim::RetryDelivery) + } + } +} + +fn shared_write_outcome( + outcome: CompensationAttemptWriteOutcome, +) -> ExecutionAttemptWatchdogResponseOutcome { + match outcome { + CompensationAttemptWriteOutcome::Applied(_) => { + ExecutionAttemptWatchdogResponseOutcome::Settled + } + CompensationAttemptWriteOutcome::Replayed(_) + | CompensationAttemptWriteOutcome::NotFound => { + ExecutionAttemptWatchdogResponseOutcome::ReplayedOrStale + } + CompensationAttemptWriteOutcome::Conflict => { + ExecutionAttemptWatchdogResponseOutcome::RetryDelivery + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // Pins: a stale or absent cancellation delivery is an idempotent no-op, but + // an invalid live state cannot silently release capacity or its watchdog. + #[test] + fn compensation_release_claim_fails_closed_for_invalid_state_offline() { + assert!(matches!( + release_claimed(CompensationAttemptReleaseClaimOutcome::NotFound) + .expect("not-found delivery should be harmless"), + SharedReleaseClaim::ReplayedOrStale + )); + assert!(matches!( + release_claimed(CompensationAttemptReleaseClaimOutcome::Stale) + .expect("stale delivery should be harmless"), + SharedReleaseClaim::ReplayedOrStale + )); + assert!(matches!( + release_claimed(CompensationAttemptReleaseClaimOutcome::InvalidState) + .expect("invalid live state must keep watchdog delivery retryable"), + SharedReleaseClaim::RetryDelivery + )); + } + + // Pins: shared cancellation delivery preserves the exact persisted release + // intent instead of misclassifying pause or watchdog as terminal cancellation. + #[test] + fn compensation_shared_release_intents_map_to_exact_finalizers_offline() { + assert_eq!( + shared_release_settlement(ExecutionCompensationReleaseIntent::Pause), + Some(SharedReleaseSettlement::Paused) + ); + assert_eq!( + shared_release_settlement(ExecutionCompensationReleaseIntent::Watchdog), + Some(SharedReleaseSettlement::WatchdogExpired) + ); + for intent in [ + ExecutionCompensationReleaseIntent::Deadline, + ExecutionCompensationReleaseIntent::RunTerminal, + ] { + assert_eq!( + shared_release_settlement(intent), + Some(SharedReleaseSettlement::Cancelled) + ); + } + for intent in [ + ExecutionCompensationReleaseIntent::Outcome, + ExecutionCompensationReleaseIntent::Retry, + ExecutionCompensationReleaseIntent::Review, + ExecutionCompensationReleaseIntent::ExternalJob, + ] { + assert_eq!(shared_release_settlement(intent), None); + } + } + + #[test] + fn compensation_watchdog_acknowledges_only_durable_settlement_offline() { + // Pins: trigger delivery may settle on an applied or stale finalizer, but + // a live exact conflict remains retryable and cannot strand capacity. + assert_eq!( + shared_write_outcome(CompensationAttemptWriteOutcome::Conflict), + ExecutionAttemptWatchdogResponseOutcome::RetryDelivery + ); + assert_eq!( + shared_write_outcome(CompensationAttemptWriteOutcome::NotFound), + ExecutionAttemptWatchdogResponseOutcome::ReplayedOrStale + ); + } +} diff --git a/crates/moa-orchestrator/src/workflows/execution_run.rs b/crates/moa-orchestrator/src/workflows/execution_run.rs deleted file mode 100644 index 0b8693583..000000000 --- a/crates/moa-orchestrator/src/workflows/execution_run.rs +++ /dev/null @@ -1,3428 +0,0 @@ -//! Durable keyed workflow that advances one persisted dynamic execution run. - -use async_trait::async_trait; -use moa_artifacts::execution_plan::{ExecutionCancelPolicy, ExecutionOperation}; -use moa_brain::execution_planning::{ - AmendmentPlanningEvidence, ExecutionAmendmentPlanningRequest, - ExecutionAmendmentPlanningResultKind, plan_amendment, -}; -use moa_core::{ - events::ExecutionInputRequired, - traits::LLMProvider, - types::{ - completion::{CompletionRequest, CompletionStream, SharedCompletionRequest}, - execution_planning::{ - EXECUTION_REPORT_MAX_BYTES, ExecutionPlanningAuditEnvelope, - ExecutionPlanningAuditPayload, - }, - identifiers::ModelId, - model::ModelCapabilities, - }, -}; -use moa_execution::{ - completion::{ - CompletionEvaluation, CompletionEvaluationRequest, CompletionStatus, evaluate_completion, - execution_terminal_reason, terminal_cause, terminal_evidence_from_evaluation, - terminal_projection_from_evaluation, terminal_projection_matches_completion, - }, - interpreter::{ScheduleRequest, ready_empty_map_nodes, schedule}, - replan::{ReplanExhaustion, replan_exhaustion_reason, replan_stop_gaps, replan_stop_status}, - repository::{ - BeginCompensationOutcome, CompensationClaimOutcome, CompensationFinalizationOutcome, - CompileAuditWriteOutcome, ExecutionNodeMaterialization, ExecutionRepository, - ExecutionRunRecord, ExecutionScope, FencedTerminalFinalizationOutcome, FinalizationOutcome, - MaterializationOutcome, PlannerCallAuditWriteOutcome, RunFinalizationRequest, - TaskOutcomeWrite, TerminalFenceOutcome, TransitionOutcome, WakeAckOutcome, - }, - state::{ - CompensationStatus, ExecutionRunStatus, ExecutionTaskId, ExecutionTaskStatus, - ExecutionTerminalCause, PendingExecutionTerminal, ScheduleDecision, TerminalProjection, - WaitingReason, cancelled_task_outcome, run_status_from_terminal_projection, - }, - wire::{ - ExecutionAmendmentRequest, ExecutionCompensationWorkflowRequest, ExecutionMutationResponse, - ExecutionPlanningContextSnapshot, ExecutionRunRequest, ExecutionRunWakeRequest, - ExecutionRunWorkflowRequest, ExecutionTaskWorkflowRequest, ExecutionTerminalDelivery, - execution_progress_from_run, - }, -}; -use moa_observability::{ - restate_observability::annotate_restate_handler_span, - runtime_metrics::{ - record_execution_dispatch_batch_size, record_execution_owned_in_flight_tasks, - }, -}; -use restate_sdk::context::macro_support::SealedDurableFuture; -use restate_sdk::prelude::*; -use serde::{Deserialize, Serialize}; -use serde_json::{Value, json}; -use std::collections::{BTreeMap, BTreeSet}; -use std::sync::atomic::{AtomicUsize, Ordering}; -use tracing_opentelemetry::OpenTelemetrySpanExt; - -use crate::objects::session::SessionClient; -use crate::services::{ - execution::ExecutionClient, - llm_gateway::{ - LLMCompletionAction, LLMCompletionOwner, LLMGatewayClient, - cancel_completion_owner_from_workflow, completion_idempotency_key, - }, -}; -use crate::workflows::execution_compensation::ExecutionCompensationClient; -use crate::workflows::execution_task::ExecutionTaskClient; - -const K_PROCESSED_WAKE_EPOCH: &str = "execution_processed_wake_epoch"; -const K_AWAITED_WAKE_EPOCH: &str = "execution_awaited_wake_epoch"; - -/// Durable workflow surface for one keyed execution run. -#[restate_sdk::workflow] -pub trait ExecutionRun { - /// Drives the run until it is terminal, parking durably between wake epochs. - async fn run(request: Json) -> Result<(), HandlerError>; - - /// Records one persisted scheduling wake and resumes the parked driver when needed. - #[shared] - async fn wake(request: Json) -> Result<(), HandlerError>; -} - -/// PostgreSQL-backed execution-run workflow implementation. -#[derive(Clone)] -pub struct ExecutionRunImpl { - repository: ExecutionRepository, - config: moa_config::ExecutionConfig, - planner_model: ModelId, -} - -impl ExecutionRunImpl { - /// Creates one durable run workflow over the shared execution repository. - #[must_use] - pub fn new( - pool: sqlx::PgPool, - config: moa_config::ExecutionConfig, - planner_model: ModelId, - ) -> Self { - Self { - repository: ExecutionRepository::new(pool), - config, - planner_model, - } - } -} - -impl ExecutionRun for ExecutionRunImpl { - #[tracing::instrument(skip(self, ctx, request))] - // SAFETY: started only by Execution/start after parent-session authorization; all recovery reads use the persisted scope in this keyed request. - async fn run( - &self, - ctx: WorkflowContext<'_>, - request: Json, - ) -> Result<(), HandlerError> { - crate::ctx::adopt_incoming_trace_parent(&ctx); - annotate_restate_handler_span("ExecutionRun", "run"); - let request = request.into_inner(); - annotate_execution_run_span(request.run_uid); - if request.run_uid.to_string() != ctx.key() { - return Err(TerminalError::new_with_code(404, "execution run id mismatch").into()); - } - if request.identity.tenant_id != request.tenant_id { - return Err(TerminalError::new_with_code( - 409, - "execution run identity tenant mismatch", - ) - .into()); - } - ctx.set(K_PROCESSED_WAKE_EPOCH, Json::from(0_u64)); - ctx.set(K_AWAITED_WAKE_EPOCH, Json::from(0_u64)); - let scope = execution_scope(&request); - let mut step_index = 0_u64; - let mut owned_task_calls = Vec::new(); - let mut owned_task_ids = BTreeSet::new(); - loop { - let repository = self.repository.clone(); - let drive_request = request.clone(); - let config = self.config.clone(); - let drive_owned_task_ids = owned_task_ids.clone(); - let step = ctx - .run(|| async move { - drive_once( - repository, - scope, - drive_request, - config, - drive_owned_task_ids, - ) - .await - .map(Json::from) - }) - .name(format!("execution_run_drive_{step_index}")) - .await? - .into_inner(); - step_index = step_index.saturating_add(1); - deliver_session_projection( - &ctx, - self.repository.clone(), - scope, - &request, - matches!(&step, RunDriveStep::Terminal { .. }), - step_index, - ) - .await?; - match step { - RunDriveStep::Continue => continue, - RunDriveStep::PlanAmendment { plan_revision } => { - if pause_automatic_amendment_planner() { - ctx.sleep(std::time::Duration::from_millis(25)).await?; - continue; - } - let amendment_step = plan_and_apply_waiting_replan( - &ctx, - AmendmentOperationContext { - repository: self.repository.clone(), - config: self.config.clone(), - planner_model: self.planner_model.clone(), - scope, - request: request.clone(), - }, - plan_revision, - ) - .await?; - deliver_session_projection( - &ctx, - self.repository.clone(), - scope, - &request, - matches!(&amendment_step, RunDriveStep::Terminal { .. }), - step_index, - ) - .await?; - match amendment_step { - RunDriveStep::Continue => continue, - RunDriveStep::Terminal { task_ids, reason } => { - if !owned_task_calls.is_empty() || !task_ids.is_empty() { - cancel_completion_owner_from_workflow( - &ctx, - LLMCompletionOwner::execution_run(request.run_uid.to_string()), - ) - .await?; - } - for (task, _) in &owned_task_calls { - signal_forward_task_cancellation(&ctx, task, &reason).await?; - } - for (_, call) in owned_task_calls.drain(..) { - call.await?; - } - for task_id in task_ids { - if !owned_task_ids.contains(&task_id) { - signal_task_cancellation(&ctx, task_id, &reason).await?; - } - } - owned_task_ids.clear(); - return Ok(()); - } - RunDriveStep::PlanAmendment { .. } - | RunDriveStep::Dispatch { .. } - | RunDriveStep::SettleForward { .. } - | RunDriveStep::Compensate { .. } - | RunDriveStep::Park { .. } => { - return Err(TerminalError::new( - "amendment operation returned an invalid driver step", - ) - .into()); - } - } - } - RunDriveStep::Dispatch { - tasks, - processed_epoch, - } => { - if owned_task_calls.len() != owned_task_ids.len() { - return Err(TerminalError::new( - "execution task call ownership index diverged", - ) - .into()); - } - if owned_task_calls.len() > self.config.max_in_flight_tasks { - return Err(TerminalError::new(format!( - "execution task call ownership {} exceeds configured window {}", - owned_task_calls.len(), - self.config.max_in_flight_tasks - )) - .into()); - } - let tasks = select_dispatch_batch( - tasks, - &owned_task_ids, - self.config - .max_in_flight_tasks - .saturating_sub(owned_task_calls.len()), - ); - record_execution_dispatch_batch_size(tasks.len()); - for task in tasks { - if !owned_task_ids.insert(task.task_id) { - return Err(TerminalError::new(format!( - "execution task {} was dispatched while its original call was still owned", - task.task_id - )) - .into()); - } - let call = crate::restate_identity::replay_safe_request( - ctx.workflow_client::(task.task_id.to_string()) - .run(Json::from(task.clone())), - ) - .call(); - owned_task_calls.push((task, call)); - } - record_execution_owned_in_flight_tasks(owned_task_calls.len()); - await_persisted_wake_or_owned_task( - &ctx, - self.repository.clone(), - scope, - request.run_uid, - processed_epoch, - &mut owned_task_calls, - &mut owned_task_ids, - ) - .await?; - } - RunDriveStep::SettleForward { tasks, reason } => { - cancel_completion_owner_from_workflow( - &ctx, - LLMCompletionOwner::execution_run(request.run_uid.to_string()), - ) - .await?; - let expected = tasks - .iter() - .map(|task| task.task_id) - .collect::>(); - if !expected.is_subset(&owned_task_ids) { - return Err(TerminalError::new(format!( - "execution run settlement task ownership mismatch: expected {expected:?}, owned {owned_task_ids:?}" - )) - .into()); - } - for task in &tasks { - signal_forward_task_cancellation(&ctx, task, &reason).await?; - } - for (_, call) in owned_task_calls.drain(..) { - call.await?; - } - owned_task_ids.clear(); - } - RunDriveStep::Compensate { request } => { - crate::restate_identity::replay_safe_request( - ctx.workflow_client::( - request.compensation_id.to_string(), - ) - .run(Json::from(request)), - ) - .call() - .await?; - } - RunDriveStep::Park { processed_epoch } => { - await_persisted_wake_or_owned_task( - &ctx, - self.repository.clone(), - scope, - request.run_uid, - processed_epoch, - &mut owned_task_calls, - &mut owned_task_ids, - ) - .await?; - } - RunDriveStep::Terminal { task_ids, reason } => { - if !owned_task_calls.is_empty() || !task_ids.is_empty() { - cancel_completion_owner_from_workflow( - &ctx, - LLMCompletionOwner::execution_run(request.run_uid.to_string()), - ) - .await?; - } - for (task, _) in &owned_task_calls { - signal_forward_task_cancellation(&ctx, task, &reason).await?; - } - for (_, call) in owned_task_calls.drain(..) { - call.await?; - } - for task_id in task_ids { - if !owned_task_ids.contains(&task_id) { - signal_task_cancellation(&ctx, task_id, &reason).await?; - } - } - owned_task_ids.clear(); - return Ok(()); - } - } - } - } - - #[tracing::instrument(skip(self, ctx, request))] - // SAFETY: invoked only after an authorized service or keyed task transaction persisted the exact wake epoch. - async fn wake( - &self, - ctx: SharedWorkflowContext<'_>, - request: Json, - ) -> Result<(), HandlerError> { - crate::ctx::adopt_incoming_trace_parent(&ctx); - annotate_restate_handler_span("ExecutionRun", "wake"); - let request = request.into_inner(); - annotate_execution_run_span(request.run_uid); - if request.run_uid.to_string() != ctx.key() { - return Err(TerminalError::new_with_code(404, "execution run id mismatch").into()); - } - let processed_epoch = ctx - .get::>(K_PROCESSED_WAKE_EPOCH) - .await? - .map(Json::into_inner) - .unwrap_or_default(); - let awaited_epoch = ctx - .get::>(K_AWAITED_WAKE_EPOCH) - .await? - .map(Json::into_inner) - .unwrap_or_default(); - if let Some(promise_epoch) = - wake_promise_epoch(processed_epoch, awaited_epoch, request.wake_epoch) - { - ctx.resolve_promise(&wake_promise_key(promise_epoch), request.wake_epoch); - } - #[cfg(feature = "integration")] - delay_test_wake_acknowledgement().await; - Ok(()) - } -} - -#[cfg(feature = "integration")] -fn pause_automatic_amendment_planner() -> bool { - std::env::var("MOA_EXECUTION_TEST_PAUSE_AMENDMENT_PLANNER").as_deref() == Ok("true") -} - -#[cfg(not(feature = "integration"))] -const fn pause_automatic_amendment_planner() -> bool { - false -} - -#[cfg(feature = "integration")] -async fn delay_test_wake_acknowledgement() { - if std::env::var("MOA_EXECUTION_TEST_DELAY_WAKE_ACK").as_deref() == Ok("true") { - // The service E2E observes this handler after its durable work but before - // its response, proving public mutations join the wake acknowledgement. - tokio::time::sleep(std::time::Duration::from_secs(2)).await; - } -} - -#[cfg(feature = "integration")] -async fn test_wake_handoff_checkpoint(ctx: &WorkflowContext<'_>) -> Result<(), HandlerError> { - let Ok(mode) = std::env::var("MOA_EXECUTION_TEST_WAKE_HANDOFF") else { - return Ok(()); - }; - match mode.as_str() { - "delay" => { - ctx.sleep(std::time::Duration::from_secs(2)).await?; - } - "crash_once" => { - static CRASHED: std::sync::atomic::AtomicBool = - std::sync::atomic::AtomicBool::new(false); - let should_crash = !CRASHED.swap(true, std::sync::atomic::Ordering::SeqCst); - ctx.sleep(std::time::Duration::from_secs(2)).await?; - if should_crash { - return Err(anyhow::anyhow!("injected execution wake handoff crash").into()); - } - } - _ => {} - } - Ok(()) -} - -async fn await_persisted_wake_or_owned_task( - ctx: &WorkflowContext<'_>, - repository: ExecutionRepository, - scope: ExecutionScope, - run_uid: uuid::Uuid, - processed_epoch: u64, - owned_task_calls: &mut Vec<(ExecutionTaskWorkflowRequest, F)>, - owned_task_ids: &mut BTreeSet, -) -> Result<(), HandlerError> -where - F: restate_sdk::context::DurableFuture> - + SealedDurableFuture, -{ - ctx.set(K_AWAITED_WAKE_EPOCH, Json::from(processed_epoch)); - let acknowledgement = ctx - .run(|| async move { - repository - .ack_run_wake(scope, run_uid, processed_epoch) - .await - .map(Json::from) - .map_err(execution_error) - }) - .name(format!("execution_run_ack_{processed_epoch}")) - .await? - .into_inner(); - let processed_epoch = match acknowledgement { - WakeAckOutcome::Acknowledged { - processed_wake_epoch, - } - | WakeAckOutcome::Replayed { - processed_wake_epoch, - } => processed_wake_epoch, - WakeAckOutcome::Changed { .. } => return Ok(()), - WakeAckOutcome::NotFound => { - return Err(TerminalError::new_with_code(404, "execution run not found").into()); - } - }; - #[cfg(feature = "integration")] - test_wake_handoff_checkpoint(ctx).await?; - ctx.set(K_PROCESSED_WAKE_EPOCH, Json::from(processed_epoch)); - let promise_key = wake_promise_key(processed_epoch); - let wake = ctx.promise::(&promise_key); - if owned_task_calls.is_empty() { - let _: u64 = wake.await?; - return Ok(()); - } - - let mut handles = Vec::with_capacity(owned_task_calls.len().saturating_add(1)); - handles.push(wake.handle()); - handles.extend(owned_task_calls.iter().map(|(_, call)| call.handle())); - let selected = wake.inner_context().select(handles).await?; - if selected == 0 { - let _: u64 = wake.await?; - return Ok(()); - } - - let call_index = selected.saturating_sub(1); - if call_index >= owned_task_calls.len() { - return Err(TerminalError::new(format!( - "execution task selector returned invalid branch {selected}" - )) - .into()); - } - let (task, call) = owned_task_calls.remove(call_index); - owned_task_ids.remove(&task.task_id); - call.await?; - record_execution_owned_in_flight_tasks(owned_task_calls.len()); - Ok(()) -} - -#[derive(Clone, Debug, Deserialize, Serialize)] -#[serde(tag = "step", rename_all = "snake_case")] -enum RunDriveStep { - Continue, - PlanAmendment { - plan_revision: u64, - }, - Dispatch { - tasks: Vec, - processed_epoch: u64, - }, - SettleForward { - tasks: Vec, - reason: String, - }, - Compensate { - request: ExecutionCompensationWorkflowRequest, - }, - Park { - processed_epoch: u64, - }, - Terminal { - task_ids: Vec, - reason: String, - }, -} - -#[derive(Clone, Debug, Deserialize, Serialize)] -struct SessionProjectionDelivery { - progress: moa_core::events::ExecutionProgress, - inputs: Vec, - terminal: Option, -} - -async fn deliver_session_projection( - ctx: &WorkflowContext<'_>, - repository: ExecutionRepository, - scope: ExecutionScope, - request: &ExecutionRunWorkflowRequest, - include_terminal: bool, - step_index: u64, -) -> Result<(), HandlerError> { - #[cfg(feature = "integration")] - if std::env::var("MOA_EXECUTION_TEST_SKIP_SESSION_DELIVERY").as_deref() == Ok("true") { - return Ok(()); - } - - let delivery_request = request.clone(); - let delivery = ctx - .run(|| async move { - let snapshot = repository - .load_scheduling_snapshot(scope, delivery_request.run_uid) - .await - .map_err(execution_error)? - .ok_or_else(|| TerminalError::new_with_code(404, "execution run not found"))?; - if snapshot.run.tenant_id != delivery_request.tenant_id - || snapshot.run.contact_id != delivery_request.contact_id - || snapshot.run.session_id != delivery_request.session_id - { - return Err(TerminalError::new_with_code( - 409, - "execution delivery scope does not match the workflow request", - ) - .into()); - } - let inputs = snapshot - .projection - .tasks - .iter() - .filter_map(|task| { - if task.status != ExecutionTaskStatus::WaitingInput { - return None; - } - let moa_artifacts::execution_plan::ExecutionTaskResult::NeedsInput { - question, - audience: moa_artifacts::execution_plan::InputAudience::User, - } = &task.outcome.as_ref()?.result - else { - return None; - }; - Some(ExecutionInputRequired { - run_uid: snapshot.run.run_uid, - originating_user_sequence_num: snapshot.run.originating_user_sequence_num, - task_id: task.task_id.as_uuid(), - generation: task.generation, - question: question.clone(), - }) - }) - .collect(); - let terminal = if include_terminal { - Some( - repository - .load_terminal_delivery(scope, delivery_request.run_uid) - .await - .map_err(execution_error)? - .ok_or_else(|| { - TerminalError::new_with_code( - 404, - "terminal execution delivery disappeared after finalization", - ) - })?, - ) - } else { - None - }; - Ok(Json::from(SessionProjectionDelivery { - progress: execution_progress_from_run(&snapshot.run), - inputs, - terminal, - })) - }) - .name(format!("execution_session_projection_{step_index}")) - .await? - .into_inner(); - - let session = ctx.object_client::(request.session_id.to_string()); - crate::restate_identity::replay_safe_request( - session.execution_progress(Json::from(delivery.progress)), - ) - .call() - .await?; - for input in delivery.inputs { - crate::restate_identity::replay_safe_request( - ctx.object_client::(request.session_id.to_string()) - .execution_input_required(Json::from(input)), - ) - .call() - .await?; - } - if let Some(terminal) = delivery.terminal { - match terminal.status { - ExecutionRunStatus::Completed | ExecutionRunStatus::Cancelled => {} - status => { - moa_execution::wire::execution_failure_disposition(status) - .map_err(execution_error)?; - } - } - crate::restate_identity::replay_safe_request( - ctx.object_client::(request.session_id.to_string()) - .execution_terminal(Json::from(terminal)), - ) - .call() - .await?; - } - Ok(()) -} - -struct RestateAmendmentPlannerProvider<'a> { - ctx: &'a WorkflowContext<'a>, - run_uid: uuid::Uuid, - plan_revision: u64, - next_attempt: AtomicUsize, -} - -#[async_trait] -impl LLMProvider for RestateAmendmentPlannerProvider<'_> { - fn name(&self) -> &'static str { - "restate-llm-gateway" - } - - fn capabilities(&self) -> ModelCapabilities { - ModelCapabilities::default() - } - - async fn complete( - &self, - request: SharedCompletionRequest, - ) -> moa_core::error::Result { - // Restate's JSON transport requires the owned durable DTO. This is the - // explicit serialization boundary after in-process shared routing. - let request = CompletionRequest::from_view(&request); - // Amendment generation and its bounded repair are sequential planner calls. - let attempt = self.next_attempt.fetch_add(1, Ordering::Relaxed); - let response = crate::restate_identity::replay_safe_request( - self.ctx - .service_client::() - .complete(Json::from(request)) - .idempotency_key(completion_idempotency_key( - self.ctx.invocation_id(), - LLMCompletionAction::ExecutionAmendment { - run_uid: self.run_uid, - plan_revision: self.plan_revision, - attempt, - }, - )), - ) - .call() - .await - .map_err(|error| moa_core::error::MoaError::ProviderError(error.to_string()))? - .into_inner(); - Ok(CompletionStream::from_response(response)) - } -} - -#[derive(Clone, Debug, Deserialize, Serialize)] -struct PreparedAmendmentPlanning { - context: ExecutionPlanningContextSnapshot, - evidence: AmendmentPlanningEvidence, - remaining_budget: moa_artifacts::execution_plan::ExecutionBudgetLimit, - now: chrono::DateTime, -} - -struct AmendmentOperationContext { - repository: ExecutionRepository, - config: moa_config::ExecutionConfig, - planner_model: ModelId, - scope: ExecutionScope, - request: ExecutionRunWorkflowRequest, -} - -async fn plan_and_apply_waiting_replan( - ctx: &WorkflowContext<'_>, - operation: AmendmentOperationContext, - plan_revision: u64, -) -> Result { - let AmendmentOperationContext { - repository, - config, - planner_model, - scope, - request, - } = operation; - let load_request = request.clone(); - let load_repository = repository.clone(); - let prepared = ctx - .run(|| async move { - prepare_amendment_planning(&load_repository, scope, &load_request, plan_revision) - .await - .map(Json::from) - }) - .name(format!( - "execution_amendment_inputs_{}_{}", - request.run_uid, plan_revision - )) - .await? - .into_inner(); - let Some(prepared) = prepared else { - return Ok(RunDriveStep::Continue); - }; - - let provider = RestateAmendmentPlannerProvider { - ctx, - run_uid: request.run_uid, - plan_revision, - next_attempt: AtomicUsize::new(0), - }; - let planned = plan_amendment( - &provider, - ExecutionAmendmentPlanningRequest { - run_uid: request.run_uid, - base_plan_revision: plan_revision, - context: prepared.context, - evidence: prepared.evidence, - remaining_budget: prepared.remaining_budget, - planner_model, - config: config.clone(), - now: prepared.now, - }, - ) - .await - .map_err(crate::workflows::errors::moa_error_to_handler_error)?; - for audit in planned.audits { - persist_amendment_audit(&repository, scope, audit).await?; - } - - let amendment = match planned.kind { - ExecutionAmendmentPlanningResultKind::Ready { amendment, .. } => amendment, - ExecutionAmendmentPlanningResultKind::NeedsInput { message } - | ExecutionAmendmentPlanningResultKind::Unsupported { message } => { - // Planner-authored verdict text is safe to carry into the replan-stop reason. - return finalize_amendment_planner_stop( - repository, - scope, - request.run_uid, - plan_revision, - message, - ) - .await; - } - ExecutionAmendmentPlanningResultKind::ProviderFailure { message } => { - // Infrastructure failure, not a semantic verdict: the raw provider string must not - // reach the user-surfaced replan-stop gaps. Record the detail for operators (the - // persisted planner audit already carries the ProviderError outcome) and stop the - // replan with a bounded, user-safe description. - tracing::error!( - run_uid = %request.run_uid, - plan_revision, - detail = %message, - "amendment planner provider failure" - ); - return finalize_amendment_planner_stop( - repository, - scope, - request.run_uid, - plan_revision, - "an internal error interrupted amendment planning".to_string(), - ) - .await; - } - }; - let response = crate::restate_identity::replay_safe_request( - ctx.service_client::() - .apply_planned_amendment(Json::from(ExecutionAmendmentRequest { - run: ExecutionRunRequest { - tenant_id: request.tenant_id, - contact_id: request.contact_id, - session_id: request.session_id, - run_uid: request.run_uid, - }, - expected_plan_revision: plan_revision, - amendment, - })), - ) - .call() - .await? - .into_inner(); - match response { - ExecutionMutationResponse::Applied { .. } - | ExecutionMutationResponse::Replayed { .. } - | ExecutionMutationResponse::Conflict { .. } => Ok(RunDriveStep::Continue), - ExecutionMutationResponse::NotFound => { - Err(TerminalError::new_with_code(404, "execution run not found").into()) - } - } -} - -async fn prepare_amendment_planning( - repository: &ExecutionRepository, - scope: ExecutionScope, - request: &ExecutionRunWorkflowRequest, - plan_revision: u64, -) -> Result, HandlerError> { - let Some(snapshot) = repository - .load_scheduling_snapshot(scope, request.run_uid) - .await - .map_err(execution_error)? - else { - return Err(TerminalError::new_with_code(404, "execution run not found").into()); - }; - if snapshot.run.tenant_id != request.tenant_id - || snapshot.run.contact_id != request.contact_id - || snapshot.run.session_id != request.session_id - { - return Err(TerminalError::new_with_code(409, "execution scope mismatch").into()); - } - if snapshot.run.plan_revision != plan_revision - || snapshot.run.status != ExecutionRunStatus::WaitingReplan - { - return Ok(None); - } - let waiting_tasks = snapshot - .projection - .tasks - .iter() - .filter(|task| task.status == ExecutionTaskStatus::WaitingReplan) - .collect::>(); - let [waiting_task] = waiting_tasks.as_slice() else { - return Err(TerminalError::new( - "amendment planning requires exactly one WaitingReplan task", - ) - .into()); - }; - let Some(outcome) = waiting_task.outcome.as_ref() else { - return Err(TerminalError::new("WaitingReplan task has no persisted outcome").into()); - }; - let moa_artifacts::execution_plan::ExecutionTaskResult::NeedsReplan { reason, evidence } = - &outcome.result - else { - return Err(TerminalError::new("WaitingReplan task has no NeedsReplan evidence").into()); - }; - let failure_evidence = bounded_failure_evidence(reason, evidence)?; - let planning_context = repository - .load_planning_context(scope, snapshot.run.planning_context_uid) - .await - .map_err(execution_error)? - .ok_or_else(|| TerminalError::new("execution planning context does not exist"))?; - if planning_context.planning_context_hash != snapshot.run.planning_context_hash - || planning_context.snapshot.tenant_id != snapshot.run.tenant_id - || planning_context.snapshot.contact_id != snapshot.run.contact_id - || planning_context.snapshot.session_id != snapshot.run.session_id - || planning_context.snapshot.originating_user_sequence_num - != snapshot.run.originating_user_sequence_num - || planning_context.snapshot.owner_user_id != snapshot.run.owner_user_id - || planning_context.snapshot.catalog != snapshot.catalog - || planning_context.snapshot.authorization != snapshot.authorization - || planning_context.snapshot.pinned_instruction_skills != snapshot.pinned_instruction_skills - { - return Err(TerminalError::new_with_code( - 409, - "persisted amendment planning authority does not match the active run", - ) - .into()); - } - let mut effective_context = planning_context.snapshot; - effective_context.budget = snapshot.run.approved_budget.clone(); - // Amendment replay is governed by the exact capability catalog persisted with - // the run. Consulting the deployment router here would make the same wake - // compile differently after a catalog refresh and would drop installed - // connector provenance before dispatch can generation-fence it. - let admitted_tool_names = effective_context - .catalog - .capabilities - .iter() - .filter_map(|capability| capability.source.model_visible_tool_name()) - .map(ToString::to_string) - .collect(); - let context = narrow_amendment_context(effective_context, &admitted_tool_names) - .map_err(execution_error)?; - let remaining_budget = snapshot - .budget_ledger - .remaining_limit() - .map_err(execution_error)?; - let waiting_task_id = waiting_task.task_id; - Ok(Some(PreparedAmendmentPlanning { - context, - evidence: AmendmentPlanningEvidence { - goal: snapshot.run.goal, - active_plan: snapshot.run.active_plan, - projection: snapshot.projection, - failure_evidence, - waiting_task: waiting_task_id, - }, - remaining_budget, - now: chrono::Utc::now(), - })) -} - -fn bounded_failure_evidence(reason: &str, evidence: &Value) -> Result { - let failure_evidence = json!({"reason": reason, "evidence": evidence}); - let encoded = moa_core::canonical_json::canonical_json_bytes(&failure_evidence) - .map_err(|error| TerminalError::new(error.to_string()))?; - if encoded.len() > EXECUTION_REPORT_MAX_BYTES { - return Err(TerminalError::new_with_code( - 422, - "WaitingReplan failure evidence exceeds the bounded planner envelope", - ) - .into()); - } - Ok(failure_evidence) -} - -fn narrow_amendment_context( - mut context: ExecutionPlanningContextSnapshot, - available_tool_names: &BTreeSet, -) -> moa_execution::Result { - use moa_execution::capability::CapabilitySource; - - let retained_refs = context - .catalog - .capabilities - .iter() - .filter(|capability| match &capability.source { - CapabilitySource::BuiltInTool { name } | CapabilitySource::HandTool { name } => { - available_tool_names.contains(name) - } - // `tool_name`, not `remote_name`: availability is membership in the - // router's registered names, and a connector tool is registered - // under its server-qualified reference. - CapabilitySource::McpTool { tool_name, .. } - | CapabilitySource::ActionArtifact { tool_name, .. } - | CapabilitySource::ConnectorAction { tool_name, .. } - | CapabilitySource::InstalledConnectorAction { tool_name, .. } - | CapabilitySource::SkillAction { tool_name, .. } - | CapabilitySource::Memory { tool_name, .. } => { - available_tool_names.contains(tool_name) - } - CapabilitySource::SkillCode { .. } - | CapabilitySource::Knowledge { .. } - | CapabilitySource::Model => true, - }) - .map(|capability| capability.reference.clone()) - .collect::>(); - narrow_authorized_capability_refs(&mut context.authorization.capability_refs, &retained_refs); - context - .validate() - .map_err(|error| moa_execution::Error::InvalidRepositoryInput { - message: error.to_string(), - })?; - Ok(context) -} - -fn narrow_authorized_capability_refs( - authorized: &mut Vec, - live: &[moa_artifacts::execution_plan::CapabilityReference], -) { - authorized.retain(|reference| live.contains(reference)); -} - -async fn persist_amendment_audit( - repository: &ExecutionRepository, - scope: ExecutionScope, - envelope: ExecutionPlanningAuditEnvelope, -) -> Result<(), HandlerError> { - match &envelope.payload { - ExecutionPlanningAuditPayload::PlannerCall { .. } => { - let result = repository - .write_planner_call_audit(scope, &envelope) - .await - .map_err(execution_error)?; - if matches!(result, PlannerCallAuditWriteOutcome::Conflict { .. }) { - return Err(TerminalError::new_with_code( - 409, - "execution amendment planner audit conflicts with first persisted evidence", - ) - .into()); - } - } - ExecutionPlanningAuditPayload::Compile { .. } => { - let result = repository - .write_compile_audit(scope, &envelope) - .await - .map_err(execution_error)?; - if matches!(result, CompileAuditWriteOutcome::Conflict { .. }) { - return Err(TerminalError::new_with_code( - 409, - "execution amendment compile audit conflicts with first persisted evidence", - ) - .into()); - } - } - ExecutionPlanningAuditPayload::Route { .. } => { - return Err(TerminalError::new_with_code( - 422, - "execution amendment planning produced a route audit", - ) - .into()); - } - } - Ok(()) -} - -async fn finalize_amendment_planner_stop( - repository: ExecutionRepository, - scope: ExecutionScope, - run_uid: uuid::Uuid, - expected_plan_revision: u64, - message: String, -) -> Result { - let Some(snapshot) = repository - .load_scheduling_snapshot(scope, run_uid) - .await - .map_err(execution_error)? - else { - return Err(TerminalError::new_with_code(404, "execution run not found").into()); - }; - if snapshot.run.plan_revision != expected_plan_revision - || snapshot.run.status != ExecutionRunStatus::WaitingReplan - { - return Ok(RunDriveStep::Continue); - } - fence_replan_stop( - &repository, - scope, - snapshot, - ReplanExhaustion { - reason: moa_execution::ReplanStopReason::NoProgress, - description: format!("amendment planner stopped: {message}"), - }, - ) - .await -} - -async fn drive_once( - repository: ExecutionRepository, - scope: ExecutionScope, - request: ExecutionRunWorkflowRequest, - config: moa_config::ExecutionConfig, - owned_task_ids: BTreeSet, -) -> Result { - let Some(snapshot) = repository - .load_scheduling_snapshot(scope, request.run_uid) - .await - .map_err(execution_error)? - else { - return Err(TerminalError::new_with_code(404, "execution run not found").into()); - }; - if snapshot.run.tenant_id != request.tenant_id - || snapshot.run.contact_id != request.contact_id - || snapshot.run.session_id != request.session_id - { - return Err(TerminalError::new_with_code(409, "execution scope mismatch").into()); - } - if snapshot.run.status.is_terminal() { - return Ok(terminal_step( - &snapshot.projection.tasks, - format!("execution run ended as {}", snapshot.run.status.as_str()), - )); - } - if snapshot.run.manual_repair_required && snapshot.run.pending_terminal.is_none() { - return finalize_internal_failure( - &repository, - scope, - snapshot, - "compensation registration requires manual repair".to_string(), - ) - .await; - } - if snapshot.run.pending_terminal.is_some() - || snapshot.run.status == ExecutionRunStatus::Compensating - { - return drive_compensation(&repository, scope, &request, snapshot, &owned_task_ids).await; - } - if snapshot.run.status == ExecutionRunStatus::AwaitingConfirmation { - return Ok(park_at_epoch(&snapshot.run)); - } - if matches!( - snapshot.run.status, - ExecutionRunStatus::Queued | ExecutionRunStatus::Running - ) { - let pending_tasks = snapshot - .projection - .tasks - .iter() - .filter(|task| task.status == ExecutionTaskStatus::Pending) - .map(|task| ExecutionTaskWorkflowRequest { - run_uid: snapshot.run.run_uid, - task_id: task.task_id, - generation: task.generation, - tenant_id: snapshot.run.tenant_id, - contact_id: snapshot.run.contact_id, - session_id: snapshot.run.session_id, - identity: request.identity.clone(), - }) - .collect::>(); - if !pending_tasks.is_empty() { - return Ok(RunDriveStep::Dispatch { - tasks: pending_tasks, - processed_epoch: snapshot.run.wake_epoch, - }); - } - } - let now = chrono::Utc::now(); - let schedule_request = ScheduleRequest { - run_uid: snapshot.run.run_uid, - goal: snapshot.run.goal.clone(), - plan: snapshot.run.active_plan.clone(), - run_input: snapshot.run.input.clone(), - catalog: snapshot.catalog.clone(), - projection: snapshot.projection.clone(), - config, - budget_ledger: snapshot.budget_ledger.clone(), - now, - }; - let empty_map_nodes = match ready_empty_map_nodes(&schedule_request) { - Ok(node_ids) => node_ids, - Err(error) => { - return finalize_internal_failure(&repository, scope, snapshot, error.to_string()) - .await; - } - }; - let mut applied_empty_map = false; - for node_id in empty_map_nodes { - let marker = ExecutionNodeMaterialization::Map { - node_id, - fanout_items: 0, - }; - match repository - .materialize_node( - scope, - snapshot.run.run_uid, - snapshot.run.plan_revision, - Some(marker), - Vec::new(), - ) - .await - .map_err(execution_error)? - { - MaterializationOutcome::Applied(evidence) => { - evidence.marker.as_ref().ok_or_else(|| { - TerminalError::new("empty map application omitted its durable marker") - })?; - applied_empty_map = true; - } - MaterializationOutcome::Replayed { tasks } => { - if !tasks.is_empty() { - return Err(TerminalError::new( - "empty map replay unexpectedly returned logical tasks", - ) - .into()); - } - } - MaterializationOutcome::Conflict => return Ok(RunDriveStep::Continue), - } - } - if applied_empty_map { - return Ok(RunDriveStep::Continue); - } - let scheduled = match schedule(schedule_request) { - Ok(scheduled) => scheduled, - Err(error) => { - return finalize_internal_failure(&repository, scope, snapshot, error.to_string()) - .await; - } - }; - let mut snapshot = snapshot; - snapshot.projection = scheduled.effective_projection; - match scheduled.decision { - ScheduleDecision::Ready(tasks) => { - let mut tasks_by_node = BTreeMap::>::new(); - for task in tasks { - tasks_by_node - .entry(task.node_id.clone()) - .or_default() - .push(task); - } - let mut records = Vec::new(); - for (node_id, tasks) in tasks_by_node { - let marker = node_materialization_marker(&snapshot.run, &node_id, &tasks)?; - match repository - .materialize_node( - scope, - snapshot.run.run_uid, - snapshot.run.plan_revision, - marker, - tasks, - ) - .await - .map_err(execution_error)? - { - MaterializationOutcome::Applied(evidence) => { - records.extend(evidence.tasks); - } - MaterializationOutcome::Replayed { tasks } => records.extend(tasks), - MaterializationOutcome::Conflict => return Ok(RunDriveStep::Continue), - } - } - let dispatch_epoch = repository - .load_run(scope, snapshot.run.run_uid) - .await - .map_err(execution_error)? - .ok_or_else(|| TerminalError::new_with_code(404, "execution run not found"))? - .wake_epoch; - Ok(RunDriveStep::Dispatch { - tasks: records - .into_iter() - .filter(|task| !task.status.is_terminal()) - .map(|task| ExecutionTaskWorkflowRequest { - run_uid: task.run_uid, - task_id: task.task_id, - generation: task.generation, - tenant_id: task.tenant_id, - contact_id: task.contact_id, - session_id: snapshot.run.session_id, - identity: request.identity.clone(), - }) - .collect(), - processed_epoch: dispatch_epoch, - }) - } - ScheduleDecision::Waiting(waiting) => { - if let Some(reason) = immediately_knowable_replan_stop(&snapshot) { - return fence_replan_stop(&repository, scope, snapshot, reason).await; - } - let waiting_status = waiting_status(&snapshot.projection.tasks, &waiting); - let transition = repository - .transition_run_wait_with_reasons( - scope, - snapshot.run.run_uid, - snapshot.run.status, - waiting_status, - waiting, - ) - .await - .map_err(execution_error)?; - Ok(wait_transition_step( - waiting_status, - matches!(transition, TransitionOutcome::RunApplied(_)), - snapshot.run.plan_revision, - snapshot.run.wake_epoch, - )) - } - ScheduleDecision::Terminal(terminal) => { - let cause = terminal_cause( - &snapshot.projection, - &snapshot.budget_ledger, - &terminal, - now, - ); - finalize(&repository, scope, snapshot, terminal, cause).await - } - ScheduleDecision::NoProgress { pending_node_ids } => { - let evaluation = evaluate_completion(CompletionEvaluationRequest { - goal: snapshot.run.goal.clone(), - plan: snapshot.run.active_plan.clone(), - run_input: snapshot.run.input.clone(), - projection: snapshot.projection.clone(), - terminal_output: snapshot.run.output.clone(), - budget_ledger: snapshot.budget_ledger.clone(), - now: chrono::Utc::now(), - }) - .map_err(execution_error)?; - let terminal = terminal_projection_from_evaluation( - &evaluation, - snapshot.run.output.clone(), - Some(format!( - "scheduler made no progress with pending nodes: {}", - pending_node_ids.join(", ") - )), - None, - None, - ) - .map_err(execution_error)?; - let terminal_evidence = terminal_evidence_from_evaluation( - ExecutionTerminalCause::SchedulerNoProgress, - &evaluation, - ) - .map_err(execution_error)?; - let terminal_reason = - execution_terminal_reason(&terminal_evidence.cause, &terminal, &evaluation) - .map_err(execution_error)?; - if fence_terminal_before_settlement( - &repository, - scope, - &snapshot.run, - TerminalFenceInput { - terminal: &terminal, - terminal_evidence: &terminal_evidence, - terminal_reason, - output: snapshot.run.output.clone(), - evaluation: &evaluation, - }, - ) - .await? - { - return Ok(RunDriveStep::Continue); - } - match repository - .finalize_run( - scope, - RunFinalizationRequest { - run_uid: snapshot.run.run_uid, - expected_revision: snapshot.run.plan_revision, - expected_wake_epoch: snapshot.run.wake_epoch, - terminal_projection: terminal, - completion_evaluation: evaluation, - terminal_evidence, - terminal_reason, - }, - ) - .await - .map_err(execution_error)? - { - FinalizationOutcome::Finalized(_) => Ok(terminal_step( - &snapshot.projection.tasks, - "execution scheduler made no progress".to_string(), - )), - FinalizationOutcome::Replayed(_) => Ok(terminal_step( - &snapshot.projection.tasks, - "execution scheduler made no progress".to_string(), - )), - FinalizationOutcome::Conflict => Ok(RunDriveStep::Continue), - FinalizationOutcome::NotFound => { - Err(TerminalError::new_with_code(404, "execution run not found").into()) - } - } - } - } -} - -async fn drive_compensation( - repository: &ExecutionRepository, - scope: ExecutionScope, - request: &ExecutionRunWorkflowRequest, - scheduling: moa_execution::repository::ExecutionSchedulingSnapshot, - owned_task_ids: &BTreeSet, -) -> Result { - let snapshot = repository - .load_compensation_snapshot(scope, request.run_uid) - .await - .map_err(execution_error)? - .ok_or_else(|| TerminalError::new_with_code(404, "execution run not found"))?; - if snapshot.run.tenant_id != request.tenant_id - || snapshot.run.contact_id != request.contact_id - || snapshot.run.session_id != request.session_id - { - return Err( - TerminalError::new_with_code(409, "execution compensation scope mismatch").into(), - ); - } - let settlement_reason = "execution run fenced forward work before compensation"; - let mut owned_forward_tasks = Vec::new(); - let mut settled_undispatched = false; - for task in &snapshot.nonterminal_forward_tasks { - if owned_task_ids.contains(&task.task_id) { - owned_forward_tasks.push(task); - continue; - } - if task.status != ExecutionTaskStatus::Pending { - return Err(TerminalError::new(format!( - "execution run lost ownership of active task {} in status {}", - task.task_id, - task.status.as_str() - )) - .into()); - } - match repository - .record_task_outcome( - scope, - task.run_uid, - task.task_id, - task.generation, - cancelled_task_outcome(settlement_reason.to_string(), task.actual.clone()), - ) - .await - .map_err(execution_error)? - { - TaskOutcomeWrite::Applied { .. } | TaskOutcomeWrite::Replayed { .. } => { - settled_undispatched = true; - } - TaskOutcomeWrite::Rejected { reason, .. } => { - return Err(TerminalError::new(format!( - "execution run could not fence undispatched task {}: {reason:?}", - task.task_id - )) - .into()); - } - TaskOutcomeWrite::NotFound => { - return Err(TerminalError::new_with_code( - 404, - format!("execution task {} not found", task.task_id), - ) - .into()); - } - } - } - if settled_undispatched { - return Ok(RunDriveStep::Continue); - } - if !owned_forward_tasks.is_empty() { - return Ok(RunDriveStep::SettleForward { - tasks: owned_forward_tasks - .into_iter() - .map(|task| task_workflow_request(&snapshot.run, task, request)) - .collect(), - reason: settlement_reason.to_string(), - }); - } - if snapshot.run.status != ExecutionRunStatus::Compensating { - let pending_status = snapshot - .run - .pending_terminal - .as_ref() - .map(|terminal| terminal.status) - .ok_or_else(|| TerminalError::new("fenced execution run omitted pending terminal"))?; - if compensation_entry_decision( - snapshot.run.active_plan.definition.cancel_policy, - pending_status, - snapshot.manual_repair_required, - ) == CompensationEntryDecision::FinalizeFenced - { - return finalize_fenced_terminal_step(repository, scope, &snapshot.run, request).await; - } - return match repository - .begin_compensation( - scope, - snapshot.run.run_uid, - snapshot.run.plan_revision, - snapshot.run.wake_epoch, - ) - .await - .map_err(execution_error)? - { - BeginCompensationOutcome::Applied(_) | BeginCompensationOutcome::Replayed(_) => { - Ok(RunDriveStep::Continue) - } - BeginCompensationOutcome::NoCompensations(run) => { - finalize_fenced_terminal_step(repository, scope, &run, request).await - } - BeginCompensationOutcome::ForwardTasksPending(tasks) => { - Ok(RunDriveStep::SettleForward { - tasks: tasks - .iter() - .map(|task| task_workflow_request(&snapshot.run, task, request)) - .collect(), - reason: "execution run fenced forward work before compensation".to_string(), - }) - } - BeginCompensationOutcome::Conflict => Ok(RunDriveStep::Continue), - BeginCompensationOutcome::NotFound => { - Err(TerminalError::new_with_code(404, "execution run not found").into()) - } - }; - } - if snapshot.manual_repair_required { - return finalize_compensation_step(repository, scope, &snapshot.run, &scheduling).await; - } - let next = snapshot.registrations.iter().find(|registration| { - compensation_registration_decision(registration.status) - != CompensationRegistrationDecision::SkipCompleted - }); - let Some(next) = next else { - return finalize_compensation_step(repository, scope, &snapshot.run, &scheduling).await; - }; - let claimed = match compensation_registration_decision(next.status) { - CompensationRegistrationDecision::Claim => match repository - .claim_next_compensation( - scope, - snapshot.run.run_uid, - next.compensation_id, - next.generation, - ) - .await - .map_err(execution_error)? - { - CompensationClaimOutcome::Claimed(registration) - | CompensationClaimOutcome::Replayed(registration) => registration, - CompensationClaimOutcome::BudgetRejected(_) => return Ok(RunDriveStep::Continue), - CompensationClaimOutcome::Conflict => return Ok(RunDriveStep::Continue), - CompensationClaimOutcome::NotFound => { - return Err( - TerminalError::new_with_code(404, "execution compensation not found").into(), - ); - } - }, - CompensationRegistrationDecision::Dispatch => next.clone(), - CompensationRegistrationDecision::Finalize => { - return finalize_compensation_step(repository, scope, &snapshot.run, &scheduling).await; - } - CompensationRegistrationDecision::SkipCompleted => return Ok(RunDriveStep::Continue), - }; - Ok(RunDriveStep::Compensate { - request: ExecutionCompensationWorkflowRequest { - run_uid: snapshot.run.run_uid, - compensation_id: claimed.compensation_id, - generation: claimed.generation, - tenant_id: snapshot.run.tenant_id, - contact_id: snapshot.run.contact_id, - session_id: snapshot.run.session_id, - identity: request.identity.clone(), - }, - }) -} - -async fn finalize_fenced_terminal_step( - repository: &ExecutionRepository, - scope: ExecutionScope, - run: &ExecutionRunRecord, - request: &ExecutionRunWorkflowRequest, -) -> Result { - match repository - .finalize_fenced_terminal(scope, run.run_uid, run.plan_revision, run.wake_epoch) - .await - .map_err(execution_error)? - { - FencedTerminalFinalizationOutcome::Finalized(finalized) - | FencedTerminalFinalizationOutcome::Replayed(finalized) - | FencedTerminalFinalizationOutcome::ManualRepairRequired(finalized) => { - Ok(RunDriveStep::Terminal { - task_ids: Vec::new(), - reason: format!("execution run ended as {}", finalized.status.as_str()), - }) - } - FencedTerminalFinalizationOutcome::ForwardTasksPending(tasks) => { - Ok(RunDriveStep::SettleForward { - tasks: tasks - .iter() - .map(|task| task_workflow_request(run, task, request)) - .collect(), - reason: "execution run fenced forward work before terminal settlement".to_string(), - }) - } - FencedTerminalFinalizationOutcome::Conflict => Ok(RunDriveStep::Continue), - FencedTerminalFinalizationOutcome::NotFound => { - Err(TerminalError::new_with_code(404, "execution run not found").into()) - } - } -} - -async fn finalize_compensation_step( - repository: &ExecutionRepository, - scope: ExecutionScope, - run: &ExecutionRunRecord, - scheduling: &moa_execution::repository::ExecutionSchedulingSnapshot, -) -> Result { - match repository - .finalize_compensation(scope, run.run_uid, run.wake_epoch) - .await - .map_err(execution_error)? - { - CompensationFinalizationOutcome::Finalized(finalized) - | CompensationFinalizationOutcome::Replayed(finalized) - | CompensationFinalizationOutcome::ManualRepairRequired(finalized) => Ok(terminal_step( - &scheduling.projection.tasks, - format!( - "execution compensation ended as {}", - finalized.status.as_str() - ), - )), - CompensationFinalizationOutcome::Conflict => Ok(RunDriveStep::Continue), - CompensationFinalizationOutcome::NotFound => { - Err(TerminalError::new_with_code(404, "execution run not found").into()) - } - } -} - -fn task_workflow_request( - run: &ExecutionRunRecord, - task: &moa_execution::repository::ExecutionTaskRecord, - request: &ExecutionRunWorkflowRequest, -) -> ExecutionTaskWorkflowRequest { - ExecutionTaskWorkflowRequest { - run_uid: run.run_uid, - task_id: task.task_id, - generation: task.generation, - tenant_id: run.tenant_id, - contact_id: run.contact_id, - session_id: run.session_id, - identity: request.identity.clone(), - } -} - -async fn signal_forward_task_cancellation( - ctx: &WorkflowContext<'_>, - task: &ExecutionTaskWorkflowRequest, - reason: &str, -) -> Result<(), HandlerError> { - signal_task_cancellation(ctx, task.task_id, reason).await -} - -async fn signal_task_cancellation( - ctx: &WorkflowContext<'_>, - task_id: ExecutionTaskId, - reason: &str, -) -> Result<(), HandlerError> { - let cancellation = crate::restate_identity::replay_safe_request( - ctx.workflow_client::(task_id.to_string()) - .cancel(Json::from(reason.to_string())), - ) - .call() - .await; - match cancellation { - Ok(()) => {} - Err(error) if error.code() == 404 => {} - Err(error) => return Err(error.into()), - } - Ok(()) -} - -async fn fence_replan_stop( - repository: &ExecutionRepository, - scope: ExecutionScope, - snapshot: moa_execution::repository::ExecutionSchedulingSnapshot, - stop: ReplanExhaustion, -) -> Result { - let ReplanExhaustion { - reason, - description, - } = stop; - if !snapshot - .projection - .tasks - .iter() - .any(|task| task.status == ExecutionTaskStatus::WaitingReplan) - { - return Err( - TerminalError::new("replan stop has no originating waiting-replan task").into(), - ); - } - let mut evaluation = evaluate_completion(CompletionEvaluationRequest { - goal: snapshot.run.goal.clone(), - plan: snapshot.run.active_plan.clone(), - run_input: snapshot.run.input.clone(), - projection: snapshot.projection.clone(), - terminal_output: snapshot.run.output.clone(), - budget_ledger: snapshot.budget_ledger.clone(), - now: chrono::Utc::now(), - }) - .map_err(execution_error)?; - evaluation.status = replan_stop_status( - snapshot.run.output.is_some(), - evaluation.satisfied_requirement_ids.len(), - ); - let stop_gaps = replan_stop_gaps(reason, Some(&description)); - evaluation.gaps.extend(stop_gaps.iter().cloned()); - evaluation.gaps.sort(); - evaluation.gaps.dedup(); - let terminal = terminal_projection_from_evaluation( - &evaluation, - snapshot.run.output.clone(), - None, - None, - None, - ) - .map_err(execution_error)?; - let terminal_evidence = terminal_evidence_from_evaluation( - ExecutionTerminalCause::ReplanStop { reason }, - &evaluation, - ) - .map_err(execution_error)?; - let terminal_reason = - execution_terminal_reason(&terminal_evidence.cause, &terminal, &evaluation) - .map_err(execution_error)?; - fence_terminal_before_settlement( - repository, - scope, - &snapshot.run, - TerminalFenceInput { - terminal: &terminal, - terminal_evidence: &terminal_evidence, - terminal_reason, - output: snapshot.run.output.clone(), - evaluation: &evaluation, - }, - ) - .await?; - Ok(RunDriveStep::Continue) -} - -fn immediately_knowable_replan_stop( - snapshot: &moa_execution::repository::ExecutionSchedulingSnapshot, -) -> Option { - if !snapshot - .projection - .tasks - .iter() - .any(|task| task.status == ExecutionTaskStatus::WaitingReplan) - { - return None; - } - replan_exhaustion_reason(&snapshot.budget_ledger, chrono::Utc::now()) -} - -async fn finalize_internal_failure( - repository: &ExecutionRepository, - scope: ExecutionScope, - snapshot: moa_execution::repository::ExecutionSchedulingSnapshot, - message: String, -) -> Result { - let mut unsatisfied_requirement_ids = snapshot - .run - .goal - .requirements - .iter() - .map(|requirement| requirement.id.clone()) - .collect::>(); - unsatisfied_requirement_ids.sort(); - let evaluation = CompletionEvaluation { - status: CompletionStatus::Failed, - limit_stop: None, - checks: Vec::new(), - satisfied_requirement_ids: Vec::new(), - unsatisfied_requirement_ids, - gaps: vec![format!("internal execution failure: {message}")], - }; - let terminal = TerminalProjection::Failed { - failure: moa_execution::state::ExecutionTaskFailure { - class: moa_artifacts::execution_plan::ExecutionFailureClass::Terminal, - message: message.clone(), - capability_ref: None, - }, - }; - let terminal_evidence = - terminal_evidence_from_evaluation(ExecutionTerminalCause::InternalFailure, &evaluation) - .map_err(execution_error)?; - let terminal_reason = - execution_terminal_reason(&terminal_evidence.cause, &terminal, &evaluation) - .map_err(execution_error)?; - if fence_terminal_before_settlement( - repository, - scope, - &snapshot.run, - TerminalFenceInput { - terminal: &terminal, - terminal_evidence: &terminal_evidence, - terminal_reason, - output: None, - evaluation: &evaluation, - }, - ) - .await? - { - return Ok(RunDriveStep::Continue); - } - match repository - .finalize_run( - scope, - RunFinalizationRequest { - run_uid: snapshot.run.run_uid, - expected_revision: snapshot.run.plan_revision, - expected_wake_epoch: snapshot.run.wake_epoch, - terminal_projection: terminal, - completion_evaluation: evaluation, - terminal_evidence, - terminal_reason, - }, - ) - .await - .map_err(execution_error)? - { - FinalizationOutcome::Finalized(_) => Ok(terminal_step( - &snapshot.projection.tasks, - format!("internal execution failure: {message}"), - )), - FinalizationOutcome::Replayed(_) => Ok(terminal_step( - &snapshot.projection.tasks, - format!("internal execution failure: {message}"), - )), - FinalizationOutcome::Conflict => Ok(RunDriveStep::Continue), - FinalizationOutcome::NotFound => { - Err(TerminalError::new_with_code(404, "execution run not found").into()) - } - } -} - -async fn finalize( - repository: &ExecutionRepository, - scope: ExecutionScope, - snapshot: moa_execution::repository::ExecutionSchedulingSnapshot, - mut terminal: TerminalProjection, - cause: ExecutionTerminalCause, -) -> Result { - let task_failure_gaps = snapshot - .projection - .tasks - .iter() - .filter_map(|task| { - task.outcome - .as_ref() - .and_then(|outcome| match &outcome.result { - moa_artifacts::execution_plan::ExecutionTaskResult::Failed { - message, .. - } - | moa_artifacts::execution_plan::ExecutionTaskResult::UnknownOutcome { - message, - } => Some(message.clone()), - _ => None, - }) - }) - .collect::>(); - let terminal_output = match &terminal { - TerminalProjection::Completed { output } => Some(output.clone()), - TerminalProjection::Partial { output, .. } | TerminalProjection::Blocked { output, .. } => { - output.clone() - } - TerminalProjection::Unsupported { .. } - | TerminalProjection::Failed { .. } - | TerminalProjection::Cancelled { .. } => None, - }; - let task_ids = snapshot - .projection - .tasks - .iter() - .map(|task| task.task_id) - .collect::>(); - let mut evaluation = evaluate_completion(CompletionEvaluationRequest { - goal: snapshot.run.goal.clone(), - plan: snapshot.run.active_plan.clone(), - run_input: snapshot.run.input.clone(), - projection: snapshot.projection, - terminal_output: terminal_output.clone(), - budget_ledger: snapshot.budget_ledger, - now: chrono::Utc::now(), - }) - .map_err(execution_error)?; - evaluation.gaps.extend(task_failure_gaps); - evaluation.gaps.sort(); - evaluation.gaps.dedup(); - if !terminal_projection_matches_completion(&terminal, evaluation.status) { - terminal = terminal_projection_from_evaluation( - &evaluation, - terminal_output.clone(), - None, - None, - None, - ) - .map_err(execution_error)?; - } - let terminal_reason = format!("execution run reached terminal projection {terminal:?}"); - let terminal_evidence = - terminal_evidence_from_evaluation(cause, &evaluation).map_err(execution_error)?; - let selected_terminal_reason = - execution_terminal_reason(&terminal_evidence.cause, &terminal, &evaluation) - .map_err(execution_error)?; - if fence_terminal_before_settlement( - repository, - scope, - &snapshot.run, - TerminalFenceInput { - terminal: &terminal, - terminal_evidence: &terminal_evidence, - terminal_reason: selected_terminal_reason, - output: terminal_output.clone(), - evaluation: &evaluation, - }, - ) - .await? - { - return Ok(RunDriveStep::Continue); - } - match repository - .finalize_run( - scope, - RunFinalizationRequest { - run_uid: snapshot.run.run_uid, - expected_revision: snapshot.run.plan_revision, - expected_wake_epoch: snapshot.run.wake_epoch, - terminal_projection: terminal, - completion_evaluation: evaluation, - terminal_evidence, - terminal_reason: selected_terminal_reason, - }, - ) - .await - .map_err(execution_error)? - { - FinalizationOutcome::Finalized(_) => Ok(RunDriveStep::Terminal { - task_ids, - reason: terminal_reason, - }), - FinalizationOutcome::Replayed(_) => Ok(RunDriveStep::Terminal { - task_ids, - reason: terminal_reason, - }), - FinalizationOutcome::Conflict => Ok(RunDriveStep::Continue), - FinalizationOutcome::NotFound => { - Err(TerminalError::new_with_code(404, "execution run not found").into()) - } - } -} - -struct TerminalFenceInput<'a> { - terminal: &'a TerminalProjection, - terminal_evidence: &'a moa_execution::state::ExecutionTerminalEvidence, - terminal_reason: moa_execution::state::ExecutionTerminalReason, - output: Option, - evaluation: &'a CompletionEvaluation, -} - -async fn fence_terminal_before_settlement( - repository: &ExecutionRepository, - scope: ExecutionScope, - run: &ExecutionRunRecord, - input: TerminalFenceInput<'_>, -) -> Result { - let status = run_status_from_terminal_projection(input.terminal); - if status == ExecutionRunStatus::Completed { - return Ok(false); - } - let cancellation_reason = match input.terminal { - TerminalProjection::Cancelled { reason } => Some(reason.clone()), - TerminalProjection::Completed { .. } - | TerminalProjection::Partial { .. } - | TerminalProjection::Blocked { .. } - | TerminalProjection::Unsupported { .. } - | TerminalProjection::Failed { .. } => None, - }; - match repository - .fence_run_for_terminal( - scope, - run.run_uid, - run.plan_revision, - run.wake_epoch, - PendingExecutionTerminal { - status, - reason: input.terminal_reason, - terminal_evidence: input.terminal_evidence.clone(), - output: input.output, - completion_check_results: input - .evaluation - .checks - .iter() - .map(|check| { - serde_json::to_value(check).map_err(|error| { - TerminalError::new(format!( - "serialize terminal completion check: {error}" - )) - }) - }) - .collect::, _>>()?, - terminal_gaps: input.evaluation.gaps.clone(), - cancellation_reason, - }, - ) - .await - .map_err(execution_error)? - { - TerminalFenceOutcome::Applied(_) | TerminalFenceOutcome::Replayed(_) => Ok(true), - TerminalFenceOutcome::Conflict => Ok(true), - TerminalFenceOutcome::NotFound => { - Err(TerminalError::new_with_code(404, "execution run not found").into()) - } - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum CompensationEntryDecision { - Begin, - FinalizeFenced, -} - -const fn compensation_entry_decision( - policy: ExecutionCancelPolicy, - status: ExecutionRunStatus, - manual_repair_required: bool, -) -> CompensationEntryDecision { - if manual_repair_required - || matches!(status, ExecutionRunStatus::Completed) - || (matches!(status, ExecutionRunStatus::Cancelled) - && matches!(policy, ExecutionCancelPolicy::RetainEffects)) - { - CompensationEntryDecision::FinalizeFenced - } else { - CompensationEntryDecision::Begin - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum CompensationRegistrationDecision { - Claim, - Dispatch, - Finalize, - SkipCompleted, -} - -const fn compensation_registration_decision( - status: CompensationStatus, -) -> CompensationRegistrationDecision { - match status { - CompensationStatus::Pending => CompensationRegistrationDecision::Claim, - CompensationStatus::Running => CompensationRegistrationDecision::Dispatch, - CompensationStatus::Failed | CompensationStatus::UnknownOutcome => { - CompensationRegistrationDecision::Finalize - } - CompensationStatus::Completed => CompensationRegistrationDecision::SkipCompleted, - } -} - -fn terminal_step( - tasks: &[moa_execution::state::ExecutionTaskProjection], - reason: String, -) -> RunDriveStep { - RunDriveStep::Terminal { - task_ids: tasks.iter().map(|task| task.task_id).collect(), - reason, - } -} - -fn node_materialization_marker( - run: &ExecutionRunRecord, - node_id: &str, - tasks: &[moa_execution::state::LogicalTask], -) -> Result, HandlerError> { - let node = run - .active_plan - .definition - .nodes - .iter() - .find(|node| node.id == node_id) - .ok_or_else(|| TerminalError::new("materialized execution node disappeared"))?; - match &node.operation { - ExecutionOperation::Map { .. } => Ok(Some(ExecutionNodeMaterialization::Map { - node_id: node_id.to_string(), - fanout_items: u64::try_from(tasks.len()) - .map_err(|_| TerminalError::new("map fanout exceeds u64"))?, - })), - ExecutionOperation::Reduce { batch_size, .. } => { - let mut item_count = 0_u64; - for task in tasks { - let count = task - .input - .get("items") - .and_then(Value::as_array) - .map(Vec::len) - .ok_or_else(|| TerminalError::new("reducer task omitted its item batch"))?; - item_count = item_count - .checked_add( - u64::try_from(count) - .map_err(|_| TerminalError::new("reducer batch exceeds u64"))?, - ) - .ok_or_else(|| TerminalError::new("reducer item count overflowed"))?; - } - Ok(Some(ExecutionNodeMaterialization::Reduce { - node_id: node_id.to_string(), - reducer_depth: reducer_depth(item_count, *batch_size), - })) - } - ExecutionOperation::Capability { .. } - | ExecutionOperation::Agent { .. } - | ExecutionOperation::Review { .. } - | ExecutionOperation::WaitSignal { .. } - | ExecutionOperation::Output { .. } => Ok(None), - } -} - -fn reducer_depth(mut item_count: u64, batch_size: u32) -> u64 { - let batch_size = u64::from(batch_size); - let mut depth = 0_u64; - while item_count > 1 { - item_count = item_count.div_ceil(batch_size); - depth = depth.saturating_add(1); - } - depth -} - -fn select_dispatch_batch( - tasks: Vec, - owned_task_ids: &BTreeSet, - available_slots: usize, -) -> Vec { - tasks - .into_iter() - .filter(|task| !owned_task_ids.contains(&task.task_id)) - .take(available_slots) - .collect() -} - -fn park_at_epoch(run: &ExecutionRunRecord) -> RunDriveStep { - RunDriveStep::Park { - processed_epoch: run.wake_epoch, - } -} - -fn wait_transition_step( - waiting_status: ExecutionRunStatus, - transition_applied: bool, - plan_revision: u64, - processed_epoch: u64, -) -> RunDriveStep { - if waiting_status == ExecutionRunStatus::WaitingReplan { - RunDriveStep::PlanAmendment { plan_revision } - } else if transition_applied { - RunDriveStep::Continue - } else { - RunDriveStep::Park { processed_epoch } - } -} - -fn waiting_status( - tasks: &[moa_execution::state::ExecutionTaskProjection], - waiting: &[WaitingReason], -) -> ExecutionRunStatus { - if tasks - .iter() - .any(|task| task.status == ExecutionTaskStatus::WaitingReplan) - { - ExecutionRunStatus::WaitingReplan - } else if waiting - .iter() - .any(|reason| matches!(reason, WaitingReason::Input { .. })) - { - ExecutionRunStatus::WaitingInput - } else if waiting.iter().any(|reason| { - matches!( - reason, - WaitingReason::Review { .. } | WaitingReason::Signal { .. } - ) - }) { - ExecutionRunStatus::WaitingReview - } else { - ExecutionRunStatus::Running - } -} - -fn execution_scope(request: &ExecutionRunWorkflowRequest) -> ExecutionScope { - request.contact_id.map_or( - ExecutionScope::Tenant { - tenant_id: request.tenant_id, - }, - |contact_id| ExecutionScope::Contact { - tenant_id: request.tenant_id, - contact_id, - }, - ) -} - -fn annotate_execution_run_span(run_uid: uuid::Uuid) { - tracing::Span::current().set_attribute("moa.execution.run_uid", run_uid.to_string()); -} - -fn wake_promise_key(processed_epoch: u64) -> String { - format!("execution_run_wake_after_{processed_epoch}") -} - -fn wake_promise_epoch( - processed_epoch: u64, - awaited_epoch: u64, - received_epoch: u64, -) -> Option { - (received_epoch > processed_epoch && received_epoch > awaited_epoch).then_some(awaited_epoch) -} - -fn execution_error(error: moa_execution::Error) -> HandlerError { - TerminalError::new(format!("execution run workflow failed: {error}")).into() -} - -#[cfg(test)] -mod tests { - use std::collections::{BTreeMap, BTreeSet}; - - use chrono::{Duration, Utc}; - use moa_artifacts::execution_plan::{ - CapabilityReference, CompletionCheck, CompletionCheckKind, ExecutionBudgetLimit, - ExecutionCancelPolicy, ExecutionFailureClass, ExecutionGoalContract, ExecutionNode, - ExecutionOperation, ExecutionPlanDefinition, ExecutionRequirement, ExecutionTaskOutcome, - ExecutionTaskResult, ExecutionUsage, GeneratedAmendmentCandidate, PlanAmendment, - PlanAmendmentOperation, RetryPolicy, - }; - use moa_core::traits::{Identity, IdentityType}; - use moa_core::types::{ - action_policy::{ActionClass, ActionPolicyEffect, RiskLevel}, - execution_planning::{ExecutionSourceProvenance, GeneratedPlanPlannerProvenance}, - identifiers::{ModelId, SessionId, TenantId, UserId}, - model::ModelCapabilities, - tools::IdempotencyClass, - }; - use moa_execution::{ - ReplanStopReason, - budget::BudgetLedger, - capability::{ - CapabilityPolicyContext, CapabilitySource, ExecutionAuthorizationEnvelope, - ExecutionCapability, ExecutionCapabilityCatalog, ExecutionClass, ExecutionEstimate, - ExecutionHash, - }, - compiler::{ - CompileExecutionRequest, ValidateAmendmentRequest, compile, validate_amendment, - }, - repository::{ - ConfirmationOutcome, NewExecutionPlanningContext, NewExecutionRun, - PlanningContextWriteOutcome, ReservationOutcome, TaskOutcomeWrite, TransitionOutcome, - }, - state::{ - CompensationStatus, ExecutionLimitStop, ExecutionNodeStatus, ExecutionProjection, - ExecutionTaskFailure, ExecutionTaskId, ExecutionTerminalCause, LogicalTask, - LogicalTaskKind, TerminalProjection, - }, - wire::{ - ExecutionAmendmentRequest, ExecutionMutationResponse, ExecutionPlanningContextSnapshot, - ExecutionRunRequest, ExecutionRunWorkflowRequest, ExecutionTaskWorkflowRequest, - planning_context_hash, - }, - }; - use moa_providers::ScriptedProvider; - use serde_json::json; - use uuid::Uuid; - - use super::{ - CompensationEntryDecision, CompensationRegistrationDecision, ExecutionRunStatus, - ReplanExhaustion, RunDriveStep, bounded_failure_evidence, compensation_entry_decision, - compensation_registration_decision, narrow_amendment_context, - narrow_authorized_capability_refs, prepare_amendment_planning, replan_exhaustion_reason, - select_dispatch_batch, terminal_cause, wait_transition_step, wake_promise_epoch, - }; - - #[tokio::test] - async fn waiting_replan_uses_confirmed_budget_for_planning_apply_and_replay_db() { - // Pins: confirmation may replace the initial planning budget, after which - // WaitingReplan must use only the persisted run ledger for planning and apply. - let test_db = ExecutionRunTestDb::new().await; - let pool = test_db.pool().clone(); - let repository = moa_execution::repository::ExecutionRepository::new(pool.clone()); - let tenant_id = TenantId::new(); - let session_id = SessionId::new(); - let owner_user_id = UserId::new("confirmed-replan-owner"); - let scope = moa_execution::repository::ExecutionScope::Tenant { tenant_id }; - let catalog = ExecutionCapabilityCatalog::build(Vec::new()) - .expect("empty capability catalog should be valid"); - let authorization = ExecutionAuthorizationEnvelope { - capability_refs: Vec::new(), - skill_refs: Vec::new(), - }; - let planning_budget = replan_budget(1_000_000, 10); - let confirmed_budget = replan_budget(2_000_000, 3); - let goal = replan_goal(); - let compile_outcome = compile(CompileExecutionRequest { - goal: goal.clone(), - plan: replan_plan(), - run_input: json!({}), - catalog: catalog.clone(), - authorization: authorization.clone(), - approved_budget: planning_budget.clone(), - config: moa_config::ExecutionConfig::default(), - now: Utc::now(), - }); - let compiled = compile_outcome.compiled.unwrap_or_else(|| { - panic!( - "replan fixture should compile within the initial planning budget: {:?}", - compile_outcome.report.issues - ) - }); - let compiled_plan = compiled.plan; - let planning_snapshot = ExecutionPlanningContextSnapshot { - schema_version: 1, - tenant_id, - contact_id: None, - session_id, - originating_user_sequence_num: 17, - originating_user_event_hash: ExecutionHash::from_bytes([17; 32]).to_string(), - owner_user_id: owner_user_id.clone(), - catalog: catalog.clone(), - authorization: authorization.clone(), - pinned_instruction_skills: Vec::new(), - execution_templates: Vec::new(), - budget: planning_budget.clone(), - }; - let planning_hash = planning_context_hash(&planning_snapshot) - .expect("planning snapshot should have a canonical hash"); - let PlanningContextWriteOutcome::Created(planning_context) = repository - .create_planning_context( - scope, - NewExecutionPlanningContext { - snapshot: planning_snapshot, - planning_context_hash: planning_hash, - }, - ) - .await - .expect("planning context should persist") - else { - panic!("fresh planning context should be created"); - }; - let run = repository - .create_run( - scope, - NewExecutionRun { - tenant_id, - contact_id: None, - session_id, - originating_user_sequence_num: 17, - planning_context_uid: planning_context.planning_context_uid, - planning_context_hash: planning_context.planning_context_hash, - owner_user_id, - goal, - plan: compiled_plan.clone(), - catalog, - authorization, - pinned_instruction_skills: Vec::new(), - source_provenance: ExecutionSourceProvenance::GeneratedPlan { - planner: GeneratedPlanPlannerProvenance { - model: "scripted-confirmed-replan".to_string(), - prompt_version: "confirmed-replan".to_string(), - candidate_hash: "a".repeat(64), - compiler_report_hash: "b".repeat(64), - final_plan_hash: compiled_plan.plan_hash.to_string(), - repair_attempts: 0, - }, - }, - input: json!({}), - status: ExecutionRunStatus::AwaitingConfirmation, - approved_budget: planning_budget.clone(), - idempotency_key: Some("confirmed-replan-budget".to_string()), - }, - ) - .await - .expect("awaiting-confirmation run should persist"); - let ConfirmationOutcome::Confirmed(confirmed) = repository - .confirm_run( - scope, - run.run_uid, - &run.active_plan_hash, - confirmed_budget.clone(), - ) - .await - .expect("confirmation write should succeed") - else { - panic!("confirmation should replace the approved budget"); - }; - assert_eq!(confirmed.approved_budget, confirmed_budget); - - let tasks = vec![ - replan_task(run.run_uid, "prepare", json!({"value": "prepared"})), - replan_task(run.run_uid, "output", json!({"value": "stale"})), - ]; - repository - .materialize_tasks(scope, run.run_uid, 1, tasks.clone()) - .await - .expect("confirmed run should materialize tasks"); - start_task(&repository, scope, run.run_uid, tasks[0].task_id).await; - assert!(matches!( - repository - .record_task_outcome( - scope, - run.run_uid, - tasks[0].task_id, - 1, - replan_outcome(ExecutionTaskResult::Completed { - output: json!({"value": "prepared"}), - citations: Vec::new(), - }), - ) - .await - .expect("prepare outcome should persist"), - TaskOutcomeWrite::Applied { .. } - )); - start_task(&repository, scope, run.run_uid, tasks[1].task_id).await; - assert!(matches!( - repository - .record_task_outcome( - scope, - run.run_uid, - tasks[1].task_id, - 1, - replan_outcome(ExecutionTaskResult::NeedsReplan { - reason: "shape changed".to_string(), - evidence: json!({"kind": "confirmed-budget"}), - }), - ) - .await - .expect("NeedsReplan outcome should persist"), - TaskOutcomeWrite::Applied { .. } - )); - - let request = ExecutionRunWorkflowRequest { - run_uid: run.run_uid, - tenant_id, - contact_id: None, - session_id, - identity: moa_core::traits::Identity { - identity_type: moa_core::traits::IdentityType::Operator, - id: Uuid::from_u128(1), - tenant_id, - api_key_id: None, - acting_on_behalf_of: None, - }, - }; - let prepared = prepare_amendment_planning(&repository, scope, &request, 1) - .await - .expect("confirmed WaitingReplan should prepare amendment planning") - .expect("active WaitingReplan revision should produce planner input"); - assert_eq!(prepared.context.budget, confirmed_budget); - assert_eq!( - repository - .load_planning_context(scope, planning_context.planning_context_uid) - .await - .expect("immutable planning context should reload") - .expect("immutable planning context should remain present") - .snapshot - .budget, - planning_budget - ); - assert_eq!(prepared.remaining_budget.max_cost_microusd, Some(1_999_997)); - assert_eq!(prepared.remaining_budget.max_tokens, Some(199_997)); - assert_eq!(prepared.remaining_budget.max_tasks, Some(1)); - - let provider = ScriptedProvider::new(ModelCapabilities::default()) - .push_text(replan_amendment_candidate()); - let planned = moa_brain::execution_planning::plan_amendment( - &provider, - moa_brain::execution_planning::ExecutionAmendmentPlanningRequest { - run_uid: run.run_uid, - base_plan_revision: 1, - context: prepared.context, - evidence: prepared.evidence, - remaining_budget: prepared.remaining_budget, - planner_model: ModelId::new("scripted-confirmed-replan"), - config: moa_config::ExecutionConfig::default(), - now: prepared.now, - }, - ) - .await - .expect("persisted confirmed budget should permit amendment planning"); - assert_eq!(provider.recorded_requests().len(), 1); - let planner_prompt = serde_json::to_string(&provider.recorded_requests()[0].messages) - .expect("recorded amendment request should serialize"); - assert!( - planner_prompt.contains("1999997"), - "amendment planner prompt must carry the reconciled confirmed budget" - ); - let moa_brain::execution_planning::ExecutionAmendmentPlanningResultKind::Ready { - amendment, - .. - } = planned.kind - else { - panic!("valid confirmed-budget amendment should be ready"); - }; - let amendment_request = ExecutionAmendmentRequest { - run: ExecutionRunRequest { - tenant_id, - contact_id: None, - session_id, - run_uid: run.run_uid, - }, - expected_plan_revision: 1, - amendment, - }; - let applied = crate::services::execution::handlers::apply_amendment_for_test( - pool.clone(), - moa_config::ExecutionConfig::default(), - amendment_request.clone(), - ) - .await - .expect("planned amendment should apply through the production service boundary"); - assert!( - matches!( - applied, - ExecutionMutationResponse::Applied { ref run } if run.plan_revision == 2 - ), - "planned amendment should apply revision two: {applied:?}" - ); - let replayed = crate::services::execution::handlers::apply_amendment_for_test( - pool, - moa_config::ExecutionConfig::default(), - amendment_request.clone(), - ) - .await - .expect("exact amendment replay should remain idempotent"); - assert!(matches!( - replayed, - ExecutionMutationResponse::Replayed { ref run } if run.plan_revision == 2 - )); - - let mut injected = - serde_json::to_value(amendment_request).expect("amendment request should serialize"); - injected - .as_object_mut() - .expect("amendment request wire shape should be an object") - .insert( - "approved_budget".to_string(), - json!({"max_tasks": 1_000_000}), - ); - serde_json::from_value::(injected) - .expect_err("caller-supplied amendment budget authority must be rejected"); - serde_json::from_value::(json!({ - "amendment": replan_amendment_value(), - "approved_budget": {"max_tasks": 1_000_000} - })) - .expect_err("model-supplied amendment budget authority must be rejected"); - } - - fn replan_budget(max_resource: u64, max_tasks: u64) -> ExecutionBudgetLimit { - // Truncated to microseconds so equality against a Postgres round-trip - // is exact: nanosecond-granular CI clocks otherwise fail assertions - // that microsecond-granular local clocks let pass. - let deadline = Utc::now() + Duration::hours(1); - let deadline = chrono::DateTime::::from_timestamp_micros(deadline.timestamp_micros()) - .expect("hour-offset deadline is representable at microsecond precision"); - ExecutionBudgetLimit { - max_cost_microusd: Some(max_resource), - max_tokens: Some(max_resource / 10), - max_tasks: Some(max_tasks), - max_tool_calls: Some(max_resource / 10_000), - max_retrieved_bytes: Some(max_resource.saturating_mul(20)), - deadline_at: Some(deadline), - } - } - - struct ExecutionRunTestDb { - pool: Option, - database_url: String, - schema_name: String, - } - - impl ExecutionRunTestDb { - async fn new() -> Self { - let (database_url, schema_name) = moa_session::testing::provision_cloned_database() - .await - .expect("execution test database should provision"); - let pool = sqlx::PgPool::connect(&database_url) - .await - .expect("execution test database should connect"); - Self { - pool: Some(pool), - database_url, - schema_name, - } - } - - fn pool(&self) -> &sqlx::PgPool { - self.pool - .as_ref() - .expect("execution test database pool should remain available") - } - } - - impl Drop for ExecutionRunTestDb { - fn drop(&mut self) { - let Some(pool) = self.pool.take() else { - return; - }; - let database_url = self.database_url.clone(); - let schema_name = self.schema_name.clone(); - let cleanup = std::thread::spawn(move || { - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .expect("execution test cleanup runtime should build"); - runtime.block_on(async move { - pool.close().await; - moa_session::testing::cleanup_test_schema(&database_url, &schema_name).await - }) - }); - cleanup - .join() - .expect("execution test cleanup thread should not panic") - .expect("execution test database should clean up"); - } - } - - fn replan_goal() -> ExecutionGoalContract { - ExecutionGoalContract { - objective: "repair with the confirmed budget".to_string(), - requirements: vec![ - ExecutionRequirement { - id: "req_inputs".to_string(), - description: "prepare report inputs".to_string(), - }, - ExecutionRequirement { - id: "req_report".to_string(), - description: "produce the repaired report".to_string(), - }, - ], - deliverables: Vec::new(), - coverage: Vec::new(), - constraints: Vec::new(), - // Both requirements are covered by the terminal output check so the - // linkage survives amendments that rename the prepare/output nodes - // (a node-referencing check would dangle after ReplacePendingNode). - completion_checks: vec![CompletionCheck { - id: "check_output".to_string(), - description: "validate the repaired output".to_string(), - requirement_ids: vec!["req_inputs".to_string(), "req_report".to_string()], - constraint_ids: Vec::new(), - kind: CompletionCheckKind::OutputSchema, - }], - } - } - - fn replan_plan() -> ExecutionPlanDefinition { - ExecutionPlanDefinition { - cancel_policy: ExecutionCancelPolicy::RetainEffects, - input_schema: json!({"type": "object"}), - output_schema: json!({"type": "object"}), - nodes: vec![ - ExecutionNode { - id: "prepare".to_string(), - requirement_ids: vec!["req_inputs".to_string()], - depends_on: Vec::new(), - when: None, - input: json!({}), - output_schema: json!({"type": "object"}), - operation: ExecutionOperation::Agent { - instructions: "prepare report inputs".to_string(), - skill_refs: Vec::new(), - capability_refs: Vec::new(), - max_turns: 1, - }, - compensation: None, - retry: RetryPolicy { - max_attempts: 1, - initial_backoff_ms: 0, - max_backoff_ms: 0, - }, - budget: None, - }, - replan_node( - "output", - vec!["prepare".to_string()], - json!({"value": "stale"}), - ), - ], - } - } - - fn replan_node(id: &str, depends_on: Vec, value: serde_json::Value) -> ExecutionNode { - ExecutionNode { - id: id.to_string(), - requirement_ids: vec!["req_report".to_string()], - depends_on, - when: None, - input: json!({}), - output_schema: json!({"type": "object"}), - operation: ExecutionOperation::Output { value }, - compensation: None, - retry: RetryPolicy { - max_attempts: 1, - initial_backoff_ms: 0, - max_backoff_ms: 0, - }, - budget: None, - } - } - - fn replan_task(run_uid: Uuid, node_id: &str, value: serde_json::Value) -> LogicalTask { - LogicalTask { - task_id: ExecutionTaskId::derive(run_uid, node_id, "") - .expect("fixture task id should derive"), - node_id: node_id.to_string(), - item_key: String::new(), - requirement_ids: if node_id == "prepare" { - vec!["req_inputs".to_string()] - } else { - vec!["req_report".to_string()] - }, - plan_revision: 1, - generation: 1, - input: json!({}), - kind: if node_id == "prepare" { - LogicalTaskKind::Agent { - instructions: "prepare report inputs".to_string(), - skill_refs: Vec::new(), - capability_refs: Vec::new(), - max_turns: 1, - } - } else { - LogicalTaskKind::Output { value } - }, - compensation: None, - retry: RetryPolicy { - max_attempts: 1, - initial_backoff_ms: 0, - max_backoff_ms: 0, - }, - reservation: ExecutionEstimate { - cost_microusd: 2, - tokens: 2, - tasks: 1, - tool_calls: 2, - retrieved_bytes: 2, - }, - } - } - - async fn start_task( - repository: &moa_execution::repository::ExecutionRepository, - scope: moa_execution::repository::ExecutionScope, - run_uid: Uuid, - task_id: ExecutionTaskId, - ) { - assert!(matches!( - repository - .reserve_task(scope, run_uid, task_id, 1) - .await - .expect("task reservation should succeed"), - ReservationOutcome::Reserved(_) - )); - assert!(matches!( - repository - .mark_task_running(scope, run_uid, task_id, 1) - .await - .expect("task start should succeed"), - TransitionOutcome::Applied(_) - )); - } - - fn replan_outcome(result: ExecutionTaskResult) -> ExecutionTaskOutcome { - ExecutionTaskOutcome { - schema_version: 1, - usage: ExecutionUsage { - cost_microusd: 1, - tokens: 1, - tool_calls: 1, - retrieved_bytes: 1, - }, - result, - } - } - - fn replan_amendment_candidate() -> String { - json!({"amendment": replan_amendment_value()}).to_string() - } - - fn replan_amendment_value() -> serde_json::Value { - json!({ - "base_plan_revision": 1, - "reason": "replace stale output", - "evidence": {"shape": "changed"}, - "operations": [ - {"kind": "remove_pending_node", "node_id": "output"}, - { - "kind": "add_node", - "node": { - "id": "replacement_output", - "requirement_ids": ["req_report"], - "depends_on": ["prepare"], - "when": null, - "input": {}, - "output_schema": {"type": "object"}, - "operation": { - "kind": "output", - "value": {"value": "repaired"} - }, - "compensation": null, - "retry": { - "max_attempts": 1, - "initial_backoff_ms": 0, - "max_backoff_ms": 0 - }, - "budget": null - } - } - ] - }) - } - - fn dispatch_task(sequence: u128) -> ExecutionTaskWorkflowRequest { - let tenant_id = TenantId::from(Uuid::from_u128(10)); - ExecutionTaskWorkflowRequest { - run_uid: Uuid::from_u128(20), - task_id: ExecutionTaskId::from_uuid(Uuid::from_u128(sequence)), - generation: 1, - tenant_id, - contact_id: None, - session_id: SessionId(Uuid::from_u128(30)), - identity: Identity { - identity_type: IdentityType::Operator, - id: Uuid::from_u128(40), - tenant_id, - api_key_id: None, - acting_on_behalf_of: None, - }, - } - } - - fn dispatch_task_ids(tasks: &[ExecutionTaskWorkflowRequest]) -> Vec { - tasks.iter().map(|task| task.task_id).collect() - } - - #[test] - fn dispatch_window_preserves_stable_candidate_order() { - // Pins: a bounded refill selects the first persisted candidates without - // reordering their stable scheduler/materialization order. - let tasks = vec![dispatch_task(3), dispatch_task(1), dispatch_task(2)]; - - let selected = select_dispatch_batch(tasks, &BTreeSet::new(), 2); - - assert_eq!( - dispatch_task_ids(&selected), - vec![ - ExecutionTaskId::from_uuid(Uuid::from_u128(3)), - ExecutionTaskId::from_uuid(Uuid::from_u128(1)), - ] - ); - } - - #[test] - fn dispatch_window_full_capacity_selects_nothing_before_parking() { - // Pins: a full owned-call window starts no additional task before the - // driver acknowledges its epoch and parks on promises/attached calls. - let owned = BTreeSet::from([ - ExecutionTaskId::from_uuid(Uuid::from_u128(1)), - ExecutionTaskId::from_uuid(Uuid::from_u128(2)), - ]); - let tasks = vec![dispatch_task(1), dispatch_task(2), dispatch_task(3)]; - - let selected = select_dispatch_batch(tasks, &owned, 0); - - assert!(selected.is_empty()); - } - - #[test] - fn dispatch_window_refills_exactly_one_open_slot() { - // Pins: one completed attached call opens exactly one physical slot, - // even when multiple durable pending task rows are ready. - let tasks = vec![dispatch_task(1), dispatch_task(2), dispatch_task(3)]; - - let selected = select_dispatch_batch(tasks, &BTreeSet::new(), 1); - - assert_eq!( - dispatch_task_ids(&selected), - vec![ExecutionTaskId::from_uuid(Uuid::from_u128(1))] - ); - } - - #[test] - fn dispatch_window_replay_skips_already_owned_task_ids() { - // Pins: replayed pending rows whose attached calls are already journal-owned - // do not consume refill slots or create duplicate task invocations. - let owned = BTreeSet::from([ - ExecutionTaskId::from_uuid(Uuid::from_u128(1)), - ExecutionTaskId::from_uuid(Uuid::from_u128(3)), - ]); - let tasks = vec![ - dispatch_task(1), - dispatch_task(2), - dispatch_task(3), - dispatch_task(4), - ]; - - let selected = select_dispatch_batch(tasks, &owned, 2); - - assert_eq!( - dispatch_task_ids(&selected), - vec![ - ExecutionTaskId::from_uuid(Uuid::from_u128(2)), - ExecutionTaskId::from_uuid(Uuid::from_u128(4)), - ] - ); - } - - #[test] - fn waiting_replan_invokes_revision_keyed_amendment_planner_instead_of_parking() { - // Pins: the Task 6 driver hands one persisted WaitingReplan revision to - // Task 7 instead of treating the wait as an externally awakened state. - let step = wait_transition_step(ExecutionRunStatus::WaitingReplan, true, 7, 11); - - assert!(matches!( - step, - RunDriveStep::PlanAmendment { plan_revision: 7 } - )); - } - - #[test] - fn persisted_waiting_replan_still_invokes_revision_keyed_amendment_planner() { - // Pins: NeedsReplan outcome persistence may transition the run before the - // scheduler observes it; that exact replay remains internally actionable. - let step = wait_transition_step(ExecutionRunStatus::WaitingReplan, false, 7, 11); - - assert!(matches!( - step, - RunDriveStep::PlanAmendment { plan_revision: 7 } - )); - } - - #[test] - fn replayed_wait_transition_parks_at_the_processed_epoch() { - // Pins: an unchanged persisted wait does not become a status-read loop; - // it suspends on the epoch promise and any attached task calls. - let step = wait_transition_step(ExecutionRunStatus::Running, false, 7, 11); - - assert!(matches!( - step, - RunDriveStep::Park { - processed_epoch: 11 - } - )); - } - - #[test] - fn amendment_context_preserves_persisted_catalog_and_only_narrows_authorization() { - // Pins: live availability narrows transient amendment authority without - // changing the immutable catalog hash pinned by the active plan. - let retained = amendment_tool_capability("persisted.retained", "retained-tool"); - let unavailable = amendment_tool_capability("persisted.unavailable", "unavailable-tool"); - let retained_ref = retained.reference.clone(); - let unavailable_ref = unavailable.reference.clone(); - let catalog = ExecutionCapabilityCatalog::build(vec![retained, unavailable]) - .expect("two tool-backed capabilities should form a valid catalog"); - let authorization = ExecutionAuthorizationEnvelope { - capability_refs: catalog - .capabilities - .iter() - .map(|capability| capability.reference.clone()) - .collect(), - skill_refs: Vec::new(), - }; - let source = ExecutionPlanningContextSnapshot { - schema_version: 1, - tenant_id: TenantId::new(), - contact_id: None, - session_id: SessionId::new(), - originating_user_sequence_num: 23, - originating_user_event_hash: ExecutionHash::from_bytes([23; 32]).to_string(), - owner_user_id: UserId::new("amendment-context-owner"), - catalog, - authorization, - pinned_instruction_skills: Vec::new(), - execution_templates: Vec::new(), - budget: replan_budget(1_000_000, 10), - }; - let persisted_source = source.clone(); - - let narrowed = narrow_amendment_context( - source.clone(), - &BTreeSet::from(["retained-tool".to_string()]), - ) - .expect("live availability should narrow a valid planning context"); - - assert_eq!(narrowed.catalog, persisted_source.catalog); - assert_eq!( - narrowed.catalog.catalog_hash, - persisted_source.catalog.catalog_hash - ); - assert_eq!( - moa_core::canonical_json::canonical_json_bytes(&narrowed.catalog) - .expect("narrowed catalog should serialize canonically"), - moa_core::canonical_json::canonical_json_bytes(&persisted_source.catalog) - .expect("persisted catalog should serialize canonically") - ); - assert_eq!( - narrowed.authorization.capability_refs, - vec![retained_ref.clone()] - ); - let live_only_ref = CapabilityReference { - name: "live.only".to_string(), - version: "v1".to_string(), - }; - assert!( - !narrowed - .authorization - .capability_refs - .contains(&live_only_ref) - ); - assert_eq!(source, persisted_source); - - let mut active_plan = replan_plan(); - active_plan.nodes[0].operation = ExecutionOperation::Capability { - reference: retained_ref.clone(), - }; - let compile_outcome = compile(CompileExecutionRequest { - goal: replan_goal(), - plan: active_plan, - run_input: json!({}), - catalog: persisted_source.catalog.clone(), - authorization: persisted_source.authorization.clone(), - approved_budget: persisted_source.budget.clone(), - config: moa_config::ExecutionConfig::default(), - now: Utc::now(), - }); - let compiled = compile_outcome.compiled.unwrap_or_else(|| { - panic!( - "active amendment fixture should compile: {:?}", - compile_outcome.report.issues - ) - }); - let mut replacement_prepare = compiled.plan.definition.nodes[0].clone(); - replacement_prepare.id = "replacement_prepare".to_string(); - let mut replacement_output = compiled.plan.definition.nodes[1].clone(); - replacement_output.id = "replacement_output".to_string(); - replacement_output.depends_on = vec!["replacement_prepare".to_string()]; - replacement_output.operation = ExecutionOperation::Output { - value: json!({"$ref": "$.nodes.replacement_prepare.output"}), - }; - let validation = ValidateAmendmentRequest { - goal: compiled.goal, - active_plan: compiled.plan, - amendment: PlanAmendment { - base_plan_revision: 1, - reason: "Use the retained live capability".to_string(), - evidence: json!({"availability": "narrowed"}), - operations: vec![ - PlanAmendmentOperation::ReplacePendingNode { - node_id: "prepare".to_string(), - node: replacement_prepare, - }, - PlanAmendmentOperation::ReplacePendingNode { - node_id: "output".to_string(), - node: replacement_output, - }, - ], - }, - projection: ExecutionProjection { - plan_revision: 1, - node_statuses: BTreeMap::from([ - ("prepare".to_string(), ExecutionNodeStatus::Pending), - ("output".to_string(), ExecutionNodeStatus::Pending), - ]), - tasks: Vec::new(), - }, - catalog: narrowed.catalog, - authorization: narrowed.authorization, - remaining_budget: replan_budget(1_000_000, 10), - config: moa_config::ExecutionConfig::default(), - now: Utc::now(), - }; - let mut unavailable_validation = validation.clone(); - let PlanAmendmentOperation::ReplacePendingNode { node, .. } = - &mut unavailable_validation.amendment.operations[0] - else { - panic!("first amendment operation should replace the prepare node"); - }; - node.operation = ExecutionOperation::Capability { - reference: unavailable_ref, - }; - - let accepted = validate_amendment(validation); - assert!( - accepted.plan.is_some(), - "retained capability amendment should validate: {:?}", - accepted.report.issues - ); - assert!( - !accepted - .report - .issues - .iter() - .any(|issue| issue.code == "catalog_hash_changed") - ); - - let rejected = validate_amendment(unavailable_validation); - assert!(rejected.plan.is_none()); - assert_eq!( - rejected - .report - .issues - .iter() - .filter(|issue| issue.code == "capability_not_authorized") - .count(), - 1 - ); - } - - #[test] - fn amendment_live_authority_check_only_removes_persisted_capabilities() { - // Pins: a live availability set is an intersection with persisted - // planning authority; it cannot introduce a caller/model-selected ref. - let persisted_a = moa_artifacts::execution_plan::CapabilityReference { - name: "persisted-a".to_string(), - version: "1".to_string(), - }; - let persisted_b = moa_artifacts::execution_plan::CapabilityReference { - name: "persisted-b".to_string(), - version: "1".to_string(), - }; - let live_only = moa_artifacts::execution_plan::CapabilityReference { - name: "live-only".to_string(), - version: "1".to_string(), - }; - let mut authorized = vec![persisted_a, persisted_b.clone()]; - - narrow_authorized_capability_refs(&mut authorized, &[persisted_b.clone(), live_only]); - - assert_eq!(authorized, vec![persisted_b]); - } - - fn amendment_tool_capability(reference_name: &str, tool_name: &str) -> ExecutionCapability { - let source = CapabilitySource::BuiltInTool { - name: tool_name.to_string(), - }; - ExecutionCapability { - reference: CapabilityReference { - name: reference_name.to_string(), - version: "v1".to_string(), - }, - contract_revision: "contract-v1".to_string(), - description: format!("Capability {reference_name}"), - input_schema: json!({"type": "object"}), - output_schema: json!({"type": "object"}), - action_class: ActionClass::Read, - risk_level: RiskLevel::Low, - default_effect: ActionPolicyEffect::Allow, - idempotency_class: IdempotencyClass::Idempotent, - execution_class: ExecutionClass::Data, - policy_context: CapabilityPolicyContext::registered(source.clone()), - source, - estimate: ExecutionEstimate { - tool_calls: 1, - tasks: 1, - ..ExecutionEstimate::default() - }, - rollback: None, - } - } - - #[test] - fn amendment_failure_evidence_is_preserved_and_bounded_before_provider_use() { - // Pins: runtime planning keeps exact structured NeedsReplan evidence, - // while rejecting an over-cap value before any model call is possible. - let evidence = json!({"shape": ["a", "b"]}); - assert_eq!( - bounded_failure_evidence("shape changed", &evidence) - .expect("small evidence should remain available"), - json!({"reason": "shape changed", "evidence": evidence}) - ); - let oversized = json!({ - "body": "x".repeat( - moa_core::types::execution_planning::EXECUTION_REPORT_MAX_BYTES - ) - }); - let error = bounded_failure_evidence("shape changed", &oversized) - .expect_err("oversized evidence should fail before planning"); - let message = >::as_ref(&error) - .to_string(); - assert!(message.contains("exceeds the bounded planner envelope")); - } - - #[test] - fn post_ack_wake_resolves_the_epoch_advertised_before_ack() { - // Pins: while K_PROCESSED_WAKE_EPOCH still contains N after the DB has - // acknowledged N+1, a wake for N+2 resolves promise N+1. A replay or - // handler restart therefore awaits the same already-resolved promise. - assert_eq!(wake_promise_epoch(7, 8, 9), Some(8)); - assert_eq!(wake_promise_epoch(8, 8, 8), None); - assert_eq!(wake_promise_epoch(8, 9, 9), None); - } - - #[test] - fn waiting_replan_exhaustion_checks_every_reserved_resource_dimension_and_deadline() { - // Pins: before parking WaitingReplan, exact consumed plus reserved - // exhaustion is terminal evidence for every configured dimension. - let now = Utc::now(); - let dimensions = [ - ("cost_microusd", 0), - ("tokens", 1), - ("tasks", 2), - ("tool_calls", 3), - ("retrieved_bytes", 4), - ]; - for (name, index) in dimensions { - let mut ledger = ledger(); - match index { - 0 => { - ledger.limit.max_cost_microusd = Some(5); - ledger.consumed.cost_microusd = 2; - ledger.reserved.cost_microusd = 3; - } - 1 => { - ledger.limit.max_tokens = Some(5); - ledger.consumed.tokens = 2; - ledger.reserved.tokens = 3; - } - 2 => { - ledger.limit.max_tasks = Some(5); - ledger.consumed.tasks = 2; - ledger.reserved.tasks = 3; - } - 3 => { - ledger.limit.max_tool_calls = Some(5); - ledger.consumed.tool_calls = 2; - ledger.reserved.tool_calls = 3; - } - 4 => { - ledger.limit.max_retrieved_bytes = Some(5); - ledger.consumed.retrieved_bytes = 2; - ledger.reserved.retrieved_bytes = 3; - } - _ => unreachable!("fixture dimension index is exhaustive"), - } - assert_eq!( - replan_exhaustion_reason(&ledger, now), - Some(ReplanExhaustion { - reason: ReplanStopReason::BudgetExhausted, - description: format!("budget exhausted: {name}"), - }) - ); - } - - let mut deadline = ledger(); - deadline.limit.deadline_at = Some(now - Duration::milliseconds(1)); - assert_eq!( - replan_exhaustion_reason(&deadline, now), - Some(ReplanExhaustion { - reason: ReplanStopReason::DeadlineExceeded, - description: "deadline exceeded".to_string(), - }) - ); - - deadline.overrun = true; - assert_eq!( - replan_exhaustion_reason(&deadline, now), - Some(ReplanExhaustion { - reason: ReplanStopReason::DeadlineExceeded, - description: "deadline exceeded".to_string(), - }), - "deadline must win when both typed limit conditions hold" - ); - - let mut available = ledger(); - available.limit.max_tokens = Some(6); - available.consumed.tokens = 2; - available.reserved.tokens = 3; - assert_eq!(replan_exhaustion_reason(&available, now), None); - } - - #[test] - fn terminal_cause_selection_covers_limits_failures_completion_and_cancellation() { - // Pins: zero-dispatch limits are distinct from ordinary completion; - // deadline wins over simultaneous budget exhaustion. - let now = Utc::now(); - let unfinished = ExecutionProjection { - plan_revision: 1, - node_statuses: BTreeMap::from([("pending".to_string(), ExecutionNodeStatus::Pending)]), - tasks: Vec::new(), - }; - let budget_failure = TerminalProjection::Failed { - failure: ExecutionTaskFailure { - class: ExecutionFailureClass::BudgetExceeded, - message: "budget".to_string(), - capability_ref: None, - }, - }; - let mut simultaneous = ledger(); - simultaneous.limit.deadline_at = Some(now - Duration::milliseconds(1)); - simultaneous.overrun = true; - assert_eq!( - terminal_cause(&unfinished, &simultaneous, &budget_failure, now), - ExecutionTerminalCause::LimitStop { - reason: ExecutionLimitStop::DeadlineExceeded - } - ); - - let finished = ExecutionProjection { - plan_revision: 1, - node_statuses: BTreeMap::from([("done".to_string(), ExecutionNodeStatus::Completed)]), - tasks: Vec::new(), - }; - assert_eq!( - terminal_cause(&finished, &ledger(), &budget_failure, now), - ExecutionTerminalCause::LimitStop { - reason: ExecutionLimitStop::BudgetExceeded - } - ); - let typed_failure = TerminalProjection::Failed { - failure: ExecutionTaskFailure { - class: ExecutionFailureClass::InvalidOutput, - message: "invalid".to_string(), - capability_ref: None, - }, - }; - assert_eq!( - terminal_cause(&finished, &ledger(), &typed_failure, now), - ExecutionTerminalCause::TaskFailure { - class: ExecutionFailureClass::InvalidOutput - } - ); - assert_eq!( - terminal_cause( - &finished, - &ledger(), - &TerminalProjection::Cancelled { - reason: "cancelled".to_string(), - }, - now, - ), - ExecutionTerminalCause::Cancellation - ); - assert_eq!( - terminal_cause( - &finished, - &ledger(), - &TerminalProjection::Completed { - output: serde_json::json!({}), - }, - now, - ), - ExecutionTerminalCause::Completion { limit_stop: None } - ); - let mut overrun = ledger(); - overrun.overrun = true; - assert_eq!( - terminal_cause( - &finished, - &overrun, - &TerminalProjection::Partial { - output: Some(serde_json::json!({})), - gaps: vec!["overrun".to_string()], - }, - now, - ), - ExecutionTerminalCause::Completion { - limit_stop: Some(ExecutionLimitStop::BudgetExceeded) - } - ); - } - - fn ledger() -> BudgetLedger { - BudgetLedger { - limit: ExecutionBudgetLimit { - max_cost_microusd: None, - max_tokens: None, - max_tasks: None, - max_tool_calls: None, - max_retrieved_bytes: None, - deadline_at: None, - }, - reserved: ExecutionEstimate::default(), - consumed: ExecutionEstimate::default(), - overrun: false, - } - } - - #[test] - fn compensation_entry_selects_direct_finalization_before_any_claim() { - // Pins: RetainEffects cancellation and manual-repair forward ambiguity - // finalize the fenced terminal directly (zero compensation claims), while - // failures and CompensateCommitted cancellation enter the reverse driver. - assert_eq!( - compensation_entry_decision( - ExecutionCancelPolicy::RetainEffects, - ExecutionRunStatus::Cancelled, - false, - ), - CompensationEntryDecision::FinalizeFenced, - ); - assert_eq!( - compensation_entry_decision( - ExecutionCancelPolicy::CompensateCommitted, - ExecutionRunStatus::Cancelled, - false, - ), - CompensationEntryDecision::Begin, - ); - assert_eq!( - compensation_entry_decision( - ExecutionCancelPolicy::RetainEffects, - ExecutionRunStatus::Failed, - false, - ), - CompensationEntryDecision::Begin, - ); - assert_eq!( - compensation_entry_decision( - ExecutionCancelPolicy::CompensateCommitted, - ExecutionRunStatus::Failed, - true, - ), - CompensationEntryDecision::FinalizeFenced, - ); - assert_eq!( - compensation_entry_decision( - ExecutionCancelPolicy::CompensateCommitted, - ExecutionRunStatus::Completed, - false, - ), - CompensationEntryDecision::FinalizeFenced, - ); - } - - #[test] - fn compensation_replay_skips_completed_and_finalizes_ambiguous_records() { - // Pins: replay never dispatches a completed undo again, and a failed or - // unknown undo routes directly to finalization without another claim. - assert_eq!( - compensation_registration_decision(CompensationStatus::Completed), - CompensationRegistrationDecision::SkipCompleted, - ); - assert_eq!( - compensation_registration_decision(CompensationStatus::Failed), - CompensationRegistrationDecision::Finalize, - ); - assert_eq!( - compensation_registration_decision(CompensationStatus::UnknownOutcome), - CompensationRegistrationDecision::Finalize, - ); - assert_eq!( - compensation_registration_decision(CompensationStatus::Pending), - CompensationRegistrationDecision::Claim, - ); - assert_eq!( - compensation_registration_decision(CompensationStatus::Running), - CompensationRegistrationDecision::Dispatch, - ); - } -} diff --git a/crates/moa-orchestrator/src/workflows/execution_task.rs b/crates/moa-orchestrator/src/workflows/execution_task.rs deleted file mode 100644 index 7b475f118..000000000 --- a/crates/moa-orchestrator/src/workflows/execution_task.rs +++ /dev/null @@ -1,2400 +0,0 @@ -//! Durable keyed workflow that executes one persisted logical task across generations. - -use std::{ - collections::{BTreeMap, BTreeSet}, - sync::Arc, -}; - -use moa_artifacts::execution_plan::{ - CapabilityReference, ExecutionFailureClass, ExecutionTaskOutcome, ExecutionTaskResult, - ExecutionUsage, -}; -use moa_config::SessionLimitsConfig; -use moa_core::{ - traits::{ChannelAdapter, SessionStore as _}, - types::{ - action_policy::{ - ActionReviewOwner, ActionRuleScope, CapabilityProvenance, ExecutionTaskOrigin, - }, - channel::Channel, - completion::{ - CompletionContent, CompletionRequest, DEFER_BRAIN_RESPONSE_METADATA_KEY, StopReason, - ToolCallContent, - }, - context::ContextMessage, - identifiers::ToolCallId, - session::SessionMeta, - tools::IdempotencyClass, - }, -}; -use moa_execution::{ - capability::{CapabilitySource, ExecutionCapability}, - interpreter::validate_task_outcome, - repository::{ - ActionReviewResolutionWrite, ExecutionRepository, ExecutionRunRecord, ExecutionScope, - ExecutionTaskRecord, ReservationOutcome, TaskOutcomeWrite, TransitionOutcome, - }, - schema::validate_instance, - state::{ - ExecutionTaskStatus, LogicalTaskKind, cancelled_task_outcome, completed_task_outcome, - exhaust_retry_outcome, failed_task_outcome, parse_agent_task_outcome, retry_delay_ms, - }, - wire::{ - ExecutionActionReviewAcknowledgement, ExecutionActionReviewResolution, - ExecutionActionReviewResolutionRequest, ExecutionInputRequest, - ExecutionReviewDecisionRequest, ExecutionRunWakeReason, ExecutionRunWakeRequest, - ExecutionSignalRequest, ExecutionTaskWorkflowRequest, ExecutionToolDispatchRejection, - }, -}; -use moa_observability::propagation::link_remote_context_from_link_headers; -use moa_observability::restate_observability::annotate_restate_handler_span; -use moa_session::PostgresSessionStore; -use restate_sdk::prelude::*; -use serde::{Deserialize, Serialize}; -use serde_json::{Value, json}; -use tracing_opentelemetry::OpenTelemetrySpanExt; - -use crate::ctx::RequestHeaders; - -use crate::workflows::child_invocation::{ChildInvocationOutcome, cancel_and_join_child_call}; -use crate::{ - services::{ - action_reviews::{ - ActionReviewsClient, ExecutionActionReviewSettlement, - SettleExecutionActionReviewRequest, - }, - llm_gateway::{ - LLMCompletionAction, LLMCompletionOwner, LLMGatewayClient, attach_completion_owner, - completion_idempotency_key, - }, - tool_executor::{ReleaseExecutionTaskHandsRequest, ToolExecutorClient}, - }, - tool_invocation::governed::{ - GovernedInvocationDisposition, GovernedInvocationOrigin, GovernedInvocationOutcome, - GovernedInvocationRequest, GovernedInvocationResult, invoke_governed_tool, - }, - workflows::execution_run::ExecutionRunClient, -}; - -const K_CANCEL_PROMISE: &str = "execution_task_cancel"; - -/// Durable workflow surface for one stable logical execution task. -#[restate_sdk::workflow] -pub trait ExecutionTask { - /// Executes and persists one task across bounded retries and input resumes. - async fn run(request: Json) -> Result<(), HandlerError>; - - /// Resumes a task after the authorized service persisted one input payload. - #[shared] - async fn input_delivered(request: Json) -> Result<(), HandlerError>; - - /// Releases a parked explicit review promise after persistence. - #[shared] - async fn review_decided( - request: Json, - ) -> Result<(), HandlerError>; - - /// Releases a parked named-signal promise after persistence. - #[shared] - async fn signal_delivered(request: Json) -> Result<(), HandlerError>; - - /// Persists and resolves one action-policy review outbox delivery. - #[shared] - async fn resolve_action_review( - request: Json, - ) -> Result, HandlerError>; - - /// Cancels any currently parked promise for terminal cleanup. - #[shared] - async fn cancel(reason: Json) -> Result<(), HandlerError>; -} - -/// Runtime dependencies for one execution-task workflow. -#[derive(Clone)] -pub struct ExecutionTaskImpl { - repository: ExecutionRepository, - pool: sqlx::PgPool, - session_store: Arc, - session_limits: SessionLimitsConfig, - channel_adapters: Arc>>, -} - -impl ExecutionTaskImpl { - /// Creates one task workflow with the exact runtime services used by governed calls. - #[must_use] - pub fn new( - pool: sqlx::PgPool, - session_store: Arc, - session_limits: SessionLimitsConfig, - channel_adapters: Arc>>, - ) -> Self { - Self { - repository: ExecutionRepository::new(pool.clone()), - pool, - session_store, - session_limits, - channel_adapters, - } - } -} - -impl ExecutionTask for ExecutionTaskImpl { - #[tracing::instrument(skip(self, ctx, request))] - // SAFETY: dispatched only by the keyed ExecutionRun workflow from persisted scoped task rows. - async fn run( - &self, - ctx: WorkflowContext<'_>, - request: Json, - ) -> Result<(), HandlerError> { - crate::ctx::adopt_incoming_trace_parent(&ctx); - annotate_restate_handler_span("ExecutionTask", "run"); - let request = request.into_inner(); - annotate_execution_task_identity_span(request.run_uid, request.task_id); - if request.task_id.to_string() != ctx.key() { - return Err(TerminalError::new_with_code(404, "execution task id mismatch").into()); - } - if request.identity.tenant_id != request.tenant_id { - return Err(TerminalError::new_with_code( - 409, - "execution task identity tenant mismatch", - ) - .into()); - } - let scope = execution_scope(&request); - let mut operation_index = 0_u64; - loop { - let repository = self.repository.clone(); - let prepare_request = request.clone(); - let enforce_dispatch_generation = operation_index == 0; - let prepared = ctx - .run(|| async move { - prepare_task( - repository, - scope, - prepare_request, - enforce_dispatch_generation, - ) - .await - .map(Json::from) - }) - .name(format!("execution_task_prepare_{operation_index}")) - .await? - .into_inner(); - operation_index = operation_index.saturating_add(1); - annotate_execution_task_record_span(&prepared.run, &prepared.task); - if prepared.wake_run { - send_run_wake( - &ctx, - prepared.run.run_uid, - prepared.run.wake_epoch, - ExecutionRunWakeReason::TaskOutcome, - ); - } - if prepared.task.status.is_terminal() || prepared.run.status.is_terminal() { - cleanup_task_hands(&ctx, &prepared.run, &prepared.task).await?; - return Ok(()); - } - if prepared.run.pending_terminal.is_some() { - persist_cancelled_task_and_finish( - &ctx, - self.repository.clone(), - scope, - prepared, - "execution run fenced forward work before compensation".to_string(), - operation_index, - ) - .await?; - return Ok(()); - } - - match &prepared.task.kind { - LogicalTaskKind::Review { .. } => { - if let ParkedTaskWake::Cancelled(reason) = - await_review_or_cancel(&ctx, &prepared.task).await? - { - persist_cancelled_task_and_finish( - &ctx, - self.repository.clone(), - scope, - prepared, - reason, - operation_index, - ) - .await?; - return Ok(()); - } - continue; - } - LogicalTaskKind::WaitSignal { signal_name } => { - if let ParkedTaskWake::Cancelled(reason) = - await_signal_or_cancel(&ctx, &prepared.task, signal_name).await? - { - persist_cancelled_task_and_finish( - &ctx, - self.repository.clone(), - scope, - prepared, - reason, - operation_index, - ) - .await?; - return Ok(()); - } - continue; - } - _ => {} - } - - let outcome = - execute_task(self, &ctx, &request.identity, &prepared.run, &prepared.task).await; - let outcome = match outcome { - Ok(outcome) => outcome, - Err(error) => ExecutionTaskOutcome { - schema_version: 1, - usage: prepared.task.actual.clone(), - result: ExecutionTaskResult::Failed { - class: ExecutionFailureClass::Terminal, - message: format!("{error:?}"), - }, - }, - }; - let outcome = validate_task_outcome( - &prepared.run.active_plan, - &prepared.task.node_id, - &prepared.task.kind, - outcome, - ); - let outcome = - exhaust_retry_outcome(prepared.task.attempt, &prepared.task.retry, outcome); - let repository = self.repository.clone(); - let persist_task = prepared.task.clone(); - let persist_outcome = outcome.clone(); - let persisted = ctx - .run(|| async move { - persist_task_outcome(repository, scope, persist_task, persist_outcome) - .await - .map(Json::from) - }) - .name(format!("execution_task_outcome_{operation_index}")) - .await? - .into_inner(); - operation_index = operation_index.saturating_add(1); - send_run_wake( - &ctx, - persisted.run.run_uid, - persisted.run.wake_epoch, - ExecutionRunWakeReason::TaskOutcome, - ); - match &outcome.result { - ExecutionTaskResult::Failed { - class: ExecutionFailureClass::Retryable, - .. - } if persisted.task.status == ExecutionTaskStatus::Running => { - let repository = self.repository.clone(); - let retry_task = persisted.task.clone(); - let retried = ctx - .run(|| async move { - retry_task_generation(repository, scope, retry_task) - .await - .map(Json::from) - }) - .name(format!("execution_task_retry_{operation_index}")) - .await? - .into_inner(); - operation_index = operation_index.saturating_add(1); - send_run_wake( - &ctx, - retried.run.run_uid, - retried.run.wake_epoch, - ExecutionRunWakeReason::TaskOutcome, - ); - if retried.task.status.is_terminal() || retried.run.status.is_terminal() { - cleanup_task_hands(&ctx, &retried.run, &retried.task).await?; - return Ok(()); - } - let delay = retry_delay_ms(retried.task.attempt, &retried.task.retry); - ctx.sleep(std::time::Duration::from_millis(delay)).await?; - } - ExecutionTaskResult::NeedsInput { .. } => { - if let ParkedTaskWake::Cancelled(reason) = - await_input_or_cancel(&ctx, &persisted.task).await? - { - persist_cancelled_task_and_finish( - &ctx, - self.repository.clone(), - scope, - persisted, - reason, - operation_index, - ) - .await?; - return Ok(()); - } - } - ExecutionTaskResult::NeedsReplan { .. } => { - let reason: String = ctx.promise(K_CANCEL_PROMISE).await?; - persist_cancelled_task_and_finish( - &ctx, - self.repository.clone(), - scope, - persisted, - reason, - operation_index, - ) - .await?; - return Ok(()); - } - ExecutionTaskResult::Completed { .. } - | ExecutionTaskResult::Cancelled { .. } - | ExecutionTaskResult::UnknownOutcome { .. } - | ExecutionTaskResult::Failed { .. } => { - cleanup_task_hands(&ctx, &persisted.run, &persisted.task).await?; - return Ok(()); - } - } - } - } - - #[tracing::instrument(skip(self, ctx, request))] - // SAFETY: invoked only by Execution/deliver_input after the generation-fenced input transaction commits. - async fn input_delivered( - &self, - ctx: SharedWorkflowContext<'_>, - request: Json, - ) -> Result<(), HandlerError> { - crate::ctx::adopt_incoming_trace_parent(&ctx); - annotate_restate_handler_span("ExecutionTask", "input_delivered"); - let request = request.into_inner(); - annotate_execution_task_identity_span(request.run_uid, request.task_id); - require_task_key(ctx.key(), request.task_id)?; - ctx.resolve_promise( - &input_promise_key(request.task_id, request.expected_generation), - Json::from(request.input), - ); - Ok(()) - } - - #[tracing::instrument(skip(self, ctx, request))] - // SAFETY: invoked only by Execution/decide_review after tenant-operator authorization and generation-fenced persistence. - async fn review_decided( - &self, - ctx: SharedWorkflowContext<'_>, - request: Json, - ) -> Result<(), HandlerError> { - crate::ctx::adopt_incoming_trace_parent(&ctx); - annotate_restate_handler_span("ExecutionTask", "review_decided"); - let request = request.into_inner(); - annotate_execution_task_identity_span(request.run_uid, request.task_id); - require_task_key(ctx.key(), request.task_id)?; - ctx.resolve_promise( - &review_promise_key(request.task_id, request.expected_generation), - Json::from(request.decision), - ); - Ok(()) - } - - #[tracing::instrument(skip(self, ctx, request))] - // SAFETY: invoked only by Execution/deliver_signal after tenant-operator authorization and exact signal persistence. - async fn signal_delivered( - &self, - ctx: SharedWorkflowContext<'_>, - request: Json, - ) -> Result<(), HandlerError> { - crate::ctx::adopt_incoming_trace_parent(&ctx); - annotate_restate_handler_span("ExecutionTask", "signal_delivered"); - let request = request.into_inner(); - annotate_execution_task_identity_span(request.run_uid, request.task_id); - require_task_key(ctx.key(), request.task_id)?; - ctx.resolve_promise( - &signal_promise_key( - request.task_id, - request.expected_generation, - &request.signal_name, - ), - Json::from(request.payload), - ); - Ok(()) - } - - #[tracing::instrument(skip(self, ctx, request))] - // SAFETY: invoked only by the bounded execution-action-review outbox dispatcher from a terminal persisted review row. - async fn resolve_action_review( - &self, - ctx: SharedWorkflowContext<'_>, - request: Json, - ) -> Result, HandlerError> { - crate::ctx::adopt_incoming_trace_parent(&ctx); - annotate_restate_handler_span("ExecutionTask", "resolve_action_review"); - let headers = ctx.request_headers(); - let _ = link_remote_context_from_link_headers(&tracing::Span::current(), |name| { - headers.get(name).cloned() - }); - let request = request.into_inner(); - annotate_execution_task_identity_span(request.run_uid, request.task_id); - require_task_key(ctx.key(), request.task_id)?; - let repository = self.repository.clone(); - let record_request = request.clone(); - let write = ctx - .run(|| async move { - repository - .record_action_review_resolution( - ExecutionScope::ControlPlane, - record_request.run_uid, - record_request.task_id, - record_request.generation, - record_request.review_uid, - &record_request.resolution, - ) - .await - .map(Json::from) - .map_err(execution_error) - }) - .name(format!( - "execution_action_review_resolution_{}", - request.review_uid - )) - .await? - .into_inner(); - let acknowledgement = match write { - ActionReviewResolutionWrite::Applied => { - ctx.resolve_promise( - &action_review_promise_key(request.review_uid, request.generation), - Json::from(request.resolution), - ); - ExecutionActionReviewAcknowledgement::Applied - } - ActionReviewResolutionWrite::Replayed => { - ctx.resolve_promise( - &action_review_promise_key(request.review_uid, request.generation), - Json::from(request.resolution), - ); - ExecutionActionReviewAcknowledgement::Replayed - } - ActionReviewResolutionWrite::AuditedStale | ActionReviewResolutionWrite::NotFound => { - ExecutionActionReviewAcknowledgement::AuditedStale - } - }; - Ok(Json::from(acknowledgement)) - } - - #[tracing::instrument(skip(self, ctx, reason))] - // SAFETY: invoked by the owning run workflow after persisted cancellation. - async fn cancel( - &self, - ctx: SharedWorkflowContext<'_>, - reason: Json, - ) -> Result<(), HandlerError> { - crate::ctx::adopt_incoming_trace_parent(&ctx); - annotate_restate_handler_span("ExecutionTask", "cancel"); - tracing::Span::current().set_attribute("moa.execution.task_id", ctx.key().to_string()); - ctx.resolve_promise(K_CANCEL_PROMISE, reason.into_inner()); - Ok(()) - } -} - -#[derive(Clone, Debug, Deserialize, Serialize)] -struct PreparedTask { - run: ExecutionRunRecord, - task: ExecutionTaskRecord, - wake_run: bool, -} - -async fn prepare_task( - repository: ExecutionRepository, - scope: ExecutionScope, - request: ExecutionTaskWorkflowRequest, - enforce_dispatch_generation: bool, -) -> Result { - let run = repository - .load_run(scope, request.run_uid) - .await - .map_err(execution_error)? - .ok_or_else(|| TerminalError::new_with_code(404, "execution run not found"))?; - if run.tenant_id != request.tenant_id - || run.contact_id != request.contact_id - || run.session_id != request.session_id - { - return Err(TerminalError::new_with_code(409, "execution task scope mismatch").into()); - } - let mut task = repository - .load_task(scope, request.run_uid, request.task_id) - .await - .map_err(execution_error)? - .ok_or_else(|| TerminalError::new_with_code(404, "execution task not found"))?; - if enforce_dispatch_generation && task.generation != request.generation { - return Err(TerminalError::new_with_code( - 409, - "execution task dispatch generation is stale", - ) - .into()); - } - if run.pending_terminal.is_some() && !task.status.is_terminal() { - return Ok(PreparedTask { - run, - task, - wake_run: false, - }); - } - if task.status.is_terminal() || run.status.is_terminal() { - let wake_run = task.status.is_terminal() && !run.status.is_terminal(); - return Ok(PreparedTask { - run, - task, - wake_run, - }); - } - if task.status == ExecutionTaskStatus::Pending { - task = match repository - .reserve_task(scope, task.run_uid, task.task_id, task.generation) - .await - .map_err(execution_error)? - { - ReservationOutcome::Reserved(task) => task, - ReservationOutcome::AlreadyReserved(task) => task, - ReservationOutcome::Terminalized(terminalized) => { - return Ok(PreparedTask { - run: terminalized.run, - task: terminalized.task, - wake_run: true, - }); - } - ReservationOutcome::AlreadyTerminalized(terminalized) => { - return Ok(PreparedTask { - run: terminalized.run, - task: terminalized.task, - wake_run: true, - }); - } - ReservationOutcome::NotFound => { - return Err(TerminalError::new_with_code(404, "execution task not found").into()); - } - ReservationOutcome::Rejected(reason) => { - return Err(TerminalError::new(format!( - "execution task reservation rejected: {reason:?}" - )) - .into()); - } - }; - } - let mut wake_run = task.status.is_terminal(); - if task.status == ExecutionTaskStatus::Reserved { - task = match repository - .mark_task_running(scope, task.run_uid, task.task_id, task.generation) - .await - .map_err(execution_error)? - { - TransitionOutcome::Applied(task) => task, - TransitionOutcome::AlreadyApplied(task) => task, - other => { - return Err(TerminalError::new(format!( - "execution task start rejected: {other:?}" - )) - .into()); - } - }; - wake_run = true; - } - let run = repository - .load_run(scope, request.run_uid) - .await - .map_err(execution_error)? - .ok_or_else(|| TerminalError::new_with_code(404, "execution run not found"))?; - Ok(PreparedTask { - run, - task, - wake_run, - }) -} - -async fn execute_task( - workflow: &ExecutionTaskImpl, - ctx: &WorkflowContext<'_>, - identity: &moa_core::traits::Identity, - run: &ExecutionRunRecord, - task: &ExecutionTaskRecord, -) -> Result { - match &task.kind { - LogicalTaskKind::Capability { reference } => { - execute_capability(workflow, ctx, identity, run, task, reference).await - } - LogicalTaskKind::Agent { - instructions, - skill_refs, - capability_refs, - max_turns, - } => { - execute_agent( - workflow, - ctx, - AgentExecutionRequest { - identity, - run, - task, - instructions, - skill_refs, - capability_refs, - max_turns: *max_turns, - }, - ) - .await - } - LogicalTaskKind::Output { value } => { - validate_instance( - &run.active_plan.definition.output_schema, - value, - "execution_task.output", - ) - .map_err(execution_error)?; - Ok(completed_task_outcome(value.clone(), task.actual.clone())) - } - LogicalTaskKind::CompletionVerifier { - instructions, - max_turns, - .. - } => { - execute_agent( - workflow, - ctx, - AgentExecutionRequest { - identity, - run, - task, - instructions, - skill_refs: &[], - capability_refs: &[], - max_turns: *max_turns, - }, - ) - .await - } - LogicalTaskKind::Review { .. } | LogicalTaskKind::WaitSignal { .. } => { - Err(TerminalError::new("parked execution task reached executable adapter").into()) - } - } -} - -async fn execute_capability( - workflow: &ExecutionTaskImpl, - ctx: &WorkflowContext<'_>, - identity: &moa_core::traits::Identity, - run: &ExecutionRunRecord, - task: &ExecutionTaskRecord, - reference: &CapabilityReference, -) -> Result { - let capability = find_capability(run, reference)?; - validate_instance( - &capability.input_schema, - &task.input, - "execution_task.capability_input", - ) - .map_err(execution_error)?; - let session = load_session(workflow, ctx, run.session_id, task).await?; - if let Some(outcome) = cancellation_outcome_before_next_agent_tool( - ctx.peek_promise::(K_CANCEL_PROMISE).await?, - &task.actual, - ) { - return Ok(outcome); - } - let invocation = invoke_capability_tool( - workflow, - ctx, - CapabilityInvocationContext { - identity, - run, - task, - capability, - session: &session, - // A direct capability task runs no model turn, so there is no system - // context holding a canary for its output to leak. The canary belongs - // to agent turns, which do have one. - active_canary: None, - }, - task.input.clone(), - 0, - ) - .await?; - let mut usage = task.actual.clone(); - usage.tool_calls = usage.tool_calls.saturating_add(1); - if let CapabilityInvocationResult::Output(output) = &invocation { - usage.retrieved_bytes = usage - .retrieved_bytes - .saturating_add(serialized_len(&output.safe_output.structured_payload())); - } - let outcome = capability_invocation_outcome( - capability.idempotency_class, - capability.action_class, - invocation, - usage, - )?; - let output_usage = outcome.usage.clone(); - let ExecutionTaskResult::Completed { output: value, .. } = &outcome.result else { - return Ok(outcome); - }; - if let Err(error) = validate_instance( - &capability.output_schema, - value, - "execution_task.capability_output", - ) { - return Ok(invalid_capability_output_outcome( - capability.action_class, - error.to_string(), - output_usage, - )); - } - Ok(outcome) -} - -struct AgentExecutionRequest<'a> { - identity: &'a moa_core::traits::Identity, - run: &'a ExecutionRunRecord, - task: &'a ExecutionTaskRecord, - instructions: &'a str, - skill_refs: &'a [moa_artifacts::reference::ArtifactRef], - capability_refs: &'a [CapabilityReference], - max_turns: u32, -} - -#[derive(Debug)] -struct AgentCapabilityBinding<'a> { - capability: &'a ExecutionCapability, - tool_name: &'a str, -} - -fn validate_agent_capability_bindings( - capabilities: Vec<&ExecutionCapability>, -) -> Result>, HandlerError> { - let mut bindings = Vec::with_capacity(capabilities.len()); - let mut references_by_tool = BTreeMap::<&str, Vec<&CapabilityReference>>::new(); - for capability in capabilities { - let Some(tool_name) = capability.source.model_visible_tool_name() else { - return Err( - TerminalError::new("capability has no governed tool owner in Task 6").into(), - ); - }; - let references = references_by_tool.entry(tool_name).or_default(); - if !references - .iter() - .any(|reference| **reference == capability.reference) - { - references.push(&capability.reference); - } - bindings.push(AgentCapabilityBinding { - capability, - tool_name, - }); - } - for references in references_by_tool.values_mut() { - references.sort_by(|left, right| { - left.name - .cmp(&right.name) - .then_with(|| left.version.cmp(&right.version)) - }); - } - if let Some((tool_name, references)) = references_by_tool - .iter() - .find(|(_, references)| references.len() > 1) - { - let references = references - .iter() - .map(|reference| format!("{}@{}", reference.name, reference.version)) - .collect::>() - .join(" and "); - return Err(TerminalError::new(format!( - "task-local agent capability references {references} resolve to ambiguous model-visible tool `{tool_name}`" - )) - .into()); - } - Ok(bindings) -} - -/// Fixed terminal message for a task the security circuit halted. -const EXECUTION_TASK_SECURITY_HALT_MESSAGE: &str = "task stopped: a capability returned output classified as a prompt-injection or \ - restricted-material result"; - -/// Fixed user-facing question for a task the security circuit suspended. -/// -/// Fixed rather than derived from the output: the output is exactly what MOA has -/// decided it cannot trust, so quoting it into a user prompt would forward the -/// attack to the human. -const EXECUTION_TASK_SECURITY_INPUT_QUESTION: &str = "A tool this task used returned output that MOA classified as a possible \ - prompt-injection attempt. Should this task continue without that capability?"; - -/// Derives the replay-stable tool-call identity for one task-local agent call. -/// -/// The circuit deduplicates by tool call, so this must be a pure function of the -/// task and the position in its loop — a fresh UUID would let a replayed turn -/// score the same output twice. -fn execution_task_tool_call_id( - task_uid: uuid::Uuid, - turn: u32, - call_index: usize, -) -> moa_core::types::identifiers::ToolCallId { - const NAMESPACE: uuid::Uuid = uuid::Uuid::from_u128(0x6d6f_615f_6574_6300_9e3a_41d5_b7c8_0002); - let name = format!("{task_uid}:{turn}:{call_index}"); - moa_core::types::identifiers::ToolCallId(uuid::Uuid::new_v5(&NAMESPACE, name.as_bytes())) -} - -/// Journals one execution-task circuit transition and its signed finding. -async fn record_execution_task_transition( - ctx: &WorkflowContext<'_>, - tenant_id: moa_core::types::identifiers::TenantId, - session_id: moa_core::types::identifiers::SessionId, - transition: moa_core::types::security::SecurityCircuitTransition, - assessment: &moa_core::types::security::ToolOutputAssessment, -) -> Result<(), HandlerError> { - let occurred_at = ctx - .run(|| async move { Ok(Json::from(chrono::Utc::now())) }) - .name("execution_task_prompt_injection_transition_timestamp") - .await? - .into_inner(); - - let dedupe_key = transition.key.clone(); - crate::restate_identity::replay_safe_request( - ctx.service_client::() - .append_event(Json::from(moa_wire::session_store::AppendEventRequest { - session_id, - event: moa_core::events::Event::PromptInjectionCircuitTransition { - transition: transition.clone(), - signals: assessment.signals.clone(), - redacted_spans: assessment.redacted_spans, - deduplicated_carriers: assessment.deduplicated_carriers, - }, - dedupe_key: Some(dedupe_key), - })), - ) - .call() - .await?; - - crate::restate_identity::replay_safe_request( - ctx.service_client::() - .record_circuit_transition(Json::from( - crate::services::security_events::RecordCircuitTransitionRequest { - tenant_id, - session_id, - transition, - signals: assessment.signals.clone(), - occurred_at, - }, - )), - ) - .call() - .await?; - Ok(()) -} - -async fn execute_agent( - workflow: &ExecutionTaskImpl, - ctx: &WorkflowContext<'_>, - request: AgentExecutionRequest<'_>, -) -> Result { - let AgentExecutionRequest { - identity, - run, - task, - instructions, - skill_refs, - capability_refs, - max_turns, - } = request; - if max_turns == 0 { - return Ok(failed_task_outcome( - ExecutionFailureClass::InvalidInput, - "agent max_turns must be positive".to_string(), - task.actual.clone(), - )); - } - // Ordinary, map, and reduce agents all materialize as `LogicalTaskKind::Agent`, - // so this one pre-I/O guard covers every task-local model/tool loop. - let capability_bindings = validate_agent_capability_bindings( - capability_refs - .iter() - .map(|reference| find_capability(run, reference)) - .collect::, _>>()?, - )?; - let session = load_session(workflow, ctx, run.session_id, task).await?; - let skills = load_pinned_skills(workflow, ctx, run, task, skill_refs).await?; - // The model sees the exact schemas persisted in the run's scoped capability - // catalog. Re-reading the deployment router here would make a replay depend - // on another tenant's catalog state and would discard installed-connector - // provenance before governed dispatch rechecks its durable pins. - let tools = capability_bindings - .iter() - .map(task_agent_tool_schema) - .collect::>(); - // One canary per task turn, journaled so a replay reproduces the same token - // rather than minting a fresh one that the already-persisted output could - // never match. It goes into the system context AND to every capability - // invocation: the system copy is what an attacker can exfiltrate, and the - // invocation copy is what lets the classifier recognize the exfiltration. - let active_canary = ctx - .run(|| async { Ok::<_, HandlerError>(Json::from(moa_security::new_canary_token())) }) - .name("execution_task_agent_canary") - .await? - .into_inner(); - let mut messages = vec![ - ContextMessage::system(agent_system_prompt(instructions, &skills)), - ContextMessage::system(moa_security::canary_system_message(&active_canary)), - ContextMessage::user( - json!({ - "resolved_input": task.input, - "resume_inputs": task.resume_input_history, - }) - .to_string(), - ), - ]; - let mut usage = task.actual.clone(); - // The task-local agent is its own circuit owner. The state lives in this - // durable workflow rather than in a virtual object because the workflow *is* - // the single writer for this owner, and its journal already makes the state - // replay-stable. Routing it through the Session VO instead would make one - // shared circuit alternate between the coordinator owner and each detached - // task owner, and each switch would clear the other's accumulated score. - let mut circuit = moa_core::types::security::SecurityCircuitState::default(); - let mut disabled_capabilities = - std::collections::BTreeMap::::new(); - let circuit_owner = moa_core::types::security::SecurityCircuitOwner::ExecutionTask { - run_uid: run.run_uid, - task_uid: task.task_id.as_uuid(), - generation: task.generation, - }; - circuit.adopt_owner(&circuit_owner); - for turn in 0..max_turns { - let mut request = CompletionRequest { - model: None, - messages: messages.clone(), - tools: tools - .iter() - .filter(|tool| { - tool.get("name") - .and_then(Value::as_str) - .is_none_or(|name| !disabled_capabilities.contains_key(name)) - }) - .cloned() - .collect(), - max_output_tokens: None, - temperature: None, - response_format: None, - native_web_search: Default::default(), - metadata: std::collections::HashMap::new(), - }; - request.metadata.insert( - DEFER_BRAIN_RESPONSE_METADATA_KEY.to_string(), - Value::Bool(true), - ); - let completion_owner = LLMCompletionOwner::execution_run(run.run_uid.to_string()); - attach_completion_owner(&mut request, &completion_owner); - let call = crate::restate_identity::replay_safe_request( - ctx.service_client::() - .complete(Json::from(request)) - .idempotency_key(completion_idempotency_key( - ctx.invocation_id(), - LLMCompletionAction::ExecutionTaskModel { - generation: task.generation, - turn, - }, - )), - ) - .call(); - let response = match cancel_and_join_child_call( - ctx.promise::(K_CANCEL_PROMISE), - call, - ) - .await? - { - ChildInvocationOutcome::Cancelled(reason) => { - return Ok(cancelled_task_outcome(reason, usage)); - } - ChildInvocationOutcome::Completed(response) => response.into_inner(), - }; - if response.stop_reason == StopReason::Cancelled { - return Ok(cancelled_task_outcome( - "execution run fenced its provider completion".to_string(), - usage, - )); - } - usage.tokens = usage - .tokens - .saturating_add(response.usage.total_input_tokens() as u64) - .saturating_add(response.usage.output_tokens as u64); - usage.cost_microusd = usage.cost_microusd.saturating_add( - moa_providers::pricing_for_model(response.model.as_str()) - .map(|pricing| pricing.cost_micros(&response.usage)) - .unwrap_or_default(), - ); - let tool_calls = response - .content - .iter() - .filter_map(|content| match content { - CompletionContent::ToolCall(call) => Some(call.clone()), - CompletionContent::Text(_) | CompletionContent::ProviderToolResult { .. } => None, - }) - .collect::>(); - if tool_calls.is_empty() { - return Ok(parse_agent_task_outcome(&response.text, usage)); - } - messages.push(ContextMessage::assistant_with_thought_signature( - response.text, - response.thought_signature, - )); - for (call_index, tool_call) in tool_calls.into_iter().enumerate() { - let cancellation_reason = ctx.peek_promise::(K_CANCEL_PROMISE).await?; - if let Some(outcome) = - cancellation_outcome_before_next_agent_tool(cancellation_reason, &usage) - { - // A prior governed effect is always joined and accounted before - // reaching this boundary. Once cancellation is durable, no later - // tool from the same model response may cross the admission fence. - return Ok(outcome); - } - let binding = capability_bindings - .iter() - .find(|binding| binding.tool_name == tool_call.invocation.name) - .ok_or_else(|| TerminalError::new("agent emitted an undeclared capability"))?; - if disabled_capabilities.contains_key(&tool_call.invocation.name) { - let tool_use_id = - tool_call.invocation.id.clone().unwrap_or_else(|| { - format!("execution-{}-{turn}-{call_index}", task.task_id) - }); - messages.push(ContextMessage::assistant_tool_call( - tool_call.invocation, - "", - )); - messages.push(ContextMessage::tool_result( - tool_use_id, - EXECUTION_TASK_DISABLED_CAPABILITY_MESSAGE, - None, - )); - continue; - } - let invocation = invoke_capability_tool( - workflow, - ctx, - CapabilityInvocationContext { - identity, - run, - task, - capability: binding.capability, - session: &session, - active_canary: Some(active_canary.as_str()), - }, - tool_call.invocation.input.clone(), - u64::from(turn) - .saturating_mul(1_000) - .saturating_add(call_index as u64), - ) - .await?; - usage.tool_calls = usage.tool_calls.saturating_add(1); - let output = match invocation { - CapabilityInvocationResult::Output(output) => output, - CapabilityInvocationResult::Terminal(result) => { - return Ok(ExecutionTaskOutcome { - schema_version: 1, - usage, - result, - }); - } - }; - usage.retrieved_bytes = usage - .retrieved_bytes - .saturating_add(serialized_len(&output.safe_output.structured_payload())); - - // Score the classified output against this task's own circuit. A halt - // is a terminal task failure and a suspend is a user-audience input - // request, which are the execution domain's equivalents of the - // coordinator's halt and NeedsInput outcomes. - if !output.assessment.is_safe() { - let applied = moa_security::apply_owner_assessment( - &mut circuit, - moa_security::CircuitTarget { - session_id: session.id, - owner: &circuit_owner, - capability: &output.capability, - tool_call_id: execution_task_tool_call_id( - task.task_id.as_uuid(), - turn, - call_index, - ), - }, - &output.assessment, - ) - .map_err(|error| { - tracing::error!( - active_owner_kind = error.active.as_ref().map(|owner| owner.kind()), - active_owner_generation = - error.active.as_ref().map(|owner| owner.generation()), - received_owner_kind = error.received.kind(), - received_owner_generation = error.received.generation(), - "execution task security assessment owner mismatch" - ); - TerminalError::new("security assessment owner mismatch") - })?; - if let Some(transition) = applied { - record_execution_task_transition( - ctx, - session.tenant_id, - session.id, - transition, - &output.assessment, - ) - .await?; - } - let stage = circuit.stage(&circuit_owner, &output.capability); - if !stage.permits_dispatch() { - disabled_capabilities - .insert(tool_call.invocation.name.clone(), output.capability.clone()); - } - match stage { - moa_core::types::security::SecurityCircuitStage::Halted => { - return Ok(failed_task_outcome( - ExecutionFailureClass::Terminal, - EXECUTION_TASK_SECURITY_HALT_MESSAGE.to_string(), - usage, - )); - } - moa_core::types::security::SecurityCircuitStage::SuspendedForInput => { - return Ok(ExecutionTaskOutcome { - schema_version: 1, - usage, - result: ExecutionTaskResult::NeedsInput { - question: EXECUTION_TASK_SECURITY_INPUT_QUESTION.to_string(), - audience: moa_artifacts::execution_plan::InputAudience::User, - }, - }); - } - _ => {} - } - } - let tool_use_id = tool_call - .invocation - .id - .clone() - .unwrap_or_else(|| format!("execution-{}-{turn}-{call_index}", task.task_id)); - messages.push(ContextMessage::assistant_tool_call( - tool_call.invocation, - "", - )); - // Only the classified output reaches the task-local agent's context. - messages.push(ContextMessage::tool_result( - tool_use_id, - output.safe_output.to_text(), - Some(output.safe_output.content.clone()), - )); - } - } - Ok(failed_task_outcome( - ExecutionFailureClass::Terminal, - format!("task-local agent exhausted max_turns={max_turns}"), - usage, - )) -} - -fn cancellation_outcome_before_next_agent_tool( - cancellation_reason: Option, - usage: &ExecutionUsage, -) -> Option { - cancellation_reason.map(|reason| cancelled_task_outcome(reason, usage.clone())) -} - -fn task_agent_tool_schema(binding: &AgentCapabilityBinding<'_>) -> Value { - json!({ - "name": binding.tool_name, - "description": binding.capability.description, - "input_schema": binding.capability.input_schema, - }) -} - -/// Fixed model-facing refusal for a capability disabled by the task circuit. -const EXECUTION_TASK_DISABLED_CAPABILITY_MESSAGE: &str = "This tool capability is disabled for the current task because its prior output triggered the security circuit."; - -struct CapabilityInvocationContext<'a> { - identity: &'a moa_core::traits::Identity, - run: &'a ExecutionRunRecord, - task: &'a ExecutionTaskRecord, - capability: &'a ExecutionCapability, - session: &'a SessionMeta, - /// Per-task-turn canary this invocation must be screened against. - /// - /// Without it `CanaryLeak` is unreachable for execution-agent turns, which - /// makes the whole clear-to-halt jump unreachable for this owner — the - /// circuit would silently top out at the lower classes. - active_canary: Option<&'a str>, -} - -enum CapabilityInvocationResult { - Output(Box), - Terminal(ExecutionTaskResult), -} - -async fn invoke_capability_tool( - workflow: &ExecutionTaskImpl, - ctx: &WorkflowContext<'_>, - invocation: CapabilityInvocationContext<'_>, - input: Value, - call_index: u64, -) -> Result { - let CapabilityInvocationContext { - identity, - run, - task, - capability, - session, - active_canary, - } = invocation; - let tool_name = capability_tool_name(capability)?; - let tool_id = ToolCallId(uuid::Uuid::new_v5( - &task.task_id.as_uuid(), - format!("{}:{call_index}", task.generation).as_bytes(), - )); - let tool_call = ToolCallContent { - invocation: moa_core::types::completion::ToolInvocation { - id: Some(tool_id.to_string()), - name: tool_name.clone(), - input, - }, - provider_metadata: None, - }; - let allowed_tools = BTreeSet::from([tool_name]); - let provenance = CapabilityProvenance { - kind: Some(capability_source_kind(&capability.source).to_string()), - id: Some(format!( - "{}@{}", - capability.reference.name, capability.reference.version - )), - step_id: Some(task.node_id.clone()), - }; - let outcome = invoke_governed_tool( - ctx, - GovernedInvocationRequest { - session, - identity, - session_id: run.session_id, - tool_id, - tool_call: &tool_call, - allowed_tools: &allowed_tools, - expected_tool_contract_revision: Some(&capability.contract_revision), - active_canary, - trusted_sandbox_manifest: None, - origin: GovernedInvocationOrigin::ExecutionTask { - run_uid: run.run_uid, - task_uid: task.task_id.as_uuid(), - generation: task.generation, - }, - capability_provenance: Some(&provenance), - capability_policy_context: Some(&capability.policy_context), - resource_budget: moa_core::types::resource::ResourceBudget::UNBOUNDED, - }, - &workflow.session_limits, - workflow.session_store.clone(), - workflow.channel_adapters.as_ref(), - ) - .await?; - let result = match classify_governed_capability_outcome(outcome)? { - GovernedCapabilityOutcome::Completed(result) => result, - GovernedCapabilityOutcome::Terminal(result) => { - return Ok(CapabilityInvocationResult::Terminal(result)); - } - }; - if result.disposition == GovernedInvocationDisposition::ReviewPending { - let promise_key = action_review_promise_key(tool_id.0, task.generation); - let resolution = restate_sdk::select! { - reason = ctx.promise::(K_CANCEL_PROMISE) => { - let reason = reason?; - let settlement = crate::restate_identity::replay_safe_request( - ctx.service_client::() - .settle_execution_owner_review(Json::from( - SettleExecutionActionReviewRequest { - tenant_id: run.tenant_id, - review_id: tool_id.0, - owner: ActionReviewOwner::ExecutionTask { - session_id: run.session_id, - origin: ExecutionTaskOrigin { - run_uid: run.run_uid, - task_uid: task.task_id.as_uuid(), - generation: task.generation, - }, - }, - }, - )), - ) - .call() - .await? - .into_inner(); - match action_review_cancellation_step(settlement, &reason) { - ActionReviewCancellationStep::Cancelled(result) => { - return Ok(CapabilityInvocationResult::Terminal(result)); - } - ActionReviewCancellationStep::JoinResolution => { - ctx.promise::>(&promise_key) - .await? - .into_inner() - } - } - }, - resolution = ctx.promise::>( - &promise_key - ) => { - resolution?.into_inner() - } - }; - return action_review_invocation_result(resolution); - } - Ok(CapabilityInvocationResult::Output(Box::new(result.output))) -} - -enum ActionReviewCancellationStep { - Cancelled(ExecutionTaskResult), - JoinResolution, -} - -fn action_review_cancellation_step( - settlement: ExecutionActionReviewSettlement, - reason: &str, -) -> ActionReviewCancellationStep { - match settlement { - ExecutionActionReviewSettlement::Revoked => { - ActionReviewCancellationStep::Cancelled(ExecutionTaskResult::Cancelled { - reason: reason.to_string(), - }) - } - ExecutionActionReviewSettlement::JoinRequired => { - ActionReviewCancellationStep::JoinResolution - } - } -} - -enum GovernedCapabilityOutcome { - Completed(Box), - Terminal(ExecutionTaskResult), -} - -fn classify_governed_capability_outcome( - outcome: GovernedInvocationOutcome, -) -> Result { - match outcome { - GovernedInvocationOutcome::Completed(result) => { - Ok(GovernedCapabilityOutcome::Completed(result)) - } - GovernedInvocationOutcome::UnknownOutcome { message, .. } => Ok( - GovernedCapabilityOutcome::Terminal(ExecutionTaskResult::UnknownOutcome { message }), - ), - GovernedInvocationOutcome::NotDispatched { reason, .. } => Ok( - GovernedCapabilityOutcome::Terminal(ExecutionTaskResult::Failed { - class: ExecutionFailureClass::Terminal, - message: execution_dispatch_rejection_message(reason), - }), - ), - GovernedInvocationOutcome::Delegation { .. } => { - Err(TerminalError::new("execution agents cannot invoke delegation tools").into()) - } - } -} - -async fn persist_task_outcome( - repository: ExecutionRepository, - scope: ExecutionScope, - task: ExecutionTaskRecord, - outcome: ExecutionTaskOutcome, -) -> Result { - let write = repository - .record_task_outcome(scope, task.run_uid, task.task_id, task.generation, outcome) - .await - .map_err(execution_error)?; - let (task, persisted_run) = match write { - TaskOutcomeWrite::Applied { run, task, .. } - | TaskOutcomeWrite::Replayed { run, task, .. } => (task, Some(run)), - TaskOutcomeWrite::Rejected { task, .. } => (task, None), - TaskOutcomeWrite::NotFound => { - return Err(TerminalError::new_with_code(404, "execution task not found").into()); - } - }; - let run = match persisted_run { - Some(run) => run, - None => repository - .load_run(scope, task.run_uid) - .await - .map_err(execution_error)? - .ok_or_else(|| TerminalError::new_with_code(404, "execution run not found"))?, - }; - Ok(PreparedTask { - run, - task, - wake_run: false, - }) -} - -async fn persist_cancelled_task_and_finish( - ctx: &WorkflowContext<'_>, - repository: ExecutionRepository, - scope: ExecutionScope, - prepared: PreparedTask, - reason: String, - operation_index: u64, -) -> Result<(), HandlerError> { - let task = prepared.task.clone(); - let outcome = cancelled_task_outcome(reason, task.actual.clone()); - let persisted = ctx - .run(|| async move { - persist_task_outcome(repository, scope, task, outcome) - .await - .map(Json::from) - }) - .name(format!( - "execution_task_cancelled_outcome_{operation_index}" - )) - .await? - .into_inner(); - send_run_wake( - ctx, - persisted.run.run_uid, - persisted.run.wake_epoch, - ExecutionRunWakeReason::TaskOutcome, - ); - cleanup_task_hands(ctx, &persisted.run, &persisted.task).await -} - -async fn retry_task_generation( - repository: ExecutionRepository, - scope: ExecutionScope, - task: ExecutionTaskRecord, -) -> Result { - let task = match repository - .retry_task(scope, task.run_uid, task.task_id, task.generation) - .await - .map_err(execution_error)? - { - TransitionOutcome::Applied(task) | TransitionOutcome::AlreadyApplied(task) => task, - other => { - return Err(TerminalError::new(format!( - "execution retry transition rejected: {other:?}" - )) - .into()); - } - }; - let run = repository - .load_run(scope, task.run_uid) - .await - .map_err(execution_error)? - .ok_or_else(|| TerminalError::new_with_code(404, "execution run not found"))?; - Ok(PreparedTask { - run, - task, - wake_run: false, - }) -} - -async fn load_session( - workflow: &ExecutionTaskImpl, - ctx: &WorkflowContext<'_>, - session_id: moa_core::types::identifiers::SessionId, - task: &ExecutionTaskRecord, -) -> Result { - let store = workflow.session_store.clone(); - Ok(ctx - .run(|| async move { - store - .get_session(session_id) - .await - .map(Json::from) - .map_err(crate::workflows::errors::moa_error_to_handler_error) - }) - .name(format!( - "execution_task_load_session_{}_{}", - task.generation, task.attempt - )) - .await? - .into_inner()) -} - -async fn load_pinned_skills( - workflow: &ExecutionTaskImpl, - ctx: &WorkflowContext<'_>, - run: &ExecutionRunRecord, - task: &ExecutionTaskRecord, - skill_refs: &[moa_artifacts::reference::ArtifactRef], -) -> Result, HandlerError> { - let mut markdown = Vec::with_capacity(skill_refs.len()); - let scope = action_scope(run.tenant_id, run.contact_id); - for (index, skill_ref) in skill_refs.iter().enumerate() { - if !run.authorization.skill_refs.contains(skill_ref) { - return Err(TerminalError::new( - "task requested an instruction skill outside the authorization envelope", - ) - .into()); - } - let pinned = run - .pinned_instruction_skills - .iter() - .find(|pinned| pinned.skill_ref == *skill_ref) - .ok_or_else(|| TerminalError::new("task requested an unpinned instruction skill"))?; - let pool = workflow.pool.clone(); - let revision_uid = pinned.revision_uid; - let loaded = ctx - .run(|| async move { - moa_skills::registry::SkillRegistry::new(pool) - .load_skill_markdown(&scope, revision_uid) - .await - .map(Json::from) - .map_err(crate::workflows::errors::moa_error_to_handler_error) - }) - .name(format!( - "execution_task_skill_{}_{}_{}", - task.generation, index, revision_uid - )) - .await? - .into_inner(); - markdown.push(loaded); - } - Ok(markdown) -} - -async fn await_input_or_cancel( - ctx: &WorkflowContext<'_>, - task: &ExecutionTaskRecord, -) -> Result { - let promise_key = input_promise_key(task.task_id, task.generation); - Ok(restate_sdk::select! { - reason = ctx.promise::(K_CANCEL_PROMISE) => { - ParkedTaskWake::Cancelled(reason?) - }, - _ = ctx.promise::>(&promise_key) => ParkedTaskWake::Resumed, - }) -} - -async fn await_review_or_cancel( - ctx: &WorkflowContext<'_>, - task: &ExecutionTaskRecord, -) -> Result { - let promise_key = review_promise_key(task.task_id, task.generation); - Ok(restate_sdk::select! { - reason = ctx.promise::(K_CANCEL_PROMISE) => { - ParkedTaskWake::Cancelled(reason?) - }, - _ = ctx.promise::>( - &promise_key - ) => ParkedTaskWake::Resumed, - }) -} - -async fn await_signal_or_cancel( - ctx: &WorkflowContext<'_>, - task: &ExecutionTaskRecord, - signal_name: &str, -) -> Result { - let promise_key = signal_promise_key(task.task_id, task.generation, signal_name); - Ok(restate_sdk::select! { - reason = ctx.promise::(K_CANCEL_PROMISE) => { - ParkedTaskWake::Cancelled(reason?) - }, - _ = ctx.promise::>( - &promise_key - ) => ParkedTaskWake::Resumed, - }) -} - -enum ParkedTaskWake { - Resumed, - Cancelled(String), -} - -fn find_capability<'a>( - run: &'a ExecutionRunRecord, - reference: &CapabilityReference, -) -> Result<&'a ExecutionCapability, HandlerError> { - if !run.authorization.capability_refs.contains(reference) { - return Err(TerminalError::new( - "capability is outside the persisted authorization envelope", - ) - .into()); - } - run.catalog - .capabilities - .iter() - .find(|capability| capability.reference == *reference) - .ok_or_else(|| TerminalError::new("capability is absent from the persisted catalog").into()) -} - -/// Returns the registered tool name a capability dispatches through. -/// -/// Every arm must yield a name the router actually knows. A connector tool's -/// published name is not one — it resolves only under its server-qualified -/// reference — which is why `McpTool` contributes `tool_name` here and its -/// `remote_name` appears nowhere in this function. -pub(crate) fn capability_tool_name( - capability: &ExecutionCapability, -) -> Result { - capability - .source - .model_visible_tool_name() - .map(str::to_string) - .ok_or_else(|| TerminalError::new("capability has no governed tool owner in Task 6").into()) -} - -const fn capability_source_kind(source: &CapabilitySource) -> &'static str { - match source { - CapabilitySource::BuiltInTool { .. } => "built_in_tool", - CapabilitySource::HandTool { .. } => "hand_tool", - CapabilitySource::McpTool { .. } => "mcp_tool", - CapabilitySource::ActionArtifact { .. } => "action_artifact", - CapabilitySource::ConnectorAction { .. } => "connector_action", - CapabilitySource::InstalledConnectorAction { .. } => "installed_connector_action", - CapabilitySource::SkillAction { .. } => "skill_action", - CapabilitySource::SkillCode { .. } => "skill_code", - CapabilitySource::Memory { .. } => "memory", - CapabilitySource::Knowledge { .. } => "knowledge", - CapabilitySource::Model => "model", - } -} - -fn action_review_invocation_result( - resolution: ExecutionActionReviewResolution, -) -> Result { - match resolution { - ExecutionActionReviewResolution::Completed { tool_output } => { - match serde_json::from_value(tool_output) { - Ok(output) => Ok(CapabilityInvocationResult::Output(Box::new(output))), - Err(error) => Ok(CapabilityInvocationResult::Terminal( - ExecutionTaskResult::UnknownOutcome { - message: format!( - "reviewed capability returned an invalid output after possible commit: {error}" - ), - }, - )), - } - } - ExecutionActionReviewResolution::Failed { class, message } => Ok( - CapabilityInvocationResult::Terminal(ExecutionTaskResult::Failed { class, message }), - ), - ExecutionActionReviewResolution::UnknownOutcome { message } => Ok( - CapabilityInvocationResult::Terminal(ExecutionTaskResult::UnknownOutcome { message }), - ), - ExecutionActionReviewResolution::NotDispatched { reason } => Ok( - CapabilityInvocationResult::Terminal(ExecutionTaskResult::Failed { - class: ExecutionFailureClass::Terminal, - message: execution_dispatch_rejection_message(reason), - }), - ), - ExecutionActionReviewResolution::Denied { reason } => Ok( - CapabilityInvocationResult::Terminal(ExecutionTaskResult::Failed { - class: ExecutionFailureClass::AuthorizationDenied, - message: reason, - }), - ), - ExecutionActionReviewResolution::TimedOut { reason } => Ok( - CapabilityInvocationResult::Terminal(ExecutionTaskResult::Failed { - class: ExecutionFailureClass::DeadlineExceeded, - message: reason, - }), - ), - } -} - -fn execution_dispatch_rejection_message(reason: ExecutionToolDispatchRejection) -> String { - let reason = match reason { - ExecutionToolDispatchRejection::OriginNotFound => "origin_not_found", - ExecutionToolDispatchRejection::StaleGeneration => "stale_generation", - ExecutionToolDispatchRejection::OperationNotRunning => "operation_not_running", - ExecutionToolDispatchRejection::RunNotDispatchable => "run_not_dispatchable", - }; - format!("execution effect was not dispatched: {reason}") -} - -fn capability_invocation_outcome( - idempotency_class: IdempotencyClass, - action_class: moa_core::types::action_policy::ActionClass, - invocation: CapabilityInvocationResult, - usage: ExecutionUsage, -) -> Result { - match invocation { - CapabilityInvocationResult::Terminal(result) => Ok(ExecutionTaskOutcome { - schema_version: 1, - usage, - result, - }), - CapabilityInvocationResult::Output(output) if output.is_error() => { - if idempotency_class == IdempotencyClass::Idempotent { - Ok(failed_task_outcome( - ExecutionFailureClass::Retryable, - output.safe_output.to_text(), - usage, - )) - } else if action_class != moa_core::types::action_policy::ActionClass::Read { - Ok(ExecutionTaskOutcome { - schema_version: 1, - usage, - result: ExecutionTaskResult::UnknownOutcome { - message: format!( - "non-idempotent side-effecting capability returned an error after possible commit: {}", - output.safe_output.to_text() - ), - }, - }) - } else { - Ok(failed_task_outcome( - ExecutionFailureClass::Terminal, - output.safe_output.to_text(), - usage, - )) - } - } - CapabilityInvocationResult::Output(output) => { - // A non-safe class already cleared `structured`, so a task whose - // capability returned attacker-shaped output completes with the safe - // replacement text rather than the raw structured payload. - let value = output - .safe_output - .structured_payload() - .cloned() - .unwrap_or_else(|| Value::String(output.safe_output.to_text())); - Ok(completed_task_outcome(value, usage)) - } - } -} - -fn invalid_capability_output_outcome( - action_class: moa_core::types::action_policy::ActionClass, - message: String, - usage: ExecutionUsage, -) -> ExecutionTaskOutcome { - if action_class == moa_core::types::action_policy::ActionClass::Read { - failed_task_outcome(ExecutionFailureClass::InvalidOutput, message, usage) - } else { - ExecutionTaskOutcome { - schema_version: 1, - usage, - result: ExecutionTaskResult::UnknownOutcome { - message: format!( - "side-effecting capability returned invalid output after possible commit: {message}" - ), - }, - } - } -} - -fn agent_system_prompt(instructions: &str, skills: &[String]) -> String { - format!( - "{instructions}\n\nPinned instruction skills:\n{}\n\nReturn only JSON. To finish normally return any JSON value. To request input or replanning, return the exact ExecutionTaskResult tagged shape with status needs_input or needs_replan.", - skills.join("\n\n---\n\n") - ) -} - -fn serialized_len(value: &T) -> u64 { - serde_json::to_vec(value) - .map(|bytes| bytes.len() as u64) - .unwrap_or_default() -} - -fn input_promise_key(task_id: moa_execution::state::ExecutionTaskId, generation: u64) -> String { - format!("execution_input:{task_id}:{generation}") -} - -fn review_promise_key(task_id: moa_execution::state::ExecutionTaskId, generation: u64) -> String { - format!("execution_review:{task_id}:{generation}") -} - -fn signal_promise_key( - task_id: moa_execution::state::ExecutionTaskId, - generation: u64, - signal_name: &str, -) -> String { - format!("execution_signal:{task_id}:{generation}:{signal_name}") -} - -fn action_review_promise_key(review_uid: uuid::Uuid, generation: u64) -> String { - format!("execution_action_review:{review_uid}:{generation}") -} - -fn require_task_key( - key: &str, - task_id: moa_execution::state::ExecutionTaskId, -) -> Result<(), HandlerError> { - if key == task_id.to_string() { - Ok(()) - } else { - Err(TerminalError::new_with_code(404, "execution task id mismatch").into()) - } -} - -fn execution_scope(request: &ExecutionTaskWorkflowRequest) -> ExecutionScope { - request.contact_id.map_or( - ExecutionScope::Tenant { - tenant_id: request.tenant_id, - }, - |contact_id| ExecutionScope::Contact { - tenant_id: request.tenant_id, - contact_id, - }, - ) -} - -fn annotate_execution_task_identity_span( - run_uid: uuid::Uuid, - task_id: moa_execution::state::ExecutionTaskId, -) { - let span = tracing::Span::current(); - span.set_attribute("moa.execution.run_uid", run_uid.to_string()); - span.set_attribute("moa.execution.task_id", task_id.to_string()); -} - -fn annotate_execution_task_record_span(run: &ExecutionRunRecord, task: &ExecutionTaskRecord) { - annotate_execution_task_identity_span(task.run_uid, task.task_id); - let span = tracing::Span::current(); - span.set_attribute("moa.execution.plan_hash", run.active_plan_hash.to_string()); - span.set_attribute( - "moa.execution.plan_revision", - task.plan_revision.to_string(), - ); - span.set_attribute("moa.execution.node_id", task.node_id.clone()); -} - -fn action_scope( - tenant_id: moa_core::types::identifiers::TenantId, - contact_id: Option, -) -> ActionRuleScope { - contact_id.map_or(ActionRuleScope::Tenant { tenant_id }, |contact_id| { - ActionRuleScope::Contact { - tenant_id, - contact_id, - } - }) -} - -fn send_run_wake( - ctx: &WorkflowContext<'_>, - run_uid: uuid::Uuid, - wake_epoch: u64, - reason: ExecutionRunWakeReason, -) { - // Detached by design: wake_epoch is a persisted generation fence and the run - // workflow ignores duplicate or superseded notifications. - crate::restate_identity::replay_safe_request( - ctx.workflow_client::(run_uid.to_string()) - .wake(Json::from(ExecutionRunWakeRequest { - run_uid, - wake_epoch, - reason, - })), - ) - .send(); -} - -async fn cleanup_task_hands( - ctx: &WorkflowContext<'_>, - run: &ExecutionRunRecord, - task: &ExecutionTaskRecord, -) -> Result<(), HandlerError> { - crate::restate_identity::replay_safe_request( - ctx.service_client::() - .release_execution_task_hands(Json::from(ReleaseExecutionTaskHandsRequest { - tenant_id: run.tenant_id, - session_id: run.session_id, - run_uid: run.run_uid, - task_id: task.task_id, - })), - ) - .call() - .await?; - Ok(()) -} - -fn execution_error(error: moa_execution::Error) -> HandlerError { - TerminalError::new(format!("execution task workflow failed: {error}")).into() -} - -#[cfg(test)] -mod tests { - use moa_artifacts::{ - execution_plan::{ - CapabilityReference, ExecutionFailureClass, ExecutionTaskResult, ExecutionUsage, - }, - reference::ArtifactRef, - }; - use moa_core::types::{ - action_policy::{ActionClass, ActionPolicyEffect, RiskLevel}, - completion::ToolInvocation, - identifiers::{ConnectorConnectionId, ToolCallId}, - tools::IdempotencyClass, - }; - use moa_execution::capability::{ - CapabilityPolicyContext, CapabilitySource, ExecutionCapability, ExecutionClass, - ExecutionEstimate, - }; - use moa_execution::wire::ExecutionActionReviewResolution; - use moa_execution::wire::ExecutionToolDispatchRejection; - use serde_json::json; - - use super::{ - ActionReviewCancellationStep, CapabilityInvocationResult, ExecutionActionReviewSettlement, - GovernedCapabilityOutcome, GovernedInvocationOutcome, action_review_cancellation_step, - action_review_invocation_result, cancellation_outcome_before_next_agent_tool, - capability_invocation_outcome, classify_governed_capability_outcome, - task_agent_tool_schema, validate_agent_capability_bindings, - }; - - fn task_agent_capability( - reference_name: &str, - source: CapabilitySource, - policy_context: CapabilityPolicyContext, - ) -> ExecutionCapability { - ExecutionCapability { - reference: CapabilityReference { - name: reference_name.to_string(), - version: "v1".to_string(), - }, - contract_revision: "contract-v1".to_string(), - description: format!("Task agent capability {reference_name}"), - input_schema: json!({"type": "object"}), - output_schema: json!({"type": "object"}), - action_class: ActionClass::ExternalWrite, - risk_level: RiskLevel::High, - default_effect: ActionPolicyEffect::Allow, - idempotency_class: IdempotencyClass::Idempotent, - execution_class: ExecutionClass::External, - source, - policy_context, - estimate: ExecutionEstimate { - tool_calls: 1, - tasks: 1, - ..ExecutionEstimate::default() - }, - rollback: None, - } - } - - fn registered_task_agent_capability(tool_name: &str) -> ExecutionCapability { - let source = CapabilitySource::McpTool { - server: "fixture".to_string(), - tool_name: tool_name.to_string(), - remote_name: "probe".to_string(), - }; - task_agent_capability( - tool_name, - source.clone(), - CapabilityPolicyContext::registered(source), - ) - } - - fn action_task_agent_capability(tool_name: &str) -> ExecutionCapability { - let action_ref = ArtifactRef::action_artifact("reviewed-operation"); - let revision_uid = uuid::Uuid::from_u128(11); - let source = CapabilitySource::ActionArtifact { - action_ref: action_ref.clone(), - revision_uid, - tool_name: tool_name.to_string(), - }; - task_agent_capability( - &action_ref.to_string(), - source.clone(), - CapabilityPolicyContext::artifact( - source, - Some(action_ref), - uuid::Uuid::from_u128(10), - revision_uid, - ActionPolicyEffect::AdminReview, - ), - ) - } - - fn skill_action_task_agent_capability(tool_name: &str) -> ExecutionCapability { - let skill_ref = ArtifactRef::artifact( - moa_artifacts::document::ArtifactKind::Skill, - "reviewed-operations", - ); - let action_ref = ArtifactRef::action_artifact("reviewed-operation"); - let revision_uid = uuid::Uuid::from_u128(21); - let source = CapabilitySource::SkillAction { - skill_ref: skill_ref.clone(), - revision_uid, - action_id: "reviewed-operation".to_string(), - tool_name: tool_name.to_string(), - }; - task_agent_capability( - &format!("{skill_ref}#reviewed-operation"), - source.clone(), - CapabilityPolicyContext::artifact( - source, - Some(action_ref), - uuid::Uuid::from_u128(20), - revision_uid, - ActionPolicyEffect::AdminReview, - ), - ) - } - - #[test] - fn task_agent_schema_uses_persisted_installed_connector_capability() { - // Pins: replayed task-agent prompts use the exact model name and schema - // already persisted with typed connector provenance; no live global - // router lookup or connector-name parsing can substitute authority. - let connector_ref = ArtifactRef::connector("support"); - let action_ref = ArtifactRef::action("support", "create_ticket"); - let source = CapabilitySource::InstalledConnectorAction { - connector_ref, - connection_id: ConnectorConnectionId(uuid::Uuid::from_u128(71)), - binding_id: uuid::Uuid::from_u128(72), - connection_generation: 9, - definition_artifact_uid: uuid::Uuid::from_u128(73), - definition_revision_uid: uuid::Uuid::from_u128(74), - action_id: "create_ticket".to_string(), - contract_hash: "ab".repeat(32), - governed_contract_revision: "governed-v9".to_string(), - minimum_effect: ActionPolicyEffect::AdminReview, - tool_name: "conn__00000000000000000000000000000047__create_ticket".to_string(), - }; - let capability = task_agent_capability( - &action_ref.to_string(), - source.clone(), - CapabilityPolicyContext::artifact( - source, - Some(action_ref), - uuid::Uuid::from_u128(73), - uuid::Uuid::from_u128(74), - ActionPolicyEffect::AdminReview, - ), - ); - let bindings = validate_agent_capability_bindings(vec![&capability]) - .expect("one installed connector capability should be unambiguous"); - - assert_eq!( - task_agent_tool_schema(&bindings[0]), - json!({ - "name": "conn__00000000000000000000000000000047__create_ticket", - "description": "Task agent capability action://support.create_ticket", - "input_schema": {"type": "object"}, - }) - ); - } - - #[test] - fn artifact_policy_floor_rejects_ambiguous_task_agent_bindings_in_both_orders() { - // Pins: model-visible backing-tool names cannot collapse raw Allow authority - // with either an Action or inherited SkillAction review floor. This pure - // production guard runs before the task performs any model or tool I/O. - let tool_name = "mcp__fixture__probe"; - let raw = registered_task_agent_capability(tool_name); - for alias in [ - action_task_agent_capability(tool_name), - skill_action_task_agent_capability(tool_name), - ] { - let mut reference_labels = [ - format!("{}@{}", alias.reference.name, alias.reference.version), - format!("{}@{}", raw.reference.name, raw.reference.version), - ]; - reference_labels.sort(); - let expected = format!( - "Terminal error [500]: task-local agent capability references {} and {} resolve to ambiguous model-visible tool `{tool_name}`", - reference_labels[0], reference_labels[1] - ); - for capabilities in [vec![&raw, &alias], vec![&alias, &raw]] { - let error = validate_agent_capability_bindings(capabilities) - .expect_err("ambiguous model-visible authority must fail before model use"); - let actual = >::as_ref(&error) - .to_string(); - assert_eq!(actual, expected); - } - } - } - - #[test] - fn idempotent_capability_action_review_terminal_results_never_become_retryable() { - // Pins: durable action-review delivery remains typed through capability - // execution; capability idempotency may classify ordinary tool errors, - // but it cannot rewrite review denial, timeout, or terminal failure. - let cases = [ - ( - ExecutionActionReviewResolution::Failed { - class: ExecutionFailureClass::Unsupported, - message: "reviewed action failed".to_string(), - }, - ExecutionFailureClass::Unsupported, - "reviewed action failed", - ), - ( - ExecutionActionReviewResolution::Denied { - reason: "tenant admin denied".to_string(), - }, - ExecutionFailureClass::AuthorizationDenied, - "tenant admin denied", - ), - ( - ExecutionActionReviewResolution::TimedOut { - reason: "review deadline elapsed".to_string(), - }, - ExecutionFailureClass::DeadlineExceeded, - "review deadline elapsed", - ), - ]; - - for (resolution, expected_class, expected_message) in cases { - let invocation = action_review_invocation_result(resolution) - .expect("terminal review resolution should be valid"); - assert!(matches!( - invocation, - CapabilityInvocationResult::Terminal(_) - )); - let outcome = capability_invocation_outcome( - IdempotencyClass::Idempotent, - ActionClass::Read, - invocation, - ExecutionUsage { - cost_microusd: 0, - tokens: 0, - tool_calls: 0, - retrieved_bytes: 0, - }, - ) - .expect("typed review result should map to a task outcome"); - assert!(matches!( - outcome.result, - ExecutionTaskResult::Failed { class, message } - if class == expected_class && message == expected_message - )); - } - } - - #[test] - fn governed_execution_ambiguity_is_a_typed_unknown_task_outcome() { - // Pins: once ToolExecutor reports that a side effect may have committed, - // the execution workflow persists UnknownOutcome and cannot resend it as - // an ordinary failed invocation. - let classified = - classify_governed_capability_outcome(GovernedInvocationOutcome::UnknownOutcome { - tool_id: ToolCallId(uuid::Uuid::from_u128(81)), - invocation: ToolInvocation { - id: Some("tool-81".to_string()), - name: "fixture_effect".to_string(), - input: json!({"value": 1}), - }, - message: "external result is ambiguous".to_string(), - }) - .expect("typed ambiguity is a valid terminal task result"); - assert!(matches!( - classified, - GovernedCapabilityOutcome::Terminal(ExecutionTaskResult::UnknownOutcome { message }) - if message == "external result is ambiguous" - )); - } - - #[test] - fn governed_execution_admission_rejection_is_definitive_failure() { - // Pins: the row-locked owner admission proved no external effect began, so - // a fenced or stale origin is terminal Failed and never UnknownOutcome. - let classified = - classify_governed_capability_outcome(GovernedInvocationOutcome::NotDispatched { - tool_id: ToolCallId(uuid::Uuid::from_u128(82)), - invocation: ToolInvocation { - id: Some("tool-82".to_string()), - name: "fixture_effect".to_string(), - input: json!({"value": 2}), - }, - reason: ExecutionToolDispatchRejection::RunNotDispatchable, - }) - .expect("definitive admission rejection is a valid task result"); - assert!(matches!( - classified, - GovernedCapabilityOutcome::Terminal(ExecutionTaskResult::Failed { - class: ExecutionFailureClass::Terminal, - message, - }) if message.ends_with("run_not_dispatchable") - )); - } - - #[test] - fn reviewed_execution_ambiguity_and_malformed_output_are_unknown() { - // Pins: an approved external effect has crossed the commit boundary, so - // both an explicit ambiguous resolution and undecodable completed output - // require reconciliation instead of retry or generic workflow failure. - for resolution in [ - ExecutionActionReviewResolution::UnknownOutcome { - message: "reviewed result is ambiguous".to_string(), - }, - ExecutionActionReviewResolution::Completed { - tool_output: json!("not a secured tool output"), - }, - ] { - let result = action_review_invocation_result(resolution) - .expect("review ambiguity remains a typed task result"); - assert!(matches!( - result, - CapabilityInvocationResult::Terminal(ExecutionTaskResult::UnknownOutcome { .. }) - )); - } - } - - #[test] - fn cancellation_revokes_unclaimed_review_but_joins_claimed_effect() { - // Pins: cancellation before tenant clear terminalizes the review before any - // tool dispatch; once the decision transaction claimed the effect, the task - // must join its definitive resolution and cannot overwrite it as Cancelled. - assert!(matches!( - action_review_cancellation_step( - ExecutionActionReviewSettlement::Revoked, - "run cancelled", - ), - ActionReviewCancellationStep::Cancelled(ExecutionTaskResult::Cancelled { reason }) - if reason == "run cancelled" - )); - assert!(matches!( - action_review_cancellation_step( - ExecutionActionReviewSettlement::JoinRequired, - "run cancelled", - ), - ActionReviewCancellationStep::JoinResolution - )); - - let definitive = - action_review_invocation_result(ExecutionActionReviewResolution::UnknownOutcome { - message: "claimed effect is ambiguous".to_string(), - }) - .expect("claimed review ambiguity must remain typed"); - assert!(matches!( - definitive, - CapabilityInvocationResult::Terminal(ExecutionTaskResult::UnknownOutcome { message }) - if message == "claimed effect is ambiguous" - )); - - let no_effect = - action_review_invocation_result(ExecutionActionReviewResolution::NotDispatched { - reason: ExecutionToolDispatchRejection::RunNotDispatchable, - }) - .expect("claimed no-effect admission rejection stays definitive"); - assert!(matches!( - no_effect, - CapabilityInvocationResult::Terminal(ExecutionTaskResult::Failed { - class: ExecutionFailureClass::Terminal, - message, - }) if message.ends_with("run_not_dispatchable") - )); - - let unfenced = - action_review_invocation_result(ExecutionActionReviewResolution::NotDispatched { - reason: ExecutionToolDispatchRejection::StaleGeneration, - }) - .expect("definitive no-effect admission rejection is typed"); - assert!(matches!( - unfenced, - CapabilityInvocationResult::Terminal(ExecutionTaskResult::Failed { - class: ExecutionFailureClass::Terminal, - message, - }) if message.ends_with("stale_generation") - )); - } - - #[test] - fn cancellation_fence_stops_the_next_agent_tool() { - // Pins: after one joined governed effect, a cancellation observed at the - // per-tool admission boundary terminates the task before a second tool - // from the same model response can be dispatched. - let outcome = cancellation_outcome_before_next_agent_tool( - Some("run fenced forward work".to_string()), - &ExecutionUsage { - cost_microusd: 0, - tokens: 0, - tool_calls: 1, - retrieved_bytes: 0, - }, - ) - .expect("durable cancellation must stop the next tool"); - assert!(matches!( - outcome.result, - ExecutionTaskResult::Cancelled { reason } - if reason == "run fenced forward work" - )); - } - - #[test] - fn a_leaked_task_canary_halts_the_task_owner_in_exactly_one_transition() { - // Pins the execution-task end of the canary contract, which nothing else - // reaches: an agent turn mints a canary, and a capability that echoes it - // back must halt THAT task owner in a single transition. - // - // Three properties, each load bearing: - // 1. The token this crate mints is the token the classifier recognizes. - // A format change on either side would silently make every task - // canary undetectable, and no other test composes the two. - // 2. `CanaryLeak` scores 4, so a clear circuit jumps straight to - // `Halted`. Exactly one transition must be produced — a walk through - // warned and disabled would mean the single-highest-stage rule broke - // for this owner. - // 3. The token never survives into `safe_output`. A halt that still - // forwarded the leaked marker to the model would defeat its purpose. - use moa_core::types::identifiers::{SessionId, ToolCallId}; - use moa_core::types::security::{ - OutputAssessmentClass, SecurityCircuitOwner, SecurityCircuitStage, - SecurityCircuitState, ToolCapabilityId, - }; - use moa_core::types::tools::ToolOutput; - - let canary = moa_security::new_canary_token(); - assert!( - moa_security::canary_system_message(&canary).contains(&canary), - "the system copy must carry the exact minted token; it is what an \ - attacker exfiltrates" - ); - - let leaked = ToolOutput::text( - format!("Here is the marker you asked for: {canary}"), - std::time::Duration::from_millis(1), - ); - let capability = ToolCapabilityId::builtin("lookup"); - let secured = moa_security::classify_tool_output( - &leaked, - moa_security::OutputClassification { - capability: &capability, - active_canary: Some(canary.as_str()), - }, - ); - - assert_eq!( - secured.assessment.class, - OutputAssessmentClass::CanaryLeak, - "a capability echoing the turn's canary is a leak, not merely suspicious" - ); - assert!(secured.assessment.class.clears_raw_carriers()); - assert!( - !serde_json::to_string(&secured) - .expect("serialize secured output") - .contains(&canary), - "the leaked marker must not survive anywhere in the envelope" - ); - - let owner = SecurityCircuitOwner::ExecutionTask { - run_uid: uuid::Uuid::from_u128(0x9001), - task_uid: uuid::Uuid::from_u128(0x9002), - generation: 3, - }; - let mut circuit = SecurityCircuitState::default(); - circuit.adopt_owner(&owner); - let transition = moa_security::apply_owner_assessment( - &mut circuit, - moa_security::CircuitTarget { - session_id: SessionId(uuid::Uuid::from_u128(0x9003)), - owner: &owner, - capability: &capability, - tool_call_id: ToolCallId(uuid::Uuid::from_u128(0x9004)), - }, - &secured.assessment, - ) - .expect("the admitted owner matches") - .expect("a first-strike canary leak must produce one transition"); - - assert_eq!(transition.prior_stage, SecurityCircuitStage::Clear); - assert_eq!( - transition.reached_stage, - SecurityCircuitStage::Halted, - "score 4 from clear halts directly; no warned or disabled step" - ); - assert_eq!(transition.prior_score, 0); - assert_eq!(transition.reached_score, 4); - assert_eq!( - circuit.stage(&owner, &capability), - SecurityCircuitStage::Halted, - "the halted stage is what drives ExecutionTaskResult::Failed{{Terminal}}" - ); - assert!( - !circuit.permits_dispatch(&owner, &capability), - "a halted capability must not dispatch again under this owner" - ); - } -} diff --git a/crates/moa-orchestrator/src/workflows/execution_task_attempt.rs b/crates/moa-orchestrator/src/workflows/execution_task_attempt.rs new file mode 100644 index 000000000..8f32de8f9 --- /dev/null +++ b/crates/moa-orchestrator/src/workflows/execution_task_attempt.rs @@ -0,0 +1,419 @@ +//! One immutable, bounded task-attempt workflow per durable dispatch identity. + +mod active; +mod external; +mod watchdog; +mod yielding; + +use std::{collections::HashMap, sync::Arc}; + +use chrono::{Duration, Utc}; +use moa_artifacts::execution_plan::{ExecutionFailureClass, ExecutionTaskResult}; +use moa_config::{ExecutionConfig, SessionLimitsConfig}; +use moa_core::{traits::ChannelAdapter, types::channel::Channel}; +use moa_execution::wire::{ + ExecutionAttemptWatchdogResponse, ExecutionTaskAttemptCancelRequest, + ExecutionTaskAttemptRequest, ExecutionTaskAttemptWatchdogRequest, +}; +use moa_execution::{ + capability::ExecutionCapability, + interpreter::validate_task_outcome, + repository::{ + ExecutionRepository, + task::{ + ReleasedTaskAttemptCapacityOutcome, TaskAttemptFence, TaskAttemptRecord, + TaskAttemptSettlementOutcome, TaskAttemptStartOutcome, + }, + }, + state::{exhaust_retry_outcome, retry_delay_ms}, +}; +use moa_observability::restate_observability::annotate_restate_handler_span; +use moa_session::PostgresSessionStore; +use restate_sdk::prelude::*; + +use crate::{ + services::execution_dispatcher::{DispatchExecutionsRequest, ExecutionDispatcherClient}, + workflows::errors::execution_error_to_handler_error, +}; + +/// Returns the catalog-owned model-visible name for one governed capability. +pub(crate) fn capability_tool_name( + capability: &ExecutionCapability, +) -> Result { + capability + .source + .model_visible_tool_name() + .map(str::to_string) + .ok_or_else(|| TerminalError::new("capability has no governed tool owner").into()) +} + +/// Durable surface for one bounded task-attempt slice. +/// +/// Both handlers are keyed by the immutable dispatch UID. `run` never waits for +/// input, review, signals, timers, or provider callbacks; those conditions are +/// persisted and resumed by a later controller activation. +#[restate_sdk::workflow] +pub trait ExecutionTaskAttempt { + /// Executes at most one admitted attempt generation and then returns. + async fn run(request: Json) -> Result<(), HandlerError>; + + /// Classifies one exact active attempt whose durable watchdog became due. + #[shared] + async fn watchdog( + request: Json, + ) -> Result, HandlerError>; + + /// Checkpoints and relinquishes one exact active attempt after a durable run fence. + #[shared] + async fn cancel(request: Json) -> Result<(), HandlerError>; +} + +/// Runtime dependencies for one immutable bounded task attempt. +#[derive(Clone)] +pub struct ExecutionTaskAttemptImpl { + repository: ExecutionRepository, + pool: sqlx::PgPool, + config: ExecutionConfig, + session_store: Arc, + session_limits: SessionLimitsConfig, + channel_adapters: Arc>>, +} + +impl ExecutionTaskAttemptImpl { + /// Creates the bounded task-attempt workflow over authoritative runtime stores. + #[must_use] + pub fn new( + pool: sqlx::PgPool, + config: ExecutionConfig, + session_store: Arc, + session_limits: SessionLimitsConfig, + channel_adapters: Arc>>, + ) -> Self { + Self { + repository: ExecutionRepository::new(pool.clone()), + pool, + config, + session_store, + session_limits, + channel_adapters, + } + } +} + +impl ExecutionTaskAttempt for ExecutionTaskAttemptImpl { + #[tracing::instrument(skip(self, ctx, request))] + // SAFETY: only the strict execution-dispatch outbox can invoke this identity-free, + // generation-fenced workflow; authoritative identity is loaded from the locked run. + async fn run( + &self, + ctx: WorkflowContext<'_>, + request: Json, + ) -> Result<(), HandlerError> { + crate::ctx::adopt_incoming_trace_parent(&ctx); + annotate_restate_handler_span("ExecutionTaskAttempt", "run"); + let request = request.into_inner(); + require_dispatch_key(ctx.key(), request.dispatch_uid)?; + let fence = task_attempt_fence(&request); + let repository = self.repository.clone(); + let started = ctx + .run(|| async move { + repository + .start_task_attempt(fence) + .await + .map(|outcome| match outcome { + TaskAttemptStartOutcome::Started(record) + | TaskAttemptStartOutcome::AlreadyStarted(record) => Some(record), + TaskAttemptStartOutcome::NotFound + | TaskAttemptStartOutcome::Stale + | TaskAttemptStartOutcome::InvalidState => None, + }) + .map(Json::from) + .map_err(execution_error_to_handler_error) + }) + .name("start_task_attempt") + .await? + .into_inner(); + let Some(started) = started else { + return Ok(()); + }; + let repository = self.repository.clone(); + let checkpoint_scope = moa_execution::repository::ExecutionScope::Tenant { + tenant_id: request.tenant_id, + }; + let checkpoint_run_uid = request.run_uid; + let checkpoint_task_id = request.task_id; + let checkpoint = ctx + .run(|| async move { + repository + .load_task_attempt_checkpoint( + checkpoint_scope, + checkpoint_run_uid, + checkpoint_task_id, + ) + .await + .map(Json::from) + .map_err(execution_error_to_handler_error) + }) + .name("load_task_attempt_checkpoint") + .await? + .into_inner(); + let exit = active::execute_task_attempt(self, &ctx, &request, &started, checkpoint).await?; + settle_active_exit(self, &ctx, &request, &started, exit).await + } + + #[tracing::instrument(skip(self, ctx, request))] + // SAFETY: only exact durable trigger delivery invokes this shared handler; all + // coordinates are revalidated against the active attempt before mutation. + async fn watchdog( + &self, + ctx: SharedWorkflowContext<'_>, + request: Json, + ) -> Result, HandlerError> { + crate::ctx::adopt_incoming_trace_parent(&ctx); + annotate_restate_handler_span("ExecutionTaskAttempt", "watchdog"); + let request = request.into_inner(); + require_dispatch_key(ctx.key(), request.dispatch_uid)?; + let watchdog = watchdog::handle_task_attempt_watchdog(self, &ctx, request).await?; + // TriggerDelivery awaits this handler, so its owning dispatcher observes every outbox row + // committed by watchdog settlement before selecting the next durable timing head. + Ok(Json::from(ExecutionAttemptWatchdogResponse { + outcome: watchdog.outcome, + })) + } + + #[tracing::instrument(skip(self, ctx, request))] + // SAFETY: only strict terminal-fence cancellation outbox delivery invokes this + // handler; exact attempt, dispatch, capacity, and watchdog identities are checked. + async fn cancel( + &self, + ctx: SharedWorkflowContext<'_>, + request: Json, + ) -> Result<(), HandlerError> { + crate::ctx::adopt_incoming_trace_parent(&ctx); + annotate_restate_handler_span("ExecutionTaskAttempt", "cancel"); + let request = request.into_inner(); + require_dispatch_key(ctx.key(), request.active_dispatch_uid)?; + let dispatch_uid = request.active_dispatch_uid; + yielding::cancel_task_attempt(self, &ctx, request).await?; + kick_dispatcher_shared(&ctx, dispatch_uid, "cancel").await + } +} + +fn task_attempt_fence(request: &ExecutionTaskAttemptRequest) -> TaskAttemptFence { + TaskAttemptFence { + tenant_id: request.tenant_id, + run_uid: request.run_uid, + task_id: request.task_id, + controller_generation: request.controller_generation, + attempt_generation: request.attempt_generation, + dispatch_uid: request.dispatch_uid, + capacity_reservation_uid: request.capacity_reservation_uid, + watchdog_trigger_uid: request.watchdog_trigger_uid, + attempt_deadline_at: request.attempt_deadline_at, + } +} + +async fn settle_active_exit( + workflow: &ExecutionTaskAttemptImpl, + ctx: &WorkflowContext<'_>, + request: &ExecutionTaskAttemptRequest, + started: &TaskAttemptRecord, + exit: active::ActiveTaskAttemptExit, +) -> Result<(), HandlerError> { + let boundary = match exit { + active::ActiveTaskAttemptExit::Outcome(outcome) => { + let outcome = validate_task_outcome( + &started.run.active_plan, + &started.task.node_id, + &started.task.kind, + outcome, + ); + let outcome = exhaust_retry_outcome(started.task.attempt, &started.task.retry, outcome); + let settled_at = journal_now(ctx, "task_attempt_settled_at").await?; + let retry_at = matches!( + outcome.result, + ExecutionTaskResult::Failed { + class: ExecutionFailureClass::Retryable, + .. + } + ) + .then(|| { + settled_at + + Duration::milliseconds( + i64::try_from(retry_delay_ms( + started.task.attempt.saturating_add(1), + &started.task.retry, + )) + .unwrap_or(i64::MAX), + ) + }); + let Some(releasing) = yielding::begin_release_workflow( + workflow, + ctx, + request, + started.task.generation, + "task_outcome", + ) + .await? + else { + return Ok(()); + }; + let release_receipt = + yielding::checkpoint_task_hands_workflow(workflow, ctx, request, &releasing) + .await?; + let Some(release_receipt) = release_receipt else { + return Err(TerminalError::new( + "normal task outcome omitted its durable hand-release receipt", + ) + .into()); + }; + let repository = workflow.repository.clone(); + let fence = task_attempt_fence(request); + let logical_generation = releasing.task.generation; + let capacity_receipt = release_receipt.clone(); + let released = ctx + .run(|| async move { + repository + .release_released_task_attempt_capacity( + fence, + logical_generation, + capacity_receipt, + ) + .await + .and_then(|outcome| match outcome { + ReleasedTaskAttemptCapacityOutcome::Applied + | ReleasedTaskAttemptCapacityOutcome::Replayed => Ok(Json::from(true)), + ReleasedTaskAttemptCapacityOutcome::NotFound + | ReleasedTaskAttemptCapacityOutcome::Stale => Ok(Json::from(false)), + ReleasedTaskAttemptCapacityOutcome::InvalidState => { + Err(moa_execution::Error::InvalidRepositoryData { + message: "normal task outcome capacity release was rejected" + .to_string(), + }) + } + }) + .map_err(execution_error_to_handler_error) + }) + .name("release_task_attempt_capacity") + .await? + .into_inner(); + if !released { + return Ok(()); + } + let repository = workflow.repository.clone(); + let config = workflow.config.clone(); + let fence = task_attempt_fence(request); + ctx.run(|| async move { + let settlement = repository + .settle_released_task_attempt( + &config, + fence, + outcome, + retry_at, + settled_at, + Some(release_receipt), + ) + .await; + settlement + .and_then(|result| match result { + TaskAttemptSettlementOutcome::Applied { .. } + | TaskAttemptSettlementOutcome::Replayed { .. } + | TaskAttemptSettlementOutcome::NotFound + | TaskAttemptSettlementOutcome::Stale => Ok(()), + TaskAttemptSettlementOutcome::InvalidState => { + Err(moa_execution::Error::InvalidRepositoryData { + message: "active task-attempt settlement was rejected".to_string(), + }) + } + }) + .map_err(execution_error_to_handler_error) + }) + .name("settle_task_attempt") + .await?; + "outcome" + } + active::ActiveTaskAttemptExit::ReviewPending { continuation } => { + yielding::park_review(workflow, ctx, request, started, continuation).await?; + "review" + } + active::ActiveTaskAttemptExit::Continue { continuation } => { + yielding::yield_continuation(workflow, ctx, request, started, continuation).await?; + "continuation" + } + active::ActiveTaskAttemptExit::InputPending { + outcome, + continuation, + } => { + yielding::park_input(workflow, ctx, request, started, outcome, continuation).await?; + "input" + } + active::ActiveTaskAttemptExit::ExternalJob { + external_job_uid, + continuation, + } => { + external::yield_external_job( + workflow, + ctx, + request, + started, + external_job_uid, + continuation, + ) + .await?; + "external" + } + active::ActiveTaskAttemptExit::OwnershipLost => return Ok(()), + }; + // Task workflows run asynchronously from the dispatcher. Wake it immediately after the + // transaction commits; it indexes the persisted future head and owns the only delayed wake. + kick_dispatcher(ctx, request.dispatch_uid, boundary).await +} + +async fn kick_dispatcher( + ctx: &WorkflowContext<'_>, + dispatch_uid: uuid::Uuid, + boundary: &'static str, +) -> Result<(), HandlerError> { + let handle = crate::restate_identity::replay_safe_request( + ctx.service_client::() + .dispatch(Json::from(DispatchExecutionsRequest::default())) + .idempotency_key(format!("task-attempt-dispatch:{dispatch_uid}:{boundary}")), + ) + .send(); + let _invocation_id = handle.invocation_id().await?; + Ok(()) +} + +async fn kick_dispatcher_shared( + ctx: &SharedWorkflowContext<'_>, + dispatch_uid: uuid::Uuid, + boundary: &'static str, +) -> Result<(), HandlerError> { + let handle = crate::restate_identity::replay_safe_request( + ctx.service_client::() + .dispatch(Json::from(DispatchExecutionsRequest::default())) + .idempotency_key(format!("task-attempt-dispatch:{dispatch_uid}:{boundary}")), + ) + .send(); + let _invocation_id = handle.invocation_id().await?; + Ok(()) +} + +async fn journal_now( + ctx: &WorkflowContext<'_>, + name: &'static str, +) -> Result, HandlerError> { + Ok(ctx + .run(|| async { Ok::<_, HandlerError>(Json::from(Utc::now())) }) + .name(name) + .await? + .into_inner()) +} + +fn require_dispatch_key(key: &str, dispatch_uid: uuid::Uuid) -> Result<(), HandlerError> { + if key == dispatch_uid.to_string() { + Ok(()) + } else { + Err(TerminalError::new_with_code(404, "execution attempt dispatch mismatch").into()) + } +} diff --git a/crates/moa-orchestrator/src/workflows/execution_task_attempt/active.rs b/crates/moa-orchestrator/src/workflows/execution_task_attempt/active.rs new file mode 100644 index 000000000..884f122cc --- /dev/null +++ b/crates/moa-orchestrator/src/workflows/execution_task_attempt/active.rs @@ -0,0 +1,1838 @@ +//! Typed exits produced by one bounded active task slice. + +use std::collections::{BTreeMap, BTreeSet}; + +use moa_artifacts::execution_plan::{ + CapabilityReference, ExecutionFailureClass, ExecutionTaskOutcome, ExecutionTaskResult, + ExecutionUsage, +}; +use moa_core::{ + traits::SessionStore as _, + types::{ + action_policy::{ActionRuleScope, CapabilityProvenance}, + completion::{CompletionContent, ToolCallContent, ToolInvocation}, + context::ContextMessage, + identifiers::ToolCallId, + resource::ResourceBudget, + security::{ + SecurityCircuitOwner, SecurityCircuitStage, SecurityCircuitState, ToolCapabilityId, + }, + tools::{AsyncToolJobTerminalOutcome, IdempotencyClass, ToolAsyncMode}, + }, +}; +use moa_execution::{ + capability::{CapabilitySource, ExecutionCapability}, + repository::task::{ + NewTaskAttemptCheckpoint, TaskAttemptCheckpointKind, TaskAttemptCheckpointRecord, + TaskAttemptCheckpointWriteOutcome, TaskAttemptRecord, + }, + schema::validate_instance, + state::{LogicalTaskKind, completed_task_outcome, failed_task_outcome}, + wire::{ExecutionTaskAttemptRequest, ExecutionToolDispatchRejection}, +}; +use restate_sdk::prelude::*; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::{ + services::llm_gateway::{ + BoundedCompletionRequest, LLMCompletionAction, LLMCompletionOwner, LLMGatewayClient, + attach_completion_owner, completion_idempotency_key, + }, + tool_invocation::governed::{ + GovernedInvocationDisposition, GovernedInvocationOrigin, GovernedInvocationOutcome, + GovernedInvocationRequest, invoke_governed_tool, + }, + workflows::{ + errors::moa_error_to_handler_error, + execution_task_attempt::{ + ExecutionTaskAttemptImpl, capability_tool_name, journal_now, task_attempt_fence, + }, + }, +}; + +/// Current durable schema for a bounded task-agent continuation. +pub(super) const TASK_ATTEMPT_CONTINUATION_SCHEMA_VERSION: u32 = 1; + +/// Maximum canonical continuation payload accepted by persistence. +pub(super) const MAX_TASK_ATTEMPT_CONTINUATION_BYTES: usize = 1024 * 1024; + +/// Canonical state needed to resume an agent without replaying an external effect. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct TaskAttemptContinuation { + /// Durable schema version. + pub schema_version: u32, + /// Exact bounded execution state. + pub state: TaskAttemptContinuationState, + /// Exact storage-only action-review resolution consumed by the next attempt. + pub review_resolution: Option, + /// Exact terminal provider outcome consumed by a resumed agent external effect. + pub external_job_resolution: Option, + /// Release receipt that proves sandbox compute is asleep before this wait was published. + pub workspace_release_receipt_id: Option, +} + +/// Supported bounded continuation points. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub(super) enum TaskAttemptContinuationState { + /// Task-local agent state after a complete model/tool boundary. + Agent { + /// Complete bounded conversation required by the next model turn. + messages: Vec, + /// Zero-based model turn to execute next. + next_turn: u32, + /// Cumulative durable task usage. + usage: moa_artifacts::execution_plan::ExecutionUsage, + /// Prompt-injection circuit state owned by this exact task generation. + security_circuit: SecurityCircuitState, + /// Capabilities fenced by the persisted circuit. + disabled_capabilities: std::collections::BTreeMap, + /// Exact effect waiting on a storage-only action review, when present. + pending_review: Option>, + /// Model-emitted tool effects not yet dispatched by a bounded slice. + pending_tool_calls: Vec, + /// Exact agent tool invocation currently owned by an asynchronous provider job. + pending_external: Option, + }, + /// Direct capability effect waiting on a storage-only action review. + CapabilityReview { + /// Exact reviewed effect; resumption consumes its persisted resolution. + pending_review: PendingReviewedToolInvocation, + /// Cumulative durable task usage. + usage: moa_artifacts::execution_plan::ExecutionUsage, + }, + /// Direct async-capable effect reserved before its provider start. + CapabilityExternalStart { + /// Stable tool-call identity reused if recovery proves the provider did not start. + tool_id: ToolCallId, + /// Cumulative durable task usage before provider dispatch. + usage: moa_artifacts::execution_plan::ExecutionUsage, + }, +} + +/// Reviewed provider effect that must never be reconstructed from a fresh model turn. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct PendingReviewedToolInvocation { + /// Stable action-review identity. + pub review_uid: Uuid, + /// Exact durable review expiry returned by action-review admission. + pub expires_at: chrono::DateTime, + /// Exact provider invocation accepted by policy. + pub invocation: ToolInvocation, + /// Compiler/catalog-pinned replay semantics for watchdog classification. + pub effect_idempotency: IdempotencyClass, +} + +/// Agent effect that was durably handed to an asynchronous provider. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct PendingExternalToolInvocation { + /// Stable MOA external-job identity bound before sandbox release. + pub external_job_uid: Option, + /// Exact model-emitted invocation awaiting the terminal provider result. + pub invocation: ToolInvocation, + /// Compiler/catalog-pinned replay semantics. + pub effect_idempotency: IdempotencyClass, +} + +struct AgentTaskSpec<'a> { + instructions: &'a str, + skill_refs: &'a [moa_artifacts::reference::ArtifactRef], + capability_refs: &'a [CapabilityReference], + max_turns: u32, +} + +struct AgentPending { + review: Option, + tool_calls: Vec, + external: Option, +} + +impl TaskAttemptContinuation { + /// Returns the exact action-review identity carried by a parked continuation. + pub(super) const fn pending_review_uid(&self) -> Option { + match &self.state { + TaskAttemptContinuationState::Agent { pending_review, .. } => match pending_review { + Some(pending) => Some(pending.review_uid), + None => None, + }, + TaskAttemptContinuationState::CapabilityReview { pending_review, .. } => { + Some(pending_review.review_uid) + } + TaskAttemptContinuationState::CapabilityExternalStart { .. } => None, + } + } + + /// Binds the deterministic MOA external-job identity before checkpoint persistence. + pub(super) fn bind_external_job(&mut self, external_job_uid: Uuid) -> Result<(), String> { + let TaskAttemptContinuationState::Agent { + pending_external: Some(pending), + .. + } = &mut self.state + else { + return Err("agent external continuation is missing its pending effect".to_string()); + }; + if pending + .external_job_uid + .is_some_and(|current| current != external_job_uid) + { + return Err("agent external continuation is bound to another job".to_string()); + } + pending.external_job_uid = Some(external_job_uid); + Ok(()) + } + + /// Serializes and enforces the hard continuation-size bound before any DB write. + pub(super) fn to_bounded_json(&self) -> Result { + if self.schema_version != TASK_ATTEMPT_CONTINUATION_SCHEMA_VERSION { + return Err(format!( + "unsupported task continuation schema version {}", + self.schema_version + )); + } + let bytes = serde_json::to_vec(self) + .map_err(|error| format!("serialize task continuation: {error}"))?; + if bytes.len() > MAX_TASK_ATTEMPT_CONTINUATION_BYTES { + return Err(format!( + "task continuation is {} bytes; maximum is {} and the task must be decomposed or replanned", + bytes.len(), + MAX_TASK_ATTEMPT_CONTINUATION_BYTES + )); + } + serde_json::from_slice(&bytes) + .map_err(|error| format!("decode canonical task continuation: {error}")) + } +} + +/// Complete set of boundaries at which an active task workflow must return. +#[derive(Clone, Debug, PartialEq)] +pub(super) enum ActiveTaskAttemptExit { + /// A terminal, retryable, input, or replan outcome is ready for settlement. + Outcome(ExecutionTaskOutcome), + /// Action policy persisted a review; no workflow promise may remain live. + ReviewPending { + /// Exact bounded state written before releasing active ownership. + continuation: TaskAttemptContinuation, + }, + /// A complete model/tool boundary must resume in a freshly admitted attempt. + Continue { + /// Exact bounded state consumed by the next slice. + continuation: TaskAttemptContinuation, + }, + /// User input is required, with exact task-local agent state persisted before parking. + InputPending { + /// Canonical NeedsInput outcome written to the logical task. + outcome: ExecutionTaskOutcome, + /// Exact bounded state resumed after input settlement. + continuation: TaskAttemptContinuation, + }, + /// Provider work was committed and must resume outside this invocation. + ExternalJob { + /// MOA-owned job identity reserved before provider dispatch. + external_job_uid: Uuid, + /// Agent state to resume after the terminal provider callback. + continuation: Option, + }, + /// Another durable owner fenced this attempt before its provider start. + OwnershipLost, +} + +/// Executes one admitted task without waiting on any future event. +pub(super) async fn execute_task_attempt( + workflow: &ExecutionTaskAttemptImpl, + ctx: &WorkflowContext<'_>, + request: &ExecutionTaskAttemptRequest, + started: &TaskAttemptRecord, + checkpoint: Option, +) -> Result { + let continuation = checkpoint + .map( + |checkpoint| -> Result { + if checkpoint.task_generation != started.task.generation + || checkpoint.controller_generation != started.run.controller_generation + { + return Err(TerminalError::new("task continuation generation is stale").into()); + } + serde_json::from_value::(checkpoint.payload).map_err( + |error| TerminalError::new(format!("decode task continuation: {error}")).into(), + ) + }, + ) + .transpose()?; + match &started.task.kind { + LogicalTaskKind::Output { value } => { + let outcome = match validate_instance( + &started.run.active_plan.definition.output_schema, + value, + "execution_task.output", + ) { + Ok(()) => completed_task_outcome(value.clone(), started.task.actual.clone()), + Err(error) => failed_task_outcome( + ExecutionFailureClass::InvalidOutput, + error.to_string(), + started.task.actual.clone(), + ), + }; + Ok(ActiveTaskAttemptExit::Outcome(outcome)) + } + LogicalTaskKind::Capability { reference } => { + execute_direct_capability( + workflow, + ctx, + request, + started, + reference, + continuation.as_ref(), + ) + .await + } + LogicalTaskKind::Agent { + instructions, + skill_refs, + capability_refs, + max_turns, + } => { + execute_agent_turn( + workflow, + ctx, + request, + started, + AgentTaskSpec { + instructions, + skill_refs, + capability_refs, + max_turns: *max_turns, + }, + continuation.as_ref(), + ) + .await + } + LogicalTaskKind::CompletionVerifier { + instructions, + max_turns, + .. + } => { + execute_agent_turn( + workflow, + ctx, + request, + started, + AgentTaskSpec { + instructions, + skill_refs: &[], + capability_refs: &[], + max_turns: *max_turns, + }, + continuation.as_ref(), + ) + .await + } + LogicalTaskKind::Review { .. } + | LogicalTaskKind::WaitSignal { .. } + | LogicalTaskKind::WaitUntil { .. } => Err(TerminalError::new( + "storage-only logical task was incorrectly admitted as an active attempt", + ) + .into()), + } +} + +async fn persist_external_start_checkpoint( + workflow: &ExecutionTaskAttemptImpl, + ctx: &WorkflowContext<'_>, + request: &ExecutionTaskAttemptRequest, + started: &TaskAttemptRecord, + kind: TaskAttemptCheckpointKind, + continuation: &TaskAttemptContinuation, +) -> Result { + let checkpoint = NewTaskAttemptCheckpoint { + fence: task_attempt_fence(request), + task_generation: started.task.generation, + kind, + schema_version: continuation.schema_version, + payload: continuation.to_bounded_json().map_err(TerminalError::new)?, + workspace_release_receipt: None, + created_at: journal_now(ctx, "task_external_start_checkpointed_at").await?, + }; + let repository = workflow.repository.clone(); + Ok(ctx + .run(|| async move { + repository + .persist_running_task_external_start_checkpoint(checkpoint) + .await + .and_then(|outcome| match outcome { + TaskAttemptCheckpointWriteOutcome::Applied(_) + | TaskAttemptCheckpointWriteOutcome::Replayed(_) => Ok(Json::from(true)), + TaskAttemptCheckpointWriteOutcome::NotFound + | TaskAttemptCheckpointWriteOutcome::Stale => Ok(Json::from(false)), + TaskAttemptCheckpointWriteOutcome::InvalidState => { + Err(moa_execution::Error::InvalidRepositoryData { + message: "active external-start checkpoint was rejected".to_string(), + }) + } + }) + .map_err(crate::workflows::errors::execution_error_to_handler_error) + }) + .name("persist_task_external_start_checkpoint") + .await? + .into_inner()) +} + +async fn execute_direct_capability( + workflow: &ExecutionTaskAttemptImpl, + ctx: &WorkflowContext<'_>, + request: &ExecutionTaskAttemptRequest, + started: &TaskAttemptRecord, + reference: &CapabilityReference, + continuation: Option<&TaskAttemptContinuation>, +) -> Result { + let capability = find_capability(&started.run, reference)?; + let (tool_id, mut usage) = match continuation { + Some( + continuation @ TaskAttemptContinuation { + state: TaskAttemptContinuationState::CapabilityReview { .. }, + .. + }, + ) => return resume_reviewed_capability(capability, continuation), + Some(TaskAttemptContinuation { + state: TaskAttemptContinuationState::CapabilityExternalStart { tool_id, usage }, + review_resolution: None, + external_job_resolution: None, + .. + }) => (*tool_id, usage.clone()), + Some(_) => { + return Err(TerminalError::new( + "direct capability received an incompatible continuation", + ) + .into()); + } + None => { + if let Err(error) = validate_instance( + &capability.input_schema, + &started.task.input, + "execution_task.capability_input", + ) { + return Ok(ActiveTaskAttemptExit::Outcome(failed_task_outcome( + ExecutionFailureClass::InvalidInput, + error.to_string(), + started.task.actual.clone(), + ))); + } + ( + ToolCallId(Uuid::new_v5( + &request.dispatch_uid, + format!("task-capability:{}", started.task.generation).as_bytes(), + )), + started.task.actual.clone(), + ) + } + }; + let session = load_session(workflow, ctx, &started.run, &started.task).await?; + let tool_name = capability_tool_name(capability)?; + let tool_call = ToolCallContent { + invocation: ToolInvocation { + id: Some(tool_id.to_string()), + name: tool_name.clone(), + input: started.task.input.clone(), + }, + provider_metadata: None, + }; + let allowed_tools = BTreeSet::from([tool_name]); + let provenance = CapabilityProvenance { + kind: Some(capability_source_kind(&capability.source).to_string()), + id: Some(format!( + "{}@{}", + capability.reference.name, capability.reference.version + )), + step_id: Some(started.task.node_id.clone()), + }; + if matches!( + capability.async_mode, + ToolAsyncMode::MayReturnExternalJob { .. } + ) { + let provisional = TaskAttemptContinuation { + schema_version: TASK_ATTEMPT_CONTINUATION_SCHEMA_VERSION, + state: TaskAttemptContinuationState::CapabilityExternalStart { + tool_id, + usage: usage.clone(), + }, + review_resolution: None, + external_job_resolution: None, + workspace_release_receipt_id: None, + }; + if !persist_external_start_checkpoint( + workflow, + ctx, + request, + started, + TaskAttemptCheckpointKind::CapabilityExternalStart, + &provisional, + ) + .await? + { + return Ok(ActiveTaskAttemptExit::OwnershipLost); + } + } + let governed = invoke_governed_tool( + ctx, + GovernedInvocationRequest { + session: &session, + identity: &started.run.admitted_identity, + session_id: started.run.session_id, + tool_id, + tool_call: &tool_call, + allowed_tools: &allowed_tools, + expected_tool_contract_revision: Some(&capability.contract_revision), + active_canary: None, + trusted_sandbox_manifest: None, + origin: GovernedInvocationOrigin::ExecutionTask { + run_uid: started.run.run_uid, + task_uid: started.task.task_id.as_uuid(), + generation: started.task.generation, + attempt_generation: request.attempt_generation, + }, + capability_provenance: Some(&provenance), + capability_policy_context: Some(&capability.policy_context), + resource_budget: ResourceBudget::until(request.attempt_deadline_at), + }, + &workflow.session_limits, + workflow.session_store.clone(), + workflow.channel_adapters.as_ref(), + ) + .await?; + usage.tool_calls = usage.tool_calls.saturating_add(1); + classify_capability_outcome(capability, governed, usage) +} + +fn classify_capability_outcome( + capability: &ExecutionCapability, + outcome: GovernedInvocationOutcome, + mut usage: ExecutionUsage, +) -> Result { + match outcome { + GovernedInvocationOutcome::Completed(result) + if result.disposition == GovernedInvocationDisposition::ReviewPending => + { + let review = result.review.ok_or_else(|| { + TerminalError::new( + "review-pending governed result is missing durable review identity", + ) + })?; + Ok(ActiveTaskAttemptExit::ReviewPending { + continuation: TaskAttemptContinuation { + schema_version: TASK_ATTEMPT_CONTINUATION_SCHEMA_VERSION, + state: TaskAttemptContinuationState::CapabilityReview { + pending_review: PendingReviewedToolInvocation { + review_uid: review.review_uid, + expires_at: review.expires_at, + invocation: result.invocation, + effect_idempotency: capability.idempotency_class, + }, + usage, + }, + review_resolution: None, + external_job_resolution: None, + workspace_release_receipt_id: None, + }, + }) + } + GovernedInvocationOutcome::Completed(result) => { + usage.retrieved_bytes = usage.retrieved_bytes.saturating_add(serialized_len( + &result.output.safe_output.structured_payload(), + )); + let task_outcome = if result.output.is_error() { + if capability.idempotency_class == IdempotencyClass::Idempotent { + failed_task_outcome( + ExecutionFailureClass::Retryable, + result.output.safe_output.to_text(), + usage, + ) + } else if capability.action_class + != moa_core::types::action_policy::ActionClass::Read + { + ExecutionTaskOutcome { + schema_version: 1, + usage, + result: ExecutionTaskResult::UnknownOutcome { + message: format!( + "non-idempotent side effect returned an error after possible commit: {}", + result.output.safe_output.to_text() + ), + }, + } + } else { + failed_task_outcome( + ExecutionFailureClass::Terminal, + result.output.safe_output.to_text(), + usage, + ) + } + } else { + let value = result + .output + .safe_output + .structured_payload() + .cloned() + .unwrap_or_else(|| Value::String(result.output.safe_output.to_text())); + if let Err(error) = validate_instance( + &capability.output_schema, + &value, + "execution_task.capability_output", + ) { + if capability.action_class == moa_core::types::action_policy::ActionClass::Read + { + failed_task_outcome( + ExecutionFailureClass::InvalidOutput, + error.to_string(), + usage, + ) + } else { + ExecutionTaskOutcome { + schema_version: 1, + usage, + result: ExecutionTaskResult::UnknownOutcome { + message: format!( + "side effect returned invalid output after possible commit: {error}" + ), + }, + } + } + } else { + completed_task_outcome(value, usage) + } + }; + Ok(ActiveTaskAttemptExit::Outcome(task_outcome)) + } + GovernedInvocationOutcome::ExternalJob { + external_job_uid, .. + } => Ok(ActiveTaskAttemptExit::ExternalJob { + external_job_uid, + continuation: None, + }), + GovernedInvocationOutcome::UnknownOutcome { message, .. } => { + Ok(ActiveTaskAttemptExit::Outcome(ExecutionTaskOutcome { + schema_version: 1, + usage, + result: ExecutionTaskResult::UnknownOutcome { message }, + })) + } + GovernedInvocationOutcome::NotDispatched { reason, .. } => { + Ok(ActiveTaskAttemptExit::Outcome(failed_task_outcome( + ExecutionFailureClass::Terminal, + execution_dispatch_rejection_message(reason), + usage, + ))) + } + GovernedInvocationOutcome::Delegation { .. } => { + Err(TerminalError::new("execution tasks cannot invoke delegation capabilities").into()) + } + } +} + +fn resume_reviewed_capability( + capability: &ExecutionCapability, + continuation: &TaskAttemptContinuation, +) -> Result { + let TaskAttemptContinuationState::CapabilityReview { + pending_review: _, + usage, + } = &continuation.state + else { + return Err(TerminalError::new( + "direct capability received an incompatible agent continuation", + ) + .into()); + }; + let resolution = continuation.review_resolution.as_ref().ok_or_else(|| { + TerminalError::new("reviewed capability continuation has no durable resolution") + })?; + let exit = match resolution { + moa_execution::wire::ExecutionActionReviewResolution::Completed { tool_output } => { + match serde_json::from_value::( + tool_output.clone(), + ) { + Ok(output) => ActiveTaskAttemptExit::Outcome(capability_output_outcome( + capability, + output, + usage.clone(), + )), + Err(error) => ActiveTaskAttemptExit::Outcome(ExecutionTaskOutcome { + schema_version: 1, + usage: usage.clone(), + result: ExecutionTaskResult::UnknownOutcome { + message: format!( + "reviewed capability returned invalid output after possible commit: {error}" + ), + }, + }), + } + } + moa_execution::wire::ExecutionActionReviewResolution::ExternalJob { + external_job_uid, + .. + } => ActiveTaskAttemptExit::ExternalJob { + external_job_uid: *external_job_uid, + continuation: None, + }, + moa_execution::wire::ExecutionActionReviewResolution::Failed { class, message } => { + ActiveTaskAttemptExit::Outcome(ExecutionTaskOutcome { + schema_version: 1, + usage: usage.clone(), + result: ExecutionTaskResult::Failed { + class: class.clone(), + message: message.clone(), + }, + }) + } + moa_execution::wire::ExecutionActionReviewResolution::UnknownOutcome { message } => { + ActiveTaskAttemptExit::Outcome(ExecutionTaskOutcome { + schema_version: 1, + usage: usage.clone(), + result: ExecutionTaskResult::UnknownOutcome { + message: message.clone(), + }, + }) + } + moa_execution::wire::ExecutionActionReviewResolution::NotDispatched { reason } => { + ActiveTaskAttemptExit::Outcome(failed_task_outcome( + ExecutionFailureClass::Terminal, + execution_dispatch_rejection_message(*reason), + usage.clone(), + )) + } + moa_execution::wire::ExecutionActionReviewResolution::Denied { reason } => { + ActiveTaskAttemptExit::Outcome(failed_task_outcome( + ExecutionFailureClass::AuthorizationDenied, + reason.clone(), + usage.clone(), + )) + } + moa_execution::wire::ExecutionActionReviewResolution::TimedOut { reason } => { + ActiveTaskAttemptExit::Outcome(failed_task_outcome( + ExecutionFailureClass::DeadlineExceeded, + reason.clone(), + usage.clone(), + )) + } + }; + Ok(exit) +} + +fn capability_output_outcome( + capability: &ExecutionCapability, + output: moa_core::types::tools::SecuredToolOutput, + usage: ExecutionUsage, +) -> ExecutionTaskOutcome { + if output.is_error() { + if capability.idempotency_class == IdempotencyClass::Idempotent { + return failed_task_outcome( + ExecutionFailureClass::Retryable, + output.safe_output.to_text(), + usage, + ); + } + if capability.action_class != moa_core::types::action_policy::ActionClass::Read { + return ExecutionTaskOutcome { + schema_version: 1, + usage, + result: ExecutionTaskResult::UnknownOutcome { + message: format!( + "non-idempotent side effect returned an error after possible commit: {}", + output.safe_output.to_text() + ), + }, + }; + } + return failed_task_outcome( + ExecutionFailureClass::Terminal, + output.safe_output.to_text(), + usage, + ); + } + let value = output + .safe_output + .structured_payload() + .cloned() + .unwrap_or_else(|| Value::String(output.safe_output.to_text())); + if let Err(error) = validate_instance( + &capability.output_schema, + &value, + "execution_task.capability_output", + ) { + if capability.action_class == moa_core::types::action_policy::ActionClass::Read { + failed_task_outcome( + ExecutionFailureClass::InvalidOutput, + error.to_string(), + usage, + ) + } else { + ExecutionTaskOutcome { + schema_version: 1, + usage, + result: ExecutionTaskResult::UnknownOutcome { + message: format!( + "side effect returned invalid output after possible commit: {error}" + ), + }, + } + } + } else { + completed_task_outcome(value, usage) + } +} + +async fn execute_agent_turn( + workflow: &ExecutionTaskAttemptImpl, + ctx: &WorkflowContext<'_>, + request: &ExecutionTaskAttemptRequest, + started: &TaskAttemptRecord, + spec: AgentTaskSpec<'_>, + continuation: Option<&TaskAttemptContinuation>, +) -> Result { + let AgentTaskSpec { + instructions, + skill_refs, + capability_refs, + max_turns, + } = spec; + if max_turns == 0 { + return Ok(ActiveTaskAttemptExit::Outcome(failed_task_outcome( + ExecutionFailureClass::InvalidInput, + "agent max_turns must be positive".to_string(), + started.task.actual.clone(), + ))); + } + let mut capabilities = BTreeMap::::new(); + for reference in capability_refs { + let capability = find_capability(&started.run, reference)?; + let tool_name = capability_tool_name(capability)?; + if capabilities.insert(tool_name.clone(), capability).is_some() { + return Err(TerminalError::new(format!( + "task-local agent has ambiguous capability tool `{tool_name}`" + )) + .into()); + } + } + let circuit_owner = SecurityCircuitOwner::ExecutionTask { + run_uid: started.run.run_uid, + task_uid: started.task.task_id.as_uuid(), + generation: started.task.generation, + }; + let ( + mut messages, + mut next_turn, + mut usage, + mut security_circuit, + mut disabled_capabilities, + mut pending_review, + mut pending_tool_calls, + mut pending_external, + ) = match continuation { + Some(TaskAttemptContinuation { + state: + TaskAttemptContinuationState::Agent { + messages, + next_turn, + usage, + security_circuit, + disabled_capabilities, + pending_review, + pending_tool_calls, + pending_external, + }, + .. + }) => ( + messages.clone(), + *next_turn, + usage.clone(), + security_circuit.clone(), + disabled_capabilities.clone(), + pending_review.as_deref().cloned(), + pending_tool_calls.clone(), + pending_external.clone(), + ), + Some(_) => { + return Err(TerminalError::new( + "task-local agent received an incompatible continuation", + ) + .into()); + } + None => { + let skills = load_pinned_skills(workflow, ctx, started, skill_refs).await?; + let mut circuit = SecurityCircuitState::default(); + circuit.adopt_owner(&circuit_owner); + ( + vec![ + ContextMessage::system(agent_system_prompt(instructions, &skills)), + ContextMessage::user( + json!({ + "resolved_input": started.task.input, + "resume_inputs": started.task.resume_input_history, + }) + .to_string(), + ), + ], + 0, + started.task.actual.clone(), + circuit, + BTreeMap::new(), + None, + Vec::new(), + None, + ) + } + }; + security_circuit.adopt_owner(&circuit_owner); + + if let Some(external) = pending_external.take() { + if let Some(external_job_uid) = external.external_job_uid { + let resolution = continuation + .and_then(|continuation| continuation.external_job_resolution.as_ref()) + .ok_or_else(|| { + TerminalError::new("agent external continuation has no terminal resolution") + })?; + let tool_use_id = external + .invocation + .id + .clone() + .unwrap_or_else(|| format!("external-job-{external_job_uid}")); + match resolution { + AsyncToolJobTerminalOutcome::Completed { output } => { + messages.push(ContextMessage::tool_result( + tool_use_id, + output.to_string(), + None, + )); + } + AsyncToolJobTerminalOutcome::Failed { error } => { + messages.push(ContextMessage::tool_result( + tool_use_id, + format!("external tool failed: {error}"), + None, + )); + } + AsyncToolJobTerminalOutcome::Cancelled => { + messages.push(ContextMessage::tool_result( + tool_use_id, + "external tool was cancelled", + None, + )); + } + AsyncToolJobTerminalOutcome::UnknownOutcome { error } => { + return Ok(ActiveTaskAttemptExit::Outcome(ExecutionTaskOutcome { + schema_version: 1, + usage, + result: ExecutionTaskResult::UnknownOutcome { + message: format!("external agent effect outcome is unknown: {error}"), + }, + })); + } + } + } else { + // Provider start recovery proved NotStarted and re-admitted the exact continuation. + // Reinsert the original model invocation so its stable tool id/idempotency key is + // dispatched again without asking the model or repeating prior tool effects. + pending_tool_calls.insert(0, external.invocation); + } + } + + if let Some(reviewed) = pending_review.take() { + let resolution = continuation + .and_then(|continuation| continuation.review_resolution.as_ref()) + .ok_or_else(|| { + TerminalError::new("agent review continuation has no durable resolution") + })?; + match resolution { + moa_execution::wire::ExecutionActionReviewResolution::Completed { tool_output } => { + let output = serde_json::from_value::( + tool_output.clone(), + ) + .map_err(|error| { + TerminalError::new(format!("decode reviewed agent capability output: {error}")) + })?; + append_agent_tool_output(&mut messages, &reviewed.invocation, &output); + usage.retrieved_bytes = usage + .retrieved_bytes + .saturating_add(serialized_len(&output.safe_output.structured_payload())); + } + moa_execution::wire::ExecutionActionReviewResolution::ExternalJob { + external_job_uid, + .. + } => { + return Ok(ActiveTaskAttemptExit::ExternalJob { + external_job_uid: *external_job_uid, + continuation: Some(agent_continuation( + messages, + next_turn, + usage, + security_circuit, + disabled_capabilities, + AgentPending { + review: None, + tool_calls: pending_tool_calls, + external: Some(PendingExternalToolInvocation { + external_job_uid: None, + invocation: reviewed.invocation, + effect_idempotency: reviewed.effect_idempotency, + }), + }, + )), + }); + } + moa_execution::wire::ExecutionActionReviewResolution::Failed { class, message } => { + return Ok(ActiveTaskAttemptExit::Outcome(failed_task_outcome( + class.clone(), + message.clone(), + usage, + ))); + } + moa_execution::wire::ExecutionActionReviewResolution::UnknownOutcome { message } => { + return Ok(ActiveTaskAttemptExit::Outcome(ExecutionTaskOutcome { + schema_version: 1, + usage, + result: ExecutionTaskResult::UnknownOutcome { + message: message.clone(), + }, + })); + } + moa_execution::wire::ExecutionActionReviewResolution::NotDispatched { reason } => { + return Ok(ActiveTaskAttemptExit::Outcome(failed_task_outcome( + ExecutionFailureClass::Terminal, + execution_dispatch_rejection_message(*reason), + usage, + ))); + } + moa_execution::wire::ExecutionActionReviewResolution::Denied { reason } => { + return Ok(ActiveTaskAttemptExit::Outcome(failed_task_outcome( + ExecutionFailureClass::AuthorizationDenied, + reason.clone(), + usage, + ))); + } + moa_execution::wire::ExecutionActionReviewResolution::TimedOut { reason } => { + return Ok(ActiveTaskAttemptExit::Outcome(failed_task_outcome( + ExecutionFailureClass::DeadlineExceeded, + reason.clone(), + usage, + ))); + } + } + } + + if pending_tool_calls.is_empty() { + if next_turn >= max_turns { + return Ok(ActiveTaskAttemptExit::Outcome(failed_task_outcome( + ExecutionFailureClass::Terminal, + format!("task-local agent exhausted max_turns={max_turns}"), + usage, + ))); + } + let mut completion = moa_core::types::completion::CompletionRequest { + model: None, + messages: messages.clone(), + tools: capabilities + .iter() + .filter(|(name, _)| !disabled_capabilities.contains_key(*name)) + .map(|(name, capability)| agent_tool_schema(name, capability)) + .collect(), + max_output_tokens: None, + temperature: None, + response_format: None, + native_web_search: Default::default(), + metadata: std::collections::HashMap::new(), + }; + let owner = LLMCompletionOwner::execution_task_attempt(request.dispatch_uid); + attach_completion_owner(&mut completion, &owner); + let response = crate::restate_identity::replay_safe_request( + ctx.service_client::() + .complete_bounded(Json::from(BoundedCompletionRequest { + request: completion, + budget: ResourceBudget::until(request.attempt_deadline_at), + })) + .idempotency_key(completion_idempotency_key( + ctx.invocation_id(), + LLMCompletionAction::ExecutionTaskModel { + generation: started.task.generation, + turn: next_turn, + }, + )), + ) + .call() + .await? + .into_inner(); + usage.tokens = usage + .tokens + .saturating_add(response.usage.total_input_tokens() as u64) + .saturating_add(response.usage.output_tokens as u64); + usage.cost_microusd = usage.cost_microusd.saturating_add( + moa_providers::pricing_for_model(response.model.as_str()) + .map(|pricing| pricing.cost_micros(&response.usage)) + .unwrap_or_default(), + ); + let tool_calls = response + .content + .iter() + .filter_map(|content| match content { + CompletionContent::ToolCall(call) => Some(call.invocation.clone()), + CompletionContent::Text(_) | CompletionContent::ProviderToolResult { .. } => None, + }) + .collect::>(); + if tool_calls.is_empty() { + let outcome = moa_execution::state::parse_agent_task_outcome(&response.text, usage); + if matches!(outcome.result, ExecutionTaskResult::NeedsInput { .. }) { + messages.push(ContextMessage::assistant_with_thought_signature( + response.text, + response.thought_signature, + )); + return Ok(ActiveTaskAttemptExit::InputPending { + continuation: agent_continuation( + messages, + next_turn.saturating_add(1), + outcome.usage.clone(), + security_circuit, + disabled_capabilities, + AgentPending { + review: None, + tool_calls: pending_tool_calls, + external: pending_external, + }, + ), + outcome, + }); + } + return Ok(ActiveTaskAttemptExit::Outcome(outcome)); + } + for (index, invocation) in tool_calls.iter().cloned().enumerate() { + messages.push(ContextMessage::assistant_tool_call_with_thought_signature( + invocation, + if index == 0 { + response.text.clone() + } else { + String::new() + }, + (index == 0) + .then(|| response.thought_signature.clone()) + .flatten(), + )); + } + pending_tool_calls = tool_calls; + next_turn = next_turn.saturating_add(1); + } + + let invocation = pending_tool_calls.remove(0); + let capability = capabilities.get(&invocation.name).copied().ok_or_else(|| { + TerminalError::new(format!( + "agent emitted undeclared capability `{}`", + invocation.name + )) + })?; + if disabled_capabilities.contains_key(&invocation.name) { + let tool_use_id = invocation + .id + .clone() + .unwrap_or_else(|| format!("execution-{}-{next_turn}", started.task.task_id)); + messages.push(ContextMessage::tool_result( + tool_use_id, + "This tool capability is disabled for this task by the security circuit.", + None, + )); + return Ok(ActiveTaskAttemptExit::Continue { + continuation: agent_continuation( + messages, + next_turn, + usage, + security_circuit, + disabled_capabilities, + AgentPending { + review: None, + tool_calls: pending_tool_calls, + external: pending_external, + }, + ), + }); + } + let session = load_session(workflow, ctx, &started.run, &started.task).await?; + let tool_id = ToolCallId(Uuid::new_v5( + &started.task.task_id.as_uuid(), + format!( + "agent-tool:{}:{}:{}", + started.task.generation, + next_turn, + invocation.id.as_deref().unwrap_or(&invocation.name) + ) + .as_bytes(), + )); + let tool_call = ToolCallContent { + invocation: invocation.clone(), + provider_metadata: None, + }; + let allowed_tools = capabilities.keys().cloned().collect::>(); + let provenance = CapabilityProvenance { + kind: Some(capability_source_kind(&capability.source).to_string()), + id: Some(format!( + "{}@{}", + capability.reference.name, capability.reference.version + )), + step_id: Some(started.task.node_id.clone()), + }; + if matches!( + capability.async_mode, + ToolAsyncMode::MayReturnExternalJob { .. } + ) { + let provisional = agent_continuation( + messages.clone(), + next_turn, + usage.clone(), + security_circuit.clone(), + disabled_capabilities.clone(), + AgentPending { + review: None, + tool_calls: pending_tool_calls.clone(), + external: Some(PendingExternalToolInvocation { + external_job_uid: None, + invocation: invocation.clone(), + effect_idempotency: capability.idempotency_class, + }), + }, + ); + if !persist_external_start_checkpoint( + workflow, + ctx, + request, + started, + TaskAttemptCheckpointKind::AgentContinuation, + &provisional, + ) + .await? + { + return Ok(ActiveTaskAttemptExit::OwnershipLost); + } + } + let governed = invoke_governed_tool( + ctx, + GovernedInvocationRequest { + session: &session, + identity: &started.run.admitted_identity, + session_id: started.run.session_id, + tool_id, + tool_call: &tool_call, + allowed_tools: &allowed_tools, + expected_tool_contract_revision: Some(&capability.contract_revision), + active_canary: None, + trusted_sandbox_manifest: None, + origin: GovernedInvocationOrigin::ExecutionTask { + run_uid: started.run.run_uid, + task_uid: started.task.task_id.as_uuid(), + generation: started.task.generation, + attempt_generation: request.attempt_generation, + }, + capability_provenance: Some(&provenance), + capability_policy_context: Some(&capability.policy_context), + resource_budget: ResourceBudget::until(request.attempt_deadline_at), + }, + &workflow.session_limits, + workflow.session_store.clone(), + workflow.channel_adapters.as_ref(), + ) + .await?; + usage.tool_calls = usage.tool_calls.saturating_add(1); + match governed { + GovernedInvocationOutcome::Completed(result) + if result.disposition == GovernedInvocationDisposition::ReviewPending => + { + let review = result.review.ok_or_else(|| { + TerminalError::new("review-pending agent result is missing durable review identity") + })?; + Ok(ActiveTaskAttemptExit::ReviewPending { + continuation: agent_continuation( + messages, + next_turn, + usage, + security_circuit, + disabled_capabilities, + AgentPending { + review: Some(PendingReviewedToolInvocation { + review_uid: review.review_uid, + expires_at: review.expires_at, + invocation: result.invocation, + effect_idempotency: capability.idempotency_class, + }), + tool_calls: pending_tool_calls, + external: pending_external, + }, + ), + }) + } + GovernedInvocationOutcome::Completed(result) => { + let output = result.output; + usage.retrieved_bytes = usage + .retrieved_bytes + .saturating_add(serialized_len(&output.safe_output.structured_payload())); + if !output.assessment.is_safe() { + moa_security::apply_owner_assessment( + &mut security_circuit, + moa_security::CircuitTarget { + session_id: session.id, + owner: &circuit_owner, + capability: &output.capability, + tool_call_id: tool_id, + }, + &output.assessment, + ) + .map_err(|_| TerminalError::new("agent security assessment owner mismatch"))?; + let stage = security_circuit.stage(&circuit_owner, &output.capability); + if !stage.permits_dispatch() { + disabled_capabilities + .insert(invocation.name.clone(), output.capability.clone()); + } + if stage == SecurityCircuitStage::Halted { + return Ok(ActiveTaskAttemptExit::Outcome(failed_task_outcome( + ExecutionFailureClass::Terminal, + "task stopped after unsafe capability output".to_string(), + usage, + ))); + } + if stage == SecurityCircuitStage::SuspendedForInput { + append_agent_tool_output(&mut messages, &invocation, &output); + let outcome = ExecutionTaskOutcome { + schema_version: 1, + usage: usage.clone(), + result: ExecutionTaskResult::NeedsInput { + question: "A capability returned potentially unsafe content. Continue?" + .to_string(), + audience: moa_artifacts::execution_plan::InputAudience::User, + }, + }; + return Ok(ActiveTaskAttemptExit::InputPending { + outcome, + continuation: agent_continuation( + messages, + next_turn, + usage, + security_circuit, + disabled_capabilities, + AgentPending { + review: None, + tool_calls: pending_tool_calls, + external: pending_external, + }, + ), + }); + } + } + append_agent_tool_output(&mut messages, &invocation, &output); + Ok(ActiveTaskAttemptExit::Continue { + continuation: agent_continuation( + messages, + next_turn, + usage, + security_circuit, + disabled_capabilities, + AgentPending { + review: None, + tool_calls: pending_tool_calls, + external: pending_external, + }, + ), + }) + } + GovernedInvocationOutcome::ExternalJob { + external_job_uid, .. + } => Ok(ActiveTaskAttemptExit::ExternalJob { + external_job_uid, + continuation: Some(agent_continuation( + messages, + next_turn, + usage, + security_circuit, + disabled_capabilities, + AgentPending { + review: None, + tool_calls: pending_tool_calls, + external: Some(PendingExternalToolInvocation { + external_job_uid: None, + invocation, + effect_idempotency: capability.idempotency_class, + }), + }, + )), + }), + GovernedInvocationOutcome::UnknownOutcome { message, .. } => { + Ok(ActiveTaskAttemptExit::Outcome(ExecutionTaskOutcome { + schema_version: 1, + usage, + result: ExecutionTaskResult::UnknownOutcome { message }, + })) + } + GovernedInvocationOutcome::NotDispatched { reason, .. } => { + Ok(ActiveTaskAttemptExit::Outcome(failed_task_outcome( + ExecutionFailureClass::Terminal, + execution_dispatch_rejection_message(reason), + usage, + ))) + } + GovernedInvocationOutcome::Delegation { .. } => Err(TerminalError::new( + "execution task agents cannot invoke delegation capabilities", + ) + .into()), + } +} + +fn agent_continuation( + messages: Vec, + next_turn: u32, + usage: ExecutionUsage, + security_circuit: SecurityCircuitState, + disabled_capabilities: BTreeMap, + pending: AgentPending, +) -> TaskAttemptContinuation { + TaskAttemptContinuation { + schema_version: TASK_ATTEMPT_CONTINUATION_SCHEMA_VERSION, + state: TaskAttemptContinuationState::Agent { + messages, + next_turn, + usage, + security_circuit, + disabled_capabilities, + pending_review: pending.review.map(Box::new), + pending_tool_calls: pending.tool_calls, + pending_external: pending.external, + }, + review_resolution: None, + external_job_resolution: None, + workspace_release_receipt_id: None, + } +} + +fn agent_tool_schema(name: &str, capability: &ExecutionCapability) -> Value { + json!({ + "name": name, + "description": capability.description, + "input_schema": capability.input_schema, + }) +} + +fn append_agent_tool_output( + messages: &mut Vec, + invocation: &ToolInvocation, + output: &moa_core::types::tools::SecuredToolOutput, +) { + let tool_use_id = invocation.id.clone().unwrap_or_else(|| { + Uuid::new_v5( + &Uuid::NAMESPACE_OID, + format!("{}:{}", invocation.name, invocation.input).as_bytes(), + ) + .to_string() + }); + messages.push(ContextMessage::tool_result( + tool_use_id, + output.safe_output.to_text(), + Some(output.safe_output.content.clone()), + )); +} + +async fn load_session( + workflow: &ExecutionTaskAttemptImpl, + ctx: &WorkflowContext<'_>, + run: &moa_execution::repository::ExecutionRunRecord, + task: &moa_execution::repository::ExecutionTaskRecord, +) -> Result { + let store = workflow.session_store.clone(); + let session_id = run.session_id; + Ok(ctx + .run(|| async move { + store + .get_session(session_id) + .await + .map(Json::from) + .map_err(moa_error_to_handler_error) + }) + .name(format!( + "task_attempt_load_session:{}:{}", + task.generation, task.attempt_generation + )) + .await? + .into_inner()) +} + +async fn load_pinned_skills( + workflow: &ExecutionTaskAttemptImpl, + ctx: &WorkflowContext<'_>, + started: &TaskAttemptRecord, + skill_refs: &[moa_artifacts::reference::ArtifactRef], +) -> Result, HandlerError> { + let mut markdown = Vec::with_capacity(skill_refs.len()); + let scope = started.run.contact_id.map_or( + ActionRuleScope::Tenant { + tenant_id: started.run.tenant_id, + }, + |contact_id| ActionRuleScope::Contact { + tenant_id: started.run.tenant_id, + contact_id, + }, + ); + for (index, skill_ref) in skill_refs.iter().enumerate() { + if !started.run.authorization.skill_refs.contains(skill_ref) { + return Err(TerminalError::new( + "task requested a skill outside its authorization envelope", + ) + .into()); + } + let pinned = started + .run + .pinned_instruction_skills + .iter() + .find(|pinned| pinned.skill_ref == *skill_ref) + .ok_or_else(|| TerminalError::new("task requested an unpinned skill"))?; + let pool = workflow.pool.clone(); + let revision_uid = pinned.revision_uid; + let loaded = ctx + .run(|| async move { + moa_skills::registry::SkillRegistry::new(pool) + .load_skill_markdown(&scope, revision_uid) + .await + .map(Json::from) + .map_err(moa_error_to_handler_error) + }) + .name(format!("task_attempt_skill:{index}:{revision_uid}")) + .await? + .into_inner(); + markdown.push(loaded); + } + Ok(markdown) +} + +fn find_capability<'a>( + run: &'a moa_execution::repository::ExecutionRunRecord, + reference: &CapabilityReference, +) -> Result<&'a ExecutionCapability, HandlerError> { + if !run.authorization.capability_refs.contains(reference) { + return Err(TerminalError::new( + "capability is outside the persisted authorization envelope", + ) + .into()); + } + run.catalog + .capabilities + .iter() + .find(|capability| capability.reference == *reference) + .ok_or_else(|| TerminalError::new("capability is absent from the persisted catalog").into()) +} + +const fn capability_source_kind(source: &CapabilitySource) -> &'static str { + match source { + CapabilitySource::BuiltInTool { .. } => "built_in_tool", + CapabilitySource::HandTool { .. } => "hand_tool", + CapabilitySource::McpTool { .. } => "mcp_tool", + CapabilitySource::ActionArtifact { .. } => "action_artifact", + CapabilitySource::ConnectorAction { .. } => "connector_action", + CapabilitySource::InstalledConnectorAction { .. } => "installed_connector_action", + CapabilitySource::SkillAction { .. } => "skill_action", + CapabilitySource::SkillCode { .. } => "skill_code", + CapabilitySource::Memory { .. } => "memory", + CapabilitySource::Knowledge { .. } => "knowledge", + CapabilitySource::Model => "model", + } +} + +fn execution_dispatch_rejection_message(reason: ExecutionToolDispatchRejection) -> String { + let label = match reason { + ExecutionToolDispatchRejection::OriginNotFound => "origin_not_found", + ExecutionToolDispatchRejection::StaleGeneration => "stale_generation", + ExecutionToolDispatchRejection::OperationNotRunning => "operation_not_running", + ExecutionToolDispatchRejection::RunNotDispatchable => "run_not_dispatchable", + }; + format!("execution effect was not dispatched: {label}") +} + +fn agent_system_prompt(instructions: &str, skills: &[String]) -> String { + format!( + "{instructions}\n\nPinned instruction skills:\n{}\n\nReturn only JSON.", + skills.join("\n\n---\n\n") + ) +} + +fn serialized_len(value: &T) -> u64 { + serde_json::to_vec(value) + .map(|bytes| bytes.len() as u64) + .unwrap_or_default() +} + +#[cfg(test)] +mod tests { + use chrono::{TimeZone, Utc}; + use moa_artifacts::execution_plan::ExecutionUsage; + use moa_core::types::{completion::ToolInvocation, context::ContextMessage}; + + use super::*; + + // Pins: a continuation that cannot fit in the bounded durable payload is rejected + // before persistence so callers must decompose or request a replan. + #[test] + fn oversized_agent_continuation_requires_decomposition_offline() { + let continuation = TaskAttemptContinuation { + schema_version: TASK_ATTEMPT_CONTINUATION_SCHEMA_VERSION, + state: TaskAttemptContinuationState::Agent { + messages: vec![ContextMessage::user( + "x".repeat(MAX_TASK_ATTEMPT_CONTINUATION_BYTES), + )], + next_turn: 1, + usage: ExecutionUsage { + cost_microusd: 0, + tokens: 0, + tool_calls: 0, + retrieved_bytes: 0, + }, + security_circuit: SecurityCircuitState::default(), + disabled_capabilities: std::collections::BTreeMap::new(), + pending_review: None, + pending_tool_calls: Vec::new(), + pending_external: None, + }, + review_resolution: None, + external_job_resolution: None, + workspace_release_receipt_id: None, + }; + + let error = continuation + .to_bounded_json() + .expect_err("oversized continuation must fail closed"); + assert!(error.contains("must be decomposed or replanned")); + } + + // Pins: once an asynchronous provider start commits, the durable checkpoint + // retains the exact model invocation, effect semantics, and MOA job identity; + // decoding the checkpoint must not reconstruct or resend that effect. + #[test] + fn agent_external_continuation_round_trips_exact_effect_owner_offline() { + let external_job_uid = Uuid::from_u128(41); + let invocation = ToolInvocation { + id: Some("provider-call-7".to_string()), + name: "render_video".to_string(), + input: json!({"prompt": "durable sunrise"}), + }; + let mut continuation = agent_continuation( + vec![ContextMessage::user("render a durable sunrise")], + 3, + zero_usage(), + SecurityCircuitState::default(), + BTreeMap::new(), + AgentPending { + review: None, + tool_calls: Vec::new(), + external: Some(PendingExternalToolInvocation { + external_job_uid: None, + invocation: invocation.clone(), + effect_idempotency: IdempotencyClass::NonIdempotent, + }), + }, + ); + + continuation + .bind_external_job(external_job_uid) + .expect("fresh external continuation must accept its durable job identity"); + let persisted = continuation + .to_bounded_json() + .expect("exact continuation must fit the durable bound"); + let decoded: TaskAttemptContinuation = + serde_json::from_value(persisted).expect("persisted continuation must decode"); + + let TaskAttemptContinuationState::Agent { + pending_external: Some(pending), + next_turn, + .. + } = decoded.state + else { + panic!("external continuation lost its exact pending effect"); + }; + assert_eq!(next_turn, 3); + assert_eq!(pending.external_job_uid, Some(external_job_uid)); + assert_eq!(pending.invocation, invocation); + assert_eq!(pending.effect_idempotency, IdempotencyClass::NonIdempotent); + } + + // Pins: a storage-only review checkpoint retains the exact reviewed + // invocation and expiry across serialization, so a resumed attempt consumes + // the decision without regenerating the provider effect. + #[test] + fn agent_review_continuation_round_trips_exact_effect_fence_offline() { + let review_uid = Uuid::from_u128(51); + let expires_at = Utc + .with_ymd_and_hms(2030, 5, 6, 7, 8, 9) + .single() + .expect("fixed review expiry"); + let invocation = ToolInvocation { + id: Some("reviewed-call-2".to_string()), + name: "publish_release".to_string(), + input: json!({"version": "2.0.0"}), + }; + let continuation = agent_continuation( + vec![ContextMessage::user("publish only after review")], + 2, + zero_usage(), + SecurityCircuitState::default(), + BTreeMap::new(), + AgentPending { + review: Some(PendingReviewedToolInvocation { + review_uid, + expires_at, + invocation: invocation.clone(), + effect_idempotency: IdempotencyClass::NonIdempotent, + }), + tool_calls: Vec::new(), + external: None, + }, + ); + + let decoded: TaskAttemptContinuation = serde_json::from_value( + continuation + .to_bounded_json() + .expect("review continuation must fit the durable bound"), + ) + .expect("persisted review continuation must decode"); + assert_eq!(decoded.pending_review_uid(), Some(review_uid)); + let TaskAttemptContinuationState::Agent { + pending_review: Some(pending), + .. + } = decoded.state + else { + panic!("review continuation lost its exact pending effect"); + }; + assert_eq!(pending.expires_at, expires_at); + assert_eq!(pending.invocation, invocation); + } + + // Pins: an input boundary keeps the already-completed model turn and circuit + // state in the bounded checkpoint; resumption starts at the following turn + // instead of calling the model again for the same prompt. + #[test] + fn agent_input_continuation_round_trips_next_turn_and_messages_offline() { + let continuation = agent_continuation( + vec![ + ContextMessage::user("inspect the unsafe payload"), + ContextMessage::assistant("May I continue with the unsafe payload?"), + ], + 4, + ExecutionUsage { + cost_microusd: 17, + tokens: 23, + tool_calls: 2, + retrieved_bytes: 31, + }, + SecurityCircuitState::default(), + BTreeMap::new(), + AgentPending { + review: None, + tool_calls: Vec::new(), + external: None, + }, + ); + + let decoded: TaskAttemptContinuation = serde_json::from_value( + continuation + .to_bounded_json() + .expect("input continuation must fit the durable bound"), + ) + .expect("persisted input continuation must decode"); + let TaskAttemptContinuationState::Agent { + messages, + next_turn, + usage, + .. + } = decoded.state + else { + panic!("input continuation changed state kind"); + }; + assert_eq!(next_turn, 4); + assert_eq!(messages.len(), 2); + assert_eq!(usage.tokens, 23); + assert_eq!(usage.tool_calls, 2); + } + + // Pins: provider recovery may prove that a reserved start never happened; + // the current checkpoint must retain the exact invocation with no job UID so + // the successor attempt can replay that call without repeating the model turn. + #[test] + fn provisional_agent_external_start_round_trips_without_a_job_uid_offline() { + let invocation = ToolInvocation { + id: Some("stable-provider-call".to_string()), + name: "render_video".to_string(), + input: json!({"prompt": "recover this exact effect"}), + }; + let continuation = agent_continuation( + vec![ContextMessage::user("render once")], + 2, + zero_usage(), + SecurityCircuitState::default(), + BTreeMap::new(), + AgentPending { + review: None, + tool_calls: Vec::new(), + external: Some(PendingExternalToolInvocation { + external_job_uid: None, + invocation: invocation.clone(), + effect_idempotency: IdempotencyClass::NonIdempotent, + }), + }, + ); + + let decoded: TaskAttemptContinuation = serde_json::from_value( + continuation + .to_bounded_json() + .expect("provisional continuation must fit"), + ) + .expect("provisional continuation must decode"); + let TaskAttemptContinuationState::Agent { + pending_external: Some(pending), + .. + } = decoded.state + else { + panic!("provisional external start lost its pending invocation"); + }; + assert_eq!(pending.external_job_uid, None); + assert_eq!(pending.invocation, invocation); + } + + // Pins: a direct async capability resumes with the same stable tool-call ID + // after a NotStarted recovery instead of creating a second provider identity. + #[test] + fn direct_external_start_checkpoint_round_trips_stable_tool_id_offline() { + let tool_id = ToolCallId(Uuid::from_u128(77)); + let continuation = TaskAttemptContinuation { + schema_version: TASK_ATTEMPT_CONTINUATION_SCHEMA_VERSION, + state: TaskAttemptContinuationState::CapabilityExternalStart { + tool_id, + usage: zero_usage(), + }, + review_resolution: None, + external_job_resolution: None, + workspace_release_receipt_id: None, + }; + + let decoded: TaskAttemptContinuation = serde_json::from_value( + continuation + .to_bounded_json() + .expect("direct provisional continuation must fit"), + ) + .expect("direct provisional continuation must decode"); + assert!(matches!( + decoded.state, + TaskAttemptContinuationState::CapabilityExternalStart { + tool_id: decoded_tool_id, + .. + } if decoded_tool_id == tool_id + )); + } + + const fn zero_usage() -> ExecutionUsage { + ExecutionUsage { + cost_microusd: 0, + tokens: 0, + tool_calls: 0, + retrieved_bytes: 0, + } + } +} diff --git a/crates/moa-orchestrator/src/workflows/execution_task_attempt/external.rs b/crates/moa-orchestrator/src/workflows/execution_task_attempt/external.rs new file mode 100644 index 000000000..739f3b583 --- /dev/null +++ b/crates/moa-orchestrator/src/workflows/execution_task_attempt/external.rs @@ -0,0 +1,118 @@ +//! Immutable persistence mapping for asynchronous provider-job starts. + +use moa_execution::{ + repository::task::{ + NewTaskAttemptCheckpoint, TaskAttemptCheckpointKind, TaskAttemptExternalOutcome, + TaskAttemptRecord, TaskAttemptReleaseClaimOutcome, + }, + wire::ExecutionTaskAttemptRequest, +}; +use restate_sdk::prelude::*; +use uuid::Uuid; + +use crate::workflows::{ + errors::execution_error_to_handler_error, + execution_task_attempt::{ + ExecutionTaskAttemptImpl, active::TaskAttemptContinuation, journal_now, task_attempt_fence, + yielding::checkpoint_task_hands_workflow, + }, +}; + +/// Publishes a pre-reserved provider job only after any sandbox compute is asleep. +pub(super) async fn yield_external_job( + workflow: &ExecutionTaskAttemptImpl, + ctx: &WorkflowContext<'_>, + request: &ExecutionTaskAttemptRequest, + started: &TaskAttemptRecord, + external_job_uid: Uuid, + mut continuation: Option, +) -> Result<(), HandlerError> { + if let Some(continuation) = &mut continuation { + continuation + .bind_external_job(external_job_uid) + .map_err(TerminalError::new)?; + } + let claimed_at = journal_now(ctx, "task_external_job_release_claimed_at").await?; + let repository = workflow.repository.clone(); + let fence = task_attempt_fence(request); + let task_generation = started.task.generation; + let started = ctx + .run(|| async move { + repository + .begin_task_attempt_external_release( + fence, + task_generation, + external_job_uid, + claimed_at, + ) + .await + .and_then(|outcome| match outcome { + TaskAttemptReleaseClaimOutcome::Applied(record) + | TaskAttemptReleaseClaimOutcome::Replayed(record) => Ok(Some(*record)), + TaskAttemptReleaseClaimOutcome::NotFound + | TaskAttemptReleaseClaimOutcome::Stale => Ok(None), + TaskAttemptReleaseClaimOutcome::InvalidState => { + Err(moa_execution::Error::InvalidRepositoryData { + message: "active task external-job release was rejected".to_string(), + }) + } + }) + .map(Json::from) + .map_err(execution_error_to_handler_error) + }) + .name("begin_task_attempt_external_release") + .await? + .into_inner(); + let Some(started) = started else { + return Ok(()); + }; + let release_receipt = checkpoint_task_hands_workflow(workflow, ctx, request, &started).await?; + let yielded_at = journal_now(ctx, "task_external_job_yielded_at").await?; + let continuation_checkpoint = continuation + .map(|mut continuation| { + continuation.workspace_release_receipt_id = + release_receipt.as_ref().map(|receipt| receipt.receipt_id); + let schema_version = continuation.schema_version; + continuation + .to_bounded_json() + .map(|payload| NewTaskAttemptCheckpoint { + fence: task_attempt_fence(request), + task_generation: started.task.generation, + kind: TaskAttemptCheckpointKind::AgentContinuation, + schema_version, + payload, + workspace_release_receipt: release_receipt.clone(), + created_at: yielded_at, + }) + }) + .transpose() + .map_err(TerminalError::new)?; + let repository = workflow.repository.clone(); + let fence = task_attempt_fence(request); + ctx.run(|| async move { + repository + .yield_task_attempt_to_external_job( + fence, + external_job_uid, + continuation_checkpoint, + release_receipt, + yielded_at, + ) + .await + .and_then(|outcome| match outcome { + TaskAttemptExternalOutcome::Applied { .. } + | TaskAttemptExternalOutcome::Replayed { .. } + | TaskAttemptExternalOutcome::NotFound + | TaskAttemptExternalOutcome::Stale => Ok(()), + TaskAttemptExternalOutcome::InvalidState => { + Err(moa_execution::Error::InvalidRepositoryData { + message: "active task external-job yield was rejected".to_string(), + }) + } + }) + .map_err(execution_error_to_handler_error) + }) + .name("yield_task_attempt_to_external_job") + .await?; + Ok(()) +} diff --git a/crates/moa-orchestrator/src/workflows/execution_task_attempt/watchdog.rs b/crates/moa-orchestrator/src/workflows/execution_task_attempt/watchdog.rs new file mode 100644 index 000000000..aeca00171 --- /dev/null +++ b/crates/moa-orchestrator/src/workflows/execution_task_attempt/watchdog.rs @@ -0,0 +1,316 @@ +//! Replay-safe classification of stale active task attempts. + +use chrono::Duration; +use moa_artifacts::execution_plan::{ + ExecutionFailureClass, ExecutionTaskOutcome, ExecutionTaskResult, +}; +use moa_core::types::tools::IdempotencyClass; +use moa_execution::{ + capability::CapabilitySource, + repository::{ + ExecutionAttemptState, ExecutionScope, + task::{ + TaskAttemptFence, TaskAttemptRecord, TaskAttemptSettlementOutcome, + UnstartedTaskAttemptDisposition, + }, + }, + state::{ExecutionTaskStatus, LogicalTaskKind, exhaust_retry_outcome, retry_delay_ms}, + wire::{ + ExecutionAttemptWatchdogResponseOutcome, ExecutionTaskAttemptRequest, + ExecutionTaskAttemptWatchdogRequest, + }, +}; +use restate_sdk::prelude::*; +use uuid::Uuid; + +use crate::{ + services::llm_gateway::{LLMCompletionOwner, cancel_completion_owner}, + workflows::{ + errors::execution_error_to_handler_error, + execution_task_attempt::{ + ExecutionTaskAttemptImpl, task_attempt_fence, + yielding::{begin_release_shared, checkpoint_task_hands_shared, journal_now_shared}, + }, + }, +}; + +/// Durable action selected when an active attempt misses its watchdog deadline. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum StaleTaskAttemptDisposition { + /// The effect is safe to redispatch behind a new attempt generation. + Retry, + /// Re-dispatch could duplicate an effect and requires explicit reconciliation. + UnknownOutcome, +} + +/// Durable watchdog result returned to the owning trigger-delivery chain. +pub(super) struct TaskAttemptWatchdogResult { + /// Typed receiver disposition returned to trigger delivery. + pub(super) outcome: ExecutionAttemptWatchdogResponseOutcome, +} + +/// Classifies a stale attempt solely from its persisted effect semantics. +#[must_use] +pub(super) const fn classify_stale_attempt( + idempotency: IdempotencyClass, +) -> StaleTaskAttemptDisposition { + match idempotency { + IdempotencyClass::Idempotent => StaleTaskAttemptDisposition::Retry, + IdempotencyClass::NonIdempotent => StaleTaskAttemptDisposition::UnknownOutcome, + } +} + +/// Applies one exact due watchdog without resending an ambiguous effect. +pub(super) async fn handle_task_attempt_watchdog( + workflow: &ExecutionTaskAttemptImpl, + ctx: &SharedWorkflowContext<'_>, + request: ExecutionTaskAttemptWatchdogRequest, +) -> Result { + let scope = ExecutionScope::Tenant { + tenant_id: request.tenant_id, + }; + let repository = workflow.repository.clone(); + let config = workflow.config.clone(); + let run_uid = request.run_uid; + let task_id = request.task_id; + let loaded = ctx + .run(|| async move { + let run = repository + .load_run(scope, run_uid) + .await + .map_err(execution_error_to_handler_error)?; + let task = repository + .load_task(scope, run_uid, task_id) + .await + .map_err(execution_error_to_handler_error)?; + Ok::<_, HandlerError>(Json::from(run.zip(task))) + }) + .name("load_task_attempt_for_watchdog") + .await? + .into_inner(); + let Some((run, task)) = loaded else { + return Ok(watchdog_result( + ExecutionAttemptWatchdogResponseOutcome::ReplayedOrStale, + )); + }; + let Some(deadline) = task.attempt_deadline_at else { + return Ok(watchdog_result( + ExecutionAttemptWatchdogResponseOutcome::RetryDelivery, + )); + }; + if run.controller_generation != request.controller_generation + || task.attempt_generation != request.attempt_generation + || task.active_dispatch_uid != Some(request.dispatch_uid) + { + return Ok(watchdog_result( + ExecutionAttemptWatchdogResponseOutcome::ReplayedOrStale, + )); + } + let now = journal_now_shared(ctx, "task_watchdog_observed_at").await?; + if deadline > now { + return Ok(watchdog_result( + ExecutionAttemptWatchdogResponseOutcome::RetryDelivery, + )); + } + cancel_completion_owner( + ctx, + LLMCompletionOwner::execution_task_attempt(request.dispatch_uid), + ) + .await?; + if task.status == ExecutionTaskStatus::Dispatching + && task.attempt_state == ExecutionAttemptState::Dispatching + { + // No receiver start committed, so exact delivery loss is safe to redispatch even for a + // non-idempotent capability. The repository releases capacity, supersedes the watchdog, + // advances attempt generation, and enqueues the controller wake in one transaction. + let repository = workflow.repository.clone(); + let fence = TaskAttemptFence { + tenant_id: request.tenant_id, + run_uid: request.run_uid, + task_id: request.task_id, + controller_generation: request.controller_generation, + attempt_generation: request.attempt_generation, + dispatch_uid: request.dispatch_uid, + capacity_reservation_uid: request.capacity_reservation_uid, + watchdog_trigger_uid: request.watchdog_trigger_uid, + attempt_deadline_at: deadline, + }; + let settlement = ctx + .run(|| async move { + repository + .settle_unstarted_task_attempt( + fence, + UnstartedTaskAttemptDisposition::DispatchDeliveryLost, + now, + ) + .await + .map(task_watchdog_settlement_response) + .map(Json::from) + .map_err(execution_error_to_handler_error) + }) + .name("settle_unstarted_task_attempt_watchdog") + .await? + .into_inner(); + return Ok(watchdog_result(settlement)); + } + if task.status != ExecutionTaskStatus::Running + || task.attempt_state != ExecutionAttemptState::Running + { + return Ok(watchdog_result( + if task.attempt_state == ExecutionAttemptState::Cancelling { + ExecutionAttemptWatchdogResponseOutcome::RetryDelivery + } else { + ExecutionAttemptWatchdogResponseOutcome::ReplayedOrStale + }, + )); + } + let attempt_request = ExecutionTaskAttemptRequest { + dispatch_uid: request.dispatch_uid, + capacity_reservation_uid: request.capacity_reservation_uid, + watchdog_trigger_uid: request.watchdog_trigger_uid, + watchdog_dispatch_uid: Uuid::nil(), + run_uid: request.run_uid, + task_id: request.task_id, + controller_generation: request.controller_generation, + attempt_generation: request.attempt_generation, + attempt_deadline_at: deadline, + tenant_id: request.tenant_id, + }; + let started = TaskAttemptRecord { run, task }; + let Some(started) = begin_release_shared( + workflow, + ctx, + &attempt_request, + started.task.generation, + "watchdog", + ) + .await? + else { + return Ok(watchdog_result( + ExecutionAttemptWatchdogResponseOutcome::RetryDelivery, + )); + }; + let receipt = checkpoint_task_hands_shared(workflow, ctx, &attempt_request, &started).await?; + let disposition = classify_stale_attempt(task_effect_idempotency(&started)); + let outcome = match disposition { + StaleTaskAttemptDisposition::Retry => ExecutionTaskOutcome { + schema_version: 1, + usage: started.task.actual.clone(), + result: ExecutionTaskResult::Failed { + class: ExecutionFailureClass::Retryable, + message: "task attempt watchdog expired before durable settlement".to_string(), + }, + }, + StaleTaskAttemptDisposition::UnknownOutcome => ExecutionTaskOutcome { + schema_version: 1, + usage: started.task.actual.clone(), + result: ExecutionTaskResult::UnknownOutcome { + message: "non-idempotent task attempt exceeded its watchdog after possible commit" + .to_string(), + }, + }, + }; + let outcome = exhaust_retry_outcome(started.task.attempt, &started.task.retry, outcome); + let retry_at = matches!( + outcome.result, + ExecutionTaskResult::Failed { + class: ExecutionFailureClass::Retryable, + .. + } + ) + .then(|| { + now + Duration::milliseconds( + i64::try_from(retry_delay_ms( + started.task.attempt.saturating_add(1), + &started.task.retry, + )) + .unwrap_or(i64::MAX), + ) + }); + let repository = workflow.repository.clone(); + let fence = task_attempt_fence(&attempt_request); + let settlement = ctx + .run(|| async move { + repository + .settle_released_task_attempt(&config, fence, outcome, retry_at, now, receipt) + .await + .map(task_watchdog_settlement_response) + .map(Json::from) + .map_err(execution_error_to_handler_error) + }) + .name("settle_task_attempt_watchdog") + .await? + .into_inner(); + Ok(watchdog_result(settlement)) +} + +fn watchdog_result(outcome: ExecutionAttemptWatchdogResponseOutcome) -> TaskAttemptWatchdogResult { + TaskAttemptWatchdogResult { outcome } +} + +fn task_effect_idempotency(started: &TaskAttemptRecord) -> IdempotencyClass { + let references: Vec<_> = match &started.task.kind { + LogicalTaskKind::Capability { reference } => vec![reference], + LogicalTaskKind::Agent { + capability_refs, .. + } => capability_refs.iter().collect(), + LogicalTaskKind::Output { .. } + | LogicalTaskKind::Review { .. } + | LogicalTaskKind::WaitSignal { .. } + | LogicalTaskKind::WaitUntil { .. } + | LogicalTaskKind::CompletionVerifier { .. } => Vec::new(), + }; + if references.into_iter().any(|reference| { + started + .run + .catalog + .capabilities + .iter() + .find(|capability| capability.reference == *reference) + .is_none_or(|capability| { + capability.idempotency_class == IdempotencyClass::NonIdempotent + || matches!(capability.source, CapabilitySource::Model) + }) + }) { + IdempotencyClass::NonIdempotent + } else { + IdempotencyClass::Idempotent + } +} + +fn task_watchdog_settlement_response( + outcome: TaskAttemptSettlementOutcome, +) -> ExecutionAttemptWatchdogResponseOutcome { + match outcome { + TaskAttemptSettlementOutcome::Applied { .. } => { + ExecutionAttemptWatchdogResponseOutcome::Settled + } + TaskAttemptSettlementOutcome::Replayed { .. } + | TaskAttemptSettlementOutcome::NotFound + | TaskAttemptSettlementOutcome::Stale => { + ExecutionAttemptWatchdogResponseOutcome::ReplayedOrStale + } + TaskAttemptSettlementOutcome::InvalidState => { + ExecutionAttemptWatchdogResponseOutcome::RetryDelivery + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // Pins: a watchdog must never automatically resend an ambiguous + // non-idempotent effect, while an idempotent effect remains recoverable. + #[test] + fn watchdog_separates_retryable_and_ambiguous_effects_offline() { + assert_eq!( + classify_stale_attempt(IdempotencyClass::Idempotent), + StaleTaskAttemptDisposition::Retry + ); + assert_eq!( + classify_stale_attempt(IdempotencyClass::NonIdempotent), + StaleTaskAttemptDisposition::UnknownOutcome + ); + } +} diff --git a/crates/moa-orchestrator/src/workflows/execution_task_attempt/yielding.rs b/crates/moa-orchestrator/src/workflows/execution_task_attempt/yielding.rs new file mode 100644 index 000000000..33420b889 --- /dev/null +++ b/crates/moa-orchestrator/src/workflows/execution_task_attempt/yielding.rs @@ -0,0 +1,639 @@ +//! Classification of logical tasks that park entirely in durable storage. + +use chrono::{Duration, Utc}; +use moa_artifacts::execution_plan::{ExecutionTaskOutcome, ExecutionTaskResult}; +use moa_core::types::action_policy::{ActionReviewOwner, ExecutionTaskOrigin}; +use moa_core::types::sandbox_workspace::ExecutionHandReleaseReceipt; +use moa_execution::{ + repository::{ + ExecutionAttemptState, ExecutionScope, + task::{ + NewTaskAttemptCheckpoint, TaskAttemptCheckpointKind, + TaskAttemptContinuationYieldOutcome, TaskAttemptRecord, TaskAttemptReleaseClaimOutcome, + TaskAttemptReviewParkOutcome, TaskAttemptSettlementOutcome, + UnstartedTaskAttemptDisposition, + }, + }, + state::ExecutionTaskStatus, + wire::{ + ExecutionAttemptCancelReason, ExecutionTaskAttemptCancelRequest, + ExecutionTaskAttemptRequest, + }, +}; + +use restate_sdk::prelude::*; +use uuid::Uuid; + +use crate::{ + services::{ + action_reviews::{AcknowledgeExecutionActionReviewRequest, ActionReviewsClient}, + llm_gateway::{LLMCompletionOwner, cancel_completion_owner}, + tool_executor::{CheckpointAndReleaseExecutionHandsRequest, ToolExecutorClient}, + }, + workflows::{ + errors::execution_error_to_handler_error, + execution_task_attempt::{ + ExecutionTaskAttemptImpl, + active::{TaskAttemptContinuation, TaskAttemptContinuationState}, + task_attempt_fence, + }, + }, +}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum TaskCancelSettlement { + Pause, + Terminal, +} + +const fn task_cancel_settlement(reason: ExecutionAttemptCancelReason) -> TaskCancelSettlement { + match reason { + ExecutionAttemptCancelReason::PauseRequested => TaskCancelSettlement::Pause, + ExecutionAttemptCancelReason::DeadlineExceeded + | ExecutionAttemptCancelReason::RunTerminal + | ExecutionAttemptCancelReason::ExternalJobStarted => TaskCancelSettlement::Terminal, + } +} + +/// Persists a review continuation after policy accepted the exact reviewed effect. +pub(super) async fn park_review( + workflow: &ExecutionTaskAttemptImpl, + ctx: &WorkflowContext<'_>, + request: &ExecutionTaskAttemptRequest, + started: &TaskAttemptRecord, + continuation: TaskAttemptContinuation, +) -> Result<(), HandlerError> { + let Some(started) = begin_release_workflow( + workflow, + ctx, + request, + started.task.generation, + "action_review", + ) + .await? + else { + return Ok(()); + }; + let workspace_release_receipt = + checkpoint_task_hands_workflow(workflow, ctx, request, &started).await?; + let review_uid = continuation + .pending_review_uid() + .ok_or_else(|| TerminalError::new("review continuation is missing its review UID"))?; + let payload = continuation.to_bounded_json().map_err(TerminalError::new)?; + let repository = workflow.repository.clone(); + let checkpoint = NewTaskAttemptCheckpoint { + fence: task_attempt_fence(request), + task_generation: started.task.generation, + kind: match &continuation.state { + TaskAttemptContinuationState::Agent { .. } => { + TaskAttemptCheckpointKind::AgentContinuation + } + TaskAttemptContinuationState::CapabilityReview { .. } => { + TaskAttemptCheckpointKind::CapabilityReview + } + TaskAttemptContinuationState::CapabilityExternalStart { .. } => { + TaskAttemptCheckpointKind::CapabilityExternalStart + } + }, + schema_version: continuation.schema_version, + payload, + workspace_release_receipt, + created_at: journal_now_workflow(ctx, "task_review_checkpointed_at").await?, + }; + let parked = ctx + .run(|| async move { + repository + .park_task_attempt_on_review(checkpoint, review_uid) + .await + .and_then(|outcome| match outcome { + TaskAttemptReviewParkOutcome::Applied { .. } + | TaskAttemptReviewParkOutcome::Replayed { .. } => Ok(Json::from(true)), + TaskAttemptReviewParkOutcome::NotFound + | TaskAttemptReviewParkOutcome::Stale => Ok(Json::from(false)), + TaskAttemptReviewParkOutcome::InvalidState => { + Err(moa_execution::Error::InvalidRepositoryData { + message: "active attempt review park was rejected".to_string(), + }) + } + }) + .map_err(execution_error_to_handler_error) + }) + .name("park_task_attempt_review") + .await? + .into_inner(); + if parked { + crate::restate_identity::replay_safe_request( + ctx.service_client::() + .acknowledge_execution_owner_review(Json::from( + AcknowledgeExecutionActionReviewRequest { + tenant_id: request.tenant_id, + review_id: review_uid, + owner: ActionReviewOwner::ExecutionTask { + session_id: started.run.session_id, + origin: ExecutionTaskOrigin { + run_uid: request.run_uid, + task_uid: request.task_id.as_uuid(), + generation: started.task.generation, + attempt_generation: request.attempt_generation, + }, + }, + }, + )), + ) + .call() + .await?; + } + Ok(()) +} + +/// Persists a complete agent boundary and relinquishes active ownership for redispatch. +pub(super) async fn yield_continuation( + workflow: &ExecutionTaskAttemptImpl, + ctx: &WorkflowContext<'_>, + request: &ExecutionTaskAttemptRequest, + started: &TaskAttemptRecord, + continuation: TaskAttemptContinuation, +) -> Result<(), HandlerError> { + let Some(started) = begin_release_workflow( + workflow, + ctx, + request, + started.task.generation, + "agent_continuation", + ) + .await? + else { + return Ok(()); + }; + let workspace_release_receipt = + checkpoint_task_hands_workflow(workflow, ctx, request, &started).await?; + let payload = continuation.to_bounded_json().map_err(TerminalError::new)?; + let repository = workflow.repository.clone(); + let checkpoint = NewTaskAttemptCheckpoint { + fence: task_attempt_fence(request), + task_generation: started.task.generation, + kind: TaskAttemptCheckpointKind::AgentContinuation, + schema_version: continuation.schema_version, + payload, + workspace_release_receipt, + created_at: journal_now_workflow(ctx, "task_agent_continuation_checkpointed_at").await?, + }; + ctx.run(|| async move { + repository + .yield_task_attempt_continuation(checkpoint) + .await + .and_then(|outcome| match outcome { + TaskAttemptContinuationYieldOutcome::Applied { .. } + | TaskAttemptContinuationYieldOutcome::Replayed { .. } + | TaskAttemptContinuationYieldOutcome::NotFound + | TaskAttemptContinuationYieldOutcome::Stale => Ok(()), + TaskAttemptContinuationYieldOutcome::InvalidState => { + Err(moa_execution::Error::InvalidRepositoryData { + message: "active attempt continuation yield was rejected".to_string(), + }) + } + }) + .map_err(execution_error_to_handler_error) + }) + .name("yield_task_attempt_continuation") + .await?; + Ok(()) +} + +/// Persists an exact agent continuation before publishing a storage-only input wait. +pub(super) async fn park_input( + workflow: &ExecutionTaskAttemptImpl, + ctx: &WorkflowContext<'_>, + request: &ExecutionTaskAttemptRequest, + started: &TaskAttemptRecord, + outcome: ExecutionTaskOutcome, + continuation: TaskAttemptContinuation, +) -> Result<(), HandlerError> { + if !matches!(outcome.result, ExecutionTaskResult::NeedsInput { .. }) { + return Err(TerminalError::new("input park requires a NeedsInput outcome").into()); + } + let Some(started) = begin_release_workflow( + workflow, + ctx, + request, + started.task.generation, + "agent_input", + ) + .await? + else { + return Ok(()); + }; + let workspace_release_receipt = + checkpoint_task_hands_workflow(workflow, ctx, request, &started).await?; + let payload = continuation.to_bounded_json().map_err(TerminalError::new)?; + let settled_at = journal_now_workflow(ctx, "task_input_checkpointed_at").await?; + let checkpoint = NewTaskAttemptCheckpoint { + fence: task_attempt_fence(request), + task_generation: started.task.generation, + kind: TaskAttemptCheckpointKind::AgentContinuation, + schema_version: continuation.schema_version, + payload, + workspace_release_receipt: workspace_release_receipt.clone(), + created_at: settled_at, + }; + let repository = workflow.repository.clone(); + let config = workflow.config.clone(); + let fence = task_attempt_fence(request); + ctx.run(|| async move { + repository + .settle_released_task_attempt_with_checkpoint( + &config, + fence, + outcome, + settled_at, + workspace_release_receipt, + checkpoint, + ) + .await + .and_then(|outcome| match outcome { + moa_execution::repository::task::TaskAttemptSettlementOutcome::Applied { + .. + } + | moa_execution::repository::task::TaskAttemptSettlementOutcome::Replayed { + .. + } + | moa_execution::repository::task::TaskAttemptSettlementOutcome::NotFound + | moa_execution::repository::task::TaskAttemptSettlementOutcome::Stale => Ok(()), + moa_execution::repository::task::TaskAttemptSettlementOutcome::InvalidState => { + Err(moa_execution::Error::InvalidRepositoryData { + message: "active attempt input park was rejected".to_string(), + }) + } + }) + .map_err(execution_error_to_handler_error) + }) + .name("park_task_attempt_input") + .await?; + Ok(()) +} + +/// Checkpoints any sandbox owned by an exact cancellation delivery, then settles it. +pub(super) async fn cancel_task_attempt( + workflow: &ExecutionTaskAttemptImpl, + ctx: &SharedWorkflowContext<'_>, + request: ExecutionTaskAttemptCancelRequest, +) -> Result<(), HandlerError> { + let repository = workflow.repository.clone(); + let scope = ExecutionScope::Tenant { + tenant_id: request.tenant_id, + }; + let run_uid = request.run_uid; + let task_id = request.task_id; + let loaded = ctx + .run(|| async move { + let run = repository + .load_run(scope, run_uid) + .await + .map_err(execution_error_to_handler_error)?; + let task = repository + .load_task(scope, run_uid, task_id) + .await + .map_err(execution_error_to_handler_error)?; + Ok::<_, HandlerError>(Json::from(run.zip(task))) + }) + .name("load_task_attempt_for_cancel") + .await? + .into_inner(); + let Some((run, task)) = loaded else { + return Ok(()); + }; + if run.controller_generation != request.controller_generation + || task.generation != request.task_generation + || task.attempt_generation != request.attempt_generation + || task.active_dispatch_uid != Some(request.active_dispatch_uid) + { + return Ok(()); + } + cancel_completion_owner( + ctx, + LLMCompletionOwner::execution_task_attempt(request.active_dispatch_uid), + ) + .await?; + if task.status == ExecutionTaskStatus::Dispatching + && task.attempt_state == ExecutionAttemptState::Cancelling + { + let now = journal_now_shared(ctx, "unstarted_task_cancel_settled_at").await?; + let attempt_deadline_at = task.attempt_deadline_at.ok_or_else(|| { + TerminalError::new("unstarted cancelled task is missing its attempt deadline") + })?; + let repository = workflow.repository.clone(); + let fence = moa_execution::repository::task::TaskAttemptFence { + tenant_id: request.tenant_id, + run_uid: request.run_uid, + task_id: request.task_id, + controller_generation: request.attempt_controller_generation, + attempt_generation: request.attempt_generation, + dispatch_uid: request.active_dispatch_uid, + capacity_reservation_uid: request.capacity_reservation_uid, + watchdog_trigger_uid: request.watchdog_trigger_uid, + attempt_deadline_at, + }; + let disposition = match task_cancel_settlement(request.reason) { + TaskCancelSettlement::Pause => UnstartedTaskAttemptDisposition::Paused { + controller_generation: request.controller_generation, + }, + TaskCancelSettlement::Terminal => UnstartedTaskAttemptDisposition::Cancelled { + reason: format!("bounded unstarted attempt cancelled: {:?}", request.reason), + }, + }; + ctx.run(|| async move { + repository + .settle_unstarted_task_attempt(fence, disposition, now) + .await + .and_then(|outcome| match outcome { + TaskAttemptSettlementOutcome::Applied { .. } + | TaskAttemptSettlementOutcome::Replayed { .. } + | TaskAttemptSettlementOutcome::NotFound + | TaskAttemptSettlementOutcome::Stale => Ok(()), + TaskAttemptSettlementOutcome::InvalidState => { + Err(moa_execution::Error::InvalidRepositoryData { + message: "unstarted task cancel settlement was rejected".to_string(), + }) + } + }) + .map_err(execution_error_to_handler_error) + }) + .name("settle_unstarted_task_attempt_cancel_or_pause") + .await?; + return Ok(()); + } + if task.status != ExecutionTaskStatus::Running + || task.attempt_state != ExecutionAttemptState::Cancelling + { + return Ok(()); + } + let attempt_request = ExecutionTaskAttemptRequest { + dispatch_uid: request.active_dispatch_uid, + capacity_reservation_uid: request.capacity_reservation_uid, + watchdog_trigger_uid: request.watchdog_trigger_uid, + watchdog_dispatch_uid: Uuid::nil(), + run_uid: request.run_uid, + task_id: request.task_id, + controller_generation: request.attempt_controller_generation, + attempt_generation: request.attempt_generation, + attempt_deadline_at: task.attempt_deadline_at.ok_or_else(|| { + TerminalError::new("active cancelled task is missing its attempt deadline") + })?, + tenant_id: request.tenant_id, + }; + // The run-level cancellation transaction already fenced this exact attempt by + // moving it to `Cancelling`. In particular, pause advances the run controller + // generation while the active resources retain their admission generation. + // Re-running the ordinary release claim here would compare those generations, + // return `Stale`, and acknowledge the cancel without draining active capacity. + let started = TaskAttemptRecord { run, task }; + let receipt = checkpoint_task_hands_shared(workflow, ctx, &attempt_request, &started).await?; + let now = journal_now_shared(ctx, "task_cancel_settled_at").await?; + let repository = workflow.repository.clone(); + let config = workflow.config.clone(); + let fence = task_attempt_fence(&attempt_request); + let settlement = task_cancel_settlement(request.reason); + let controller_generation = request.controller_generation; + let terminal_reason = format!("bounded attempt cancelled: {:?}", request.reason); + let usage = started.task.actual.clone(); + ctx.run(|| async move { + let outcome = match settlement { + TaskCancelSettlement::Pause => { + repository + .finalize_paused_task_attempt_release( + controller_generation, + fence, + now, + receipt, + ) + .await + } + TaskCancelSettlement::Terminal => { + repository + .settle_released_task_attempt( + &config, + fence, + ExecutionTaskOutcome { + schema_version: 1, + usage, + result: ExecutionTaskResult::Cancelled { + reason: terminal_reason, + }, + }, + None, + now, + receipt, + ) + .await + } + } + .map_err(execution_error_to_handler_error)?; + match outcome { + TaskAttemptSettlementOutcome::Applied { .. } + | TaskAttemptSettlementOutcome::Replayed { .. } + | TaskAttemptSettlementOutcome::NotFound + | TaskAttemptSettlementOutcome::Stale => Ok(()), + TaskAttemptSettlementOutcome::InvalidState => { + Err(TerminalError::new("task cancel or pause settlement was rejected").into()) + } + } + }) + .name("settle_cancelled_or_paused_task_attempt") + .await?; + Ok(()) +} + +pub(super) async fn begin_release_workflow( + workflow: &ExecutionTaskAttemptImpl, + ctx: &WorkflowContext<'_>, + request: &ExecutionTaskAttemptRequest, + task_generation: u64, + reason: &'static str, +) -> Result, HandlerError> { + let claimed_at = journal_now_workflow(ctx, "task_attempt_release_claimed_at").await?; + let repository = workflow.repository.clone(); + let fence = task_attempt_fence(request); + let outcome = ctx + .run(|| async move { + repository + .begin_task_attempt_release(fence, task_generation, reason, claimed_at) + .await + .map(Json::from) + .map_err(execution_error_to_handler_error) + }) + .name("begin_task_attempt_release") + .await? + .into_inner(); + Ok(match outcome { + TaskAttemptReleaseClaimOutcome::Applied(record) + | TaskAttemptReleaseClaimOutcome::Replayed(record) => Some(*record), + TaskAttemptReleaseClaimOutcome::NotFound + | TaskAttemptReleaseClaimOutcome::Stale + | TaskAttemptReleaseClaimOutcome::InvalidState => None, + }) +} + +pub(super) async fn begin_release_shared( + workflow: &ExecutionTaskAttemptImpl, + ctx: &SharedWorkflowContext<'_>, + request: &ExecutionTaskAttemptRequest, + task_generation: u64, + reason: &'static str, +) -> Result, HandlerError> { + let claimed_at = journal_now_shared(ctx, "task_attempt_release_claimed_at").await?; + let repository = workflow.repository.clone(); + let fence = task_attempt_fence(request); + let outcome = ctx + .run(|| async move { + repository + .begin_task_attempt_release(fence, task_generation, reason, claimed_at) + .await + .map(Json::from) + .map_err(execution_error_to_handler_error) + }) + .name("begin_task_attempt_release") + .await? + .into_inner(); + Ok(match outcome { + TaskAttemptReleaseClaimOutcome::Applied(record) + | TaskAttemptReleaseClaimOutcome::Replayed(record) => Some(*record), + TaskAttemptReleaseClaimOutcome::NotFound + | TaskAttemptReleaseClaimOutcome::Stale + | TaskAttemptReleaseClaimOutcome::InvalidState => None, + }) +} + +pub(super) async fn checkpoint_task_hands_workflow( + _workflow: &ExecutionTaskAttemptImpl, + ctx: &WorkflowContext<'_>, + request: &ExecutionTaskAttemptRequest, + started: &TaskAttemptRecord, +) -> Result, HandlerError> { + let release_started_at = journal_now_workflow(ctx, "task_hand_release_started_at").await?; + let receipt = crate::restate_identity::replay_safe_request( + ctx.service_client::() + .checkpoint_and_release_execution_hands(Json::from(checkpoint_request( + request, + started, + task_hand_release_deadline(release_started_at), + ))), + ) + .call() + .await? + .into_inner(); + Ok(Some(receipt)) +} + +pub(super) async fn checkpoint_task_hands_shared( + _workflow: &ExecutionTaskAttemptImpl, + ctx: &SharedWorkflowContext<'_>, + request: &ExecutionTaskAttemptRequest, + started: &TaskAttemptRecord, +) -> Result, HandlerError> { + let release_started_at = journal_now_shared(ctx, "task_hand_release_started_at").await?; + let receipt = crate::restate_identity::replay_safe_request( + ctx.service_client::() + .checkpoint_and_release_execution_hands(Json::from(checkpoint_request( + request, + started, + task_hand_release_deadline(release_started_at), + ))), + ) + .call() + .await? + .into_inner(); + Ok(Some(receipt)) +} + +fn checkpoint_request( + request: &ExecutionTaskAttemptRequest, + started: &TaskAttemptRecord, + release_deadline_at: chrono::DateTime, +) -> CheckpointAndReleaseExecutionHandsRequest { + CheckpointAndReleaseExecutionHandsRequest { + tenant_id: request.tenant_id, + session_id: started.run.session_id, + run_uid: request.run_uid, + owner: moa_core::types::sandbox_workspace::ExecutionHandReleaseOwner::Task { + task_id: moa_core::types::identifiers::ExecutionTaskScopeId(request.task_id.as_uuid()), + logical_generation: started.task.generation, + }, + attempt_generation: request.attempt_generation, + release_deadline_at, + } +} + +fn task_hand_release_deadline(release_started_at: chrono::DateTime) -> chrono::DateTime { + release_started_at + Duration::minutes(5) +} + +async fn journal_now_workflow( + ctx: &WorkflowContext<'_>, + name: &'static str, +) -> Result, HandlerError> { + Ok(ctx + .run(|| async { Ok::<_, HandlerError>(Json::from(Utc::now())) }) + .name(name) + .await? + .into_inner()) +} + +pub(super) async fn journal_now_shared( + ctx: &SharedWorkflowContext<'_>, + name: &'static str, +) -> Result, HandlerError> { + Ok(ctx + .run(|| async { Ok::<_, HandlerError>(Json::from(Utc::now())) }) + .name(name) + .await? + .into_inner()) +} + +#[cfg(test)] +mod tests { + use chrono::{Duration, TimeZone, Utc}; + use moa_execution::wire::ExecutionAttemptCancelReason; + + use super::{TaskCancelSettlement, task_cancel_settlement, task_hand_release_deadline}; + + #[test] + fn pause_cancel_uses_nonterminal_release_finalizer() { + // Pins: pause drains exact attempt ownership but must preserve the logical task for resume. + assert_eq!( + task_cancel_settlement(ExecutionAttemptCancelReason::PauseRequested), + TaskCancelSettlement::Pause, + ); + } + + #[test] + fn non_pause_cancel_reasons_remain_terminal() { + // Pins: deadline, terminal-run, and external-job ownership cancellation retain their + // terminal settlement classification rather than silently requeueing as a pause. + for reason in [ + ExecutionAttemptCancelReason::DeadlineExceeded, + ExecutionAttemptCancelReason::RunTerminal, + ExecutionAttemptCancelReason::ExternalJobStarted, + ] { + assert_eq!( + task_cancel_settlement(reason), + TaskCancelSettlement::Terminal, + ); + } + } + + #[test] + fn overdue_watchdog_gets_a_fresh_bounded_sandbox_release_deadline() { + // Pins: an expired compute deadline still permits one bounded checkpoint/destroy cycle; + // reusing that expired deadline would make ToolExecutor reject teardown admission. + let observed_at = Utc + .with_ymd_and_hms(2026, 8, 11, 12, 0, 0) + .single() + .expect("fixture timestamp is valid"); + let expired_compute_deadline = observed_at - Duration::seconds(1); + let release_deadline = task_hand_release_deadline(observed_at); + + assert!(expired_compute_deadline < observed_at); + assert_eq!(release_deadline, observed_at + Duration::minutes(5)); + assert!(release_deadline > observed_at); + } +} diff --git a/crates/moa-orchestrator/src/workflows/experiment_trial_run/target_execution.rs b/crates/moa-orchestrator/src/workflows/experiment_trial_run/target_execution.rs index 566d0ff95..1328994e9 100644 --- a/crates/moa-orchestrator/src/workflows/experiment_trial_run/target_execution.rs +++ b/crates/moa-orchestrator/src/workflows/experiment_trial_run/target_execution.rs @@ -38,7 +38,7 @@ use moa_eval_core::types::TEST_CASE_SCHEMA_VERSION; use moa_execution::{ CompileExecutionOutcome, CompileExecutionRequest, ExecutionValidationReport, ExecutionValidationSeverity, compile, - repository::{CompileAuditWriteOutcome, ExecutionRepository, ExecutionScope}, + repository::{ExecutionRepository, ExecutionScope, audit::CompileAuditWriteOutcome}, schema::validate_instance, state::ExecutionRunStatus, wire::{ @@ -1004,6 +1004,14 @@ pub(super) async fn run_execution_template_trial( trial.trial_uid, ) .await?; + let now = durable_utc_now(ctx, "experiment_trial_execution_compile_now").await?; + let horizon_seconds = i64::try_from(config.execution.maximum_horizon_seconds) + .map_err(|_| TerminalError::new("execution maximum horizon does not fit i64"))?; + let horizon = chrono::TimeDelta::try_seconds(horizon_seconds) + .ok_or_else(|| TerminalError::new("execution maximum horizon does not fit chrono"))?; + let deadline_at = now + .checked_add_signed(horizon) + .ok_or_else(|| TerminalError::new("execution maximum horizon exceeds timestamp range"))?; let planning_call = ctx .service_client::() .planning_context(Json::from(ExecutionPlanningContextRequest { @@ -1011,6 +1019,7 @@ pub(super) async fn run_execution_template_trial( contact_id: effective.contact_id, session_id: effective.session_id, originating_user_sequence_num: origin.sequence_num, + deadline_at, requested_template: Some(template.clone()), })); let planning_context = with_identity_headers(planning_call, &request.identity) @@ -1019,7 +1028,6 @@ pub(super) async fn run_execution_template_trial( .into_inner(); let operation_key = experiment_trial_operation_key(trial.run_uid, trial.score_run_id, trial.trial_uid); - let now = durable_utc_now(ctx, "experiment_trial_execution_compile_now").await?; let compiled = compile_experiment_template(ExperimentTemplateCompileRequest { context: &planning_context.snapshot, requested: &template, @@ -2349,7 +2357,13 @@ fn trial_stop_for_execution_run_status(status: ExecutionRunStatus) -> Option None, } } diff --git a/crates/moa-orchestrator/src/workflows/mod.rs b/crates/moa-orchestrator/src/workflows/mod.rs index 6a5ebfafc..7c66badfe 100644 --- a/crates/moa-orchestrator/src/workflows/mod.rs +++ b/crates/moa-orchestrator/src/workflows/mod.rs @@ -7,9 +7,8 @@ pub mod artifact_release_evaluation; pub(crate) mod child_invocation; pub mod consolidate; pub(crate) mod errors; -pub mod execution_compensation; -pub mod execution_run; -pub mod execution_task; +pub mod execution_compensation_attempt; +pub mod execution_task_attempt; pub(crate) mod experiment_cancel; pub(crate) mod experiment_errors; pub mod experiment_run; diff --git a/crates/moa-orchestrator/src/workflows/progress_delivery.rs b/crates/moa-orchestrator/src/workflows/progress_delivery.rs index 6c5433226..f7ee1162a 100644 --- a/crates/moa-orchestrator/src/workflows/progress_delivery.rs +++ b/crates/moa-orchestrator/src/workflows/progress_delivery.rs @@ -197,6 +197,23 @@ fn compact_status_summary(summary: &str) -> String { compact } +/// Builds the compact product summary emitted by bounded execution controllers. +pub(crate) fn execution_controller_summary( + phase: &str, + ready_tasks: u64, + active_tasks: u64, + parked_tasks: u64, + completed_tasks: u64, + next_wake_at: Option>, +) -> String { + let next_wake = next_wake_at + .map(|value| value.to_rfc3339()) + .unwrap_or_else(|| "none".to_string()); + compact_status_summary(&format!( + "Execution {phase}: ready={ready_tasks}, active={active_tasks}, parked={parked_tasks}, completed={completed_tasks}, next_wake={next_wake}" + )) +} + async fn live_delivery_enabled(ctx: &WorkflowContext<'_>) -> Result { Ok(ctx .get::>(K_PROGRESS_LIVE_DELIVERY_ENABLED) @@ -509,6 +526,20 @@ mod tests { ); } + #[test] + fn controller_summary_exposes_storage_only_wait_and_next_wake() { + // Pins: product progress distinguishes parked work from active compute and reports the + // exact durable wake instead of implying that a handler remains alive. + let wake = chrono::DateTime::parse_from_rfc3339("2026-08-12T03:04:05Z") + .expect("test timestamp parses") + .with_timezone(&chrono::Utc); + + assert_eq!( + execution_controller_summary("waiting_timer", 2, 1, 3, 8, Some(wake)), + "Execution waiting_timer: ready=2, active=1, parked=3, completed=8, next_wake=2026-08-12T03:04:05+00:00" + ); + } + #[tokio::test] async fn status_delivery_edits_existing_message_when_supported() { // Pins: Slack progress updates edit the existing status line instead of posting duplicates. diff --git a/crates/moa-orchestrator/src/workflows/turn_execution/mod.rs b/crates/moa-orchestrator/src/workflows/turn_execution/mod.rs index f4bc87cfe..5004ef3e7 100644 --- a/crates/moa-orchestrator/src/workflows/turn_execution/mod.rs +++ b/crates/moa-orchestrator/src/workflows/turn_execution/mod.rs @@ -56,10 +56,10 @@ use moa_core::{ types::session::SessionMeta, types::session::TurnOutcome as CoreTurnOutcome, }; -use moa_execution::repository::{ - CompileAuditWriteOutcome, ExecutionRepository, ExecutionScope, PlannerCallAuditWriteOutcome, - RouteAuditWriteOutcome, +use moa_execution::repository::audit::{ + CompileAuditWriteOutcome, PlannerCallAuditWriteOutcome, RouteAuditWriteOutcome, }; +use moa_execution::repository::{ExecutionRepository, ExecutionScope}; use moa_lineage_core::TurnId; use moa_observability::restate_observability::{ emit_turn_coordination_summary, emit_turn_latency_summary, emit_turn_replay_summary, @@ -1114,6 +1114,18 @@ async fn execute_durable_admission( .into()); } let contact_id = meta.contact.as_ref().map(|contact| contact.contact_id); + let planning_now = durable_utc_now(ctx, "execution_planning_now").await?; + let horizon_seconds = i64::try_from(workflow.config.execution.maximum_horizon_seconds) + .map_err(|_| TerminalError::new("execution maximum horizon does not fit i64"))?; + let horizon = chrono::TimeDelta::try_seconds(horizon_seconds) + .ok_or_else(|| TerminalError::new("execution maximum horizon does not fit chrono"))?; + let maximum_deadline = planning_now + .checked_add_signed(horizon) + .ok_or_else(|| TerminalError::new("execution maximum horizon exceeds timestamp range"))?; + let deadline_at = request + .resource_budget + .deadline + .map_or(maximum_deadline, |deadline| deadline.min(maximum_deadline)); let planning_call = ctx .service_client::() .planning_context(Json::from( @@ -1122,6 +1134,7 @@ async fn execute_durable_admission( contact_id, session_id, originating_user_sequence_num, + deadline_at, requested_template: request .execution_template .as_ref() @@ -1139,7 +1152,6 @@ async fn execute_durable_admission( .auxiliary .clone() .unwrap_or_else(|| workflow.config.models.main.clone()); - let planning_now = durable_utc_now(ctx, "execution_planning_now").await?; let provider = RestateExecutionModelProvider::new( ctx, per_model_call_budget(request.resource_budget), diff --git a/crates/moa-orchestrator/src/workflows/turn_execution/tools.rs b/crates/moa-orchestrator/src/workflows/turn_execution/tools.rs index 13ed91252..50dac70bb 100644 --- a/crates/moa-orchestrator/src/workflows/turn_execution/tools.rs +++ b/crates/moa-orchestrator/src/workflows/turn_execution/tools.rs @@ -659,7 +659,8 @@ async fn handle_tool_call( *tool_context.delegated_worker = true; } } - GovernedInvocationOutcome::UnknownOutcome { .. } + GovernedInvocationOutcome::ExternalJob { .. } + | GovernedInvocationOutcome::UnknownOutcome { .. } | GovernedInvocationOutcome::NotDispatched { .. } => { return Err(TerminalError::new( "root-turn governed invocation returned an execution-only outcome", diff --git a/crates/moa-orchestrator/src/workflows/worker_turn_execution.rs b/crates/moa-orchestrator/src/workflows/worker_turn_execution.rs index 3638571f5..cc84d2ab2 100644 --- a/crates/moa-orchestrator/src/workflows/worker_turn_execution.rs +++ b/crates/moa-orchestrator/src/workflows/worker_turn_execution.rs @@ -837,7 +837,8 @@ async fn handle_tool_call( ) .await?; } - GovernedInvocationOutcome::UnknownOutcome { .. } + GovernedInvocationOutcome::ExternalJob { .. } + | GovernedInvocationOutcome::UnknownOutcome { .. } | GovernedInvocationOutcome::NotDispatched { .. } => { return Err(TerminalError::new( "worker-origin governed invocation returned an execution-only outcome", diff --git a/crates/moa-orchestrator/tests/coordinator_worker_behavior_provider_e2e.rs b/crates/moa-orchestrator/tests/coordinator_worker_behavior_provider_e2e.rs index 9767a6b9d..74ffd2c77 100644 --- a/crates/moa-orchestrator/tests/coordinator_worker_behavior_provider_e2e.rs +++ b/crates/moa-orchestrator/tests/coordinator_worker_behavior_provider_e2e.rs @@ -2037,6 +2037,7 @@ async fn assert_generated_plan_audits_and_authorization( contact_id: None, session_id: harness.session.session_id, originating_user_sequence_num: originating_sequence, + deadline_at: chrono::Utc::now() + chrono::TimeDelta::days(1), requested_template: None, }, ) @@ -2491,6 +2492,30 @@ async fn recovery_matrix_blocked_llm_invocation( bail!("{workflow_service} has no incomplete LLMGateway child: {parents:?}") } +async fn recovery_matrix_execution_task_attempt_key( + fixture: &OrchestratorTestFixture, + run_uid: uuid::Uuid, +) -> Result { + let pool = sqlx::PgPool::connect(&fixture.postgres_url) + .await + .context("connect recovery-matrix execution database")?; + let dispatch_uids: Vec = sqlx::query_scalar( + "SELECT dispatch_uid FROM moa.execution_dispatch_outbox \ + WHERE run_uid = $1 AND dispatch_kind = 'task_attempt' \ + ORDER BY created_at, dispatch_uid", + ) + .bind(run_uid) + .fetch_all(&pool) + .await + .context("load recovery-matrix task-attempt dispatch identity")?; + let [dispatch_uid] = dispatch_uids.as_slice() else { + bail!( + "expected exactly one task-attempt dispatch for run {run_uid}, got {dispatch_uids:?}" + ); + }; + Ok(dispatch_uid.to_string()) +} + async fn recovery_matrix_assert_child_joined( fixture: &OrchestratorTestFixture, parent_id: &str, @@ -3053,6 +3078,12 @@ fn recovery_matrix_execution_candidate( }, plan: ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, + input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { + expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { + delay_seconds: 86_400, + }, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + }, input_schema: json!({"type": "object", "additionalProperties": false}), output_schema: output_schema.clone(), nodes: vec![ @@ -3180,7 +3211,7 @@ async fn recovery_matrix_wait_for_execution_status( fixture, "SELECT id, invoked_by_id, target_service_name, target_service_key, status \ FROM sys_invocation WHERE target_service_name IN \ - ('ExecutionRun', 'ExecutionTask', 'LLMGateway') \ + ('ExecutionRunController', 'ExecutionTaskAttempt', 'ExecutionDispatcher', 'LLMGateway') \ ORDER BY target_service_name, id", ) .await @@ -3225,9 +3256,9 @@ async fn recovery_matrix_wait_for_session_unfenced( #[ignore = "requires Docker for the Postgres/Restate/OpenFGA/Redis scripted-provider fixture"] async fn recovery_matrix_execution_task_llm_cancel_crash_restart_fences_budget_service_e2e() -> Result<()> { - // Pins: cancelling a durable run joins its blocked ExecutionTask LLM child across a hard - // orchestrator crash, records one cancelled run/task with zero actual usage, and permits a - // fresh replacement durable run to execute its task exactly once. + // Pins: cancelling a durable run joins its blocked ExecutionTaskAttempt LLM child across a + // hard orchestrator crash, records one cancelled run/task with zero actual usage, and permits + // a fresh replacement durable run to execute its task exactly once. let fixture = OrchestratorTestFixture::with_script(recovery_matrix_blocked_execution_script()?) .await .context("boot execution-task recovery-matrix fixture")?; @@ -3268,8 +3299,14 @@ async fn recovery_matrix_execution_task_llm_cancel_crash_restart_fences_budget_s Duration::from_secs(60), ) .await?; - let (parent_invocation_id, child_invocation_id) = - recovery_matrix_blocked_llm_invocation(&fixture, "ExecutionTask", None).await?; + let task_attempt_key = + recovery_matrix_execution_task_attempt_key(&fixture, execution_run_uid).await?; + let (parent_invocation_id, child_invocation_id) = recovery_matrix_blocked_llm_invocation( + &fixture, + "ExecutionTaskAttempt", + Some(&task_attempt_key), + ) + .await?; let session_snapshot = session.snapshot().await?; ensure!( diff --git a/crates/moa-orchestrator/tests/execution_execution_support/evaluation.rs b/crates/moa-orchestrator/tests/execution_execution_support/evaluation.rs index 16e67b4f2..7aeff4b15 100644 --- a/crates/moa-orchestrator/tests/execution_execution_support/evaluation.rs +++ b/crates/moa-orchestrator/tests/execution_execution_support/evaluation.rs @@ -1,8 +1,9 @@ //! Shared redacted execution-eval snapshot collection over production-owned state. -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; use anyhow::{Context, Result, bail}; +use moa_artifacts::execution_plan::ExecutionOperation; use moa_core::events::Event; use moa_core::types::events_stream::EventRange; use moa_eval::execution::{ @@ -10,8 +11,13 @@ use moa_eval::execution::{ ExecutionSessionEventSummary, }; use moa_execution::{ + budget::BudgetLedger, repository::{ - ExecutionRepository, ExecutionScope, ExecutionTaskCursor, ExecutionTaskPageRequest, + ExecutionRepository, ExecutionRunRecord, ExecutionSchedulingSnapshot, ExecutionScope, + ExecutionTaskCursor, ExecutionTaskPageRequest, ExecutionTaskRecord, + }, + state::{ + ExecutionNodeStatus, ExecutionProjection, ExecutionTaskProjection, ExecutionTaskStatus, }, wire::ExecutionRunRequest, }; @@ -29,12 +35,8 @@ pub(crate) async fn collect_execution_eval_snapshot( request: &ExecutionRunRequest, capability_controller: Option<&FixtureCapabilityController>, ) -> Result { - let scheduling_snapshot = repository - .load_scheduling_snapshot(scope, request.run_uid) - .await - .context("load repeatable-read execution scheduling snapshot")? - .with_context(|| format!("execution run {} is not visible", request.run_uid))?; - let task_records = list_all_task_records(repository, scope, request.run_uid).await?; + let (scheduling_snapshot, task_records) = + load_eval_runtime_state(repository, scope, request.run_uid).await?; let audits = load_execution_planning_audits(postgres_url, request.session_id).await?; let events = client .get_events(request.session_id, EventRange::all()) @@ -62,12 +64,8 @@ pub(crate) async fn collect_repository_execution_eval_snapshot( session_id: moa_core::types::identifiers::SessionId, run_uid: uuid::Uuid, ) -> Result { - let scheduling_snapshot = repository - .load_scheduling_snapshot(scope, run_uid) - .await - .context("load repeatable-read repository execution snapshot")? - .with_context(|| format!("execution run {run_uid} is not visible"))?; - let task_records = list_all_task_records(repository, scope, run_uid).await?; + let (scheduling_snapshot, task_records) = + load_eval_runtime_state(repository, scope, run_uid).await?; let audits = load_execution_planning_audits(postgres_url, session_id).await?; ExecutionEvalSnapshot::from_parts( scheduling_snapshot, @@ -78,6 +76,135 @@ pub(crate) async fn collect_repository_execution_eval_snapshot( .context("assemble repository-only execution eval snapshot") } +async fn load_eval_runtime_state( + repository: &ExecutionRepository, + scope: ExecutionScope, + run_uid: uuid::Uuid, +) -> Result<(ExecutionSchedulingSnapshot, Vec)> { + let run = repository + .load_run(scope, run_uid) + .await + .context("load bounded execution run projection")? + .with_context(|| format!("execution run {run_uid} is not visible"))?; + let task_records = list_all_task_records(repository, scope, run_uid).await?; + let snapshot = ExecutionSchedulingSnapshot { + catalog: run.catalog.clone(), + authorization: run.authorization.clone(), + pinned_instruction_skills: run.pinned_instruction_skills.clone(), + budget_ledger: BudgetLedger { + limit: run.approved_budget.clone(), + reserved: run.reserved, + consumed: run.consumed, + overrun: run.budget_overrun, + }, + projection: scheduling_projection(&run, &task_records), + run, + }; + Ok((snapshot, task_records)) +} + +fn scheduling_projection( + run: &ExecutionRunRecord, + tasks: &[ExecutionTaskRecord], +) -> ExecutionProjection { + let task_projections = tasks + .iter() + .map(|task| ExecutionTaskProjection { + task_id: task.task_id, + node_id: task.node_id.clone(), + item_key: task.item_key.clone(), + status: task.status, + attempt: task.attempt, + generation: task.generation, + input: task.input.clone(), + outcome: task.current_outcome.clone(), + }) + .collect::>(); + let node_statuses = run + .active_plan + .definition + .nodes + .iter() + .map(|node| { + let node_tasks = tasks + .iter() + .filter(|task| task.node_id == node.id) + .collect::>(); + ( + node.id.clone(), + persisted_node_status(&node.operation, &node_tasks), + ) + }) + .collect::>(); + ExecutionProjection { + plan_revision: run.plan_revision, + node_statuses, + tasks: task_projections, + } +} + +fn persisted_node_status( + operation: &ExecutionOperation, + tasks: &[&ExecutionTaskRecord], +) -> ExecutionNodeStatus { + if tasks.is_empty() { + return ExecutionNodeStatus::Pending; + } + if tasks.iter().any(|task| { + matches!( + task.status, + ExecutionTaskStatus::WaitingInput + | ExecutionTaskStatus::WaitingReview + | ExecutionTaskStatus::WaitingSignal + | ExecutionTaskStatus::WaitingTimer + | ExecutionTaskStatus::WaitingExternal + | ExecutionTaskStatus::WaitingReplan + ) + }) { + return ExecutionNodeStatus::Waiting; + } + if tasks.iter().any(|task| { + matches!( + task.status, + ExecutionTaskStatus::Pending + | ExecutionTaskStatus::Ready + | ExecutionTaskStatus::Reserved + | ExecutionTaskStatus::Dispatching + | ExecutionTaskStatus::Running + ) + }) { + return ExecutionNodeStatus::Running; + } + if tasks.iter().any(|task| { + matches!( + task.status, + ExecutionTaskStatus::Failed | ExecutionTaskStatus::UnknownOutcome + ) + }) { + return ExecutionNodeStatus::Failed; + } + if tasks + .iter() + .any(|task| task.status == ExecutionTaskStatus::Cancelled) + { + return ExecutionNodeStatus::Cancelled; + } + if matches!( + operation, + ExecutionOperation::Map { .. } | ExecutionOperation::Reduce { .. } + ) { + return ExecutionNodeStatus::Pending; + } + if tasks + .iter() + .all(|task| task.status == ExecutionTaskStatus::Skipped) + { + ExecutionNodeStatus::Skipped + } else { + ExecutionNodeStatus::Completed + } +} + async fn list_all_task_records( repository: &ExecutionRepository, scope: ExecutionScope, diff --git a/crates/moa-orchestrator/tests/execution_run_service_e2e.rs b/crates/moa-orchestrator/tests/execution_run_service_e2e.rs index 6444c21c3..a5132f85a 100644 --- a/crates/moa-orchestrator/tests/execution_run_service_e2e.rs +++ b/crates/moa-orchestrator/tests/execution_run_service_e2e.rs @@ -9,6 +9,8 @@ mod admission_replay; mod bulk_and_recovery; #[path = "execution_run_service_e2e/compensation_recovery.rs"] mod compensation_recovery; +#[path = "execution_run_service_e2e/controller_activation.rs"] +mod controller_activation; #[path = "execution_run_service_e2e/evaluation.rs"] mod evaluation; #[path = "execution_run_service_e2e/observability.rs"] @@ -46,15 +48,16 @@ use moa_execution::{ capability::{ExecutionAuthorizationEnvelope, ExecutionCapabilityCatalog, ExecutionHash}, compiler::{CompileExecutionRequest, compile}, repository::{ - ConfirmationOutcome, ExecutionRepository, ExecutionScope, NewExecutionPlanningContext, - NewExecutionRun, PlanningContextWriteOutcome, + ExecutionRepository, ExecutionScope, NewExecutionRun, + audit::{NewExecutionPlanningContext, PlanningContextWriteOutcome}, + run::RunAdmissionOutcome, }, state::{ExecutionRunStatus, ExecutionTerminalCause}, wire::{ - ExecutionCancelRequest, ExecutionConfirmRequest, ExecutionMutationResponse, - ExecutionPlanningContextRequest, ExecutionPlanningContextResponse, - ExecutionPlanningContextSnapshot, ExecutionRunRequest, ExecutionStartRequest, - ExecutionStartResponse, ExecutionStatusResponse, planning_context_hash, + ExecutionCancelRequest, ExecutionMutationResponse, ExecutionPlanningContextRequest, + ExecutionPlanningContextResponse, ExecutionPlanningContextSnapshot, ExecutionRunRequest, + ExecutionStartRequest, ExecutionStartResponse, ExecutionStatusResponse, + planning_context_hash, }, }; use moa_orchestrator::objects::session::ExecutionRunStartedDelivery; @@ -101,6 +104,7 @@ async fn output_only_run_is_durable_detached_and_reaches_terminal_state() -> Res contact_id: None, session_id, originating_user_sequence_num, + deadline_at: chrono::Utc::now() + chrono::TimeDelta::days(1), requested_template: None, }, ) @@ -128,6 +132,12 @@ async fn output_only_run_is_durable_detached_and_reaches_terminal_state() -> Res }, plan: ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, + input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { + expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { + delay_seconds: 86_400, + }, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + }, input_schema: json!({"type": "object", "additionalProperties": false}), output_schema: json!({ "type": "object", @@ -341,6 +351,12 @@ async fn cancellation_preserves_preconfirmation_null_and_postqueue_timestamp() - }, plan: ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, + input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { + expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { + delay_seconds: 86_400, + }, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + }, input_schema: json!({"type": "object"}), output_schema: json!({"type": "object"}), nodes: vec![ExecutionNode { @@ -410,6 +426,7 @@ async fn cancellation_preserves_preconfirmation_null_and_postqueue_timestamp() - let awaiting = repository .create_run( scope, + &ExecutionConfig::default(), NewExecutionRun { tenant_id: session.tenant_id, contact_id: None, @@ -418,6 +435,11 @@ async fn cancellation_preserves_preconfirmation_null_and_postqueue_timestamp() - planning_context_uid: awaiting_context_uid, planning_context_hash: awaiting_context_hash, owner_user_id: owner_user_id.clone(), + admitted_identity: test + .client() + .identity() + .cloned() + .context("fixture client must carry an admitted identity")?, goal: compiled.goal.clone(), plan: compiled.plan.clone(), catalog: catalog.clone(), @@ -431,6 +453,9 @@ async fn cancellation_preserves_preconfirmation_null_and_postqueue_timestamp() - }, ) .await?; + let RunAdmissionOutcome::Admitted(awaiting) = awaiting else { + anyhow::bail!("preconfirmation run was not admitted: {awaiting:?}") + }; let preconfirm_request = ExecutionCancelRequest { run: ExecutionRunRequest { tenant_id: session.tenant_id, @@ -524,6 +549,7 @@ async fn cancellation_preserves_preconfirmation_null_and_postqueue_timestamp() - let queued = repository .create_run( scope, + &ExecutionConfig::default(), NewExecutionRun { tenant_id: session.tenant_id, contact_id: None, @@ -532,6 +558,11 @@ async fn cancellation_preserves_preconfirmation_null_and_postqueue_timestamp() - planning_context_uid: queued_context_uid, planning_context_hash: queued_context_hash, owner_user_id, + admitted_identity: test + .client() + .identity() + .cloned() + .context("fixture client must carry an admitted identity")?, goal: compiled.goal, plan: compiled.plan, catalog, @@ -545,6 +576,9 @@ async fn cancellation_preserves_preconfirmation_null_and_postqueue_timestamp() - }, ) .await?; + let RunAdmissionOutcome::Admitted(queued) = queued else { + anyhow::bail!("postqueue run was not admitted: {queued:?}") + }; let queued_at = queued .queued_at .context("direct queued run must have a queue timestamp")?; @@ -607,262 +641,3 @@ async fn cancellation_preserves_preconfirmation_null_and_postqueue_timestamp() - assert!(postqueue.started_at.is_none()); Ok(()) } - -#[tokio::test] -#[ignore = "requires local Restate, Postgres, OpenFGA, and the service-e2e feature lane"] -async fn wake_after_db_ack_before_workflow_state_advance_is_not_lost() -> Result<()> { - // Pins: a persisted wake delivered in the exact ack-to-workflow-state gap - // either prevents parking or resolves the promise advertised before the CAS. - run_wake_handoff_case("delay").await -} - -#[tokio::test] -#[ignore = "requires local Restate, Postgres, OpenFGA, and the service-e2e feature lane"] -async fn wake_after_db_ack_survives_execution_run_failure_and_restart() -> Result<()> { - // Pins: a wake committed after DB acknowledgement remains attached to the - // same promise across one forced handler failure and Restate replay/restart. - run_wake_handoff_case("crash_once").await -} - -async fn run_wake_handoff_case(mode: &str) -> Result<()> { - let fixture = OrchestratorTestFixture::with_script_and_env( - json!({ - "default": { - "completion": { - "content": "ok", - "duration_ms": 1, - "input_tokens": 1, - "cached_input_tokens": 0, - "cache_write_input_tokens": 0, - "tool_calls": [] - } - } - }), - vec![( - "MOA_EXECUTION_TEST_WAKE_HANDOFF".to_string(), - mode.to_string(), - )], - ) - .await?; - let test = fixture.isolated().await; - let session_id = test - .create_session(&format!("execution-wake-handoff-{mode}")) - .await?; - let session = test.client().get_session(session_id).await?; - let originating_user_sequence_num = test - .client() - .append_event( - session_id, - Event::UserMessage { - text: "complete after a wake handoff".to_string(), - attachments: Vec::new(), - }, - ) - .await?; - let owner_user_id = match session.created_by { - Some(SessionActorRef::Identity { id }) => UserId::new(id.to_string()), - other => anyhow::bail!("fixture session has no identity owner: {other:?}"), - }; - let catalog = ExecutionCapabilityCatalog::build(Vec::new())?; - let authorization = ExecutionAuthorizationEnvelope { - capability_refs: Vec::new(), - skill_refs: Vec::new(), - }; - let approved_budget = ExecutionBudgetLimit { - max_cost_microusd: Some(1), - max_tokens: Some(1), - max_tasks: Some(1), - max_tool_calls: Some(1), - max_retrieved_bytes: Some(1), - deadline_at: Some(moa_test_support::fixtures::pg_now() + chrono::Duration::minutes(5)), - }; - let compiled = compile(CompileExecutionRequest { - goal: ExecutionGoalContract { - objective: "complete after a wake handoff".to_string(), - requirements: vec![ExecutionRequirement { - id: "result".to_string(), - description: "persist the handoff result".to_string(), - }], - deliverables: Vec::new(), - coverage: Vec::new(), - constraints: Vec::new(), - completion_checks: vec![CompletionCheck { - id: "output-schema".to_string(), - description: "terminal output matches its schema".to_string(), - requirement_ids: vec!["result".to_string()], - constraint_ids: Vec::new(), - kind: CompletionCheckKind::OutputSchema, - }], - }, - plan: ExecutionPlanDefinition { - cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, - input_schema: json!({"type": "object"}), - output_schema: json!({"type": "object"}), - nodes: vec![ExecutionNode { - id: "output".to_string(), - requirement_ids: vec!["result".to_string()], - depends_on: Vec::new(), - when: None, - input: json!({}), - output_schema: json!({"type": "object"}), - operation: ExecutionOperation::Output { - value: json!({"handoff": "completed"}), - }, - compensation: None, - retry: RetryPolicy { - max_attempts: 1, - initial_backoff_ms: 0, - max_backoff_ms: 0, - }, - budget: None, - }], - }, - run_input: json!({}), - catalog: catalog.clone(), - authorization: authorization.clone(), - approved_budget: approved_budget.clone(), - config: ExecutionConfig::default(), - now: moa_test_support::fixtures::pg_now(), - }) - .compiled - .context("wake handoff plan should compile")?; - let pool = sqlx::PgPool::connect(&fixture.postgres_url).await?; - let repository = ExecutionRepository::new(pool); - let scope = ExecutionScope::Tenant { - tenant_id: session.tenant_id, - }; - let (planning_context_uid, planning_context_hash) = create_test_planning_context( - &repository, - scope, - session.tenant_id, - session_id, - originating_user_sequence_num, - owner_user_id.clone(), - catalog.clone(), - authorization.clone(), - approved_budget.clone(), - ) - .await?; - let source_provenance = test_source_provenance(&compiled.plan.plan_hash.to_string()); - let run = repository - .create_run( - scope, - NewExecutionRun { - tenant_id: session.tenant_id, - contact_id: None, - session_id, - originating_user_sequence_num, - planning_context_uid, - planning_context_hash, - owner_user_id, - goal: compiled.goal, - plan: compiled.plan, - catalog, - authorization, - pinned_instruction_skills: Vec::new(), - source_provenance, - input: json!({}), - status: ExecutionRunStatus::AwaitingConfirmation, - approved_budget: approved_budget.clone(), - idempotency_key: Some(format!("wake-handoff-{mode}-{session_id}")), - }, - ) - .await?; - assert!(run.queued_at.is_none()); - let initial_epoch = run.wake_epoch; - test.client() - .post_void( - &format!("/Session/{session_id}/execution_run_started"), - &ExecutionRunStartedDelivery { - started: ExecutionRunStarted { - run_uid: run.run_uid, - originating_user_sequence_num, - plan_revision: run.plan_revision, - status: ExecutionRunAdmissionStatus::AwaitingConfirmation, - confirmation: Some(ExecutionConfirmationEvidence { - active_plan_hash: run.active_plan_hash.to_string(), - estimate: ExecutionAdmissionEstimate { - cost_microusd: run.active_plan.estimate.cost_microusd, - tokens: run.active_plan.estimate.tokens, - tasks: run.active_plan.estimate.tasks, - tool_calls: run.active_plan.estimate.tool_calls, - retrieved_bytes: run.active_plan.estimate.retrieved_bytes, - }, - methodology: ExecutionEstimateMethodology::ConservativeWorstCase, - }), - }, - approved_budget: run.approved_budget.clone(), - }, - ) - .await - .context("activate the wake-handoff run through Session")?; - - tokio::time::timeout(Duration::from_secs(10), async { - loop { - let observed = repository - .load_run(scope, run.run_uid) - .await? - .context("wake handoff run disappeared")?; - if observed.processed_wake_epoch == initial_epoch { - return Ok::<(), anyhow::Error>(()); - } - tokio::time::sleep(Duration::from_millis(20)).await; - } - }) - .await - .context("driver never reached the post-ack injection checkpoint")??; - - let ConfirmationOutcome::Confirmed(confirmed) = repository - .confirm_run( - scope, - run.run_uid, - &run.active_plan_hash, - approved_budget.clone(), - ) - .await? - else { - anyhow::bail!("wake handoff confirmation did not apply"); - }; - let queued_at = confirmed - .queued_at - .context("confirmation must persist queued_at")?; - let replay: ExecutionMutationResponse = test - .client() - .post_call( - "/Execution/confirm", - &ExecutionConfirmRequest { - run: ExecutionRunRequest { - tenant_id: session.tenant_id, - contact_id: None, - session_id, - run_uid: run.run_uid, - }, - expected_plan_hash: run.active_plan_hash, - approved_budget, - }, - ) - .await?; - assert!(matches!( - replay, - ExecutionMutationResponse::Replayed { ref run } - if run.run_uid == confirmed.run_uid && run.queued_at == Some(queued_at) - )); - - let terminal = tokio::time::timeout(Duration::from_secs(30), async { - loop { - let current = repository - .load_run(scope, run.run_uid) - .await? - .context("completed wake handoff run disappeared")?; - if current.status.is_terminal() { - return Ok::<_, anyhow::Error>(current); - } - tokio::time::sleep(Duration::from_millis(20)).await; - } - }) - .await - .context("wake was lost across the handoff checkpoint")??; - assert_eq!(terminal.status, ExecutionRunStatus::Completed); - assert_eq!(terminal.output, Some(json!({"handoff": "completed"}))); - Ok(()) -} diff --git a/crates/moa-orchestrator/tests/execution_run_service_e2e/admission_replay.rs b/crates/moa-orchestrator/tests/execution_run_service_e2e/admission_replay.rs index 9f75886ac..e42adbb6f 100644 --- a/crates/moa-orchestrator/tests/execution_run_service_e2e/admission_replay.rs +++ b/crates/moa-orchestrator/tests/execution_run_service_e2e/admission_replay.rs @@ -354,6 +354,12 @@ fn template_skill_source() -> String { }, plan: ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, + input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { + expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { + delay_seconds: 86_400, + }, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + }, input_schema: io_schema.clone(), output_schema: io_schema.clone(), nodes: vec![ExecutionNode { diff --git a/crates/moa-orchestrator/tests/execution_run_service_e2e/bulk_and_recovery.rs b/crates/moa-orchestrator/tests/execution_run_service_e2e/bulk_and_recovery.rs index 8bc338ed4..5783eab6a 100644 --- a/crates/moa-orchestrator/tests/execution_run_service_e2e/bulk_and_recovery.rs +++ b/crates/moa-orchestrator/tests/execution_run_service_e2e/bulk_and_recovery.rs @@ -810,6 +810,12 @@ fn bulk_candidate( }, plan: ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, + input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { + expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { + delay_seconds: 86_400, + }, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + }, input_schema: json!({"type": "object", "additionalProperties": false}), output_schema: report_schema.clone(), nodes: vec![ diff --git a/crates/moa-orchestrator/tests/execution_run_service_e2e/compensation_recovery.rs b/crates/moa-orchestrator/tests/execution_run_service_e2e/compensation_recovery.rs index ff8f178c5..180c1d13c 100644 --- a/crates/moa-orchestrator/tests/execution_run_service_e2e/compensation_recovery.rs +++ b/crates/moa-orchestrator/tests/execution_run_service_e2e/compensation_recovery.rs @@ -27,6 +27,9 @@ use moa_execution::{ ExecutionStartRequest, ExecutionStartResponse, }, }; +use moa_orchestrator::services::execution::{ + ExecutionRunControlRequest, ExecutionRunControlResponse, +}; use moa_test_support::{ FixtureCapabilityOptions, FixtureCapabilityOutcome, FixtureCapabilityTool, IsolatedTest, OrchestratorTestFixture, @@ -154,6 +157,127 @@ async fn third_forward_failure_restarts_after_first_completed_reverse_undo_servi Ok(()) } +#[tokio::test] +#[ignore = "requires the local Restate/Postgres/OpenFGA/Redis service fixture"] +async fn public_pause_drains_running_compensation_and_resumes_once_service_e2e() -> Result<()> { + // Pins: the public control surface accepts Compensating, emits an exact old-attempt/new-run + // cancellation fence, waits for verified provider teardown, preserves terminal intent, and + // creates one activation when the fully Paused run resumes. + let fixture = compensation_fixture( + vec![ + FixtureCapabilityOutcome::SuccessWithInput { + output: json!({"applied": true}), + }, + FixtureCapabilityOutcome::TerminalFailure { + message: "force reverse execution".to_string(), + }, + ], + true, + ) + .await?; + let test = fixture.isolated().await; + let started = start_compensated_run( + &fixture, + &test, + ExecutionCancelPolicy::CompensateCommitted, + 2, + "public-pause-running-compensation", + ) + .await?; + let controller = fixture + .fixture_capability() + .context("compensation fixture omitted its capability controller")?; + + for expected in 1..=2 { + controller.wait_for_calls(expected, SERVICE_TIMEOUT).await?; + controller.release(1); + } + let calls = controller.wait_for_calls(3, SERVICE_TIMEOUT).await?; + assert_eq!(calls[2].capability, REVERSIBLE_FIXTURE_COMPENSATOR_TOOL); + let (attempt_controller_generation, pending_terminal_status) = + await_running_compensation(&fixture.postgres_url, started.response.run.run_uid).await?; + + let pause: ExecutionRunControlResponse = test + .client() + .post_call( + "/Execution/pause", + &ExecutionRunControlRequest { + run: started.request.clone(), + expected_controller_generation: attempt_controller_generation, + }, + ) + .await?; + let paused_generation = match pause { + ExecutionRunControlResponse::Applied { + run, + controller_generation, + .. + } => { + assert_eq!(run.status, ExecutionRunStatus::Pausing); + assert_eq!(controller_generation, attempt_controller_generation + 1); + controller_generation + } + other => bail!("running compensation pause was not applied: {other:?}"), + }; + let cancellation_generations: (i64, i64) = sqlx::query_as( + "SELECT (payload->>'controller_generation')::BIGINT, \ + (payload->>'attempt_controller_generation')::BIGINT \ + FROM moa.execution_dispatch_outbox WHERE run_uid=$1 \ + AND dispatch_kind='compensation_attempt_cancel'", + ) + .bind(started.response.run.run_uid) + .fetch_one(&sqlx::PgPool::connect(&fixture.postgres_url).await?) + .await?; + assert_eq!( + cancellation_generations, + ( + i64::try_from(paused_generation)?, + i64::try_from(attempt_controller_generation)? + ) + ); + + controller.release(1); + await_paused_compensation( + &fixture.postgres_url, + started.response.run.run_uid, + &pending_terminal_status, + ) + .await?; + let resume: ExecutionRunControlResponse = test + .client() + .post_call( + "/Execution/resume", + &ExecutionRunControlRequest { + run: started.request, + expected_controller_generation: paused_generation, + }, + ) + .await?; + let (resumed_generation, resumed_wake_epoch) = match resume { + ExecutionRunControlResponse::Applied { + run, + controller_generation, + wake_epoch, + } => { + assert_eq!(run.status, ExecutionRunStatus::Compensating); + assert_eq!(controller_generation, paused_generation + 1); + (controller_generation, wake_epoch) + } + other => bail!("drained compensation resume was not applied: {other:?}"), + }; + let activation_count: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM moa.execution_dispatch_outbox WHERE run_uid=$1 \ + AND controller_generation=$2 AND wake_epoch=$3 AND dispatch_kind='run_activation'", + ) + .bind(started.response.run.run_uid) + .bind(i64::try_from(resumed_generation)?) + .bind(i64::try_from(resumed_wake_epoch)?) + .fetch_one(&sqlx::PgPool::connect(&fixture.postgres_url).await?) + .await?; + assert_eq!(activation_count, 1); + Ok(()) +} + #[tokio::test] #[ignore = "requires the local Restate/Postgres/OpenFGA/Redis service fixture"] async fn both_cancel_policies_join_admitted_late_effects_before_terminal_service_e2e() -> Result<()> @@ -503,6 +627,12 @@ fn compensated_plan( }); ExecutionPlanDefinition { cancel_policy, + input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { + expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { + delay_seconds: 60, + }, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + }, input_schema: json!({ "type": "object", "additionalProperties": false @@ -598,6 +728,69 @@ async fn await_pending_terminal(database_url: &str, run_uid: uuid::Uuid) -> Resu Ok(()) } +async fn await_running_compensation( + database_url: &str, + run_uid: uuid::Uuid, +) -> Result<(u64, String)> { + let pool = sqlx::PgPool::connect(database_url).await?; + tokio::time::timeout(SERVICE_TIMEOUT, async { + loop { + let observed: Option<(i64, String)> = sqlx::query_as( + "SELECT run.controller_generation,run.pending_terminal_status \ + FROM moa.execution_run AS run \ + JOIN moa.execution_compensation AS compensation ON compensation.run_uid=run.run_uid \ + WHERE run.run_uid=$1 AND run.status='compensating' \ + AND compensation.attempt_state='running'", + ) + .bind(run_uid) + .fetch_optional(&pool) + .await?; + if let Some((generation, pending_terminal_status)) = observed { + return Ok::<_, anyhow::Error>((u64::try_from(generation)?, pending_terminal_status)); + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .with_context(|| format!("run {run_uid} never entered a running compensation slice"))? +} + +async fn await_paused_compensation( + database_url: &str, + run_uid: uuid::Uuid, + expected_pending_terminal_status: &str, +) -> Result<()> { + let pool = sqlx::PgPool::connect(database_url).await?; + tokio::time::timeout(SERVICE_TIMEOUT, async { + loop { + let observed: Option<(String, String, i64, String)> = sqlx::query_as( + "SELECT run.status,run.activation_state,run.active_task_count, \ + run.pending_terminal_status \ + FROM moa.execution_run AS run WHERE run.run_uid=$1", + ) + .bind(run_uid) + .fetch_optional(&pool) + .await?; + if let Some((status, activation_state, active_task_count, pending_status)) = observed + && status == "paused" + && activation_state == "paused" + && active_task_count == 0 + { + if pending_status != expected_pending_terminal_status { + bail!( + "pause changed pending terminal status from {expected_pending_terminal_status} to {pending_status}" + ); + } + return Ok::<(), anyhow::Error>(()); + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .with_context(|| format!("run {run_uid} did not drain its compensation into Paused"))??; + Ok(()) +} + async fn await_transport_attempts( controller: &moa_test_support::FixtureCapabilityController, expected: usize, diff --git a/crates/moa-orchestrator/tests/execution_run_service_e2e/controller_activation.rs b/crates/moa-orchestrator/tests/execution_run_service_e2e/controller_activation.rs new file mode 100644 index 000000000..6e2192fbc --- /dev/null +++ b/crates/moa-orchestrator/tests/execution_run_service_e2e/controller_activation.rs @@ -0,0 +1,46 @@ +//! Service-E2E coverage for the bounded execution-run controller boundary. + +use anyhow::Result; +use moa_core::types::identifiers::TenantId; +use moa_orchestrator::objects::execution_run_controller::{ + ExecutionRunAdvanceRequest, ExecutionRunAdvanceResponse, +}; +use moa_test_support::OrchestratorTestFixture; +use uuid::Uuid; + +#[tokio::test] +#[ignore = "requires local Restate, Postgres, OpenFGA, and the service-e2e feature lane"] +async fn controller_rejects_a_payload_for_another_virtual_object_key() -> Result<()> { + // Pins: the deployed Restate object validates its key before any Postgres lookup, so a + // dispatch payload cannot redirect an activation to a different tenant-owned run. + let fixture = OrchestratorTestFixture::shared().await?; + let test = fixture.isolated().await; + let object_run_uid = Uuid::now_v7(); + let payload_run_uid = Uuid::now_v7(); + let result = test + .client() + .post_call::<_, ExecutionRunAdvanceResponse>( + &format!("/ExecutionRunController/{object_run_uid}/advance"), + &ExecutionRunAdvanceRequest { + dispatch_uid: Uuid::now_v7(), + tenant_id: TenantId::new(), + run_uid: payload_run_uid, + controller_generation: 1, + wake_epoch: 1, + }, + ) + .await; + + match result { + Ok(response) => anyhow::bail!( + "mismatched controller key unexpectedly returned a response: {response:?}" + ), + Err(error) => { + assert!( + error.to_string().contains("does not match run_uid"), + "unexpected controller rejection: {error:#}" + ); + Ok(()) + } + } +} diff --git a/crates/moa-orchestrator/tests/execution_run_service_e2e/observability.rs b/crates/moa-orchestrator/tests/execution_run_service_e2e/observability.rs index 2e2b2de74..2d39d0590 100644 --- a/crates/moa-orchestrator/tests/execution_run_service_e2e/observability.rs +++ b/crates/moa-orchestrator/tests/execution_run_service_e2e/observability.rs @@ -29,9 +29,9 @@ use crate::execution_execution_support::{ #[ignore = "requires the local Restate/Postgres/OpenFGA/Redis service fixture"] async fn execution_observability_exports_stable_identity_and_replay_safe_service_spans() -> Result<()> { - // Pins: the real Execution/start -> Session activation -> ExecutionRun -> ExecutionTask path - // exports one stable run identity on every durable hop without putting attempt-local trace - // headers into replayed Restate commands. + // Pins: the real Execution/start -> Session activation -> ExecutionRunController -> + // ExecutionTaskAttempt path exports one stable run identity on every durable hop without + // putting attempt-local trace headers into replayed Restate commands. let fixture = OrchestratorTestFixture::with_execution_fixture( json!({ "default": { @@ -67,6 +67,7 @@ async fn execution_observability_exports_stable_identity_and_replay_safe_service contact_id: None, session_id, originating_user_sequence_num, + deadline_at: chrono::Utc::now() + chrono::TimeDelta::days(1), requested_template: None, }, ) @@ -91,6 +92,12 @@ async fn execution_observability_exports_stable_identity_and_replay_safe_service }, plan: ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, + input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { + expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { + delay_seconds: 86_400, + }, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + }, input_schema: json!({"type": "object", "additionalProperties": false}), output_schema: json!({ "type": "object", @@ -258,21 +265,21 @@ async fn execution_observability_exports_stable_identity_and_replay_safe_service let plan_revision = persisted_task.plan_revision.to_string(); let run_span = capture .wait_for_span(SERVICE_TIMEOUT, |span| { - span.attribute("restate.service") == Some("ExecutionRun") - && span.attribute("restate.handler") == Some("run") + span.attribute("restate.service") == Some("ExecutionRunController") + && span.attribute("restate.handler") == Some("advance") && span.attribute("moa.execution.run_uid") == Some(run_uid.as_str()) }) .await - .context("wait for exported ExecutionRun handler span")?; + .context("wait for exported ExecutionRunController/advance handler span")?; let task_span = capture .wait_for_span(SERVICE_TIMEOUT, |span| { - span.attribute("restate.service") == Some("ExecutionTask") + span.attribute("restate.service") == Some("ExecutionTaskAttempt") && span.attribute("restate.handler") == Some("run") && span.attribute("moa.execution.run_uid") == Some(run_uid.as_str()) && span.attribute("moa.execution.task_id") == Some(task_id.as_str()) }) .await - .context("wait for exported ExecutionTask handler span")?; + .context("wait for exported ExecutionTaskAttempt/run handler span")?; let activation_span = capture .wait_for_span(SERVICE_TIMEOUT, |span| { span.attribute("restate.service") == Some("Session") diff --git a/crates/moa-orchestrator/tests/execution_run_service_e2e/replan_and_completion.rs b/crates/moa-orchestrator/tests/execution_run_service_e2e/replan_and_completion.rs index e1f6688a8..53615a934 100644 --- a/crates/moa-orchestrator/tests/execution_run_service_e2e/replan_and_completion.rs +++ b/crates/moa-orchestrator/tests/execution_run_service_e2e/replan_and_completion.rs @@ -17,7 +17,10 @@ use moa_execution::compiler::{ CompileExecutionRequest, ValidateAmendmentRequest, compile, validate_amendment, }; use moa_execution::completion::CompletionCheckResult; -use moa_execution::repository::{ExecutionRepository, ExecutionScope}; +use moa_execution::repository::{ + ExecutionRepository, ExecutionScope, + amendment::{AmendmentProjectionOutcome, AmendmentProjectionRequest}, +}; use moa_execution::state::{ ExecutionRunStatus, ExecutionTaskProjection, ExecutionTaskStatus, ExecutionTerminalCause, ExecutionTerminalEvidence, @@ -618,9 +621,9 @@ async fn useful_amendment_preserves_completed_work_service_e2e() -> Result<()> { #[tokio::test] #[ignore = "requires the local Restate/Postgres/OpenFGA/Redis service fixture"] -async fn amendment_retries_after_persisted_epoch_before_wake_ack_service_e2e() -> Result<()> { +async fn amendment_retries_after_persisted_epoch_before_dispatch_service_e2e() -> Result<()> { // Pins: a public amendment interrupted after its revision and wake epoch commit - // retries through the joined run wake without applying or resuming the plan twice. + // retries through the outbox dispatcher without applying or resuming the plan twice. let agent_a = agent_node("agent_a", "AMENDMENT_HANDOFF_AGENT_A"); let amendment = replacement_amendment( 1, @@ -650,7 +653,7 @@ async fn amendment_retries_after_persisted_epoch_before_wake_ack_service_e2e() - &fixture, &test, "amendment-handoff-recovery", - "recover a committed amendment before its wake acknowledgement", + "recover a committed amendment before its dispatcher handoff", None, move |_| Ok(useful_replan_contract(agent_a)), ) @@ -695,13 +698,13 @@ async fn amendment_retries_after_persisted_epoch_before_wake_ack_service_e2e() - .await?; assert!( !interrupted.is_finished(), - "public amendment returned before its joined run wake acknowledgement" + "public amendment returned before its dispatcher handoff" ); fixture .hard_crash_and_restart_orchestrator() .await - .context("crash in the persisted-amendment/pre-wake acknowledgement window")?; + .context("crash in the persisted-amendment/pre-dispatch window")?; let recovered = tokio::time::timeout(SERVICE_TIMEOUT, interrupted) .await .context("amendment request did not retry after orchestrator recovery")???; @@ -1472,11 +1475,11 @@ async fn execution_eval_amendment_cannot_broaden_authorization_service_e2e() -> let scope = ExecutionScope::Tenant { tenant_id: started.run.tenant_id, }; - let snapshot = repository - .load_scheduling_snapshot(scope, started.run.run_uid) + let persisted_run = repository + .load_run(scope, started.run.run_uid) .await? - .context("load authorization-escalation scheduling snapshot")?; - let forbidden_reference = snapshot + .context("load authorization-escalation run")?; + let forbidden_reference = persisted_run .catalog .capabilities .iter() @@ -1486,8 +1489,8 @@ async fn execution_eval_amendment_cannot_broaden_authorization_service_e2e() -> }) .map(|capability| capability.reference.clone()) .context("fixture catalog omitted forbidden amendment capability")?; - let completed_before = snapshot - .projection + let tasks_before = list_execution_tasks(test.client(), started.run.clone()).await?; + let completed_before = tasks_before .tasks .iter() .find(|task| task.node_id == USEFUL_OUTPUT_NODE) @@ -1508,16 +1511,31 @@ async fn execution_eval_amendment_cannot_broaden_authorization_service_e2e() -> replacement, "attempt to broaden active-plan capability authorization", ); + let config = ExecutionConfig::default(); + let AmendmentProjectionOutcome::Ready(snapshot) = repository + .load_amendment_projection_for_session( + scope, + &config, + AmendmentProjectionRequest { + run_uid: started.run.run_uid, + session_id: started.run.session_id, + expected_plan_revision: amendment.base_plan_revision, + }, + ) + .await? + else { + bail!("authorization-escalation projection was not ready") + }; let remaining_budget = snapshot.budget_ledger.remaining_limit()?; let validated = validate_amendment(ValidateAmendmentRequest { - goal: snapshot.run.goal, - active_plan: snapshot.run.active_plan, + goal: snapshot.run.goal.clone(), + active_plan: snapshot.run.active_plan.clone(), amendment: amendment.clone(), - projection: snapshot.projection, - catalog: snapshot.catalog, - authorization: snapshot.authorization, + projection: snapshot.projection.clone(), + catalog: snapshot.run.catalog.clone(), + authorization: snapshot.run.authorization.clone(), remaining_budget, - config: ExecutionConfig::default(), + config, now: moa_test_support::fixtures::pg_now(), }); assert!(validated.plan.is_none()); @@ -1652,6 +1670,7 @@ where contact_id: None, session_id, originating_user_sequence_num, + deadline_at: chrono::Utc::now() + chrono::TimeDelta::days(1), requested_template: None, }, ) @@ -1829,20 +1848,31 @@ async fn assert_valid_amendment( contact_id, }, ); - let snapshot = repository - .load_scheduling_snapshot(scope, started.run.run_uid) + let config = moa_config::ExecutionConfig::default(); + let AmendmentProjectionOutcome::Ready(snapshot) = repository + .load_amendment_projection_for_session( + scope, + &config, + AmendmentProjectionRequest { + run_uid: started.run.run_uid, + session_id: started.run.session_id, + expected_plan_revision: amendment.base_plan_revision, + }, + ) .await? - .context("load replan validation snapshot")?; + else { + bail!("replan validation projection was not ready") + }; let remaining_budget = snapshot.budget_ledger.remaining_limit()?; let validated = validate_amendment(ValidateAmendmentRequest { - goal: snapshot.run.goal, - active_plan: snapshot.run.active_plan, + goal: snapshot.run.goal.clone(), + active_plan: snapshot.run.active_plan.clone(), amendment: amendment.clone(), - projection: snapshot.projection, - catalog: snapshot.catalog, - authorization: snapshot.authorization, + projection: snapshot.projection.clone(), + catalog: snapshot.run.catalog.clone(), + authorization: snapshot.run.authorization.clone(), remaining_budget, - config: moa_config::ExecutionConfig::default(), + config, now: moa_test_support::fixtures::pg_now(), }); if validated.plan.is_none() { @@ -2014,6 +2044,12 @@ fn useful_replan_contract( }, ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, + input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { + expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { + delay_seconds: 86_400, + }, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + }, input_schema: empty_input_schema(), output_schema: output_schema.clone(), nodes: vec![ @@ -2266,6 +2302,12 @@ fn map_then_output_plan(spec: MapThenOutputPlan<'_>) -> ExecutionPlanDefinition } = spec; ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, + input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { + expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { + delay_seconds: 86_400, + }, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + }, input_schema: empty_input_schema(), output_schema: output_schema.clone(), nodes: vec![ @@ -2335,6 +2377,12 @@ fn missing_deliverable_contract() -> (ExecutionGoalContract, ExecutionPlanDefini }, ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, + input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { + expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { + delay_seconds: 86_400, + }, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + }, input_schema: json!({ "type": "object", "additionalProperties": false, @@ -2437,6 +2485,12 @@ fn declared_contradiction_contract() -> (ExecutionGoalContract, ExecutionPlanDef }, ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, + input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { + expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { + delay_seconds: 86_400, + }, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + }, input_schema: empty_input_schema(), output_schema: report_schema.clone(), nodes: vec![ @@ -2540,6 +2594,12 @@ fn injected_content_contract( }, ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, + input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { + expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { + delay_seconds: 86_400, + }, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + }, input_schema: empty_input_schema(), output_schema: output_schema.clone(), nodes: vec![ diff --git a/crates/moa-orchestrator/tests/execution_run_service_e2e/routing.rs b/crates/moa-orchestrator/tests/execution_run_service_e2e/routing.rs index 44303e063..d955a8bf7 100644 --- a/crates/moa-orchestrator/tests/execution_run_service_e2e/routing.rs +++ b/crates/moa-orchestrator/tests/execution_run_service_e2e/routing.rs @@ -1299,6 +1299,12 @@ fn research_candidate( goal: goal_contract(objective), plan: ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, + input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { + expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { + delay_seconds: 86_400, + }, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + }, input_schema: empty_input_schema(), output_schema: output_schema.clone(), nodes: vec![ @@ -1400,6 +1406,12 @@ fn template_skill_source() -> String { }, plan: ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, + input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { + expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { + delay_seconds: 86_400, + }, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + }, input_schema: template_io_schema(), output_schema: template_io_schema(), nodes: vec![ExecutionNode { diff --git a/crates/moa-orchestrator/tests/execution_run_service_e2e/task_lifecycle.rs b/crates/moa-orchestrator/tests/execution_run_service_e2e/task_lifecycle.rs index bb931e846..1a6f32684 100644 --- a/crates/moa-orchestrator/tests/execution_run_service_e2e/task_lifecycle.rs +++ b/crates/moa-orchestrator/tests/execution_run_service_e2e/task_lifecycle.rs @@ -30,8 +30,8 @@ use moa_execution::{ compiler::{CompileExecutionRequest, CompiledExecution, compile}, repository::{ ExecutionRepository, ExecutionRunRecord, ExecutionScope, ExecutionTaskRecord, - FencedTerminalFinalizationOutcome, NewExecutionRun, ReservationOutcome, - TaskOutcomeRejection, TaskOutcomeWrite, TransitionOutcome, + NewExecutionRun, ReservationOutcome, TaskOutcomeRejection, TaskOutcomeWrite, + TransitionOutcome, run::RunAdmissionOutcome, }, state::{ ExecutionRunStatus, ExecutionTaskId, ExecutionTaskStatus, ExecutionTerminalCause, @@ -318,9 +318,9 @@ async fn terminal_failure_does_not_retry_service_e2e() -> Result<()> { #[tokio::test] #[ignore = "requires the local Restate/Postgres/OpenFGA/Redis service fixture"] -async fn confirmation_retries_after_persisted_epoch_before_wake_ack_service_e2e() -> Result<()> { - // Pins: confirmation cannot return after its run transition commits but before the exact - // run wake is accepted; process recovery replays the confirmation without another epoch. +async fn confirmation_retries_after_persisted_epoch_before_dispatch_service_e2e() -> Result<()> { + // Pins: confirmation cannot return after its run transition commits but before the dispatcher + // kick is accepted; outbox recovery replays the confirmation without another epoch. let tool_name = "lifecycle_confirmation_handoff_probe"; let fixture = confirmation_execution_fixture(tool_name).await?; let prepared = prepare_capability_run( @@ -367,12 +367,12 @@ async fn confirmation_retries_after_persisted_epoch_before_wake_ack_service_e2e( .await?; assert!( !interrupted.is_finished(), - "public confirmation returned before its joined run wake acknowledgement" + "public confirmation returned before its dispatcher handoff" ); fixture .hard_crash_and_restart_orchestrator() .await - .context("crash in the persisted-confirmation/pre-wake acknowledgement window")?; + .context("crash in the persisted-confirmation/pre-dispatch window")?; let recovered = tokio::time::timeout(SERVICE_TIMEOUT, interrupted) .await .context("confirmation request did not retry after orchestrator recovery")???; @@ -420,80 +420,6 @@ async fn confirmation_retries_after_persisted_epoch_before_wake_ack_service_e2e( Ok(()) } -#[tokio::test] -#[ignore = "requires the local Restate/Postgres/OpenFGA/Redis service fixture"] -async fn wake_ack_before_park_failure_loses_no_wake_or_task_service_e2e() -> Result<()> { - // Pins: a task outcome committed while ExecutionRun fails after acknowledging its current - // epoch but before parking is recovered from the persisted epoch/owned-call paths exactly once. - let tool_name = "lifecycle_wake_ack_before_park_probe"; - let fixture = execution_fixture( - tool_name, - success_outcomes(), - vec![ - ( - "MOA_EXECUTION_TEST_SKIP_SESSION_DELIVERY".to_string(), - "true".to_string(), - ), - ( - "MOA_EXECUTION_TEST_WAKE_HANDOFF".to_string(), - "crash_once".to_string(), - ), - ], - ) - .await?; - let prepared = prepare_capability_run( - &fixture, - "wake-ack-before-park-recovery", - tool_name, - no_retry(), - ActionPolicyEffect::Allow, - ) - .await?; - let run = start_service_run(&fixture, &prepared, false).await?; - let controller = fixture_capability(&fixture)?; - controller.wait_for_calls(1, SERVICE_TIMEOUT).await?; - let acknowledged_epoch = await_positive_processed_wake_epoch(&run).await?; - - controller.release(1); - let terminal = await_execution_terminal(&fixture.client, &run.request).await?; - assert_terminal( - &terminal, - ExecutionRunStatus::Completed, - ExecutionTerminalCause::Completion { limit_stop: None }, - 1, - 1, - ); - let persisted = load_run(&run).await?; - assert!(persisted.processed_wake_epoch >= acknowledged_epoch); - let tasks = list_execution_tasks(&fixture.client, run.request.clone()).await?; - assert_eq!(tasks.tasks.len(), 2); - assert_eq!( - tasks - .tasks - .iter() - .filter(|task| task.node_id == CAPABILITY_NODE_ID) - .count(), - 1 - ); - assert_eq!( - tasks - .tasks - .iter() - .filter(|task| task.node_id == OUTPUT_NODE_ID) - .count(), - 1 - ); - assert!( - tasks.tasks.iter().all(|task| { - task.status == ExecutionTaskStatus::Completed && task.outcome.is_some() - }) - ); - assert_eq!(controller.calls().len(), 1); - assert_eq!(controller.transport_attempts().len(), 1); - assert_eq!(controller.current_live_calls(), 0); - Ok(()) -} - #[tokio::test] #[ignore = "requires the local Restate/Postgres/OpenFGA/Redis service fixture"] async fn cancellation_releases_reservations_and_prevents_dispatch_service_e2e() -> Result<()> { @@ -573,18 +499,6 @@ async fn cancellation_releases_reservations_and_prevents_dispatch_service_e2e() .await?, TaskOutcomeWrite::Applied { .. } )); - let settled = load_run(&run).await?; - assert!(matches!( - run.repository - .finalize_fenced_terminal( - run.scope, - run.run_uid, - settled.plan_revision, - settled.wake_epoch, - ) - .await?, - FencedTerminalFinalizationOutcome::Finalized(_) - )); let terminal = await_execution_terminal(&fixture.client, &run.request).await?; assert_terminal( &terminal, @@ -631,9 +545,9 @@ async fn cancellation_releases_reservations_and_prevents_dispatch_service_e2e() #[tokio::test] #[ignore = "requires the local Restate/Postgres/OpenFGA/Redis service fixture"] -async fn cancellation_retries_after_persisted_epoch_before_wake_ack_service_e2e() -> Result<()> { - // Pins: a process crash after the cancellation fence commits but before the - // joined run wake acknowledges cannot return success or advance the epoch twice; +async fn cancellation_retries_after_persisted_epoch_before_dispatch_service_e2e() -> Result<()> { + // Pins: a process crash after the cancellation fence commits but before the dispatcher + // accepts the kick cannot return success or advance the epoch twice; // an already-admitted late effect remains authoritative under RetainEffects. let tool_name = "lifecycle_cancel_handoff_probe"; let fixture = direct_execution_fixture(tool_name, success_outcomes()).await?; @@ -672,13 +586,13 @@ async fn cancellation_retries_after_persisted_epoch_before_wake_ack_service_e2e( await_committed_cancellation_fence(&run, before_cancel.wake_epoch).await?; assert!( !interrupted.is_finished(), - "public cancellation returned before its joined run wake acknowledgement" + "public cancellation returned before its dispatcher handoff" ); fixture .hard_crash_and_restart_orchestrator() .await - .context("crash in the persisted-epoch/pre-wake acknowledgement window")?; + .context("crash in the persisted-epoch/pre-dispatch window")?; let recovered = tokio::time::timeout(SERVICE_TIMEOUT, interrupted) .await .context("cancel request did not retry after orchestrator recovery")???; @@ -723,78 +637,11 @@ async fn cancellation_retries_after_persisted_epoch_before_wake_ack_service_e2e( Ok(()) } -#[tokio::test] -#[ignore = "requires the local Restate/Postgres/OpenFGA/Redis service fixture"] -async fn public_mutation_waits_for_run_wake_handler_ack_service_e2e() -> Result<()> { - // Pins: public mutations join ExecutionRun/wake through its handler acknowledgement; - // replacing the shared `.call()` with `.send()` returns while this gate is still held. - let tool_name = "lifecycle_joined_wake_ack_probe"; - let fixture = direct_execution_fixture(tool_name, success_outcomes()).await?; - let prepared = prepare_capability_run( - &fixture, - "joined-wake-acknowledgement", - tool_name, - no_retry(), - ActionPolicyEffect::Allow, - ) - .await?; - let run = start_service_run(&fixture, &prepared, false).await?; - let controller = fixture_capability(&fixture)?; - controller.wait_for_calls(1, SERVICE_TIMEOUT).await?; - let before_cancel = load_run(&run).await?; - fixture - .restart_orchestrator_with_env(vec![( - "MOA_EXECUTION_TEST_DELAY_WAKE_ACK".to_string(), - "true".to_string(), - )]) - .await - .context("arm the integration-only run wake acknowledgement gate")?; - - let request = ExecutionCancelRequest { - run: run.request.clone(), - reason: "prove the public mutation joins its run wake".to_string(), - }; - let client = fixture.client.clone(); - let request_for_call = request.clone(); - let response = tokio::spawn(async move { - client - .post_call::<_, ExecutionMutationResponse>("/Execution/cancel", &request_for_call) - .await - }); - let committed_epoch = - await_committed_cancellation_fence(&run, before_cancel.wake_epoch).await?; - tokio::time::sleep(Duration::from_millis(500)).await; - assert!( - !response.is_finished(), - "public cancellation returned before ExecutionRun/wake acknowledged the call" - ); - let accepted = tokio::time::timeout(SERVICE_TIMEOUT, response) - .await - .context("cancellation did not cross the run wake acknowledgement gate")???; - assert!(matches!( - accepted, - ExecutionMutationResponse::Applied { .. } | ExecutionMutationResponse::Replayed { .. } - )); - - controller.release(1); - let terminal = await_execution_terminal(&fixture.client, &run.request).await?; - assert_terminal( - &terminal, - ExecutionRunStatus::Cancelled, - ExecutionTerminalCause::Cancellation, - 0, - 1, - ); - assert!(load_run(&run).await?.wake_epoch >= committed_epoch); - assert_eq!(controller.calls().len(), 1); - Ok(()) -} - #[tokio::test] #[ignore = "requires the local Restate/Postgres/OpenFGA/Redis service fixture"] async fn input_resume_preserves_attempt_and_history_service_e2e() -> Result<()> { - // Pins: the public input mutation acknowledges the exact parked ExecutionTask and run wake, - // preserves attempt one across a post-commit crash, and replays without duplicating history. + // Pins: the public input mutation persists the exact parked task transition and dispatch + // outbox row, preserves attempt one across a crash, and never duplicates history. let tool_name = "lifecycle_input_probe"; let fixture = input_execution_fixture(tool_name).await?; let prepared = prepare_capability_run( @@ -853,12 +700,12 @@ async fn input_resume_preserves_attempt_and_history_service_e2e() -> Result<()> assert_eq!(resumed.generation_history.len(), 2); assert!( !interrupted.is_finished(), - "public input delivery returned before its task and run acknowledgements" + "public input delivery returned before its dispatcher handoff" ); fixture .hard_crash_and_restart_orchestrator() .await - .context("crash in the persisted-input/pre-wake acknowledgement window")?; + .context("crash in the persisted-input/pre-dispatch window")?; let recovered = tokio::time::timeout(SERVICE_TIMEOUT, interrupted) .await .context("input request did not retry after orchestrator recovery")???; @@ -920,9 +767,9 @@ async fn input_resume_preserves_attempt_and_history_service_e2e() -> Result<()> #[tokio::test] #[ignore = "requires the local Restate/Postgres/OpenFGA/Redis service fixture"] -async fn explicit_review_retries_after_persisted_epoch_before_ack_service_e2e() -> Result<()> { - // Pins: an approved Review node returns only after its exact task promise and run wake are - // acknowledged; a post-commit crash neither loses the decision nor duplicates its outcome. +async fn explicit_review_retries_after_persisted_epoch_before_dispatch_service_e2e() -> Result<()> { + // Pins: an approved Review node persists its exact task transition and dispatch outbox row; + // a post-commit crash neither loses the decision nor duplicates its outcome. let tool_name = "lifecycle_explicit_review_handoff_probe"; let fixture = direct_execution_fixture(tool_name, success_outcomes()).await?; let prepared = prepare_capability_run( @@ -971,12 +818,12 @@ async fn explicit_review_retries_after_persisted_epoch_before_ack_service_e2e() await_task_completion(&run, before_review.wake_epoch, &payload).await?; assert!( !interrupted.is_finished(), - "public review decision returned before its task and run acknowledgements" + "public review decision returned before its dispatcher handoff" ); fixture .hard_crash_and_restart_orchestrator() .await - .context("crash in the persisted-review/pre-acknowledgement window")?; + .context("crash in the persisted-review/pre-dispatch window")?; let recovered = tokio::time::timeout(SERVICE_TIMEOUT, interrupted) .await .context("review decision did not retry after orchestrator recovery")???; @@ -1013,9 +860,9 @@ async fn explicit_review_retries_after_persisted_epoch_before_ack_service_e2e() #[tokio::test] #[ignore = "requires the local Restate/Postgres/OpenFGA/Redis service fixture"] -async fn external_signal_retries_after_persisted_epoch_before_ack_service_e2e() -> Result<()> { - // Pins: a named WaitSignal node returns only after its exact task promise and run wake are - // acknowledged; a post-commit crash neither loses the payload nor duplicates its outcome. +async fn external_signal_retries_after_persisted_epoch_before_dispatch_service_e2e() -> Result<()> { + // Pins: a named WaitSignal node persists its exact task transition and dispatch outbox row; + // a post-commit crash neither loses the payload nor duplicates its outcome. let tool_name = "lifecycle_external_signal_handoff_probe"; let fixture = direct_execution_fixture(tool_name, success_outcomes()).await?; let prepared = prepare_capability_run( @@ -1063,12 +910,12 @@ async fn external_signal_retries_after_persisted_epoch_before_ack_service_e2e() await_task_completion(&run, before_signal.wake_epoch, &payload).await?; assert!( !interrupted.is_finished(), - "public signal delivery returned before its task and run acknowledgements" + "public signal delivery returned before its dispatcher handoff" ); fixture .hard_crash_and_restart_orchestrator() .await - .context("crash in the persisted-signal/pre-acknowledgement window")?; + .context("crash in the persisted-signal/pre-dispatch window")?; let recovered = tokio::time::timeout(SERVICE_TIMEOUT, interrupted) .await .context("signal delivery did not retry after orchestrator recovery")???; @@ -1115,7 +962,8 @@ async fn artifact_required_admin_review_reaches_durable_dispatch() -> Result<()> let pool = sqlx::PgPool::connect(&fixture.postgres_url) .await .context("connect artifact-floor review reaper")?; - let reaper = ActionReviewReaper::with_restate_ingress(pool, fixture.ingress_url.clone()); + let reaper = + ActionReviewReaper::with_restate_ingress(pool.clone(), fixture.ingress_url.clone()); for (completed_calls, route) in [ ArtifactCapabilityRoute::DirectAction, @@ -1217,7 +1065,13 @@ async fn artifact_required_admin_review_reaches_durable_dispatch() -> Result<()> .await .with_context(|| format!("clearing {label} artifact-floor review timed out"))???; - assert_eq!(reaper.trigger_execution_review_dispatch().await?, 1); + let delivered = await_outbox_delivered(&pool, review.id, 1).await?; + assert_eq!(delivered.resolution_status, "completed"); + assert_eq!( + reaper.trigger_execution_review_dispatch().await?, + 0, + "the decision handler must durably enqueue delivery; a duplicate drain is idle" + ); let terminal = await_review_terminal(label, &fixture, &run, Duration::from_secs(15)).await?; assert_terminal( @@ -1278,10 +1132,12 @@ async fn action_review_terminal_states_deliver_once_service_e2e() -> Result<()> tokio::time::timeout(SERVICE_TIMEOUT, clear) .await .context("cleared action-review decision timed out")???; + let cleared_outbox = await_outbox_delivered(&pool, cleared_review.id, 1).await?; + assert_eq!(cleared_outbox.resolution_status, "completed"); assert_eq!( reaper.trigger_execution_review_dispatch().await?, - 1, - "the cleared review resolution must be dispatched through the production outbox" + 0, + "the cleared decision must durably wake the production outbox dispatcher" ); let cleared_terminal = await_review_terminal("cleared", &fixture, &cleared_run, Duration::from_secs(15)).await?; @@ -1293,10 +1149,29 @@ async fn action_review_terminal_states_deliver_once_service_e2e() -> Result<()> 1, ); assert_review_status(&pool, cleared_review.id, "cleared").await?; - let cleared_outbox = await_outbox_delivered(&pool, cleared_review.id, 1).await?; - assert_eq!(cleared_outbox.resolution_status, "completed"); assert_review_audit_once(&cleared_run, cleared_review.id, 1).await?; + fixture + .client + .post_void( + "/ActionReviews/decide", + &DecideActionReviewRequest { + tenant_id: cleared_run.tenant_id, + review_id: cleared_review.id, + decision: ActionReviewDecisionKind::Cleared, + reason: None, + }, + ) + .await?; + assert_eq!( + reaper.trigger_execution_review_dispatch().await?, + 0, + "replaying the exact decision must not enqueue another resolution" + ); + let cleared_replay = await_outbox_delivered(&pool, cleared_review.id, 1).await?; + assert_eq!(cleared_replay.attempt_count, 1); + assert_eq!(controller.calls().len(), 1); + let denied = prepare_capability_run( &fixture, "review-denied", @@ -1320,10 +1195,12 @@ async fn action_review_terminal_states_deliver_once_service_e2e() -> Result<()> }, ) .await?; + let denied_outbox = await_outbox_delivered(&pool, denied_review.id, 1).await?; + assert_eq!(denied_outbox.resolution_status, "denied"); assert_eq!( reaper.trigger_execution_review_dispatch().await?, - 1, - "the denied review resolution must be dispatched through the production outbox" + 0, + "the denied decision must durably wake the production outbox dispatcher" ); let denied_terminal = await_review_terminal("denied", &fixture, &denied_run, Duration::from_secs(15)).await?; @@ -1337,8 +1214,6 @@ async fn action_review_terminal_states_deliver_once_service_e2e() -> Result<()> 1, ); assert_review_status(&pool, denied_review.id, "denied").await?; - let denied_outbox = await_outbox_delivered(&pool, denied_review.id, 1).await?; - assert_eq!(denied_outbox.resolution_status, "denied"); assert_review_audit_once(&denied_run, denied_review.id, 1).await?; assert_eq!(controller.calls().len(), 1); @@ -1605,6 +1480,7 @@ async fn duplicate_completion_does_not_double_account_service_e2e() -> Result<() struct PreparedCapabilityRun { tenant_id: TenantId, session_id: SessionId, + admitted_identity: moa_core::traits::Identity, originating_user_sequence_num: u64, planning: ExecutionPlanningContextResponse, compiled: CompiledExecution, @@ -1864,6 +1740,7 @@ async fn prepare_capability_run_inner( contact_id: None, session_id, originating_user_sequence_num, + deadline_at: chrono::Utc::now() + chrono::TimeDelta::days(1), requested_template: None, }, ) @@ -1907,6 +1784,11 @@ async fn prepare_capability_run_inner( Ok(PreparedCapabilityRun { tenant_id: session.tenant_id, session_id, + admitted_identity: fixture + .client + .identity() + .cloned() + .context("fixture client must carry an admitted identity")?, originating_user_sequence_num, planning, compiled, @@ -2106,6 +1988,12 @@ fn recompile_as_agent( }); let plan = ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, + input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { + expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { + delay_seconds: 86_400, + }, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + }, input_schema: json!({"type": "object", "additionalProperties": false}), output_schema: output_schema.clone(), nodes: vec![ @@ -2171,13 +2059,31 @@ fn recompile_as_external_wait( let operation = match wait { ExternalWaitKind::Review => ExecutionOperation::Review { prompt: "Approve the deterministic fixture result".to_string(), + wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { + expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { + delay_seconds: 86_400, + }, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + }, }, ExternalWaitKind::Signal => ExecutionOperation::WaitSignal { signal_name: "fixture-ready".to_string(), + wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { + expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { + delay_seconds: 86_400, + }, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + }, }, }; let plan = ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, + input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { + expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { + delay_seconds: 86_400, + }, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + }, input_schema: json!({"type": "object", "additionalProperties": false}), output_schema: output_schema.clone(), nodes: vec![ @@ -2292,6 +2198,7 @@ async fn create_direct_run( .repository .create_run( prepared.scope, + &moa_config::ExecutionConfig::default(), NewExecutionRun { tenant_id: prepared.tenant_id, contact_id: None, @@ -2300,6 +2207,7 @@ async fn create_direct_run( planning_context_uid: prepared.planning.planning_context_uid, planning_context_hash: prepared.planning.planning_context_hash.parse()?, owner_user_id: prepared.planning.snapshot.owner_user_id.clone(), + admitted_identity: prepared.admitted_identity.clone(), goal: prepared.compiled.goal.clone(), plan: prepared.compiled.plan.clone(), catalog: prepared.planning.snapshot.catalog.clone(), @@ -2317,6 +2225,9 @@ async fn create_direct_run( }, ) .await?; + let RunAdmissionOutcome::Admitted(run) = run else { + bail!("direct task lifecycle run was not admitted: {run:?}") + }; let task_id = ExecutionTaskId::derive(run.run_uid, CAPABILITY_NODE_ID, "")?; prepared .repository @@ -2389,6 +2300,12 @@ fn lifecycle_plan_with_output_schema( ) -> ExecutionPlanDefinition { ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, + input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { + expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { + delay_seconds: 86_400, + }, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + }, input_schema: json!({"type": "object", "additionalProperties": false}), output_schema: output_schema.clone(), nodes: vec![ @@ -2549,25 +2466,6 @@ async fn await_run_status( } } -async fn await_positive_processed_wake_epoch(run: &RunningCapabilityRun) -> Result { - let deadline = Instant::now() + SERVICE_TIMEOUT; - loop { - let persisted = load_run(run).await?; - if persisted.processed_wake_epoch > 0 { - return Ok(persisted.processed_wake_epoch); - } - if persisted.status.is_terminal() { - bail!("run terminalized before acknowledging a wake epoch: {persisted:#?}"); - } - if Instant::now() >= deadline { - bail!( - "run did not acknowledge a positive wake epoch within {SERVICE_TIMEOUT:?}: {persisted:#?}" - ); - } - tokio::time::sleep(POLL_INTERVAL).await; - } -} - async fn await_task_generation( run: &RunningCapabilityRun, expected_generation: u64, diff --git a/crates/moa-orchestrator/tests/execution_run_service_e2e/terminal_matrix.rs b/crates/moa-orchestrator/tests/execution_run_service_e2e/terminal_matrix.rs index 71d0bb7ff..6a64a7979 100644 --- a/crates/moa-orchestrator/tests/execution_run_service_e2e/terminal_matrix.rs +++ b/crates/moa-orchestrator/tests/execution_run_service_e2e/terminal_matrix.rs @@ -1,58 +1,31 @@ -//! Strict terminal-cause and remaining planning-audit service coverage. +//! Strict routing, planning-audit, and bounded terminal service coverage. //! -//! Natural routing, planner, amendment, cancellation, and status reads use the -//! production service boundaries. Defensive scheduler no-progress, infrastructure -//! failure and alternate terminal-write conflicts have no -//! public mutation API; those cases deliberately enter through [`ExecutionRepository`] -//! and then assert the public `Execution/status` projection. Experiment-template and -//! skill-regression compile producers remain owned by their dedicated binaries; this -//! module does not duplicate those full workflows. +//! Natural routing, planner, amendment, and status reads use the production service +//! boundaries. Experiment-template and skill-regression compile producers remain owned by +//! their dedicated binaries; this module does not duplicate those full workflows. use anyhow::{Context, Result, bail}; use moa_artifacts::execution_plan::{ - CompletionCheck, CompletionCheckKind, ExecutionBudgetLimit, ExecutionFailureClass, - ExecutionGoalContract, ExecutionNode, ExecutionOperation, ExecutionPlanDefinition, - ExecutionRequirement, ExecutionTaskOutcome, ExecutionTaskResult, ExecutionUsage, + CompletionCheck, CompletionCheckKind, ExecutionGoalContract, ExecutionNode, ExecutionOperation, + ExecutionPlanDefinition, ExecutionRequirement, ExecutionTaskResult, GeneratedAmendmentCandidate, GeneratedExecutionCandidate, PlanAmendment, PlanAmendmentOperation, RetryPolicy, }; -use moa_config::ExecutionConfig; use moa_core::{ events::Event, types::{ - contact::SessionActorRef, execution_planning::{ ExecutionCompileOutcome, ExecutionCompileSource, ExecutionPlannerCallKind, ExecutionPlannerOutcome, ExecutionPlanningAuditEnvelope, ExecutionPlanningAuditPayload, ExecutionRouteKind, ExecutionRouteStage, ExecutionStrategy, }, - identifiers::{SessionId, TenantId, UserId}, + identifiers::SessionId, session::SessionStatus, }, }; use moa_execution::{ - capability::{ExecutionAuthorizationEnvelope, ExecutionCapabilityCatalog, ExecutionEstimate}, - compiler::{CompileExecutionRequest, CompiledExecution, compile}, - completion::{ - CompletionEvaluation, CompletionStatus, execution_terminal_reason, - terminal_evidence_from_evaluation, - }, - replan::ReplanStopReason, - repository::{ - ExecutionRepository, ExecutionRunRecord, ExecutionScope, FencedTerminalFinalizationOutcome, - FinalizationOutcome, NewExecutionRun, ReservationOutcome, RunFinalizationRequest, - TaskOutcomeWrite, TerminalFenceCommit, TerminalFenceOutcome, TransitionOutcome, - }, - state::{ - ExecutionLimitStop, ExecutionRunStatus, ExecutionSourceKind, ExecutionTaskFailure, - ExecutionTaskId, ExecutionTerminalCause, ExecutionTerminalEvidence, - ExecutionTerminalReason, LogicalTask, LogicalTaskKind, PendingExecutionTerminal, - TerminalProjection, - }, - wire::{ - ExecutionCancelRequest, ExecutionMutationResponse, ExecutionRunRequest, - ExecutionStatusResponse, - }, + state::{ExecutionRunStatus, ExecutionSourceKind}, + wire::ExecutionStatusResponse, }; use moa_test_support::{ FixtureCapabilityOptions, FixtureCapabilityOutcome, FixtureCapabilityTool, @@ -87,9 +60,6 @@ const DURABLE_UPGRADE_CONTROL_BARRIER_MS: u64 = 15_000; const REPLAN_SEED_AGENT_SENTINEL: &str = "TERMINAL_MATRIX_REPLAN_SEED_AGENT"; const REPLAN_AGENT_SENTINEL: &str = "TERMINAL_MATRIX_REPLAN_AGENT"; const REPAIRED_AGENT_SENTINEL: &str = "TERMINAL_MATRIX_REPAIRED_AGENT"; -const REQ_USEFUL: &str = "useful"; -const REQ_REMAINING: &str = "remaining"; - #[tokio::test] #[ignore = "requires the local Restate/Postgres/OpenFGA/Redis service fixture"] async fn no_run_needs_input_persists_strict_route_audit_service_e2e() -> Result<()> { @@ -911,1161 +881,6 @@ async fn amendment_planning_persists_revision_fenced_strict_audits_service_e2e() assert_eq!(planning_calls, vec!["route", "initial", "amendment"]); Ok(()) } - -#[tokio::test] -#[ignore = "requires local Restate, Postgres, OpenFGA, and the service-e2e feature lane"] -async fn strict_terminal_cause_matrix_is_exhaustive_service_e2e() -> Result<()> { - // Pins: the repository replay boundary and public status API retain every - // runtime terminal cohort, useful/empty limit distinction, and exact counts. - let fixture = OrchestratorTestFixture::shared().await?; - let test = fixture.isolated().await; - let session_id = test.create_session("strict-terminal-matrix").await?; - let session = test.client().get_session(session_id).await?; - let owner_user_id = session_owner(&session.created_by)?; - let pool = sqlx::PgPool::connect(&fixture.postgres_url) - .await - .context("connect strict terminal matrix repository")?; - let repository = ExecutionRepository::new(pool); - let scope = ExecutionScope::Tenant { - tenant_id: session.tenant_id, - }; - let blueprint = terminal_blueprint()?; - - for (index, case) in ordinary_terminal_cases().into_iter().enumerate() { - assert_repository_terminal_case( - &repository, - scope, - test.client(), - session.tenant_id, - session_id, - owner_user_id.clone(), - &blueprint, - 20 + index as u64, - case, - ) - .await?; - } - - for (index, reason) in replan_reasons().into_iter().enumerate() { - assert_replan_terminal_case( - &repository, - scope, - test.client(), - session.tenant_id, - session_id, - owner_user_id.clone(), - &blueprint, - 80 + index as u64, - reason, - ) - .await?; - } - - assert_cancellation_terminal_case( - &repository, - scope, - test.client(), - session.tenant_id, - session_id, - owner_user_id, - &blueprint, - 100, - ) - .await?; - Ok(()) -} - -#[tokio::test] -#[ignore = "requires local Restate, Postgres, OpenFGA, and the service-e2e feature lane"] -async fn internal_failure_persists_typed_cause_service_e2e() -> Result<()> { - // Pins: infrastructure-failure finalization is not inferred from failure - // prose; the closed InternalFailure cause survives repository restart and status reads. - let fixture = OrchestratorTestFixture::shared().await?; - let test = fixture.isolated().await; - let session_id = test.create_session("typed-internal-failure").await?; - let session = test.client().get_session(session_id).await?; - let owner_user_id = session_owner(&session.created_by)?; - let repository = ExecutionRepository::new( - sqlx::PgPool::connect(&fixture.postgres_url) - .await - .context("connect internal-failure repository")?, - ); - let scope = ExecutionScope::Tenant { - tenant_id: session.tenant_id, - }; - let blueprint = terminal_blueprint()?; - assert_repository_terminal_case( - &repository, - scope, - test.client(), - session.tenant_id, - session_id, - owner_user_id, - &blueprint, - 120, - TerminalCase { - label: "internal-failure", - cause: ExecutionTerminalCause::InternalFailure, - status: ExecutionRunStatus::Failed, - completion_status: CompletionStatus::Failed, - projection: terminal_failure_projection( - ExecutionFailureClass::Terminal, - "injected infrastructure failure", - ), - output: None, - gaps: vec!["internal execution failure".to_string()], - satisfied: Vec::new(), - unsatisfied: vec![REQ_USEFUL.to_string(), REQ_REMAINING.to_string()], - }, - ) - .await -} - -#[derive(Clone)] -struct RunBlueprint { - compiled: CompiledExecution, - catalog: ExecutionCapabilityCatalog, - authorization: ExecutionAuthorizationEnvelope, - budget: ExecutionBudgetLimit, -} - -#[derive(Clone)] -struct TerminalCase { - label: &'static str, - cause: ExecutionTerminalCause, - status: ExecutionRunStatus, - completion_status: CompletionStatus, - projection: TerminalProjection, - output: Option, - gaps: Vec, - satisfied: Vec, - unsatisfied: Vec, -} - -fn ordinary_terminal_cases() -> Vec { - vec![ - TerminalCase { - label: "completion", - cause: ExecutionTerminalCause::Completion { limit_stop: None }, - status: ExecutionRunStatus::Completed, - completion_status: CompletionStatus::Completed, - projection: TerminalProjection::Completed { - output: json!({"result": "complete"}), - }, - output: Some(json!({"result": "complete"})), - gaps: Vec::new(), - satisfied: vec![REQ_USEFUL.to_string(), REQ_REMAINING.to_string()], - unsatisfied: Vec::new(), - }, - TerminalCase { - label: "task-failure", - cause: ExecutionTerminalCause::TaskFailure { - class: ExecutionFailureClass::InvalidOutput, - }, - status: ExecutionRunStatus::Failed, - completion_status: CompletionStatus::Failed, - projection: terminal_failure_projection( - ExecutionFailureClass::InvalidOutput, - "task output violated its schema", - ), - output: None, - gaps: vec!["task output violated its schema".to_string()], - satisfied: Vec::new(), - unsatisfied: vec![REQ_USEFUL.to_string(), REQ_REMAINING.to_string()], - }, - limit_case(ExecutionLimitStop::DeadlineExceeded, true), - limit_case(ExecutionLimitStop::DeadlineExceeded, false), - limit_case(ExecutionLimitStop::BudgetExceeded, true), - limit_case(ExecutionLimitStop::BudgetExceeded, false), - TerminalCase { - label: "scheduler-no-progress", - cause: ExecutionTerminalCause::SchedulerNoProgress, - status: ExecutionRunStatus::Failed, - completion_status: CompletionStatus::Failed, - projection: terminal_failure_projection( - ExecutionFailureClass::Terminal, - "scheduler made no progress", - ), - output: None, - gaps: vec!["scheduler made no progress".to_string()], - satisfied: Vec::new(), - unsatisfied: vec![REQ_USEFUL.to_string(), REQ_REMAINING.to_string()], - }, - ] -} - -fn limit_case(reason: ExecutionLimitStop, useful: bool) -> TerminalCase { - let (label, failure_class, gap) = match reason { - ExecutionLimitStop::DeadlineExceeded => ( - if useful { - "deadline-useful" - } else { - "deadline-empty" - }, - ExecutionFailureClass::DeadlineExceeded, - "execution deadline exceeded", - ), - ExecutionLimitStop::BudgetExceeded => ( - if useful { - "budget-useful" - } else { - "budget-empty" - }, - ExecutionFailureClass::BudgetExceeded, - "execution budget exceeded", - ), - }; - TerminalCase { - label, - cause: ExecutionTerminalCause::LimitStop { reason }, - status: if useful { - ExecutionRunStatus::Partial - } else { - ExecutionRunStatus::Failed - }, - completion_status: if useful { - CompletionStatus::Partial - } else { - CompletionStatus::Failed - }, - projection: if useful { - TerminalProjection::Partial { - output: Some(json!({"useful": true})), - gaps: vec![gap.to_string()], - } - } else { - terminal_failure_projection(failure_class, gap) - }, - output: useful.then(|| json!({"useful": true})), - gaps: vec![gap.to_string()], - satisfied: useful.then(|| REQ_USEFUL.to_string()).into_iter().collect(), - unsatisfied: if useful { - vec![REQ_REMAINING.to_string()] - } else { - vec![REQ_USEFUL.to_string(), REQ_REMAINING.to_string()] - }, - } -} - -fn replan_reasons() -> [ReplanStopReason; 6] { - [ - ReplanStopReason::DuplicatePlan, - ReplanStopReason::DuplicateAmendment, - ReplanStopReason::RepeatedFailure, - ReplanStopReason::NoProgress, - ReplanStopReason::DeadlineExceeded, - ReplanStopReason::BudgetExhausted, - ] -} - -#[allow( - clippy::too_many_arguments, - reason = "the service scenario keeps tenant, session, owner, and immutable run cohort explicit" -)] -async fn assert_repository_terminal_case( - repository: &ExecutionRepository, - scope: ExecutionScope, - client: &TestApiClient, - tenant_id: TenantId, - session_id: SessionId, - owner_user_id: UserId, - blueprint: &RunBlueprint, - origin: u64, - case: TerminalCase, -) -> Result<()> { - let run = create_active_run( - repository, - scope, - tenant_id, - session_id, - owner_user_id, - blueprint, - origin, - case.label, - ) - .await?; - let evaluation = CompletionEvaluation { - status: case.completion_status, - limit_stop: match &case.cause { - ExecutionTerminalCause::Completion { limit_stop } => *limit_stop, - ExecutionTerminalCause::TaskFailure { .. } - | ExecutionTerminalCause::LimitStop { .. } - | ExecutionTerminalCause::SchedulerNoProgress - | ExecutionTerminalCause::ReplanStop { .. } - | ExecutionTerminalCause::Cancellation - | ExecutionTerminalCause::InternalFailure - | ExecutionTerminalCause::CompensationFailure { .. } => None, - }, - checks: Vec::new(), - satisfied_requirement_ids: case.satisfied.clone(), - unsatisfied_requirement_ids: case.unsatisfied.clone(), - gaps: case.gaps.clone(), - }; - let evidence = terminal_evidence_from_evaluation(case.cause.clone(), &evaluation)?; - let terminal_reason = execution_terminal_reason(&case.cause, &case.projection, &evaluation)?; - if case.status != ExecutionRunStatus::Completed { - let pending_terminal = PendingExecutionTerminal { - status: case.status, - reason: terminal_reason, - terminal_evidence: evidence.clone(), - output: case.output.clone(), - completion_check_results: evaluation - .checks - .iter() - .map(serde_json::to_value) - .collect::, _>>()?, - terminal_gaps: case.gaps.clone(), - cancellation_reason: None, - }; - let first = repository - .fence_run_for_terminal( - scope, - run.run_uid, - run.plan_revision, - run.wake_epoch, - pending_terminal.clone(), - ) - .await?; - let TerminalFenceOutcome::Applied(first_commit) = first else { - bail!( - "{} did not enter the compensation fence: {first:?}", - case.label - ); - }; - let replay = repository - .fence_run_for_terminal( - scope, - run.run_uid, - run.plan_revision, - run.wake_epoch, - pending_terminal.clone(), - ) - .await?; - let TerminalFenceOutcome::Replayed(replayed_commit) = replay else { - bail!( - "{} did not replay its terminal fence: {replay:?}", - case.label - ); - }; - assert_eq!(replayed_commit, first_commit); - let mut conflict = pending_terminal.clone(); - conflict.terminal_evidence.satisfied_requirement_count = - conflicting_satisfied_count(&evidence); - assert_eq!( - repository - .fence_run_for_terminal( - scope, - run.run_uid, - run.plan_revision, - run.wake_epoch, - conflict, - ) - .await?, - TerminalFenceOutcome::Conflict, - "{} accepted conflicting terminal-fence evidence", - case.label - ); - assert_pending_terminal_projection( - repository, - scope, - run.run_uid, - run.status, - &pending_terminal, - ) - .await?; - settle_fenced_terminal(repository, scope, &first_commit).await?; - return assert_status_projection( - client, - tenant_id, - session_id, - run.run_uid, - case.status, - case.output, - case.gaps, - evidence, - ) - .await; - } - let request = RunFinalizationRequest { - run_uid: run.run_uid, - expected_revision: run.plan_revision, - expected_wake_epoch: run.wake_epoch, - terminal_projection: case.projection.clone(), - completion_evaluation: evaluation, - terminal_evidence: evidence.clone(), - terminal_reason, - }; - let first = repository.finalize_run(scope, request.clone()).await?; - let FinalizationOutcome::Finalized(first_record) = first else { - bail!("{} did not finalize on first write: {first:?}", case.label); - }; - assert_eq!(first_record.status, case.status, "{} status", case.label); - assert_eq!( - first_record.terminal_evidence, - Some(evidence.clone()), - "{} evidence", - case.label - ); - - let replay = repository.finalize_run(scope, request.clone()).await?; - let FinalizationOutcome::Replayed(replayed_record) = replay else { - bail!("{} did not replay exactly: {replay:?}", case.label); - }; - assert_eq!( - replayed_record, first_record, - "{} replay changed persisted bytes", - case.label - ); - - let mut count_conflict = request.clone(); - count_conflict.terminal_evidence.satisfied_requirement_count = - conflicting_satisfied_count(&evidence); - assert_eq!( - repository.finalize_run(scope, count_conflict).await?, - FinalizationOutcome::Conflict, - "{} accepted conflicting requirement counts", - case.label - ); - if let Some(alternate_cause) = - alternate_terminal_cause(&case.cause, &request.terminal_projection) - { - let mut cause_conflict = request; - cause_conflict.terminal_evidence.cause = alternate_cause; - cause_conflict.terminal_reason = execution_terminal_reason( - &cause_conflict.terminal_evidence.cause, - &cause_conflict.terminal_projection, - &cause_conflict.completion_evaluation, - )?; - assert_eq!( - repository.finalize_run(scope, cause_conflict).await?, - FinalizationOutcome::Conflict, - "{} accepted a conflicting typed cause", - case.label - ); - } - - assert_status_projection( - client, - tenant_id, - session_id, - run.run_uid, - case.status, - case.output, - case.gaps, - evidence, - ) - .await -} - -#[allow( - clippy::too_many_arguments, - reason = "each replan matrix row keeps its persisted run and task fences explicit" -)] -async fn assert_replan_terminal_case( - repository: &ExecutionRepository, - scope: ExecutionScope, - client: &TestApiClient, - tenant_id: TenantId, - session_id: SessionId, - owner_user_id: UserId, - blueprint: &RunBlueprint, - origin: u64, - reason: ReplanStopReason, -) -> Result<()> { - let label = reason.as_str(); - let run = create_active_run( - repository, - scope, - tenant_id, - session_id, - owner_user_id, - blueprint, - origin, - label, - ) - .await?; - let completed_task = logical_output_task( - run.run_uid, - "useful_output", - vec![REQ_USEFUL.to_string()], - json!({"useful": true}), - )?; - let waiting_task = logical_agent_task( - run.run_uid, - "waiting_replan", - vec![REQ_REMAINING.to_string()], - )?; - repository - .materialize_tasks( - scope, - run.run_uid, - 1, - vec![completed_task.clone(), waiting_task.clone()], - ) - .await?; - reserve_and_start(repository, scope, run.run_uid, completed_task.task_id).await?; - assert!(matches!( - repository - .record_task_outcome( - scope, - run.run_uid, - completed_task.task_id, - 1, - completed_outcome(json!({"useful": true})), - ) - .await?, - TaskOutcomeWrite::Applied { .. } - )); - reserve_and_start(repository, scope, run.run_uid, waiting_task.task_id).await?; - assert!(matches!( - repository - .record_task_outcome( - scope, - run.run_uid, - waiting_task.task_id, - 1, - needs_replan_outcome(label), - ) - .await?, - TaskOutcomeWrite::Applied { .. } - )); - let waiting_run = repository - .load_run(scope, run.run_uid) - .await? - .context("waiting-replan run disappeared")?; - assert_eq!(waiting_run.status, ExecutionRunStatus::WaitingReplan); - - let gap = format!("replan stopped: {label}"); - let projection = TerminalProjection::Partial { - output: Some(json!({"useful": true})), - gaps: vec![gap.clone()], - }; - let evaluation = CompletionEvaluation { - status: CompletionStatus::Partial, - limit_stop: None, - checks: Vec::new(), - satisfied_requirement_ids: vec![REQ_USEFUL.to_string()], - unsatisfied_requirement_ids: vec![REQ_REMAINING.to_string()], - gaps: vec![gap.clone()], - }; - let evidence = terminal_evidence_from_evaluation( - ExecutionTerminalCause::ReplanStop { reason }, - &evaluation, - )?; - let terminal_reason = execution_terminal_reason( - &ExecutionTerminalCause::ReplanStop { reason }, - &projection, - &evaluation, - )?; - let pending_terminal = PendingExecutionTerminal { - status: ExecutionRunStatus::Partial, - reason: terminal_reason, - terminal_evidence: evidence.clone(), - output: Some(json!({"useful": true})), - completion_check_results: Vec::new(), - terminal_gaps: evaluation.gaps, - cancellation_reason: None, - }; - let first = repository - .fence_run_for_terminal( - scope, - run.run_uid, - 1, - waiting_run.wake_epoch, - pending_terminal.clone(), - ) - .await?; - let TerminalFenceOutcome::Applied(first_commit) = first else { - bail!("{label} did not enter the compensation fence: {first:?}"); - }; - let replay = repository - .fence_run_for_terminal( - scope, - run.run_uid, - 1, - waiting_run.wake_epoch, - pending_terminal.clone(), - ) - .await?; - let TerminalFenceOutcome::Replayed(replayed_commit) = replay else { - bail!("{label} did not replay through the compensation fence: {replay:?}"); - }; - assert_eq!( - replayed_commit, first_commit, - "{label} replay changed persisted bytes" - ); - - let mut count_conflict = pending_terminal.clone(); - count_conflict.terminal_evidence.satisfied_requirement_count = 0; - assert_eq!( - repository - .fence_run_for_terminal( - scope, - run.run_uid, - 1, - waiting_run.wake_epoch, - count_conflict, - ) - .await?, - TerminalFenceOutcome::Conflict, - "{label} accepted conflicting requirement counts" - ); - let mut cause_conflict = pending_terminal.clone(); - cause_conflict.terminal_evidence.cause = ExecutionTerminalCause::ReplanStop { - reason: alternate_replan_reason(reason), - }; - assert_eq!( - repository - .fence_run_for_terminal( - scope, - run.run_uid, - 1, - waiting_run.wake_epoch, - cause_conflict, - ) - .await?, - TerminalFenceOutcome::Conflict, - "{label} accepted a conflicting replan cause" - ); - - assert_pending_terminal_projection( - repository, - scope, - run.run_uid, - waiting_run.status, - &pending_terminal, - ) - .await?; - settle_fenced_terminal(repository, scope, &first_commit).await?; - assert_status_projection( - client, - tenant_id, - session_id, - run.run_uid, - ExecutionRunStatus::Partial, - Some(json!({"useful": true})), - vec![gap], - evidence, - ) - .await -} - -#[allow( - clippy::too_many_arguments, - reason = "the cancellation service case keeps its complete parent and run scope explicit" -)] -async fn assert_cancellation_terminal_case( - repository: &ExecutionRepository, - scope: ExecutionScope, - client: &TestApiClient, - tenant_id: TenantId, - session_id: SessionId, - owner_user_id: UserId, - blueprint: &RunBlueprint, - origin: u64, -) -> Result<()> { - let run = create_active_run( - repository, - scope, - tenant_id, - session_id, - owner_user_id, - blueprint, - origin, - "cancellation", - ) - .await?; - let reason = "cancel strict terminal matrix".to_string(); - let request = ExecutionCancelRequest { - run: ExecutionRunRequest { - tenant_id, - contact_id: None, - session_id, - run_uid: run.run_uid, - }, - reason: reason.clone(), - }; - let first: ExecutionMutationResponse = client.post_call("/Execution/cancel", &request).await?; - let ExecutionMutationResponse::Applied { run: first_run } = first else { - bail!("cancellation did not apply: {first:?}"); - }; - let evidence = ExecutionTerminalEvidence { - cause: ExecutionTerminalCause::Cancellation, - satisfied_requirement_count: 0, - requirement_count: 2, - }; - assert_eq!(first_run.status, ExecutionRunStatus::Running); - assert!(first_run.terminal_evidence.is_none()); - - let replay: ExecutionMutationResponse = client.post_call("/Execution/cancel", &request).await?; - assert_eq!( - replay, - ExecutionMutationResponse::Replayed { - run: first_run.clone() - } - ); - let pending_terminal = PendingExecutionTerminal { - status: ExecutionRunStatus::Cancelled, - reason: ExecutionTerminalReason::Cancelled, - terminal_evidence: evidence.clone(), - output: None, - completion_check_results: Vec::new(), - terminal_gaps: Vec::new(), - cancellation_reason: Some(reason.clone()), - }; - assert_pending_terminal_projection( - repository, - scope, - run.run_uid, - ExecutionRunStatus::Running, - &pending_terminal, - ) - .await?; - let persisted = repository - .load_run(scope, run.run_uid) - .await? - .context("cancellation-fenced run disappeared")?; - let finalized = settle_fenced_terminal( - repository, - scope, - &TerminalFenceCommit { - run: persisted, - tasks_to_settle: Vec::new(), - }, - ) - .await?; - assert_eq!(finalized.cancellation_reason, Some(reason)); - assert_status_projection( - client, - tenant_id, - session_id, - run.run_uid, - ExecutionRunStatus::Cancelled, - None, - Vec::new(), - evidence, - ) - .await -} - -#[allow( - clippy::too_many_arguments, - reason = "run creation mirrors the explicit immutable production persistence cohort" -)] -async fn create_active_run( - repository: &ExecutionRepository, - scope: ExecutionScope, - tenant_id: TenantId, - session_id: SessionId, - owner_user_id: UserId, - blueprint: &RunBlueprint, - origin: u64, - label: &str, -) -> Result { - let (planning_context_uid, planning_context_hash) = crate::create_test_planning_context( - repository, - scope, - tenant_id, - session_id, - origin, - owner_user_id.clone(), - blueprint.catalog.clone(), - blueprint.authorization.clone(), - blueprint.budget.clone(), - ) - .await?; - let created = repository - .create_run( - scope, - NewExecutionRun { - tenant_id, - contact_id: None, - session_id, - originating_user_sequence_num: origin, - planning_context_uid, - planning_context_hash, - owner_user_id, - goal: blueprint.compiled.goal.clone(), - plan: blueprint.compiled.plan.clone(), - catalog: blueprint.catalog.clone(), - authorization: blueprint.authorization.clone(), - pinned_instruction_skills: Vec::new(), - source_provenance: crate::test_source_provenance( - &blueprint.compiled.plan.plan_hash.to_string(), - ), - input: json!({}), - status: ExecutionRunStatus::Queued, - approved_budget: blueprint.budget.clone(), - idempotency_key: Some(format!("terminal-matrix-{session_id}-{origin}-{label}")), - }, - ) - .await?; - match repository - .transition_run_wait( - scope, - created.run_uid, - ExecutionRunStatus::Queued, - ExecutionRunStatus::Running, - ) - .await? - { - TransitionOutcome::RunApplied(running) => Ok(running), - other => bail!("{label} did not enter running state: {other:?}"), - } -} - -async fn reserve_and_start( - repository: &ExecutionRepository, - scope: ExecutionScope, - run_uid: uuid::Uuid, - task_id: ExecutionTaskId, -) -> Result<()> { - assert!(matches!( - repository.reserve_task(scope, run_uid, task_id, 1).await?, - ReservationOutcome::Reserved(_) - )); - assert!(matches!( - repository - .mark_task_running(scope, run_uid, task_id, 1) - .await?, - TransitionOutcome::Applied(_) - )); - Ok(()) -} - -#[allow( - clippy::too_many_arguments, - reason = "the status assertion names every externally visible terminal field" -)] -async fn assert_status_projection( - client: &TestApiClient, - tenant_id: TenantId, - session_id: SessionId, - run_uid: uuid::Uuid, - expected_status: ExecutionRunStatus, - expected_output: Option, - expected_gaps: Vec, - expected_evidence: ExecutionTerminalEvidence, -) -> Result<()> { - let status = await_execution_terminal( - client, - &ExecutionRunRequest { - tenant_id, - contact_id: None, - session_id, - run_uid, - }, - ) - .await?; - assert_eq!(status.run.status, expected_status); - assert_eq!(status.output, expected_output); - assert_eq!(status.gaps, expected_gaps); - assert_eq!(status.run.terminal_evidence, Some(expected_evidence)); - Ok(()) -} - -async fn assert_pending_terminal_projection( - repository: &ExecutionRepository, - scope: ExecutionScope, - run_uid: uuid::Uuid, - expected_status: ExecutionRunStatus, - expected_pending: &PendingExecutionTerminal, -) -> Result<()> { - let run = repository.load_run(scope, run_uid).await?; - let run = run.context("fenced run disappeared before its pending projection was asserted")?; - assert_eq!(run.status, expected_status); - assert_eq!(run.pending_terminal.as_ref(), Some(expected_pending)); - assert!(run.terminal_evidence.is_none()); - Ok(()) -} - -async fn settle_fenced_terminal( - repository: &ExecutionRepository, - scope: ExecutionScope, - fence: &TerminalFenceCommit, -) -> Result { - for task in &fence.tasks_to_settle { - let outcome = ExecutionTaskOutcome { - schema_version: 1, - usage: task.actual.clone(), - result: ExecutionTaskResult::Cancelled { - reason: "terminal-matrix forward settlement".to_string(), - }, - }; - assert!(matches!( - repository - .record_task_outcome( - scope, - fence.run.run_uid, - task.task_id, - task.generation, - outcome, - ) - .await?, - TaskOutcomeWrite::Applied { .. } | TaskOutcomeWrite::Replayed { .. } - )); - } - let settled = repository - .load_run(scope, fence.run.run_uid) - .await? - .context("fenced run disappeared before terminal settlement")?; - let finalized = repository - .finalize_fenced_terminal( - scope, - settled.run_uid, - settled.plan_revision, - settled.wake_epoch, - ) - .await?; - let FencedTerminalFinalizationOutcome::Finalized(finalized) = finalized else { - bail!("fenced terminal did not finalize after forward settlement: {finalized:?}"); - }; - let replay = repository - .finalize_fenced_terminal( - scope, - finalized.run_uid, - finalized.plan_revision, - finalized.wake_epoch, - ) - .await?; - assert_eq!( - replay, - FencedTerminalFinalizationOutcome::Replayed(finalized.clone()), - "terminal-fence finalization did not replay exactly" - ); - Ok(finalized) -} - -fn terminal_blueprint() -> Result { - let catalog = ExecutionCapabilityCatalog::build(Vec::new())?; - let authorization = ExecutionAuthorizationEnvelope { - capability_refs: Vec::new(), - skill_refs: Vec::new(), - }; - let budget = generous_budget(); - let output_schema = json!({"type": "object"}); - let outcome = compile(CompileExecutionRequest { - goal: ExecutionGoalContract { - objective: "persist strict terminal evidence".to_string(), - requirements: vec![ - ExecutionRequirement { - id: REQ_USEFUL.to_string(), - description: "preserve useful work".to_string(), - }, - ExecutionRequirement { - id: REQ_REMAINING.to_string(), - description: "represent remaining work".to_string(), - }, - ], - deliverables: Vec::new(), - coverage: Vec::new(), - constraints: Vec::new(), - completion_checks: vec![CompletionCheck { - id: "terminal_output_schema".to_string(), - description: "terminal output satisfies its declared schema".to_string(), - requirement_ids: vec![REQ_USEFUL.to_string(), REQ_REMAINING.to_string()], - constraint_ids: Vec::new(), - kind: CompletionCheckKind::OutputSchema, - }], - }, - plan: ExecutionPlanDefinition { - cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, - input_schema: empty_object_schema(), - output_schema: output_schema.clone(), - nodes: vec![ExecutionNode { - id: "terminal_output".to_string(), - requirement_ids: vec![REQ_USEFUL.to_string(), REQ_REMAINING.to_string()], - depends_on: Vec::new(), - when: None, - input: json!({}), - output_schema, - operation: ExecutionOperation::Output { - value: json!({"result": "terminal-matrix"}), - }, - compensation: None, - retry: no_retry(), - budget: None, - }], - }, - run_input: json!({}), - catalog: catalog.clone(), - authorization: authorization.clone(), - approved_budget: budget.clone(), - config: ExecutionConfig::default(), - now: moa_test_support::fixtures::pg_now(), - }); - let compiled = outcome.compiled.with_context(|| { - format!( - "terminal matrix plan should compile: {:?}", - outcome.report.issues - ) - })?; - Ok(RunBlueprint { - compiled, - catalog, - authorization, - budget, - }) -} - -fn logical_output_task( - run_uid: uuid::Uuid, - node_id: &str, - requirement_ids: Vec, - value: Value, -) -> Result { - Ok(LogicalTask { - task_id: ExecutionTaskId::derive(run_uid, node_id, "")?, - node_id: node_id.to_string(), - item_key: String::new(), - requirement_ids, - plan_revision: 1, - generation: 1, - input: json!({}), - kind: LogicalTaskKind::Output { value }, - compensation: None, - retry: no_retry(), - reservation: one_task_estimate(), - }) -} - -fn logical_agent_task( - run_uid: uuid::Uuid, - node_id: &str, - requirement_ids: Vec, -) -> Result { - Ok(LogicalTask { - task_id: ExecutionTaskId::derive(run_uid, node_id, "")?, - node_id: node_id.to_string(), - item_key: String::new(), - requirement_ids, - plan_revision: 1, - generation: 1, - input: json!({}), - kind: LogicalTaskKind::Agent { - instructions: "wait for deterministic replan".to_string(), - skill_refs: Vec::new(), - capability_refs: Vec::new(), - max_turns: 1, - }, - compensation: None, - retry: no_retry(), - reservation: one_task_estimate(), - }) -} - -fn completed_outcome(output: Value) -> ExecutionTaskOutcome { - ExecutionTaskOutcome { - schema_version: 1, - usage: empty_usage(), - result: ExecutionTaskResult::Completed { - output, - citations: Vec::new(), - }, - } -} - -fn needs_replan_outcome(reason: &str) -> ExecutionTaskOutcome { - ExecutionTaskOutcome { - schema_version: 1, - usage: empty_usage(), - result: ExecutionTaskResult::NeedsReplan { - reason: reason.to_string(), - evidence: json!({"reason": reason}), - }, - } -} - -fn empty_usage() -> ExecutionUsage { - ExecutionUsage { - cost_microusd: 0, - tokens: 0, - tool_calls: 0, - retrieved_bytes: 0, - } -} - -fn one_task_estimate() -> ExecutionEstimate { - ExecutionEstimate { - cost_microusd: 0, - tokens: 0, - tasks: 1, - tool_calls: 0, - retrieved_bytes: 0, - } -} - -fn terminal_failure_projection(class: ExecutionFailureClass, message: &str) -> TerminalProjection { - TerminalProjection::Failed { - failure: ExecutionTaskFailure { - class, - message: message.to_string(), - capability_ref: None, - }, - } -} - -fn conflicting_satisfied_count(evidence: &ExecutionTerminalEvidence) -> u64 { - if evidence.satisfied_requirement_count == 0 { - 1 - } else { - 0 - } -} - -fn alternate_terminal_cause( - cause: &ExecutionTerminalCause, - projection: &TerminalProjection, -) -> Option { - match projection { - TerminalProjection::Completed { .. } | TerminalProjection::Cancelled { .. } => None, - TerminalProjection::Failed { .. } => { - Some(if *cause == ExecutionTerminalCause::InternalFailure { - ExecutionTerminalCause::SchedulerNoProgress - } else { - ExecutionTerminalCause::InternalFailure - }) - } - TerminalProjection::Partial { .. } - | TerminalProjection::Blocked { .. } - | TerminalProjection::Unsupported { .. } => { - Some(if *cause == ExecutionTerminalCause::SchedulerNoProgress { - ExecutionTerminalCause::TaskFailure { - class: ExecutionFailureClass::Terminal, - } - } else { - ExecutionTerminalCause::SchedulerNoProgress - }) - } - } -} - -fn alternate_replan_reason(reason: ReplanStopReason) -> ReplanStopReason { - if reason == ReplanStopReason::DuplicatePlan { - ReplanStopReason::NoProgress - } else { - ReplanStopReason::DuplicatePlan - } -} - -fn session_owner(created_by: &Option) -> Result { - match created_by { - Some(SessionActorRef::Identity { id }) => Ok(UserId::new(id.to_string())), - other => bail!("fixture session has no identity owner: {other:?}"), - } -} - -fn generous_budget() -> ExecutionBudgetLimit { - ExecutionBudgetLimit { - max_cost_microusd: Some(100_000_000), - max_tokens: Some(1_000_000), - max_tasks: Some(100), - max_tool_calls: Some(100), - max_retrieved_bytes: Some(1_000_000), - deadline_at: Some(moa_test_support::fixtures::pg_now() + chrono::Duration::hours(1)), - } -} - fn output_candidate( objective: &str, max_attempts: u32, @@ -2076,6 +891,12 @@ fn output_candidate( goal: single_requirement_goal(objective), plan: ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, + input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { + expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { + delay_seconds: 86_400, + }, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + }, input_schema: empty_object_schema(), output_schema: schema.clone(), nodes: vec![ExecutionNode { @@ -2105,6 +926,12 @@ fn replan_candidate(objective: &str) -> GeneratedExecutionCandidate { goal: replan_goal(objective), plan: ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, + input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { + expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { + delay_seconds: 86_400, + }, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + }, input_schema: empty_object_schema(), output_schema: schema.clone(), nodes: vec![ diff --git a/crates/moa-orchestrator/tests/integration/action_policy_flow_e2e.rs b/crates/moa-orchestrator/tests/integration/action_policy_flow_e2e.rs index f25c67cc1..316599874 100644 --- a/crates/moa-orchestrator/tests/integration/action_policy_flow_e2e.rs +++ b/crates/moa-orchestrator/tests/integration/action_policy_flow_e2e.rs @@ -1046,6 +1046,12 @@ async fn insert_execution_review_task( let plan = CanonicalExecutionPlan { definition: ExecutionPlanDefinition { cancel_policy: ExecutionCancelPolicy::RetainEffects, + input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { + expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::At { + at: chrono::Utc::now() + chrono::TimeDelta::hours(1), + }, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + }, input_schema: json!({ "type": "object" }), output_schema: json!({ "type": "object" }), nodes: Vec::new(), @@ -1169,6 +1175,7 @@ async fn insert_execution_review_task( run_uid, task_uid, generation: 1, + attempt_generation: 1, }) } diff --git a/crates/moa-orchestrator/tests/long_horizon_execution_canary_live.rs b/crates/moa-orchestrator/tests/long_horizon_execution_canary_live.rs new file mode 100644 index 000000000..c45f16b76 --- /dev/null +++ b/crates/moa-orchestrator/tests/long_horizon_execution_canary_live.rs @@ -0,0 +1,267 @@ +//! Opt-in, unbilled 24-hour and seven-day long-horizon invariant canaries. + +use std::time::Duration; + +use anyhow::{Context, Result, bail}; +use moa_test_support::OrchestratorTestFixture; +use serde_json::{Value, json}; +use sqlx::PgPool; +use tokio::time::Instant; +use uuid::Uuid; + +const SAMPLE_PERIOD: Duration = Duration::from_secs(60); +const RESTATE_KEY_BATCH_SIZE: usize = 250; + +#[tokio::test] +#[ignore = "requires MOA_RUN_LONG_HORIZON_CANARY=1 and MOA_LONG_HORIZON_CANARY_WINDOW=24h"] +async fn deployed_long_horizon_invariants_hold_for_24_hours_live() -> Result<()> { + // Pins: an explicitly selected external deployment is sampled for a full + // 24 hours; an instantaneous healthy sample cannot satisfy this canary. + if !canary_selected("24h")? { + return Ok(()); + } + run_canary(Duration::from_secs(24 * 60 * 60)).await +} + +#[tokio::test] +#[ignore = "requires MOA_RUN_LONG_HORIZON_CANARY=1 and MOA_LONG_HORIZON_CANARY_WINDOW=7d"] +async fn deployed_long_horizon_invariants_hold_for_seven_days_live() -> Result<()> { + // Pins: the seven-day deployment soak continuously rejects overdue runs, + // parked compute ownership, and still-live attempt invocations. + if !canary_selected("7d")? { + return Ok(()); + } + run_canary(Duration::from_secs(7 * 24 * 60 * 60)).await +} + +fn canary_selected(expected: &str) -> Result { + if std::env::var("MOA_RUN_LONG_HORIZON_CANARY").as_deref() != Ok("1") { + return Ok(false); + } + let selected = std::env::var("MOA_LONG_HORIZON_CANARY_WINDOW").context( + "MOA_RUN_LONG_HORIZON_CANARY=1 requires MOA_LONG_HORIZON_CANARY_WINDOW=24h or 7d", + )?; + if selected != "24h" && selected != "7d" { + bail!("MOA_LONG_HORIZON_CANARY_WINDOW must be exactly 24h or 7d"); + } + Ok(selected == expected) +} + +async fn run_canary(window: Duration) -> Result<()> { + if let Some((provider_flag, _)) = std::env::vars() + .find(|(name, value)| name.starts_with("MOA_RUN_LIVE_") && value.trim() == "1") + { + bail!("long-horizon canary refuses live integration flag {provider_flag}=1"); + } + for required in [ + "MOA_DATABASE_URL", + "MOA_RESTATE_INGRESS_URL", + "RESTATE_ADMIN_URL", + ] { + if std::env::var(required) + .ok() + .is_none_or(|value| value.trim().is_empty()) + { + bail!("long-horizon canary requires external deployment variable {required}"); + } + } + // Resolve the fixture only after proving external discovery is configured; + // otherwise `shared` could create an empty disposable stack and false-pass. + let fixture = OrchestratorTestFixture::shared().await?; + let pool = PgPool::connect(&fixture.postgres_url).await?; + let deadline = Instant::now() + window; + loop { + assert_deployment_invariants(&fixture, &pool).await?; + let now = Instant::now(); + if now >= deadline { + return Ok(()); + } + tokio::time::sleep(SAMPLE_PERIOD.min(deadline - now)).await; + } +} + +async fn assert_deployment_invariants( + fixture: &OrchestratorTestFixture, + pool: &PgPool, +) -> Result<()> { + let has_overdue: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM moa.execution_run \ + WHERE status NOT IN ('completed', 'partial', 'blocked', 'unsupported', 'failed', 'cancelled') \ + AND budget_deadline_at <= now())", + ) + .fetch_one(pool) + .await?; + let has_parked_with_invalid_receipt_count: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM ( \ + SELECT run.run_uid \ + FROM moa.execution_run AS run \ + LEFT JOIN moa.execution_capacity_reservation AS reservation \ + ON reservation.run_uid = run.run_uid \ + AND reservation.resource_dimension = 'parked_runs' \ + AND reservation.state <> 'released' \ + WHERE run.status IN ('waiting_input', 'waiting_review', 'waiting_signal', \ + 'waiting_timer', 'waiting_external', 'paused') \ + GROUP BY run.run_uid HAVING COUNT(reservation.reservation_uid) <> 1 \ + ) AS invalid_parked_receipts)", + ) + .fetch_one(pool) + .await?; + let has_parked_with_attempt_capacity: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM moa.execution_run AS run \ + JOIN moa.execution_capacity_reservation AS reservation \ + ON reservation.run_uid = run.run_uid \ + WHERE run.status IN ('waiting_input', 'waiting_review', 'waiting_signal', \ + 'waiting_timer', 'waiting_external', 'paused') \ + AND reservation.resource_dimension = 'active_tasks' \ + AND reservation.state <> 'released')", + ) + .fetch_one(pool) + .await?; + let has_parked_with_active_hands: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM moa.execution_run AS run \ + JOIN moa.sandbox_workspaces AS workspace \ + ON workspace.tenant_id = run.tenant_id \ + AND workspace.scope_kind = 'execution_task' \ + AND workspace.scope_run_id = run.run_uid \ + JOIN moa.sandbox_capacity_reservations AS reservation \ + ON reservation.tenant_id = workspace.tenant_id \ + AND reservation.workspace_id = workspace.workspace_id \ + WHERE run.status IN ('waiting_input', 'waiting_review', 'waiting_signal', \ + 'waiting_timer', 'waiting_external', 'paused') \ + AND reservation.resource_dimension = 'active_hands' \ + AND reservation.reservation_state <> 'released')", + ) + .fetch_one(pool) + .await?; + let has_parked_with_active_dispatch: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM moa.execution_run AS run \ + JOIN moa.execution_task AS task ON task.run_uid = run.run_uid \ + WHERE run.status IN ('waiting_input', 'waiting_review', 'waiting_signal', \ + 'waiting_timer', 'waiting_external', 'paused') \ + AND (task.active_dispatch_uid IS NOT NULL \ + OR task.attempt_state IN ('dispatching', 'running', 'cancelling')))", + ) + .fetch_one(pool) + .await?; + assert_no_live_parked_attempts(fixture, pool).await?; + assert_no_live_parked_controllers(fixture, pool).await?; + assert!(!has_overdue, "nonterminal execution runs exceeded deadline"); + assert!( + !has_parked_with_invalid_receipt_count, + "parked execution runs did not own exactly one ParkedRuns receipt" + ); + assert!( + !has_parked_with_attempt_capacity, + "parked execution runs retained active task capacity" + ); + assert!( + !has_parked_with_active_hands, + "parked execution runs retained active sandbox hands" + ); + assert!( + !has_parked_with_active_dispatch, + "parked execution runs retained active task dispatch state" + ); + Ok(()) +} + +async fn assert_no_live_parked_attempts( + fixture: &OrchestratorTestFixture, + pool: &PgPool, +) -> Result<()> { + let mut cursor: Option = None; + loop { + let keys: Vec = sqlx::query_scalar( + "SELECT DISTINCT dispatch.dispatch_uid \ + FROM moa.execution_dispatch_outbox AS dispatch \ + JOIN moa.execution_run AS run ON run.run_uid = dispatch.run_uid \ + WHERE run.status IN ('waiting_input', 'waiting_review', 'waiting_signal', \ + 'waiting_timer', 'waiting_external', 'paused') \ + AND dispatch.dispatch_kind IN ('task_attempt', 'compensation_attempt') \ + AND ($1::UUID IS NULL OR dispatch.dispatch_uid > $1) \ + ORDER BY dispatch.dispatch_uid LIMIT $2", + ) + .bind(cursor) + .bind(i64::try_from(RESTATE_KEY_BATCH_SIZE)?) + .fetch_all(pool) + .await?; + if keys.is_empty() { + return Ok(()); + } + assert_no_live_restate_invocations( + fixture, + &keys, + "target_service_name IN ('ExecutionTaskAttempt', 'ExecutionCompensationAttempt')", + "parked attempt", + ) + .await?; + cursor = keys.last().copied(); + } +} + +async fn assert_no_live_parked_controllers( + fixture: &OrchestratorTestFixture, + pool: &PgPool, +) -> Result<()> { + let mut cursor: Option = None; + loop { + let keys: Vec = sqlx::query_scalar( + "SELECT run_uid FROM moa.execution_run \ + WHERE status IN ('waiting_input', 'waiting_review', 'waiting_signal', \ + 'waiting_timer', 'waiting_external', 'paused') \ + AND ($1::UUID IS NULL OR run_uid > $1) \ + ORDER BY run_uid LIMIT $2", + ) + .bind(cursor) + .bind(i64::try_from(RESTATE_KEY_BATCH_SIZE)?) + .fetch_all(pool) + .await?; + if keys.is_empty() { + return Ok(()); + } + assert_no_live_restate_invocations( + fixture, + &keys, + "target_service_name = 'ExecutionRunController'", + "parked run controller", + ) + .await?; + cursor = keys.last().copied(); + } +} + +async fn assert_no_live_restate_invocations( + fixture: &OrchestratorTestFixture, + keys: &[Uuid], + service_predicate: &str, + owner_kind: &str, +) -> Result<()> { + let quoted_keys = keys + .iter() + .map(|key| format!("'{key}'")) + .collect::>() + .join(", "); + let query = format!( + "SELECT id, target_service_name, target_service_key, status \ + FROM sys_invocation WHERE {service_predicate} \ + AND target_service_key IN ({quoted_keys}) \ + AND status NOT IN ('completed', 'killed') LIMIT 1" + ); + let response = reqwest::Client::new() + .post(format!("{}/query", fixture.admin_url.trim_end_matches('/'))) + .header(reqwest::header::ACCEPT, "application/json") + .json(&json!({"query": query})) + .send() + .await? + .error_for_status()? + .json::() + .await?; + let rows = response + .get("rows") + .and_then(Value::as_array) + .context("Restate canary query omitted rows")?; + if !rows.is_empty() { + bail!("{owner_kind} retained a nonterminal Restate invocation: {rows:?}"); + } + Ok(()) +} diff --git a/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e.rs b/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e.rs new file mode 100644 index 000000000..5a3a71bb1 --- /dev/null +++ b/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e.rs @@ -0,0 +1,963 @@ +//! Deterministic accelerated long-horizon execution coverage over real services. +//! +//! Eight logical days are compressed to sixteen real seconds. All temporal work +//! still uses the production compiler, PostgreSQL trigger/outbox rows, Restate +//! delayed delivery, and database clock; no fake clock or direct state mutation +//! advances a run. + +#[path = "long_horizon_execution_service_e2e/accelerated_week.rs"] +mod accelerated_week; +#[path = "long_horizon_execution_service_e2e/burst_admission.rs"] +mod burst_admission; +#[path = "long_horizon_execution_service_e2e/deadline_and_waits.rs"] +mod deadline_and_waits; +#[path = "long_horizon_execution_service_e2e/deployment_drain.rs"] +mod deployment_drain; +#[path = "long_horizon_execution_service_e2e/disaster_recovery.rs"] +mod disaster_recovery; +#[path = "long_horizon_execution_service_e2e/pause_and_external.rs"] +mod pause_and_external; + +use std::time::Duration; + +use anyhow::{Context, Result, bail}; +use chrono::{DateTime, TimeDelta, Utc}; +use moa_artifacts::execution_plan::{ + CapabilityReference, CompletionCheck, CompletionCheckKind, ExecutionBudgetLimit, + ExecutionCancelPolicy, ExecutionGoalContract, ExecutionNode, ExecutionOperation, + ExecutionPlanDefinition, ExecutionRequirement, ExecutionTemporalTarget, + ExecutionWaitExpiryAction, ExecutionWaitPolicy, RetryPolicy, +}; +use moa_config::ExecutionConfig; +use moa_core::{ + events::Event, + types::{ + action_policy::ActionPolicyEffect, + execution_planning::{ExecutionSourceProvenance, GeneratedPlanPlannerProvenance}, + identifiers::TenantId, + }, +}; +use moa_execution::{ + capability::{ + CapabilitySource, ExecutionAuthorizationEnvelope, ExecutionCapability, + ExecutionCapabilityCatalog, + }, + compiler::{CompileExecutionRequest, compile}, + state::{ExecutionRunStatus, ExecutionTaskId, ExecutionTaskProjection, ExecutionTaskStatus}, + wire::{ + ExecutionPlanningContextRequest, ExecutionPlanningContextResponse, ExecutionRunRequest, + ExecutionStartRequest, ExecutionStartResponse, ExecutionStatusResponse, + ExecutionTaskListRequest, ExecutionTaskListResponse, + }, +}; +use moa_orchestrator::services::action_policy::UpsertActionPolicyRuleRequest; +use moa_test_support::{ + FixtureCapabilityOptions, FixtureCapabilityOutcome, FixtureCapabilityTool, IsolatedTest, + OrchestratorTestFixture, +}; +use serde_json::{Value, json}; +use sqlx::{PgPool, Row}; +use tokio::time::Instant; +use uuid::Uuid; + +/// Two real seconds represent one logical day in this deterministic lane. +const LOGICAL_DAY: Duration = Duration::from_secs(2); +/// Maximum real wait for one compressed scenario transition. +const SCENARIO_TIMEOUT: Duration = Duration::from_secs(45); +/// Poll cadence used only to observe durable state transitions. +const POLL_INTERVAL: Duration = Duration::from_millis(50); +const FIXTURE_CAPABILITY_VERSION: &str = "__fixture_current__"; +const HAND_CAPABILITY_VERSION: &str = "__hand_current__"; +const SANDBOX_TENANT_UUID: Uuid = Uuid::from_u128(0x2000_0000_0000_0000_0000_0000_0000_0001); +const SANDBOX_PROVIDER_ACCOUNT_UUID: Uuid = + Uuid::from_u128(0x3000_0000_0000_0000_0000_0000_0000_0012); + +#[derive(Clone)] +struct StartedRun { + request: ExecutionRunRequest, + tenant_id: TenantId, + run_uid: Uuid, +} + +fn fixture_script() -> Value { + json!({ + "default": { + "completion": { + "content": "fixture-only", + "duration_ms": 1, + "input_tokens": 1, + "cached_input_tokens": 0, + "cache_write_input_tokens": 0, + "tool_calls": [] + } + } + }) +} + +async fn execution_fixture(extra_env: Vec<(String, String)>) -> Result { + execution_fixture_with_script(fixture_script(), extra_env).await +} + +async fn execution_fixture_with_script( + script: Value, + extra_env: Vec<(String, String)>, +) -> Result { + OrchestratorTestFixture::with_execution_fixture( + script, + FixtureCapabilityOptions { + tools: Vec::new(), + orchestrator_env: extra_env, + }, + ) + .await +} + +async fn execution_fixture_with_tools( + tools: Vec, + extra_env: Vec<(String, String)>, +) -> Result { + execution_fixture_with_script_and_tools(fixture_script(), tools, extra_env).await +} + +async fn external_job_execution_fixture( + extra_env: Vec<(String, String)>, +) -> Result { + OrchestratorTestFixture::with_external_job_execution_fixture(fixture_script(), extra_env).await +} + +async fn execution_fixture_with_script_and_tools( + script: Value, + tools: Vec, + extra_env: Vec<(String, String)>, +) -> Result { + OrchestratorTestFixture::with_execution_fixture( + script, + FixtureCapabilityOptions { + tools, + orchestrator_env: extra_env, + }, + ) + .await +} + +async fn sandbox_execution_fixture() -> Result { + let provider_account_id = SANDBOX_PROVIDER_ACCOUNT_UUID; + let tenant_id = SANDBOX_TENANT_UUID; + OrchestratorTestFixture::with_sandbox_workspace_execution_fixture( + fixture_script(), + FixtureCapabilityOptions { + tools: Vec::new(), + orchestrator_env: vec![ + ( + "MOA_LOCAL_PROVIDER_ACCOUNT_JSON".to_string(), + json!({ + "provider_account_id": provider_account_id, + "generation": 1, + "isolation_cell": "long-horizon-task12" + }) + .to_string(), + ), + ("MOA_LOCAL_DOCKER_ENABLED".to_string(), "false".to_string()), + ( + "MOA_SANDBOX_WORKSPACE_MODE".to_string(), + "admit".to_string(), + ), + ( + "MOA_SANDBOX_WORKSPACE_CANARY_JSON".to_string(), + json!({ + "provider_account_id": provider_account_id, + "provider_account_generation": 1, + "isolation_cell": "long-horizon-task12", + "tenant_allowlist": [tenant_id] + }) + .to_string(), + ), + ( + "MOA_SANDBOX_WORKSPACE_QUOTA_ROUTES_JSON".to_string(), + json!([{ + "tenant_id": tenant_id, + "provider_account_id": provider_account_id, + "provider_account_generation": 1, + "max_workspaces": 8, + "max_active_hands": 2, + "max_checkpoints": 32, + "max_logical_bytes": 268_435_456_u64 + }]) + .to_string(), + ), + ( + "MOA_AUTHZ_OPENFGA_MODEL_VERSION".to_string(), + "7".to_string(), + ), + ], + }, + ) + .await +} + +fn after_logical_days(days: u64) -> ExecutionTemporalTarget { + ExecutionTemporalTarget::After { + delay_seconds: LOGICAL_DAY.as_secs().saturating_mul(days), + } +} + +fn continue_wait(days: u64, output: Value) -> ExecutionWaitPolicy { + ExecutionWaitPolicy { + expiry: after_logical_days(days), + on_expiry: ExecutionWaitExpiryAction::ContinueWith { output }, + } +} + +fn fixture_input_wait_policy( + compile_now: DateTime, + admitted_deadline_at: DateTime, +) -> Result { + let remaining_seconds = admitted_deadline_at + .signed_duration_since(compile_now) + .to_std() + .context("fixture execution deadline already elapsed before compilation")? + .as_secs(); + if remaining_seconds < 2 { + bail!( + "fixture execution deadline leaves no whole-second input wait strictly inside its horizon" + ); + } + Ok(ExecutionWaitPolicy { + expiry: ExecutionTemporalTarget::After { + delay_seconds: remaining_seconds / 2, + }, + on_expiry: ExecutionWaitExpiryAction::FailRun, + }) +} + +fn node( + id: &str, + depends_on: &[&str], + operation: ExecutionOperation, + output_schema: Value, +) -> ExecutionNode { + ExecutionNode { + id: id.to_string(), + requirement_ids: vec!["result".to_string()], + depends_on: depends_on.iter().map(|id| (*id).to_string()).collect(), + when: None, + input: json!({}), + output_schema, + operation, + compensation: None, + retry: RetryPolicy { + max_attempts: 2, + initial_backoff_ms: 50, + max_backoff_ms: 50, + }, + budget: None, + } +} + +fn output_node(depends_on: &[&str], value: Value) -> ExecutionNode { + node( + "output", + depends_on, + ExecutionOperation::Output { + value: value.clone(), + }, + json!({"type": "object"}), + ) +} + +fn fixture_capability_node(id: &str, tool_name: &str, input: Value) -> ExecutionNode { + let mut capability = node( + id, + &[], + ExecutionOperation::Capability { + reference: CapabilityReference { + name: moa_hands::mcp_tool_reference("fixture-capability", tool_name), + version: FIXTURE_CAPABILITY_VERSION.to_string(), + }, + }, + json!({"type": "object"}), + ); + capability.input = input; + capability +} + +fn external_job_capability_node(id: &str, input: Value) -> ExecutionNode { + let mut capability = node( + id, + &[], + ExecutionOperation::Capability { + reference: CapabilityReference { + name: "fixture_external_job".to_string(), + version: FIXTURE_CAPABILITY_VERSION.to_string(), + }, + }, + json!({"type": "object"}), + ); + capability.input = input; + capability +} + +fn hand_capability_node( + id: &str, + depends_on: &[&str], + tool_name: &str, + input: Value, +) -> ExecutionNode { + let mut capability = node( + id, + depends_on, + ExecutionOperation::Capability { + reference: CapabilityReference { + name: tool_name.to_string(), + version: HAND_CAPABILITY_VERSION.to_string(), + }, + }, + json!({"type": "string"}), + ); + capability.input = input; + capability +} + +async fn start_plan( + test: &IsolatedTest<'_>, + label: &str, + nodes: Vec, + deadline_after: Duration, +) -> Result { + start_plan_with_capability_policy( + test, + label, + nodes, + deadline_after, + Some(ActionPolicyEffect::Allow), + ) + .await +} + +async fn start_plan_with_policy( + test: &IsolatedTest<'_>, + label: &str, + nodes: Vec, + deadline_after: Duration, + configure_capability_policy: bool, +) -> Result { + start_plan_with_capability_policy( + test, + label, + nodes, + deadline_after, + configure_capability_policy.then_some(ActionPolicyEffect::Allow), + ) + .await +} + +async fn start_plan_with_capability_policy( + test: &IsolatedTest<'_>, + label: &str, + mut nodes: Vec, + deadline_after: Duration, + capability_policy: Option, +) -> Result { + let session_id = test.create_session(label).await?; + let session = test.client().get_session(session_id).await?; + let objective = format!("deterministic long-horizon scenario {label}"); + let origin = test + .client() + .append_event( + session_id, + Event::UserMessage { + text: objective.clone(), + attachments: Vec::new(), + }, + ) + .await?; + let requested_at = moa_test_support::fixtures::pg_now(); + let deadline = requested_at + + TimeDelta::from_std(deadline_after).context("convert scenario deadline to chrono")?; + let mut policy_tool_names = nodes + .iter() + .flat_map(|node| match &node.operation { + ExecutionOperation::Capability { reference } + if reference.version == FIXTURE_CAPABILITY_VERSION + || reference.version == HAND_CAPABILITY_VERSION => + { + vec![reference.name.clone()] + } + ExecutionOperation::Agent { + capability_refs, .. + } => capability_refs + .iter() + .filter(|reference| { + reference.version == FIXTURE_CAPABILITY_VERSION + || reference.version == HAND_CAPABILITY_VERSION + }) + .map(|reference| reference.name.clone()) + .collect(), + _ => Vec::new(), + }) + .collect::>(); + policy_tool_names.sort(); + policy_tool_names.dedup(); + if capability_policy.is_some() && !policy_tool_names.is_empty() { + test.fixture + .grant_default_tenant_admin(session.tenant_id) + .await + .context("grant tenant admin before Task 12 capability policy upsert")?; + } + if let Some(effect) = capability_policy { + for capability_name in &policy_tool_names { + set_fixture_capability_policy( + test.fixture, + session.tenant_id, + capability_name, + effect, + label, + ) + .await?; + } + } + let planning: ExecutionPlanningContextResponse = test + .client() + .post_call( + "/Execution/planning_context", + &ExecutionPlanningContextRequest { + tenant_id: session.tenant_id, + contact_id: None, + session_id, + originating_user_sequence_num: origin, + deadline_at: deadline, + requested_template: None, + }, + ) + .await?; + for node in &mut nodes { + match &mut node.operation { + ExecutionOperation::Capability { reference } => { + resolve_capability_placeholder(reference, &planning.snapshot.catalog.capabilities)? + } + ExecutionOperation::Agent { + capability_refs, .. + } => { + for reference in capability_refs { + resolve_capability_placeholder( + reference, + &planning.snapshot.catalog.capabilities, + )?; + } + } + _ => {} + } + } + let compile_now = moa_test_support::fixtures::pg_now(); + let admitted_deadline_at = planning + .snapshot + .budget + .deadline_at + .context("planning context omitted its admitted execution deadline")?; + let goal = ExecutionGoalContract { + objective, + requirements: vec![ExecutionRequirement { + id: "result".to_string(), + description: "produce the deterministic terminal object".to_string(), + }], + deliverables: Vec::new(), + coverage: Vec::new(), + constraints: Vec::new(), + completion_checks: vec![CompletionCheck { + id: "output-schema".to_string(), + description: "terminal output matches the declared schema".to_string(), + requirement_ids: vec!["result".to_string()], + constraint_ids: Vec::new(), + kind: CompletionCheckKind::OutputSchema, + }], + }; + let plan = ExecutionPlanDefinition { + cancel_policy: ExecutionCancelPolicy::RetainEffects, + input_wait_policy: fixture_input_wait_policy(compile_now, admitted_deadline_at)?, + input_schema: json!({"type": "object", "additionalProperties": false}), + output_schema: json!({"type": "object"}), + nodes, + }; + let outcome = compile(CompileExecutionRequest { + goal, + plan, + run_input: json!({}), + catalog: planning.snapshot.catalog.clone(), + authorization: planning.snapshot.authorization.clone(), + approved_budget: planning.snapshot.budget.clone(), + config: ExecutionConfig::default(), + now: compile_now, + }); + let compiled = outcome.compiled.with_context(|| { + format!( + "long-horizon plan `{label}` should compile: {:?}", + outcome.report.issues + ) + })?; + let source_provenance = ExecutionSourceProvenance::GeneratedPlan { + planner: GeneratedPlanPlannerProvenance { + model: "fixture-only".to_string(), + prompt_version: "long-horizon-execution-v1".to_string(), + candidate_hash: "a".repeat(64), + compiler_report_hash: "b".repeat(64), + final_plan_hash: compiled.plan.plan_hash.to_string(), + repair_attempts: 0, + }, + }; + let started: ExecutionStartResponse = test + .client() + .post_call( + "/Execution/start", + &ExecutionStartRequest { + tenant_id: session.tenant_id, + contact_id: None, + session_id, + originating_user_sequence_num: origin, + planning_context_uid: planning.planning_context_uid, + planning_context_hash: planning.planning_context_hash, + idempotency_key: Some(format!("long-horizon-{label}-{session_id}")), + compiled, + run_input: json!({}), + source_provenance, + }, + ) + .await?; + // Restate may replay this exact request after `Execution.start` committed its DB transaction + // but the handler suspended before returning. The stable idempotency key then returns the same + // durable run with `created=false`; confirmation is the only non-admitted response here. + require_unconfirmed_start_admission(label, started.created, started.confirmation_required)?; + let request = ExecutionRunRequest { + tenant_id: session.tenant_id, + contact_id: None, + session_id, + run_uid: started.run.run_uid, + }; + Ok(StartedRun { + request, + tenant_id: session.tenant_id, + run_uid: started.run.run_uid, + }) +} + +fn require_unconfirmed_start_admission( + label: &str, + created: bool, + confirmation_required: bool, +) -> Result<()> { + if confirmation_required { + bail!( + "scenario `{label}` unexpectedly required confirmation: created={created}, \ + confirmation_required={confirmation_required}" + ); + } + Ok(()) +} + +fn resolve_capability_placeholder( + reference: &mut CapabilityReference, + capabilities: &[ExecutionCapability], +) -> Result<()> { + if reference.version != FIXTURE_CAPABILITY_VERSION + && reference.version != HAND_CAPABILITY_VERSION + { + return Ok(()); + } + let resolved = capabilities + .iter() + .find(|capability| match reference.version.as_str() { + FIXTURE_CAPABILITY_VERSION => capability.reference.name == reference.name, + HAND_CAPABILITY_VERSION => matches!( + &capability.source, + CapabilitySource::HandTool { name } if name == &reference.name + ), + _ => false, + }) + .with_context(|| { + format!( + "planning catalog omitted Task 12 capability `{}` with placeholder version `{}`", + reference.name, reference.version + ) + })?; + *reference = resolved.reference.clone(); + Ok(()) +} + +async fn allow_fixture_capability( + fixture: &OrchestratorTestFixture, + tenant_id: TenantId, + capability_name: &str, + reason: &str, +) -> Result<()> { + set_fixture_capability_policy( + fixture, + tenant_id, + capability_name, + ActionPolicyEffect::Allow, + reason, + ) + .await +} + +async fn set_fixture_capability_policy( + fixture: &OrchestratorTestFixture, + tenant_id: TenantId, + capability_name: &str, + effect: ActionPolicyEffect, + reason: &str, +) -> Result<()> { + fixture + .client + .post_void( + "/ActionPolicy/upsert_rule", + &UpsertActionPolicyRuleRequest { + tenant_id, + contact_id: None, + tool_name: capability_name.to_string(), + pattern: "*".to_string(), + effect, + reason: Some(format!("Task 12 deterministic capability for {reason}")), + }, + ) + .await +} + +async fn status(test: &IsolatedTest<'_>, run: &StartedRun) -> Result { + test.client() + .post_call("/Execution/status", &run.request) + .await +} + +async fn await_run_status( + test: &IsolatedTest<'_>, + run: &StartedRun, + expected: ExecutionRunStatus, +) -> Result { + let deadline = Instant::now() + SCENARIO_TIMEOUT; + loop { + let current = status(test, run).await?; + if current.run.status == expected { + return Ok(current); + } + if current.run.status.is_terminal() || Instant::now() >= deadline { + bail!( + "run {} did not reach {expected:?}; current={:?}, waiting={:?}", + run.run_uid, + current.run.status, + current.waiting + ); + } + tokio::time::sleep(POLL_INTERVAL).await; + } +} + +async fn tasks(test: &IsolatedTest<'_>, run: &StartedRun) -> Result> { + let response: ExecutionTaskListResponse = test + .client() + .post_call( + "/Execution/list_tasks", + &ExecutionTaskListRequest { + run: run.request.clone(), + limit: Some(100), + cursor: None, + }, + ) + .await?; + Ok(response.tasks) +} + +async fn await_task_status( + test: &IsolatedTest<'_>, + run: &StartedRun, + node_id: &str, + expected: ExecutionTaskStatus, +) -> Result { + let deadline = Instant::now() + SCENARIO_TIMEOUT; + loop { + let current = tasks(test, run).await?; + if let Some(task) = current + .iter() + .find(|task| task.node_id == node_id && task.status == expected) + { + return Ok(task.clone()); + } + if Instant::now() >= deadline { + bail!( + "node `{node_id}` in run {} did not reach {expected:?}; tasks={current:?}", + run.run_uid + ); + } + tokio::time::sleep(POLL_INTERVAL).await; + } +} + +async fn assert_parked_has_no_active_compute( + fixture: &OrchestratorTestFixture, + pool: &PgPool, + run: &StartedRun, +) -> Result<()> { + let deadline = Instant::now() + SCENARIO_TIMEOUT; + let (parked_run_reservations, active_attempt_reservations, active_dispatches, active_hands) = loop { + let parked_run_reservations: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM moa.execution_capacity_reservation \ + WHERE run_uid = $1 AND resource_dimension = 'parked_runs' AND state <> 'released'", + ) + .bind(run.run_uid) + .fetch_one(pool) + .await?; + let active_attempt_reservations: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM moa.execution_capacity_reservation \ + WHERE run_uid = $1 AND resource_dimension = 'active_tasks' AND state <> 'released'", + ) + .bind(run.run_uid) + .fetch_one(pool) + .await?; + let active_dispatches: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM moa.execution_task \ + WHERE run_uid = $1 AND (active_dispatch_uid IS NOT NULL OR attempt_state IN ('dispatching', 'running'))", + ) + .bind(run.run_uid) + .fetch_one(pool) + .await?; + let active_hands: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM moa.sandbox_capacity_reservations AS reservation \ + JOIN moa.sandbox_workspaces AS workspace \ + ON workspace.tenant_id = reservation.tenant_id \ + AND workspace.workspace_id = reservation.workspace_id \ + WHERE workspace.scope_kind = 'execution_task' \ + AND workspace.scope_run_id = $1 \ + AND reservation.resource_dimension = 'active_hands' \ + AND reservation.reservation_state <> 'released'", + ) + .bind(run.run_uid) + .fetch_one(pool) + .await?; + if parked_run_reservations == 1 + && active_attempt_reservations == 0 + && active_dispatches == 0 + && active_hands == 0 + { + break ( + parked_run_reservations, + active_attempt_reservations, + active_dispatches, + active_hands, + ); + } + if Instant::now() >= deadline { + bail!( + "run {} did not finish parking; parked_runs={parked_run_reservations}, \ + active_tasks={active_attempt_reservations}, active_dispatches={active_dispatches}, \ + active_hands={active_hands}", + run.run_uid + ); + } + tokio::time::sleep(POLL_INTERVAL).await; + }; + let attempt_dispatches: Vec = sqlx::query_scalar( + "SELECT dispatch_uid FROM moa.execution_dispatch_outbox \ + WHERE run_uid = $1 AND dispatch_kind = 'task_attempt'", + ) + .bind(run.run_uid) + .fetch_all(pool) + .await?; + let dispatch_keys = attempt_dispatches + .iter() + .map(|dispatch_uid| format!("'{dispatch_uid}'")) + .collect::>() + .join(", "); + let attempt_clause = if dispatch_keys.is_empty() { + "false".to_string() + } else { + format!( + "target_service_name = 'ExecutionTaskAttempt' AND target_service_key IN ({dispatch_keys})" + ) + }; + let invocation_query = format!( + "SELECT id FROM sys_invocation WHERE \ + ((target_service_name = 'ExecutionRunController' AND target_service_key = '{}') \ + OR ({attempt_clause})) AND status NOT IN ('completed', 'killed')", + run.run_uid, + ); + let invocations = loop { + let invocations = restate_rows(fixture, &invocation_query).await?; + if invocations.is_empty() || Instant::now() >= deadline { + break invocations; + } + tokio::time::sleep(POLL_INTERVAL).await; + }; + assert_eq!( + parked_run_reservations, 1, + "a durably parked run must own exactly one ParkedRuns receipt" + ); + assert_eq!( + active_attempt_reservations, 0, + "parked run retained attempt capacity" + ); + assert_eq!( + active_dispatches, 0, + "parked run retained an active dispatch" + ); + assert_eq!( + active_hands, 0, + "parked tenant retained a live sandbox hand" + ); + assert!( + invocations.is_empty(), + "parked run retained continuing compute invocations: {invocations:?}" + ); + Ok(()) +} + +async fn restate_rows(fixture: &OrchestratorTestFixture, query: &str) -> Result> { + let response = reqwest::Client::new() + .post(format!("{}/query", fixture.admin_url.trim_end_matches('/'))) + .header(reqwest::header::ACCEPT, "application/json") + .json(&json!({"query": query})) + .send() + .await? + .error_for_status()? + .json::() + .await?; + response + .get("rows") + .and_then(Value::as_array) + .cloned() + .context("Restate query omitted rows") +} + +async fn persisted_trigger_rows( + pool: &PgPool, + run_uid: Uuid, +) -> Result, i64)>> { + let rows = sqlx::query( + "SELECT trigger_kind, due_at, COALESCE(attempt_generation, controller_generation, 0) AS generation \ + FROM moa.execution_trigger WHERE run_uid = $1 ORDER BY due_at, trigger_uid", + ) + .bind(run_uid) + .fetch_all(pool) + .await?; + rows.into_iter() + .map(|row| { + Ok(( + row.try_get("trigger_kind")?, + row.try_get("due_at")?, + row.try_get("generation")?, + )) + }) + .collect() +} + +fn task_id(task: &ExecutionTaskProjection) -> ExecutionTaskId { + task.task_id +} + +#[cfg(test)] +mod fixture_contract_tests { + use super::*; + + fn fixed_now() -> DateTime { + DateTime::parse_from_rfc3339("2026-08-11T12:00:00Z") + .expect("fixture timestamp should parse") + .with_timezone(&Utc) + } + + fn output_only_compile_request( + now: DateTime, + deadline_at: DateTime, + ) -> CompileExecutionRequest { + let goal = ExecutionGoalContract { + objective: "deterministic fixture output".to_string(), + requirements: vec![ExecutionRequirement { + id: "result".to_string(), + description: "produce deterministic output".to_string(), + }], + deliverables: Vec::new(), + coverage: Vec::new(), + constraints: Vec::new(), + completion_checks: vec![CompletionCheck { + id: "output-schema".to_string(), + description: "validate terminal output".to_string(), + requirement_ids: vec!["result".to_string()], + constraint_ids: Vec::new(), + kind: CompletionCheckKind::OutputSchema, + }], + }; + CompileExecutionRequest { + goal, + plan: ExecutionPlanDefinition { + cancel_policy: ExecutionCancelPolicy::RetainEffects, + input_wait_policy: fixture_input_wait_policy(now, deadline_at) + .expect("fixture horizon should admit an input wait"), + input_schema: json!({"type": "object", "additionalProperties": false}), + output_schema: json!({"type": "object"}), + nodes: vec![output_node(&[], json!({"status": "complete"}))], + }, + run_input: json!({}), + catalog: ExecutionCapabilityCatalog::build(Vec::new()) + .expect("empty fixture catalog should build"), + authorization: ExecutionAuthorizationEnvelope { + capability_refs: Vec::new(), + skill_refs: Vec::new(), + }, + approved_budget: ExecutionBudgetLimit { + max_cost_microusd: Some(1_000_000), + max_tokens: Some(100_000), + max_tasks: Some(100), + max_tool_calls: Some(100), + max_retrieved_bytes: Some(1_000_000), + deadline_at: Some(deadline_at), + }, + config: ExecutionConfig::default(), + now, + } + } + + #[test] + fn short_fixture_horizon_gets_a_strictly_bounded_input_wait_offline() { + // Pins: the shared fixture must not copy a day-scale default into a + // short plan whose admitted deadline is only three seconds away. + let now = fixed_now(); + let policy = fixture_input_wait_policy(now, now + TimeDelta::seconds(3)) + .expect("three-second fixture horizon should admit a wait"); + + assert_eq!( + policy.expiry, + ExecutionTemporalTarget::After { delay_seconds: 1 } + ); + } + + #[test] + fn fixture_compile_matches_server_validation_after_setup_skew_offline() { + // Pins: client compilation and server revalidation produce the exact + // same canonical plan/hash even after setup consumes part of the horizon. + let client_now = fixed_now(); + let deadline_at = client_now + TimeDelta::seconds(15); + let request = output_only_compile_request(client_now, deadline_at); + let client = compile(request.clone()) + .compiled + .expect("client fixture should compile"); + let mut server_request = request; + server_request.now = client_now + TimeDelta::seconds(3); + let server = compile(server_request) + .compiled + .expect("server validation should retain enough temporal slack"); + + assert_eq!(server, client); + assert_eq!(server.plan.plan_hash, client.plan.plan_hash); + } + + #[test] + fn start_fixture_accepts_created_or_idempotent_replayed_admission_offline() { + // Pins: `Execution.start` can return `created=false` when Restate re-enters the handler + // after the idempotent DB admission committed; both responses retain the admitted run, + // while either response still fails when explicit confirmation is required. + for created in [true, false] { + require_unconfirmed_start_admission("start-replay", created, false) + .expect("created and replayed starts should both be admitted"); + assert!( + require_unconfirmed_start_admission("start-replay", created, true).is_err(), + "confirmation-required start must not be treated as admitted when created={created}" + ); + } + } +} diff --git a/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/accelerated_week.rs b/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/accelerated_week.rs new file mode 100644 index 000000000..daa7098f0 --- /dev/null +++ b/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/accelerated_week.rs @@ -0,0 +1,313 @@ +//! Accelerated eight-logical-day wait and restart scenario. + +use super::*; + +#[tokio::test] +#[ignore = "requires Docker for the real Restate/Postgres/Valkey execution fixture"] +async fn accelerated_eight_day_waits_survive_repeated_process_and_valkey_restarts_service_e2e() +-> Result<()> { + // Pins: timer, review-expiry, and signal-expiry waits persist exact due order, + // release active compute, and resume once across repeated process/cache loss. + let fixture = execution_fixture(vec![( + "MOA_EXECUTION_TRIGGER_RECONCILIATION_CADENCE_SECONDS".to_string(), + "1".to_string(), + )]) + .await?; + let test = fixture.isolated().await; + let pool = PgPool::connect(&fixture.postgres_url).await?; + let run = start_plan( + &test, + "accelerated-week", + vec![ + node( + "day-two-timer", + &[], + ExecutionOperation::WaitUntil { + wake: after_logical_days(2), + result: json!({"day": 2}), + }, + json!({"type": "object"}), + ), + node( + "day-five-review", + &["day-two-timer"], + ExecutionOperation::Review { + prompt: "approve the logical-day-five checkpoint".to_string(), + wait_policy: continue_wait(3, json!({"approved_by_expiry": true})), + }, + json!({"type": "object"}), + ), + node( + "day-eight-signal", + &["day-five-review"], + ExecutionOperation::WaitSignal { + signal_name: "logical-week-close".to_string(), + wait_policy: continue_wait(3, json!({"closed_by_expiry": true})), + }, + json!({"type": "object"}), + ), + output_node(&["day-eight-signal"], json!({"week": "complete"})), + ], + Duration::from_secs(40), + ) + .await?; + + if let Err(error) = await_run_status(&test, &run, ExecutionRunStatus::WaitingTimer).await { + let run_state: (String, String, i64, i64, i64) = sqlx::query_as( + "SELECT status, activation_state, controller_generation, wake_epoch, \ + processed_wake_epoch FROM moa.execution_run WHERE run_uid=$1", + ) + .bind(run.run_uid) + .fetch_one(&pool) + .await?; + let dispatches: Vec<(String, String, i32, Option)> = sqlx::query_as( + "SELECT dispatch_kind, state, delivery_attempts, last_error \ + FROM moa.execution_dispatch_outbox WHERE run_uid=$1 ORDER BY created_at", + ) + .bind(run.run_uid) + .fetch_all(&pool) + .await?; + let nodes: Value = sqlx::query_scalar( + "SELECT COALESCE(jsonb_agg(jsonb_build_object( \ + 'node_id', node_id, 'status', node_status, \ + 'cursor', materialization_cursor, \ + 'remaining_dependencies', remaining_dependency_count) \ + ORDER BY node_order), '[]'::jsonb) \ + FROM moa.execution_node_state WHERE run_uid=$1", + ) + .bind(run.run_uid) + .fetch_one(&pool) + .await?; + let controller_invocations = restate_rows( + &fixture, + &format!( + "SELECT * FROM sys_invocation WHERE target_service_name = \ + 'ExecutionRunController' AND target_service_key = '{}'", + run.run_uid + ), + ) + .await + .unwrap_or_default(); + let controller_journal = restate_rows( + &fixture, + &format!( + "SELECT * FROM sys_journal WHERE id IN (SELECT id FROM sys_invocation \ + WHERE target_service_name = 'ExecutionRunController' \ + AND target_service_key = '{}')", + run.run_uid + ), + ) + .await + .unwrap_or_default(); + bail!( + "{error:#}; run_state={run_state:?}; dispatches={dispatches:?}; nodes={nodes}; \ + controller_invocations={controller_invocations:?}; \ + controller_journal={controller_journal:?}" + ); + } + await_task_status( + &test, + &run, + "day-two-timer", + ExecutionTaskStatus::WaitingTimer, + ) + .await?; + assert_parked_has_no_active_compute(&fixture, &pool, &run).await?; + fixture.hard_crash_and_restart_orchestrator().await?; + fixture.recreate_valkey_after_loss().await?; + + await_run_status(&test, &run, ExecutionRunStatus::WaitingReview).await?; + await_task_status( + &test, + &run, + "day-five-review", + ExecutionTaskStatus::WaitingReview, + ) + .await?; + assert_parked_has_no_active_compute(&fixture, &pool, &run).await?; + fixture.restart_orchestrator().await?; + fixture.stop_valkey().await?; + fixture.restart_valkey().await?; + + await_run_status(&test, &run, ExecutionRunStatus::WaitingSignal).await?; + await_task_status( + &test, + &run, + "day-eight-signal", + ExecutionTaskStatus::WaitingSignal, + ) + .await?; + assert_parked_has_no_active_compute(&fixture, &pool, &run).await?; + fixture.hard_crash_and_restart_orchestrator().await?; + + let terminal = match await_run_status(&test, &run, ExecutionRunStatus::Completed).await { + Ok(terminal) => terminal, + Err(error) => { + let run_state: Value = sqlx::query_scalar( + "SELECT jsonb_build_object( \ + 'status', status, 'activation_state', activation_state, \ + 'controller_generation', controller_generation, 'wake_epoch', wake_epoch, \ + 'processed_wake_epoch', processed_wake_epoch, \ + 'ready_task_count', ready_task_count, \ + 'active_task_count', active_task_count, \ + 'waiting_task_count', waiting_task_count, \ + 'pending_terminal_status', pending_terminal_status) \ + FROM moa.execution_run WHERE run_uid=$1", + ) + .bind(run.run_uid) + .fetch_one(&pool) + .await?; + let dispatches: Value = sqlx::query_scalar( + "SELECT COALESCE(jsonb_agg(jsonb_build_object( \ + 'dispatch_uid', dispatch_uid, 'kind', dispatch_kind, 'state', state, \ + 'delivery_attempts', delivery_attempts, 'last_error', last_error) \ + ORDER BY created_at), '[]'::jsonb) \ + FROM moa.execution_dispatch_outbox WHERE run_uid=$1", + ) + .bind(run.run_uid) + .fetch_one(&pool) + .await?; + let nodes: Value = sqlx::query_scalar( + "SELECT COALESCE(jsonb_agg(jsonb_build_object( \ + 'node_id', node_id, 'status', node_status, \ + 'remaining_dependencies', remaining_dependency_count, \ + 'ready_tasks', ready_task_count, 'active_tasks', active_task_count, \ + 'waiting_tasks', waiting_task_count, 'terminal_tasks', terminal_task_count) \ + ORDER BY node_order), '[]'::jsonb) \ + FROM moa.execution_node_state WHERE run_uid=$1", + ) + .bind(run.run_uid) + .fetch_one(&pool) + .await?; + let tasks: Value = sqlx::query_scalar( + "SELECT COALESCE(jsonb_agg(jsonb_build_object( \ + 'node_id', node_id, 'status', status, 'attempt_state', attempt_state, \ + 'generation', generation, 'attempt_generation', attempt_generation, \ + 'active_dispatch_uid', active_dispatch_uid) ORDER BY created_at), '[]'::jsonb) \ + FROM moa.execution_task WHERE run_uid=$1", + ) + .bind(run.run_uid) + .fetch_one(&pool) + .await?; + let attempt_dispatches: Vec = sqlx::query_scalar( + "SELECT dispatch_uid FROM moa.execution_dispatch_outbox \ + WHERE run_uid=$1 AND dispatch_kind='task_attempt'", + ) + .bind(run.run_uid) + .fetch_all(&pool) + .await?; + let attempt_keys = attempt_dispatches + .iter() + .map(|dispatch_uid| format!("'{dispatch_uid}'")) + .collect::>() + .join(", "); + let attempt_filter = if attempt_keys.is_empty() { + "false".to_string() + } else { + format!( + "target_service_name = 'ExecutionTaskAttempt' AND \ + target_service_key IN ({attempt_keys})" + ) + }; + let invocation_filter = format!( + "(target_service_name = 'ExecutionRunController' AND target_service_key = '{}') \ + OR ({attempt_filter}) OR (target_service_name = 'ToolExecutor' AND \ + invoked_by_id IN (SELECT id FROM sys_invocation WHERE {attempt_filter}))", + run.run_uid + ); + let invocations = restate_rows( + &fixture, + &format!( + "SELECT id, status, target_service_name, target_service_key \ + FROM sys_invocation WHERE {invocation_filter}" + ), + ) + .await + .unwrap_or_default(); + let journal = restate_rows( + &fixture, + &format!( + "SELECT id, index, entry_type, name, entry_json FROM sys_journal \ + WHERE id IN (SELECT id FROM sys_invocation WHERE {invocation_filter}) \ + AND entry_type IN ('Command: Output', 'Notification: Call') \ + ORDER BY id, index" + ), + ) + .await + .unwrap_or_default(); + bail!( + "{error:#}; run_state={run_state}; dispatches={dispatches}; nodes={nodes}; \ + tasks={tasks}; invocations={invocations:?}; journal={journal:?}" + ); + } + }; + assert_eq!(terminal.output, Some(json!({"week": "complete"}))); + let trigger_rows = sqlx::query( + "SELECT task.node_id, trigger.trigger_kind, trigger.created_at, \ + trigger.due_at, trigger.delivered_at, trigger.state, \ + trigger.attempt_generation \ + FROM moa.execution_trigger AS trigger \ + JOIN moa.execution_task AS task \ + ON task.run_uid = trigger.run_uid AND task.task_id = trigger.task_id \ + WHERE trigger.run_uid = $1 \ + AND trigger.trigger_kind IN ('task_timer', 'wait_expiry')", + ) + .bind(run.run_uid) + .fetch_all(&pool) + .await?; + assert_eq!(trigger_rows.len(), 3); + let mut observed = std::collections::BTreeMap::new(); + for row in trigger_rows { + let node_id: String = row.try_get("node_id")?; + assert_eq!(row.try_get::("state")?, "delivered"); + assert_eq!( + row.try_get::, _>("attempt_generation")?, + Some(1) + ); + let created_at: DateTime = row.try_get("created_at")?; + let due_at: DateTime = row.try_get("due_at")?; + let delivered_at: DateTime = row + .try_get::>, _>("delivered_at")? + .with_context(|| format!("{node_id} trigger omitted delivered_at"))?; + observed.insert( + node_id, + ( + row.try_get::("trigger_kind")?, + due_at.signed_duration_since(created_at), + delivered_at, + ), + ); + } + let timer = observed + .get("day-two-timer") + .context("day-two timer trigger missing")?; + let review = observed + .get("day-five-review") + .context("day-five review trigger missing")?; + let signal = observed + .get("day-eight-signal") + .context("day-eight signal trigger missing")?; + assert_eq!(timer.0, "task_timer"); + assert_eq!(review.0, "wait_expiry"); + assert_eq!(signal.0, "wait_expiry"); + let within = |actual: TimeDelta, expected: TimeDelta| { + (actual - expected).num_milliseconds().unsigned_abs() <= 250 + }; + assert!( + within(timer.1, TimeDelta::seconds(4)), + "timer due={:?}", + timer.1 + ); + assert!( + within(review.1, TimeDelta::seconds(6)), + "review due={:?}", + review.1 + ); + assert!( + within(signal.1, TimeDelta::seconds(6)), + "signal due={:?}", + signal.1 + ); + assert!(timer.2 < review.2 && review.2 < signal.2); + Ok(()) +} diff --git a/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/burst_admission.rs b/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/burst_admission.rs new file mode 100644 index 000000000..d5e4c532f --- /dev/null +++ b/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/burst_admission.rs @@ -0,0 +1,632 @@ +//! Burst admission, parked-capacity, and deterministic ordering coverage. + +use super::*; +use futures_util::{StreamExt, TryStreamExt, stream}; +use moa_core::traits::{Identity, IdentityType}; + +#[tokio::test] +#[ignore = "requires Docker and a bounded three-minute common-wake admission window"] +async fn one_thousand_common_wakes_bound_capacity_invocations_and_oldest_ready_age_service_e2e() +-> Result<()> { + // Pins: 1,000 independently admitted runs wake on one absolute instant; the production + // dispatcher drains timer delivery and run activation as one bounded chain rather than an + // unkeyed kick storm, never exceeds 32 DB receipts or Restate attempts, and exports its + // bounded batch plus checked oldest-ready age through OTLP. + const RUN_COUNT: usize = 1_000; + const FLEET_CAP: usize = 32; + const ADMISSION_CONCURRENCY: usize = 8; + const BURST_TIMEOUT: Duration = Duration::from_secs(360); + let tool_name = "long_horizon_thousand_wake_probe"; + let fixture = execution_fixture_with_tools( + vec![FixtureCapabilityTool { + name: tool_name.to_string(), + description: "Deterministic Task 12 thousand-wake barrier".to_string(), + input_schema: json!({ + "type": "object", + "additionalProperties": false, + "required": ["index"], + "properties": {"index": {"type": "integer"}} + }), + item_key_pointer: None, + idempotent: true, + outcomes: vec![FixtureCapabilityOutcome::Success { + output: json!({"completed": true}), + }], + }], + vec![ + ( + "MOA_EXECUTION_MAX_TENANT_ACTIVE_RUNS".to_string(), + RUN_COUNT.to_string(), + ), + ( + "MOA_EXECUTION_MAX_FLEET_ACTIVE_RUNS".to_string(), + RUN_COUNT.to_string(), + ), + ( + "MOA_EXECUTION_MAX_TENANT_PARKED_RUNS".to_string(), + RUN_COUNT.to_string(), + ), + ( + "MOA_EXECUTION_MAX_FLEET_PARKED_RUNS".to_string(), + RUN_COUNT.to_string(), + ), + ( + "MOA_EXECUTION_MAX_TENANT_ACTIVE_TASKS".to_string(), + FLEET_CAP.to_string(), + ), + ( + "MOA_EXECUTION_MAX_FLEET_ACTIVE_TASKS".to_string(), + FLEET_CAP.to_string(), + ), + ], + ) + .await?; + let tenant_id = fixture + .client + .identity() + .context("thousand-wake fixture omitted identity")? + .tenant_id; + fixture.grant_default_tenant_admin(tenant_id).await?; + let capability_name = moa_hands::mcp_tool_reference("fixture-capability", tool_name); + allow_fixture_capability(&fixture, tenant_id, &capability_name, "thousand-wake").await?; + let test = fixture.isolated().await; + let common_wake = Utc::now() + TimeDelta::seconds(180); + let runs = stream::iter(0..RUN_COUNT) + .map(|index| { + let test = &test; + async move { + let mut capability = + fixture_capability_node("burst-capability", tool_name, json!({"index": index})); + capability.depends_on = vec!["burst-timer".to_string()]; + start_plan_with_policy( + test, + &format!("thousand-wake-{index}"), + vec![ + node( + "burst-timer", + &[], + ExecutionOperation::WaitUntil { + wake: ExecutionTemporalTarget::At { at: common_wake }, + result: json!({"ready": true}), + }, + json!({"type": "object"}), + ), + capability, + output_node(&["burst-capability"], json!({"completed": true})), + ], + BURST_TIMEOUT, + false, + ) + .await + } + }) + // Admission locks the shared tenant capacity buckets. Keep that setup traffic below + // Restate's suspension threshold; this scenario stresses the common wake, not admission. + .buffer_unordered(ADMISSION_CONCURRENCY) + .try_collect::>() + .await?; + assert_eq!(runs.len(), RUN_COUNT); + let pool = PgPool::connect(&fixture.postgres_url).await?; + await_tenant_run_count(&pool, tenant_id, "waiting_timer", RUN_COUNT, BURST_TIMEOUT).await?; + await_capacity_quantity_before( + &pool, + tenant_id, + "parked_runs", + RUN_COUNT, + common_wake - TimeDelta::seconds(30), + ) + .await?; + let admission_margin = common_wake.signed_duration_since(Utc::now()); + assert!( + admission_margin > TimeDelta::seconds(30), + "1,000 runs were not fully parked before the shared wake: {admission_margin:?}" + ); + fixture.otlp_capture()?.clear().await; + + let controller = fixture + .fixture_capability() + .context("thousand-wake fixture omitted capability controller")?; + controller.wait_for_calls(FLEET_CAP, BURST_TIMEOUT).await?; + tokio::time::sleep(Duration::from_millis(500)).await; + assert_eq!(controller.calls().len(), FLEET_CAP); + let active: i64 = sqlx::query_scalar( + "SELECT COALESCE(SUM(quantity), 0)::BIGINT FROM moa.execution_capacity_reservation \ + WHERE tenant_id = $1 AND resource_dimension = 'active_tasks' AND state <> 'released'", + ) + .bind(tenant_id.0) + .fetch_one(&pool) + .await?; + assert_eq!(active, FLEET_CAP as i64); + let invocations = restate_rows( + &fixture, + "SELECT id FROM sys_invocation WHERE target_service_name = 'ExecutionTaskAttempt' \ + AND status NOT IN ('completed', 'killed')", + ) + .await?; + assert!(invocations.len() <= FLEET_CAP); + let dispatch_metric = fixture + .otlp_capture()? + .wait_for_metric(BURST_TIMEOUT, |metric| { + metric.name() == "moa_execution_dispatch_batch_size" + && metric.data_points().iter().any(|point| { + point.count() > 0 + && point.value() >= FLEET_CAP as f64 + && point.value() / point.count() as f64 <= FLEET_CAP as f64 + }) + }) + .await + .context("observe bounded production execution-dispatch batch metric")?; + assert!(dispatch_metric.data_points().iter().any(|point| { + point.count() > 0 + && point.value() >= FLEET_CAP as f64 + && point.value() / point.count() as f64 <= FLEET_CAP as f64 + })); + let oldest_ready_metric = fixture + .otlp_capture()? + .wait_for_metric(BURST_TIMEOUT, |metric| { + metric.name() == "moa_execution_oldest_ready_age_seconds" + && metric + .data_points() + .iter() + .any(|point| point.value() > 0.0 && point.value() <= 60.0) + }) + .await + .context("observe checked production oldest-ready-age metric")?; + assert!( + oldest_ready_metric + .data_points() + .iter() + .any(|point| point.value() > 0.0 && point.value() <= 60.0) + ); + + let mut released = 0; + let mut maximum_oldest_ready_seconds = 0.0_f64; + while released < RUN_COUNT { + let next = (released + FLEET_CAP).min(RUN_COUNT); + controller.wait_for_calls(next, BURST_TIMEOUT).await?; + let wave_active: i64 = sqlx::query_scalar( + "SELECT COALESCE(SUM(quantity), 0)::BIGINT FROM moa.execution_capacity_reservation \ + WHERE tenant_id = $1 AND resource_dimension = 'active_tasks' AND state <> 'released'", + ) + .bind(tenant_id.0) + .fetch_one(&pool) + .await?; + assert!(wave_active <= FLEET_CAP as i64); + let oldest: f64 = sqlx::query_scalar( + "SELECT COALESCE(EXTRACT(EPOCH FROM (now() - MIN(ready_at))), 0)::DOUBLE PRECISION \ + FROM moa.execution_task WHERE tenant_id = $1 AND status = 'ready'", + ) + .bind(tenant_id.0) + .fetch_one(&pool) + .await?; + maximum_oldest_ready_seconds = maximum_oldest_ready_seconds.max(oldest); + controller.release(next - released); + released = next; + } + assert!( + maximum_oldest_ready_seconds <= 60.0, + "oldest ready task exceeded bounded age: {maximum_oldest_ready_seconds}s" + ); + await_tenant_run_count(&pool, tenant_id, "completed", RUN_COUNT, BURST_TIMEOUT).await?; + let first = status(&test, &runs[0]).await?; + let last = status(&test, &runs[RUN_COUNT - 1]).await?; + assert_eq!(first.output, Some(json!({"completed": true}))); + assert_eq!(last.output, Some(json!({"completed": true}))); + Ok(()) +} + +#[tokio::test] +#[ignore = "requires Docker for the real Restate/Postgres/Valkey execution fixture"] +async fn timer_burst_obeys_parked_cap_and_preserves_fifo_due_order_service_e2e() -> Result<()> { + // Pins: a same-tenant burst converts active attempts into bounded parked + // ownership without exceeding the configured resident-run entitlement; + // cap+1 admission is rejected until one entitlement is released, and due + // work retains FIFO order after the rejected admission is retried. + let fixture = execution_fixture(vec![ + ( + "MOA_EXECUTION_MAX_TENANT_ACTIVE_RUNS".to_string(), + "4".to_string(), + ), + ( + "MOA_EXECUTION_MAX_FLEET_ACTIVE_RUNS".to_string(), + "4".to_string(), + ), + ( + "MOA_EXECUTION_MAX_TENANT_PARKED_RUNS".to_string(), + "4".to_string(), + ), + ( + "MOA_EXECUTION_MAX_FLEET_PARKED_RUNS".to_string(), + "4".to_string(), + ), + ( + "MOA_EXECUTION_MAX_TENANT_ACTIVE_TASKS".to_string(), + "2".to_string(), + ), + ( + "MOA_EXECUTION_MAX_FLEET_ACTIVE_TASKS".to_string(), + "2".to_string(), + ), + ]) + .await?; + let test = fixture.isolated().await; + let pool = PgPool::connect(&fixture.postgres_url).await?; + let mut runs = Vec::new(); + for index in 0..4_u64 { + runs.push( + start_plan( + &test, + &format!("burst-{index}"), + vec![ + node( + "burst-timer", + &[], + ExecutionOperation::WaitUntil { + wake: after_logical_days(3 + index), + result: json!({"index": index}), + }, + json!({"type": "object"}), + ), + output_node(&["burst-timer"], json!({"burst": index})), + ], + Duration::from_secs(15), + ) + .await?, + ); + } + let overflow_admission = start_plan( + &test, + "burst-overflow-rejected", + vec![ + node( + "overflow-timer", + &[], + ExecutionOperation::WaitUntil { + wake: after_logical_days(7), + result: json!({"overflow": true}), + }, + json!({"type": "object"}), + ), + output_node(&["overflow-timer"], json!({"overflow": "completed"})), + ], + Duration::from_secs(30), + ) + .await; + let overflow_error = match overflow_admission { + Ok(_) => bail!("cap+1 run was admitted without resident capacity"), + Err(error) => error, + }; + let overflow_error = format!("{overflow_error:#}"); + assert!( + overflow_error + .contains("execution parked_runs capacity is exhausted; retry admission later"), + "cap+1 admission returned the wrong error: {overflow_error}" + ); + + for run in &runs { + await_run_status(&test, run, ExecutionRunStatus::WaitingTimer).await?; + assert_parked_has_no_active_compute(&fixture, &pool, run).await?; + } + let active_task_capacity: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM moa.execution_capacity_reservation \ + WHERE tenant_id = $1 AND resource_dimension = 'active_tasks' AND state <> 'released'", + ) + .bind(runs[0].tenant_id.0) + .fetch_one(&pool) + .await?; + let parked_capacity: i64 = sqlx::query_scalar( + "SELECT COALESCE(SUM(quantity), 0)::BIGINT FROM moa.execution_capacity_reservation \ + WHERE tenant_id = $1 AND resource_dimension = 'parked_runs' AND state <> 'released'", + ) + .bind(runs[0].tenant_id.0) + .fetch_one(&pool) + .await?; + let admitted_runs: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM moa.execution_run WHERE tenant_id = $1") + .bind(runs[0].tenant_id.0) + .fetch_one(&pool) + .await?; + assert_eq!(active_task_capacity, 0); + assert_eq!(parked_capacity, 4); + assert_eq!(admitted_runs, 4); + + let first_terminal = await_run_status(&test, &runs[0], ExecutionRunStatus::Completed).await?; + assert_eq!(first_terminal.output, Some(json!({"burst": 0}))); + let overflow = start_plan( + &test, + "burst-overflow-retry", + vec![ + node( + "overflow-timer", + &[], + ExecutionOperation::WaitUntil { + wake: after_logical_days(7), + result: json!({"overflow": true}), + }, + json!({"type": "object"}), + ), + output_node(&["overflow-timer"], json!({"overflow": "completed"})), + ], + Duration::from_secs(30), + ) + .await?; + await_run_status(&test, &overflow, ExecutionRunStatus::WaitingTimer).await?; + assert_parked_has_no_active_compute(&fixture, &pool, &overflow).await?; + + for (index, run) in runs.iter().enumerate().skip(1) { + let terminal = await_run_status(&test, run, ExecutionRunStatus::Completed).await?; + assert_eq!(terminal.output, Some(json!({"burst": index as u64}))); + } + let overflow_terminal = + await_run_status(&test, &overflow, ExecutionRunStatus::Completed).await?; + assert_eq!( + overflow_terminal.output, + Some(json!({"overflow": "completed"})) + ); + let mut delivered_at = Vec::new(); + for (index, run) in runs.iter().enumerate() { + let row = sqlx::query( + "SELECT due_at, created_at, delivered_at FROM moa.execution_trigger \ + WHERE run_uid = $1 AND trigger_kind = 'task_timer'", + ) + .bind(run.run_uid) + .fetch_one(&pool) + .await?; + let due_at: DateTime = row.try_get("due_at")?; + let created_at: DateTime = row.try_get("created_at")?; + delivered_at.push( + row.try_get::>, _>("delivered_at")? + .with_context(|| format!("burst-{index} timer omitted delivered_at"))?, + ); + let expected = TimeDelta::from_std(LOGICAL_DAY * (3 + index as u32))?; + let error = due_at.signed_duration_since(created_at) - expected; + assert!( + error.num_milliseconds().unsigned_abs() <= 250, + "burst-{index} persisted the wrong relative due time: {error:?}" + ); + } + assert!( + delivered_at.windows(2).all(|pair| pair[0] < pair[1]), + "timer delivery did not preserve increasing due order: {delivered_at:?}" + ); + + Ok(()) +} + +#[tokio::test] +#[ignore = "requires Docker for the real Restate/Postgres/Valkey execution fixture"] +async fn two_tenant_burst_admits_one_attempt_each_before_second_wave_service_e2e() -> Result<()> { + // Pins: fleet capacity two and tenant capacity one produce one concurrent + // attempt per tenant; one tenant cannot consume both slots ahead of its peer. + let tool_name = "long_horizon_fairness_probe"; + let mut fixture = execution_fixture_with_tools( + vec![FixtureCapabilityTool { + name: tool_name.to_string(), + description: "Deterministic Task 12 fairness barrier".to_string(), + input_schema: json!({ + "type": "object", + "additionalProperties": false, + "required": ["case"], + "properties": {"case": {"type": "string"}} + }), + item_key_pointer: None, + idempotent: true, + outcomes: vec![FixtureCapabilityOutcome::Success { + output: json!({"result": "fair"}), + }], + }], + vec![ + ( + "MOA_EXECUTION_MAX_TENANT_ACTIVE_TASKS".to_string(), + "1".to_string(), + ), + ( + "MOA_EXECUTION_MAX_FLEET_ACTIVE_TASKS".to_string(), + "2".to_string(), + ), + ], + ) + .await?; + let pool = PgPool::connect(&fixture.postgres_url).await?; + let identity_a = fixture + .client + .identity() + .cloned() + .context("fixture client omitted tenant A identity")?; + let tenant_a = identity_a.tenant_id; + let common_wake = Utc::now() + TimeDelta::seconds(30); + let test_a = fixture.isolated().await; + let runs_a = vec![ + start_slow_fair_run(&test_a, tool_name, "tenant-a-1", common_wake).await?, + start_slow_fair_run(&test_a, tool_name, "tenant-a-2", common_wake).await?, + ]; + drop(test_a); + + let identity_b = Identity { + identity_type: IdentityType::Operator, + id: Uuid::now_v7(), + tenant_id: TenantId::from(Uuid::now_v7()), + api_key_id: None, + acting_on_behalf_of: None, + }; + let tenant_b = identity_b.tenant_id; + fixture.client = fixture.client.clone().with_identity(identity_b.clone()); + let test_b = fixture.isolated().await; + let runs_b = vec![ + start_slow_fair_run(&test_b, tool_name, "tenant-b-1", common_wake).await?, + start_slow_fair_run(&test_b, tool_name, "tenant-b-2", common_wake).await?, + ]; + + let controller = fixture + .fixture_capability() + .context("fairness fixture omitted capability controller")?; + let first_wave = controller.wait_for_calls(2, SCENARIO_TIMEOUT).await?; + assert_eq!(first_wave.len(), 2); + let first_cases = first_wave + .iter() + .filter_map(|call| call.input.get("case").and_then(Value::as_str)) + .collect::>(); + assert!(first_cases.iter().any(|case| case.starts_with("tenant-a"))); + assert!(first_cases.iter().any(|case| case.starts_with("tenant-b"))); + + let deadline = Instant::now() + SCENARIO_TIMEOUT; + loop { + let rows = sqlx::query( + "SELECT tenant_id, SUM(quantity)::BIGINT AS quantity \ + FROM moa.execution_capacity_reservation \ + WHERE resource_dimension = 'active_tasks' AND state <> 'released' \ + GROUP BY tenant_id ORDER BY tenant_id", + ) + .fetch_all(&pool) + .await?; + let observed = rows + .iter() + .map(|row| { + Ok(( + row.try_get::("tenant_id")?, + row.try_get::("quantity")?, + )) + }) + .collect::, sqlx::Error>>()?; + if observed.len() == 2 { + assert_eq!( + observed.iter().map(|(_, quantity)| *quantity).sum::(), + 2 + ); + assert!(observed.iter().all(|(_, quantity)| *quantity == 1)); + assert!(observed.iter().any(|(tenant, _)| *tenant == tenant_a.0)); + assert!(observed.iter().any(|(tenant, _)| *tenant == tenant_b.0)); + break; + } + if Instant::now() >= deadline { + bail!("two-tenant burst never admitted one active attempt per tenant: {observed:?}"); + } + tokio::time::sleep(POLL_INTERVAL).await; + } + tokio::time::sleep(Duration::from_millis(500)).await; + assert_eq!( + controller.calls().len(), + 2, + "fleet cap admitted a third attempt while the first wave was held" + ); + let active_invocations = restate_rows( + &fixture, + "SELECT id FROM sys_invocation \ + WHERE target_service_name = 'ExecutionTaskAttempt' \ + AND status NOT IN ('completed', 'killed')", + ) + .await?; + assert_eq!(active_invocations.len(), 2); + controller.release(2); + let second_wave = controller.wait_for_calls(4, SCENARIO_TIMEOUT).await?; + assert_eq!(second_wave.len(), 4); + let second_cases = second_wave[2..] + .iter() + .filter_map(|call| call.input.get("case").and_then(Value::as_str)) + .collect::>(); + assert!(second_cases.iter().any(|case| case.starts_with("tenant-a"))); + assert!(second_cases.iter().any(|case| case.starts_with("tenant-b"))); + controller.release(2); + + for run in &runs_b { + let terminal = await_run_status(&test_b, run, ExecutionRunStatus::Completed).await?; + assert_eq!(terminal.output, Some(json!({"fair": true}))); + } + drop(test_b); + fixture.client = fixture.client.clone().with_identity(identity_a); + let test_a = fixture.isolated().await; + for run in &runs_a { + let terminal = await_run_status(&test_a, run, ExecutionRunStatus::Completed).await?; + assert_eq!(terminal.output, Some(json!({"fair": true}))); + } + Ok(()) +} + +async fn start_slow_fair_run( + test: &IsolatedTest<'_>, + tool_name: &str, + label: &str, + common_wake: DateTime, +) -> Result { + let mut capability = + fixture_capability_node("fair-capability", tool_name, json!({"case": label})); + capability.depends_on = vec!["fair-timer".to_string()]; + start_plan( + test, + label, + vec![ + node( + "fair-timer", + &[], + ExecutionOperation::WaitUntil { + wake: ExecutionTemporalTarget::At { at: common_wake }, + result: json!({"ready": true}), + }, + json!({"type": "object"}), + ), + capability, + output_node(&["fair-capability"], json!({"fair": true})), + ], + Duration::from_secs(60), + ) + .await +} + +async fn await_tenant_run_count( + pool: &PgPool, + tenant_id: TenantId, + status: &str, + expected: usize, + timeout: Duration, +) -> Result<()> { + let deadline = Instant::now() + timeout; + loop { + let count: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM moa.execution_run WHERE tenant_id = $1 AND status = $2", + ) + .bind(tenant_id.0) + .bind(status) + .fetch_one(pool) + .await?; + if count == expected as i64 { + return Ok(()); + } + if Instant::now() >= deadline { + bail!( + "tenant {} reached {count}/{expected} runs in status {status} within {timeout:?}", + tenant_id.0 + ); + } + tokio::time::sleep(POLL_INTERVAL).await; + } +} + +async fn await_capacity_quantity_before( + pool: &PgPool, + tenant_id: TenantId, + dimension: &str, + expected: usize, + deadline: DateTime, +) -> Result<()> { + loop { + let quantity: i64 = sqlx::query_scalar( + "SELECT COALESCE(SUM(quantity), 0)::BIGINT \ + FROM moa.execution_capacity_reservation \ + WHERE tenant_id = $1 AND resource_dimension = $2 AND state <> 'released'", + ) + .bind(tenant_id.0) + .bind(dimension) + .fetch_one(pool) + .await?; + if quantity == expected as i64 { + return Ok(()); + } + if Utc::now() >= deadline { + bail!( + "tenant {} reached {quantity}/{expected} active {dimension} capacity before {deadline}", + tenant_id.0 + ); + } + tokio::time::sleep(POLL_INTERVAL).await; + } +} diff --git a/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/deadline_and_waits.rs b/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/deadline_and_waits.rs new file mode 100644 index 000000000..6f1d7d753 --- /dev/null +++ b/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/deadline_and_waits.rs @@ -0,0 +1,571 @@ +//! Deadline, absolute target, and wait-expiry coverage. + +use super::*; +use moa_artifacts::execution_plan::ExecutionTaskResult; +use moa_execution::state::ExecutionTerminalReason; +use moa_execution::wire::{ + ExecutionConflictReason, ExecutionMutationResponse, ExecutionSignalRequest, + ExecutionTaskAttemptRequest, +}; + +#[tokio::test] +#[ignore = "requires Docker for the real Restate/Postgres/Valkey execution fixture"] +async fn run_deadline_terminalizes_slow_active_attempt_and_releases_capacity_service_e2e() +-> Result<()> { + // Pins: the absolute admitted run deadline wins over a slow provider slice, + // terminalizes once with DeadlineExceeded, and releases the active-attempt reservation. + let completed = serde_json::to_string(&json!({"result": "too late"}))?; + let fixture = execution_fixture_with_script( + json!({ + "default": { + "content": completed, + "tool_calls": [], + "latency_ms": 10_000, + "ttft_ms": 10_000 + } + }), + Vec::new(), + ) + .await?; + let test = fixture.isolated().await; + let pool = PgPool::connect(&fixture.postgres_url).await?; + let run = start_plan( + &test, + "run-deadline", + vec![ + node( + "slow-agent", + &[], + ExecutionOperation::Agent { + instructions: "Return after the provider delay.".to_string(), + skill_refs: Vec::new(), + capability_refs: Vec::new(), + max_turns: 1, + }, + json!({"type": "object"}), + ), + output_node(&["slow-agent"], json!({"unexpected": true})), + ], + Duration::from_secs(3), + ) + .await?; + await_task_status(&test, &run, "slow-agent", ExecutionTaskStatus::Running).await?; + let terminal = await_run_status(&test, &run, ExecutionRunStatus::Failed).await?; + assert!(terminal.output.is_none()); + assert_eq!(terminal.run.failed_tasks, 0); + assert_eq!( + terminal.run.terminal_reason, + Some(ExecutionTerminalReason::DeadlineExceeded) + ); + let slow_agent = tasks(&test, &run) + .await? + .into_iter() + .find(|task| task.node_id == "slow-agent") + .context("deadline run omitted its slow agent task")?; + // The deadline fence makes the run result authoritative before cancellation delivery. A + // provider result already in flight may still win the task-level settlement race and is + // retained as terminal evidence; neither disposition may revive or complete the run. + assert!(matches!( + ( + slow_agent.status, + slow_agent.outcome.as_ref().map(|outcome| &outcome.result) + ), + ( + ExecutionTaskStatus::Completed, + Some(ExecutionTaskResult::Completed { .. }) + ) | ( + ExecutionTaskStatus::Cancelled, + Some(ExecutionTaskResult::Cancelled { .. }) + ) + )); + let active: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM moa.execution_capacity_reservation \ + WHERE run_uid = $1 AND resource_dimension = 'active_tasks' AND state <> 'released'", + ) + .bind(run.run_uid) + .fetch_one(&pool) + .await?; + assert_eq!( + active, 0, + "deadline terminalization leaked attempt capacity" + ); + let deadline_boundary: (String, String, String) = sqlx::query_as( + "SELECT trigger.state, dispatch.state, capacity.state \ + FROM moa.execution_trigger AS trigger \ + JOIN moa.execution_dispatch_outbox AS dispatch USING (trigger_uid) \ + JOIN moa.execution_capacity_reservation AS capacity USING (trigger_uid) \ + WHERE trigger.run_uid = $1 AND trigger.trigger_kind = 'run_deadline'", + ) + .bind(run.run_uid) + .fetch_one(&pool) + .await?; + assert_eq!( + deadline_boundary, + ( + "superseded".to_string(), + "cancelled".to_string(), + "released".to_string(), + ) + ); + Ok(()) +} + +#[tokio::test] +#[ignore = "requires Docker for the real Restate/Postgres/Valkey execution fixture"] +async fn exact_wait_expiry_fails_once_and_late_delivery_cannot_revive_run_service_e2e() -> Result<()> +{ + // Pins: one storage-only signal expiry uses its persisted absolute deadline, + // fails the run once, and leaves no active attempt capacity while parked. + let fixture = execution_fixture(Vec::new()).await?; + let test = fixture.isolated().await; + let pool = PgPool::connect(&fixture.postgres_url).await?; + let run = start_plan( + &test, + "wait-expiry", + vec![ + node( + "expiring-signal", + &[], + ExecutionOperation::WaitSignal { + signal_name: "never-arrives".to_string(), + wait_policy: ExecutionWaitPolicy { + expiry: after_logical_days(2), + on_expiry: ExecutionWaitExpiryAction::FailRun, + }, + }, + json!({"type": "object"}), + ), + output_node(&["expiring-signal"], json!({"unexpected": true})), + ], + Duration::from_secs(10), + ) + .await?; + await_run_status(&test, &run, ExecutionRunStatus::WaitingSignal).await?; + let waiting = await_task_status( + &test, + &run, + "expiring-signal", + ExecutionTaskStatus::WaitingSignal, + ) + .await?; + assert_parked_has_no_active_compute(&fixture, &pool, &run).await?; + let rows = persisted_trigger_rows(&pool, run.run_uid).await?; + let expiry = rows + .iter() + .find(|(kind, _, _)| kind == "wait_expiry") + .context("signal wait did not persist its expiry trigger")?; + let waiting_since: DateTime = sqlx::query_scalar( + "SELECT waiting_since FROM moa.execution_task WHERE run_uid = $1 AND task_id = $2", + ) + .bind(run.run_uid) + .bind(task_id(&waiting).as_uuid()) + .fetch_one(&pool) + .await?; + assert_eq!( + expiry.1.signed_duration_since(waiting_since), + TimeDelta::from_std(LOGICAL_DAY * 2)?, + "persisted expiry did not preserve the exact compressed two-day wait" + ); + + let failed = await_run_status(&test, &run, ExecutionRunStatus::Failed).await?; + assert!(failed.output.is_none()); + let failed_at = failed + .run + .completed_at + .context("failed run omitted terminal time")?; + tokio::time::sleep(LOGICAL_DAY).await; + let replay = status(&test, &run).await?; + assert_eq!(replay.run.status, ExecutionRunStatus::Failed); + assert_eq!(replay.run.completed_at, Some(failed_at)); + assert_eq!(replay.run.failed_tasks, 1); + let late: ExecutionMutationResponse = test + .client() + .post_call( + "/Execution/deliver_signal", + &ExecutionSignalRequest { + tenant_id: run.tenant_id, + contact_id: None, + run_uid: run.run_uid, + task_id: task_id(&waiting), + expected_generation: waiting.generation, + signal_name: "never-arrives".to_string(), + payload: json!({"late": true}), + }, + ) + .await?; + assert_eq!( + late, + ExecutionMutationResponse::Conflict { + reason: ExecutionConflictReason::AlreadyTerminal, + } + ); + let immutable = status(&test, &run).await?; + assert_eq!(immutable.run.status, ExecutionRunStatus::Failed); + assert_eq!(immutable.run.completed_at, Some(failed_at)); + Ok(()) +} + +#[tokio::test] +#[ignore = "requires Docker for real watchdog delivery and held MCP effects"] +async fn watchdog_retries_idempotent_but_never_resends_ambiguous_effect_service_e2e() -> Result<()> +{ + // Pins: the same durable watchdog boundary retries a catalog-idempotent + // effect under attempt generation two, but terminalizes a possibly-applied + // non-idempotent effect as UnknownOutcome without a second logical send. + let idempotent_tool = "long_horizon_watchdog_idempotent"; + let ambiguous_tool = "long_horizon_watchdog_ambiguous"; + let fixture = execution_fixture_with_tools( + vec![ + FixtureCapabilityTool { + name: idempotent_tool.to_string(), + description: "Task 12 idempotent watchdog barrier".to_string(), + input_schema: watchdog_input_schema(), + item_key_pointer: None, + idempotent: true, + outcomes: vec![FixtureCapabilityOutcome::Success { + output: json!({"watchdog": "retried"}), + }], + }, + FixtureCapabilityTool { + name: ambiguous_tool.to_string(), + description: "Task 12 ambiguous watchdog effect".to_string(), + input_schema: watchdog_input_schema(), + item_key_pointer: None, + idempotent: false, + outcomes: vec![FixtureCapabilityOutcome::ApplyThenDisconnect], + }, + ], + vec![ + ( + "MOA_EXECUTION_ACTIVE_ATTEMPT_TIMEOUT_SECONDS".to_string(), + "2".to_string(), + ), + ( + "MOA_EXECUTION_TRIGGER_RECONCILIATION_CADENCE_SECONDS".to_string(), + "1".to_string(), + ), + ], + ) + .await?; + let test = fixture.isolated().await; + let pool = PgPool::connect(&fixture.postgres_url).await?; + let controller = fixture + .fixture_capability() + .context("watchdog fixture omitted capability controller")?; + + let idempotent = start_plan( + &test, + "watchdog-idempotent", + vec![ + fixture_capability_node( + "watchdog-idempotent", + idempotent_tool, + json!({"case": "retry-safe"}), + ), + output_node(&["watchdog-idempotent"], json!({"watchdog": "safe"})), + ], + Duration::from_secs(20), + ) + .await?; + controller.wait_for_calls(1, SCENARIO_TIMEOUT).await?; + let first_request = await_attempt_request(&pool, idempotent.run_uid, 1).await?; + let second = controller.wait_for_calls(2, SCENARIO_TIMEOUT).await?; + assert_eq!(second.len(), 2); + assert_eq!(second[0].capability, idempotent_tool); + assert_eq!(second[1].capability, idempotent_tool); + assert_ne!(second[0].invocation_id, second[1].invocation_id); + let retrying = await_task_attempt( + &test, + &idempotent, + "watchdog-idempotent", + 2, + ExecutionTaskStatus::Running, + ) + .await?; + assert_eq!(retrying.attempt, 2); + controller.release(2); + let safe_terminal = await_run_status(&test, &idempotent, ExecutionRunStatus::Completed).await?; + assert_eq!(safe_terminal.output, Some(json!({"watchdog": "safe"}))); + let safe_task = await_task_status( + &test, + &idempotent, + "watchdog-idempotent", + ExecutionTaskStatus::Completed, + ) + .await?; + assert_eq!(safe_task.attempt, 2); + assert_eq!(controller.effect_count(), 2); + assert_watchdog_settled_once(&pool, &first_request).await?; + + let ambiguous = start_plan( + &test, + "watchdog-ambiguous", + vec![ + fixture_capability_node( + "watchdog-ambiguous", + ambiguous_tool, + json!({"case": "possible-commit"}), + ), + output_node( + &["watchdog-ambiguous"], + json!({"unexpected": "ambiguous resend"}), + ), + ], + Duration::from_secs(20), + ) + .await?; + controller.wait_for_calls(3, SCENARIO_TIMEOUT).await?; + let ambiguous_request = await_attempt_request(&pool, ambiguous.run_uid, 1).await?; + controller.release(1); + let ambiguous_task = await_task_status( + &test, + &ambiguous, + "watchdog-ambiguous", + ExecutionTaskStatus::UnknownOutcome, + ) + .await?; + assert_eq!(ambiguous_task.attempt, 1); + assert!(matches!( + ambiguous_task.outcome.as_ref().map(|outcome| &outcome.result), + Some(ExecutionTaskResult::UnknownOutcome { message }) + if message.contains("possible commit") + )); + let ambiguous_terminal = + await_run_status(&test, &ambiguous, ExecutionRunStatus::Failed).await?; + assert!(ambiguous_terminal.output.is_none()); + assert_eq!(ambiguous_terminal.run.failed_tasks, 0); + let effect_count = controller.effect_count(); + assert_eq!(effect_count, 3, "ambiguous effect was logically sent twice"); + tokio::time::sleep(Duration::from_secs(1)).await; + assert_eq!( + controller.effect_count(), + effect_count, + "watchdog replay resent an ambiguous effect" + ); + assert_watchdog_settled_once(&pool, &ambiguous_request).await?; + let active: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM moa.execution_capacity_reservation \ + WHERE run_uid = $1 AND task_id = $2 AND resource_dimension = 'active_tasks' \ + AND state <> 'released'", + ) + .bind(ambiguous.run_uid) + .bind(ambiguous_request.task_id.as_uuid()) + .fetch_one(&pool) + .await?; + assert_eq!(active, 0, "ambiguous watchdog retained active capacity"); + Ok(()) +} + +fn watchdog_input_schema() -> Value { + json!({ + "type": "object", + "additionalProperties": false, + "required": ["case"], + "properties": {"case": {"type": "string"}} + }) +} + +async fn await_attempt_request( + pool: &PgPool, + run_uid: Uuid, + attempt_generation: u64, +) -> Result { + let deadline = Instant::now() + SCENARIO_TIMEOUT; + loop { + let payload: Option = sqlx::query_scalar( + "SELECT payload FROM moa.execution_dispatch_outbox \ + WHERE run_uid = $1 AND dispatch_kind = 'task_attempt' \ + AND attempt_generation = $2", + ) + .bind(run_uid) + .bind(i64::try_from(attempt_generation)?) + .fetch_optional(pool) + .await?; + if let Some(payload) = payload { + return serde_json::from_value(payload) + .context("decode immutable task-attempt dispatch"); + } + if Instant::now() >= deadline { + bail!("run {run_uid} did not persist attempt generation {attempt_generation}") + } + tokio::time::sleep(POLL_INTERVAL).await; + } +} + +async fn await_task_attempt( + test: &IsolatedTest<'_>, + run: &StartedRun, + node_id: &str, + expected_attempt: u32, + expected_status: ExecutionTaskStatus, +) -> Result { + let deadline = Instant::now() + SCENARIO_TIMEOUT; + loop { + let current = tasks(test, run).await?; + if let Some(task) = current.iter().find(|task| { + task.node_id == node_id + && task.attempt == expected_attempt + && task.status == expected_status + }) { + return Ok(task.clone()); + } + if Instant::now() >= deadline { + bail!( + "node `{node_id}` did not reach attempt {expected_attempt} {expected_status:?}; tasks={current:?}" + ) + } + tokio::time::sleep(POLL_INTERVAL).await; + } +} + +async fn assert_watchdog_settled_once( + pool: &PgPool, + request: &ExecutionTaskAttemptRequest, +) -> Result<()> { + let row = sqlx::query( + "SELECT trigger.state, dispatch.state AS dispatch_state, \ + (SELECT COUNT(*) FROM moa.execution_trigger AS duplicate \ + WHERE duplicate.run_uid = trigger.run_uid \ + AND duplicate.task_id = trigger.task_id \ + AND duplicate.trigger_kind = 'task_watchdog' \ + AND duplicate.attempt_generation = trigger.attempt_generation) AS exact_count \ + FROM moa.execution_trigger AS trigger \ + JOIN moa.execution_dispatch_outbox AS dispatch \ + ON dispatch.trigger_uid = trigger.trigger_uid \ + AND dispatch.dispatch_kind = 'trigger_delivery' \ + WHERE trigger.trigger_uid = $1", + ) + .bind(request.watchdog_trigger_uid) + .fetch_one(pool) + .await?; + assert_eq!(row.try_get::("state")?, "superseded"); + assert_eq!(row.try_get::("dispatch_state")?, "cancelled"); + assert_eq!(row.try_get::("exact_count")?, 1); + Ok(()) +} + +#[tokio::test] +#[ignore = "requires Docker for the real Restate/Postgres/Valkey execution fixture"] +async fn retry_backoff_releases_attempt_capacity_and_redispatches_new_generation_service_e2e() +-> Result<()> { + // Pins: a retryable provider failure persists a future retry, releases the + // first attempt's capacity during backoff, and dispatches generation two once. + let tool_name = "long_horizon_retry_probe"; + let fixture = execution_fixture_with_tools( + vec![FixtureCapabilityTool { + name: tool_name.to_string(), + description: "Deterministic Task 12 retry barrier".to_string(), + input_schema: json!({ + "type": "object", + "additionalProperties": false, + "required": ["case"], + "properties": {"case": {"type": "string"}} + }), + item_key_pointer: None, + idempotent: true, + outcomes: vec![ + FixtureCapabilityOutcome::HttpFailure { + status: 429, + retry_after_ms: Some(100), + message: "fixture rate limit".to_string(), + }, + FixtureCapabilityOutcome::Success { + output: json!({"result": "retried"}), + }, + ], + }], + Vec::new(), + ) + .await?; + let test = fixture.isolated().await; + let pool = PgPool::connect(&fixture.postgres_url).await?; + let mut retry_node = fixture_capability_node( + "retry-capability", + tool_name, + json!({"case": "long-horizon-retry"}), + ); + retry_node.retry = RetryPolicy { + max_attempts: 2, + initial_backoff_ms: 1_000, + max_backoff_ms: 1_000, + }; + let run = start_plan( + &test, + "retry-backoff", + vec![ + retry_node, + output_node(&["retry-capability"], json!({"retry": "complete"})), + ], + Duration::from_secs(12), + ) + .await?; + + let controller = fixture + .fixture_capability() + .context("retry fixture omitted capability controller")?; + let first = controller.wait_for_calls(1, SCENARIO_TIMEOUT).await?; + assert_eq!(first.len(), 1); + assert_eq!(first[0].capability, tool_name); + controller.release(1); + + let deadline = Instant::now() + SCENARIO_TIMEOUT; + loop { + let row = sqlx::query( + "SELECT attempt, attempt_generation, ready_at, active_dispatch_uid, attempt_state \ + FROM moa.execution_task WHERE run_uid = $1 AND node_id = 'retry-capability'", + ) + .bind(run.run_uid) + .fetch_optional(&pool) + .await?; + if let Some(row) = row + && let Some(ready_at) = row.try_get::>, _>("ready_at")? + { + assert_eq!(row.try_get::("attempt")?, 2); + assert_eq!(row.try_get::("attempt_generation")?, 2); + assert_eq!(row.try_get::, _>("active_dispatch_uid")?, None); + assert_eq!(row.try_get::("attempt_state")?, "idle"); + let active: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM moa.execution_capacity_reservation \ + WHERE run_uid = $1 AND resource_dimension = 'active_tasks' AND state <> 'released'", + ) + .bind(run.run_uid) + .fetch_one(&pool) + .await?; + assert_eq!(active, 0, "retry backoff retained active attempt capacity"); + let database_now: DateTime = + sqlx::query_scalar("SELECT now()").fetch_one(&pool).await?; + assert!( + ready_at > database_now, + "retry backoff was not persisted in the future: ready={ready_at}, now={database_now}" + ); + assert_eq!( + controller.calls().len(), + 1, + "retry generation two dispatched before its persisted ready_at" + ); + break; + } + if Instant::now() >= deadline { + bail!("retry-agent never persisted its storage-only backoff"); + } + tokio::time::sleep(POLL_INTERVAL).await; + } + + let second = controller.wait_for_calls(2, SCENARIO_TIMEOUT).await?; + assert_eq!(second[0].input, second[1].input); + assert_ne!(second[0].invocation_id, second[1].invocation_id); + controller.release(1); + + let terminal = await_run_status(&test, &run, ExecutionRunStatus::Completed).await?; + assert_eq!(terminal.output, Some(json!({"retry": "complete"}))); + let task = await_task_status( + &test, + &run, + "retry-capability", + ExecutionTaskStatus::Completed, + ) + .await?; + assert_eq!(task.attempt, 2); + assert_eq!(task.generation, 2); + Ok(()) +} diff --git a/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/deployment_drain.rs b/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/deployment_drain.rs new file mode 100644 index 000000000..f028c31bf --- /dev/null +++ b/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/deployment_drain.rs @@ -0,0 +1,148 @@ +//! Real Restate handler revision registration, latest routing, and drain coverage. + +use super::*; + +#[tokio::test] +#[ignore = "requires Docker for three real orchestrator handler deployments"] +async fn three_handler_revisions_route_latest_then_drain_from_one_to_zero_service_e2e() -> Result<()> +{ + // Pins: each newest real deployment owns one held bounded attempt, then the + // old deployment drains from one pinned invocation to zero and is stopped. + let tool_name = "long_horizon_revision_probe"; + let fixture = execution_fixture_with_tools( + vec![FixtureCapabilityTool { + name: tool_name.to_string(), + description: "Deterministic Task 12 deployment barrier".to_string(), + input_schema: json!({ + "type": "object", + "additionalProperties": false, + "required": ["revision"], + "properties": {"revision": {"type": "integer"}} + }), + item_key_pointer: None, + idempotent: true, + outcomes: vec![FixtureCapabilityOutcome::Success { + output: json!({"drained": true}), + }], + }], + Vec::new(), + ) + .await?; + let first = fixture.current_handler_revision().await?; + let test = fixture.isolated().await; + let pool = PgPool::connect(&fixture.postgres_url).await?; + let controller = fixture + .fixture_capability() + .context("deployment fixture omitted capability controller")?; + + let first_run = start_revision_run(&test, tool_name, 1).await?; + controller.wait_for_calls(1, SCENARIO_TIMEOUT).await?; + let first_dispatch = active_dispatch_uid(&pool, first_run.run_uid).await?; + assert_dispatch_pinned_to(&fixture, first_dispatch, &first).await?; + assert!(fixture.handler_revision_pinned_invocations(&first).await? >= 1); + + let second = fixture + .start_handler_revision("long-horizon-revision-2") + .await?; + assert_ne!(first.deployment_id, second.deployment_id); + assert_ne!(first.deployment_uri, second.deployment_uri); + controller.release(1); + await_run_status(&test, &first_run, ExecutionRunStatus::Completed).await?; + fixture + .wait_for_handler_revision_drained(&first, Duration::from_secs(10)) + .await?; + fixture.stop_drained_handler_revision(&first).await?; + + let second_run = start_revision_run(&test, tool_name, 2).await?; + controller.wait_for_calls(2, SCENARIO_TIMEOUT).await?; + let second_dispatch = active_dispatch_uid(&pool, second_run.run_uid).await?; + assert_dispatch_pinned_to(&fixture, second_dispatch, &second).await?; + assert!(fixture.handler_revision_pinned_invocations(&second).await? >= 1); + + let third = fixture + .start_handler_revision("long-horizon-revision-3") + .await?; + assert_ne!(second.deployment_id, third.deployment_id); + assert_ne!(second.deployment_uri, third.deployment_uri); + controller.release(1); + await_run_status(&test, &second_run, ExecutionRunStatus::Completed).await?; + fixture + .wait_for_handler_revision_drained(&second, Duration::from_secs(10)) + .await?; + fixture.stop_drained_handler_revision(&second).await?; + + let third_run = start_revision_run(&test, tool_name, 3).await?; + controller.wait_for_calls(3, SCENARIO_TIMEOUT).await?; + let third_dispatch = active_dispatch_uid(&pool, third_run.run_uid).await?; + assert_dispatch_pinned_to(&fixture, third_dispatch, &third).await?; + assert!(fixture.handler_revision_pinned_invocations(&third).await? >= 1); + controller.release(1); + let terminal = await_run_status(&test, &third_run, ExecutionRunStatus::Completed).await?; + assert_eq!(terminal.output, Some(json!({"revision": 3}))); + fixture + .wait_for_handler_revision_drained(&third, Duration::from_secs(10)) + .await?; + fixture.stop_drained_handler_revision(&third).await?; + Ok(()) +} + +async fn start_revision_run( + test: &IsolatedTest<'_>, + tool_name: &str, + revision: u64, +) -> Result { + start_plan( + test, + &format!("deployment-revision-{revision}"), + vec![ + fixture_capability_node( + "revision-capability", + tool_name, + json!({"revision": revision}), + ), + output_node(&["revision-capability"], json!({"revision": revision})), + ], + Duration::from_secs(30), + ) + .await +} + +async fn active_dispatch_uid(pool: &PgPool, run_uid: Uuid) -> Result { + sqlx::query_scalar( + "SELECT active_dispatch_uid FROM moa.execution_task \ + WHERE run_uid = $1 AND node_id = 'revision-capability'", + ) + .bind(run_uid) + .fetch_one(pool) + .await + .context("revision capability omitted active dispatch UID") +} + +async fn assert_dispatch_pinned_to( + fixture: &OrchestratorTestFixture, + dispatch_uid: Uuid, + revision: &moa_test_support::FixtureHandlerRevision, +) -> Result<()> { + let rows = restate_rows( + fixture, + &format!( + "SELECT pinned_deployment_id, last_attempt_deployment_id, status \ + FROM sys_invocation WHERE target_service_name = 'ExecutionTaskAttempt' \ + AND target_service_key = '{dispatch_uid}'" + ), + ) + .await?; + let row = rows + .first() + .with_context(|| format!("Restate omitted task-attempt invocation {dispatch_uid}"))?; + let pinned = row + .get("pinned_deployment_id") + .and_then(Value::as_str) + .or_else(|| { + row.get("last_attempt_deployment_id") + .and_then(Value::as_str) + }) + .context("Restate invocation omitted deployment identity")?; + assert_eq!(pinned, revision.deployment_id); + Ok(()) +} diff --git a/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/disaster_recovery.rs b/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/disaster_recovery.rs new file mode 100644 index 000000000..bc911de9e --- /dev/null +++ b/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/disaster_recovery.rs @@ -0,0 +1,397 @@ +//! Postgres, outbox, Restate-loss, and Valkey-loss recovery coverage. + +use super::*; +use moa_core::types::tools::{AsyncToolJobCallbackOutcome, AsyncToolJobTerminalOutcome}; + +#[tokio::test] +#[ignore = "requires Docker for destructive empty-Restate recovery"] +async fn unbound_external_start_recovers_provider_job_without_replaying_attempt_service_e2e() +-> Result<()> { + // Pins: provider start is committed under the pre-reserved key, then all Restate state is + // lost before bind. The expired Unbound intent calls recover_start with the same identity, + // binds and releases its owner without replaying TaskAttempt, and sparse reconciliation + // completes the one provider effect. + let fixture = external_job_execution_fixture(vec![ + ( + "MOA_EXECUTION_ACTIVE_ATTEMPT_TIMEOUT_SECONDS".to_string(), + "5".to_string(), + ), + ( + "MOA_EXECUTION_TRIGGER_RECONCILIATION_CADENCE_SECONDS".to_string(), + "1".to_string(), + ), + ]) + .await?; + let test = fixture.isolated().await; + let pool = PgPool::connect(&fixture.postgres_url).await?; + let run = start_plan( + &test, + "external-start-total-restate-loss", + vec![ + external_job_capability_node("external-job", json!({"value": "recover"})), + output_node(&["external-job"], json!({"external_recovery": "complete"})), + ], + Duration::from_secs(55), + ) + .await?; + let controller = fixture + .fixture_external_job() + .context("external-job recovery fixture omitted provider controller")?; + let starts = controller.wait_for_starts(1, SCENARIO_TIMEOUT).await?; + let start = &starts[0]; + let before_loss = sqlx::query( + "SELECT job.state AS job_state, job.idempotency_key, task.status AS task_status, \ + task.attempt_state, task.attempt_generation, dispatch.dispatch_uid, \ + dispatch.delivered_at, dispatch.delivery_attempts, recovery.trigger_uid, \ + recovery.due_at, recovery.state AS recovery_state \ + FROM moa.execution_external_job AS job \ + JOIN moa.execution_task AS task \ + ON task.run_uid = job.run_uid AND task.task_id = job.task_id \ + JOIN moa.execution_dispatch_outbox AS dispatch \ + ON dispatch.run_uid = task.run_uid AND dispatch.task_id = task.task_id \ + AND dispatch.dispatch_kind = 'task_attempt' \ + AND dispatch.attempt_generation = task.attempt_generation \ + JOIN moa.execution_trigger AS recovery \ + ON recovery.payload ->> 'external_job_uid' = job.external_job_uid::TEXT \ + AND (recovery.payload ->> 'job_generation')::BIGINT = job.job_generation \ + AND recovery.trigger_kind = 'external_start_recovery' \ + WHERE job.run_uid = $1 AND job.external_job_uid = $2", + ) + .bind(run.run_uid) + .bind(start.context.external_job_uid) + .fetch_one(&pool) + .await?; + assert_eq!(before_loss.try_get::("job_state")?, "unbound"); + assert_eq!( + before_loss.try_get::("idempotency_key")?, + start.context.idempotency_key + ); + assert_eq!(before_loss.try_get::("task_status")?, "running"); + assert_eq!( + before_loss.try_get::("attempt_state")?, + "running" + ); + assert_eq!(before_loss.try_get::("attempt_generation")?, 1); + assert_eq!( + before_loss.try_get::("recovery_state")?, + "pending" + ); + let task_dispatch_uid: Uuid = before_loss.try_get("dispatch_uid")?; + let task_delivered_at: DateTime = before_loss + .try_get::>, _>("delivered_at")? + .context("provider-start TaskAttempt omitted delivered_at")?; + let recovery_trigger_uid: Uuid = before_loss.try_get("trigger_uid")?; + + fixture.recreate_restate_after_loss().await?; + controller.queue_reconcile_outcomes([AsyncToolJobCallbackOutcome::Terminal { + outcome: AsyncToolJobTerminalOutcome::Completed { + output: json!({"provider": "recovered"}), + }, + }]); + fixture.hard_crash_and_restart_orchestrator().await?; + // The provider committed its job before this gate. Release only after the + // pre-loss handler is gone so it cannot bind and settle the recovery trigger. + controller.release_starts(1); + + let recoveries = controller.wait_for_recoveries(1, SCENARIO_TIMEOUT).await?; + assert_eq!(recoveries.len(), 1); + assert_eq!(recoveries[0].context, start.context); + let completed = await_run_status(&test, &run, ExecutionRunStatus::Completed).await?; + assert_eq!( + completed.output, + Some(json!({"external_recovery": "complete"})) + ); + let after_recovery = sqlx::query( + "SELECT job.state AS job_state, job.provider_job_id, job.idempotency_key, \ + dispatch.state AS task_dispatch_state, dispatch.delivered_at, \ + dispatch.delivery_attempts, recovery.state AS recovery_state, \ + recovery_dispatch.state AS recovery_dispatch_state, \ + recovery_dispatch.delivery_attempts AS recovery_delivery_attempts, \ + (SELECT COUNT(*) FROM moa.execution_capacity_reservation AS capacity \ + WHERE capacity.external_job_uid = job.external_job_uid \ + AND capacity.resource_dimension = 'external_jobs' \ + AND capacity.state = 'released') AS released_external_receipts \ + FROM moa.execution_external_job AS job \ + JOIN moa.execution_dispatch_outbox AS dispatch ON dispatch.dispatch_uid = $2 \ + JOIN moa.execution_trigger AS recovery ON recovery.trigger_uid = $3 \ + JOIN moa.execution_dispatch_outbox AS recovery_dispatch \ + ON recovery_dispatch.trigger_uid = recovery.trigger_uid \ + AND recovery_dispatch.dispatch_kind = 'trigger_delivery' \ + WHERE job.external_job_uid = $1", + ) + .bind(start.context.external_job_uid) + .bind(task_dispatch_uid) + .bind(recovery_trigger_uid) + .fetch_one(&pool) + .await?; + assert_eq!( + after_recovery.try_get::("job_state")?, + "completed" + ); + assert_eq!( + after_recovery.try_get::("provider_job_id")?, + start.provider_job_id + ); + assert_eq!( + after_recovery.try_get::("idempotency_key")?, + start.context.idempotency_key + ); + assert_eq!( + after_recovery.try_get::("task_dispatch_state")?, + "delivered" + ); + assert_eq!( + after_recovery.try_get::>, _>("delivered_at")?, + Some(task_delivered_at) + ); + assert_eq!(after_recovery.try_get::("delivery_attempts")?, 1); + assert_eq!( + after_recovery.try_get::("recovery_state")?, + "superseded" + ); + assert_eq!( + after_recovery.try_get::("recovery_dispatch_state")?, + "cancelled" + ); + assert_eq!( + after_recovery.try_get::("recovery_delivery_attempts")?, + 1 + ); + assert_eq!( + after_recovery.try_get::("released_external_receipts")?, + 1 + ); + assert_eq!(controller.starts().len(), 1); + assert_eq!(controller.recoveries().len(), 1); + assert_eq!(controller.reconciliations().len(), 1); + Ok(()) +} + +#[tokio::test] +#[ignore = "requires Docker for destructive disposable dependency recovery"] +async fn postgres_outbox_redrives_after_total_restate_and_valkey_loss_service_e2e() -> Result<()> { + // Pins: a committed timer survives empty Restate-state replacement and Valkey loss via + // generation-fenced exact-dispatch re-drive from PostgreSQL's authoritative outbox. + let fixture = execution_fixture(vec![( + "MOA_EXECUTION_TRIGGER_RECONCILIATION_CADENCE_SECONDS".to_string(), + "1".to_string(), + )]) + .await?; + let test = fixture.isolated().await; + let pool = PgPool::connect(&fixture.postgres_url).await?; + let run = start_plan( + &test, + "disaster-recovery", + vec![ + node( + "recovery-timer", + &[], + ExecutionOperation::WaitUntil { + wake: after_logical_days(5), + result: json!({"recovered": true}), + }, + json!({"type": "object"}), + ), + output_node(&["recovery-timer"], json!({"recovered": true})), + ], + Duration::from_secs(20), + ) + .await?; + await_run_status(&test, &run, ExecutionRunStatus::WaitingTimer).await?; + assert_parked_has_no_active_compute(&fixture, &pool, &run).await?; + let before = persisted_trigger_rows(&pool, run.run_uid).await?; + assert_eq!( + before + .iter() + .filter(|(kind, _, _)| kind == "task_timer") + .count(), + 1 + ); + + fixture.recreate_valkey_after_loss().await?; + fixture.recreate_restate_after_loss().await?; + fixture.restart_orchestrator().await?; + + let recovered_pool = PgPool::connect(&fixture.postgres_url).await?; + let after = persisted_trigger_rows(&recovered_pool, run.run_uid).await?; + assert_eq!( + after, before, + "dependency recovery rewrote the immutable trigger" + ); + let terminal = await_run_status(&test, &run, ExecutionRunStatus::Completed).await?; + assert_eq!(terminal.output, Some(json!({"recovered": true}))); + let delivered: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM moa.execution_trigger \ + WHERE run_uid = $1 AND trigger_kind = 'task_timer' AND state = 'delivered'", + ) + .bind(run.run_uid) + .fetch_one(&recovered_pool) + .await?; + assert_eq!( + delivered, 1, + "recovery delivered the immutable timer more than once" + ); + let delivery = sqlx::query( + "SELECT dispatch.state, dispatch.delivery_attempts \ + FROM moa.execution_trigger AS trigger \ + JOIN moa.execution_dispatch_outbox AS dispatch \ + ON dispatch.trigger_uid = trigger.trigger_uid \ + WHERE trigger.run_uid = $1 AND trigger.trigger_kind = 'task_timer' \ + AND dispatch.dispatch_kind = 'trigger_delivery'", + ) + .bind(run.run_uid) + .fetch_one(&recovered_pool) + .await?; + assert_eq!(delivery.try_get::("state")?, "delivered"); + assert_eq!(delivery.try_get::("delivery_attempts")?, 1); + Ok(()) +} + +#[tokio::test] +#[ignore = "requires Docker for destructive empty-Restate recovery"] +async fn running_ambiguous_attempt_is_not_redriven_after_total_restate_loss_service_e2e() +-> Result<()> { + // Pins: Restate loss never turns a Running possibly-committed effect back + // into a dispatchable attempt. Its exact TaskAttempt delivery timestamp is + // immutable, while the re-driven watchdog alone settles UnknownOutcome. + let tool_name = "long_horizon_restate_loss_ambiguous"; + let fixture = execution_fixture_with_tools( + vec![FixtureCapabilityTool { + name: tool_name.to_string(), + description: "Task 12 total-Restate-loss ambiguous effect".to_string(), + input_schema: json!({ + "type": "object", + "additionalProperties": false, + "required": ["case"], + "properties": {"case": {"type": "string"}} + }), + item_key_pointer: None, + idempotent: false, + outcomes: vec![FixtureCapabilityOutcome::ApplyThenDisconnect], + }], + vec![ + ( + "MOA_EXECUTION_ACTIVE_ATTEMPT_TIMEOUT_SECONDS".to_string(), + "5".to_string(), + ), + ( + "MOA_EXECUTION_TRIGGER_RECONCILIATION_CADENCE_SECONDS".to_string(), + "1".to_string(), + ), + ], + ) + .await?; + let test = fixture.isolated().await; + let pool = PgPool::connect(&fixture.postgres_url).await?; + let run = start_plan( + &test, + "restate-loss-running-ambiguous", + vec![ + fixture_capability_node( + "lost-running-attempt", + tool_name, + json!({"case": "possible-commit-before-loss"}), + ), + output_node( + &["lost-running-attempt"], + json!({"unexpected": "ambiguous effect replayed"}), + ), + ], + Duration::from_secs(55), + ) + .await?; + let controller = fixture + .fixture_capability() + .context("Restate-loss fixture omitted capability controller")?; + controller.wait_for_calls(1, SCENARIO_TIMEOUT).await?; + let running = await_task_status( + &test, + &run, + "lost-running-attempt", + ExecutionTaskStatus::Running, + ) + .await?; + let accepted = sqlx::query( + "SELECT dispatch_uid, delivered_at FROM moa.execution_dispatch_outbox \ + WHERE run_uid = $1 AND task_id = $2 AND dispatch_kind = 'task_attempt' \ + AND attempt_generation = 1 AND state = 'delivered'", + ) + .bind(run.run_uid) + .bind(task_id(&running).as_uuid()) + .fetch_one(&pool) + .await?; + let dispatch_uid: Uuid = accepted.try_get("dispatch_uid")?; + let delivered_at: DateTime = accepted + .try_get::>, _>("delivered_at")? + .context("accepted TaskAttempt omitted delivered_at")?; + + fixture.recreate_restate_after_loss().await?; + fixture.hard_crash_and_restart_orchestrator().await?; + // The fixture effect was committed before this gate. Release only after the + // old handler is gone so the recovered watchdog owns ambiguity settlement. + controller.release(1); + + let ambiguous = await_task_status( + &test, + &run, + "lost-running-attempt", + ExecutionTaskStatus::UnknownOutcome, + ) + .await?; + assert_eq!(ambiguous.attempt, 1); + assert!(matches!( + ambiguous.outcome.as_ref().map(|outcome| &outcome.result), + Some(moa_artifacts::execution_plan::ExecutionTaskResult::UnknownOutcome { message }) + if message.contains("possible commit") + )); + let unchanged = sqlx::query( + "SELECT state, delivered_at, delivery_attempts FROM moa.execution_dispatch_outbox \ + WHERE dispatch_uid = $1", + ) + .bind(dispatch_uid) + .fetch_one(&pool) + .await?; + assert_eq!(unchanged.try_get::("state")?, "delivered"); + assert_eq!( + unchanged.try_get::>, _>("delivered_at")?, + Some(delivered_at), + "reconciliation blindly re-drove a Running TaskAttempt" + ); + assert_eq!( + unchanged.try_get::("delivery_attempts")?, + 1, + "Running TaskAttempt delivery was retried after Restate loss" + ); + assert_eq!( + controller.effect_count(), + 1, + "Running ambiguous effect was logically sent more than once" + ); + let watchdogs = sqlx::query( + "SELECT trigger.state AS trigger_state, dispatch.state AS dispatch_state, \ + dispatch.delivery_attempts \ + FROM moa.execution_trigger AS trigger \ + JOIN moa.execution_dispatch_outbox AS dispatch \ + ON dispatch.trigger_uid = trigger.trigger_uid \ + AND dispatch.dispatch_kind = 'trigger_delivery' \ + WHERE trigger.run_uid = $1 AND trigger.task_id = $2 \ + AND trigger.trigger_kind = 'task_watchdog' \ + AND trigger.attempt_generation = 1", + ) + .bind(run.run_uid) + .bind(task_id(&running).as_uuid()) + .fetch_all(&pool) + .await?; + assert_eq!(watchdogs.len(), 1, "recovery created duplicate watchdogs"); + let watchdog = &watchdogs[0]; + assert_eq!( + watchdog.try_get::("trigger_state")?, + "superseded" + ); + assert_eq!( + watchdog.try_get::("dispatch_state")?, + "cancelled" + ); + assert_eq!(watchdog.try_get::("delivery_attempts")?, 1); + Ok(()) +} diff --git a/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/pause_and_external.rs b/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/pause_and_external.rs new file mode 100644 index 000000000..b4c48ca91 --- /dev/null +++ b/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/pause_and_external.rs @@ -0,0 +1,1858 @@ +//! Pause/resume, parked wait settlement, and runtime-cache loss coverage. + +use moa_artifacts::execution_plan::{ExecutionCitation, ExecutionTaskResult, InputAudience}; +use moa_core::types::action_policy::ActionReviewStatus; +use moa_core::types::tools::{AsyncToolJobCallbackOutcome, AsyncToolJobTerminalOutcome}; +use moa_execution::wire::{ + ExecutionConflictReason, ExecutionInputRequest, ExecutionMutationResponse, + ExecutionSignalRequest, +}; +use moa_orchestrator::services::{ + action_reviews::{ + ActionReviewDecisionKind, ActionReviewSummary, DecideActionReviewRequest, + ListActionReviewsRequest, + }, + execution::{ExecutionRunControlRequest, ExecutionRunControlResponse}, +}; + +use super::*; + +async fn post_external_job_callback( + fixture: &OrchestratorTestFixture, + external_job_uid: Uuid, + job_generation: u64, + provider_event_id: &str, + body: &Value, +) -> Result { + let response = reqwest::Client::new() + .post(fixture.external_job_callback_url( + external_job_uid, + job_generation, + provider_event_id, + )?) + .bearer_auth(moa_test_support::FIXTURE_EXTERNAL_JOB_CALLBACK_TOKEN) + .json(body) + .send() + .await?; + Ok(response.status()) +} + +#[tokio::test] +#[ignore = "requires Docker for the real Restate/Postgres/Valkey execution fixture"] +async fn true_external_job_reserves_before_start_parks_rearms_and_dedupes_callbacks_service_e2e() +-> Result<()> { + // Pins: an async-capable production catalog tool reserves its stable job identity before the + // provider start, binds the same idempotency key, releases attempt compute while waiting, + // rearms sparse reconciliation from progress, and accepts each callback effect exactly once. + let fixture = external_job_execution_fixture(Vec::new()).await?; + let test = fixture.isolated().await; + let pool = PgPool::connect(&fixture.postgres_url).await?; + let run = start_plan( + &test, + "true-external-job", + vec![ + external_job_capability_node("external-job", json!({"value": "task12"})), + output_node(&["external-job"], json!({"external": "complete"})), + ], + Duration::from_secs(30), + ) + .await?; + let controller = fixture + .fixture_external_job() + .context("external-job fixture omitted provider controller")?; + let starts = controller.wait_for_starts(1, SCENARIO_TIMEOUT).await?; + let start = &starts[0]; + assert_eq!( + start.context.provider, + moa_test_support::FIXTURE_EXTERNAL_JOB_PROVIDER + ); + let reserved = sqlx::query( + "SELECT job.state, job.idempotency_key, job.provider, job.provider_job_id, \ + capacity.state AS capacity_state, task.attempt_state, task.active_dispatch_uid \ + FROM moa.execution_external_job AS job \ + JOIN moa.execution_capacity_reservation AS capacity \ + ON capacity.external_job_uid = job.external_job_uid \ + AND capacity.resource_dimension = 'external_jobs' \ + JOIN moa.execution_task AS task \ + ON task.run_uid = job.run_uid AND task.task_id = job.task_id \ + WHERE job.run_uid = $1 AND job.external_job_uid = $2", + ) + .bind(run.run_uid) + .bind(start.context.external_job_uid) + .fetch_one(&pool) + .await?; + assert_eq!(reserved.try_get::("state")?, "unbound"); + assert_eq!( + reserved.try_get::("idempotency_key")?, + start.context.idempotency_key + ); + assert_eq!(reserved.try_get::, _>("provider")?, None); + assert_eq!( + reserved.try_get::, _>("provider_job_id")?, + None + ); + assert_eq!(reserved.try_get::("capacity_state")?, "reserved"); + assert!(matches!( + reserved.try_get::("attempt_state")?.as_str(), + "dispatching" | "running" + )); + assert!( + reserved + .try_get::, _>("active_dispatch_uid")? + .is_some() + ); + + controller.release_starts(1); + let after_bind = controller.wait_for_after_bind(1, SCENARIO_TIMEOUT).await?; + assert_eq!(after_bind[0].context, start.context); + controller.release_after_bind(1); + await_run_status(&test, &run, ExecutionRunStatus::WaitingExternal).await?; + let waiting = await_task_status( + &test, + &run, + "external-job", + ExecutionTaskStatus::WaitingExternal, + ) + .await?; + assert_parked_has_no_active_compute(&fixture, &pool, &run).await?; + let bound = sqlx::query( + "SELECT state, provider, provider_job_id, idempotency_key, job_generation, \ + progress_phase, next_reconcile_at, \ + (SELECT COUNT(*) FROM moa.execution_capacity_reservation \ + WHERE external_job_uid = job.external_job_uid \ + AND resource_dimension = 'external_jobs' AND state <> 'released') \ + AS active_external_receipts \ + FROM moa.execution_external_job AS job \ + WHERE run_uid = $1 AND task_id = $2", + ) + .bind(run.run_uid) + .bind(task_id(&waiting).as_uuid()) + .fetch_one(&pool) + .await?; + assert_eq!(bound.try_get::("state")?, "running"); + assert_eq!( + bound.try_get::("provider")?, + moa_test_support::FIXTURE_EXTERNAL_JOB_PROVIDER + ); + assert_eq!( + bound.try_get::("provider_job_id")?, + start.provider_job_id + ); + assert_eq!( + bound.try_get::("idempotency_key")?, + start.context.idempotency_key + ); + assert_eq!(bound.try_get::("active_external_receipts")?, 1); + let job_generation = u64::try_from(bound.try_get::("job_generation")?)?; + + let next_reconcile_at = moa_test_support::fixtures::pg_now() + TimeDelta::seconds(10); + let progress_event_id = "task12-progress-1"; + let progress = controller.callback_body( + start.provider_job_id.clone(), + progress_event_id, + AsyncToolJobCallbackOutcome::Progress { + progress_phase: "halfway".to_string(), + next_reconcile_at, + }, + ); + assert_eq!( + post_external_job_callback( + &fixture, + start.context.external_job_uid, + job_generation, + progress_event_id, + &progress, + ) + .await?, + reqwest::StatusCode::NO_CONTENT + ); + assert_eq!( + post_external_job_callback( + &fixture, + start.context.external_job_uid, + job_generation, + progress_event_id, + &progress, + ) + .await?, + reqwest::StatusCode::NO_CONTENT + ); + let progressed = sqlx::query( + "SELECT job.state, job.progress_phase, job.next_reconcile_at, \ + (SELECT COUNT(*) FROM moa.execution_external_job_callback_receipt AS receipt \ + WHERE receipt.external_job_uid = job.external_job_uid \ + AND receipt.provider_event_id = $2) AS receipt_count, \ + (SELECT COUNT(*) FROM moa.execution_trigger AS trigger \ + WHERE trigger.payload->>'external_job_uid' = job.external_job_uid::TEXT \ + AND (trigger.payload->>'job_generation')::BIGINT = job.job_generation \ + AND trigger.trigger_kind = 'external_reconcile' \ + AND trigger.state = 'pending' \ + AND trigger.due_at = $3) AS exact_reconcile_triggers \ + FROM moa.execution_external_job AS job WHERE job.external_job_uid = $1", + ) + .bind(start.context.external_job_uid) + .bind(progress_event_id) + .bind(next_reconcile_at) + .fetch_one(&pool) + .await?; + assert_eq!( + progressed.try_get::("state")?, + "waiting_reconcile" + ); + assert_eq!( + progressed.try_get::("progress_phase")?, + "halfway" + ); + assert_eq!( + progressed.try_get::, _>("next_reconcile_at")?, + next_reconcile_at + ); + assert_eq!(progressed.try_get::("receipt_count")?, 1); + assert_eq!(progressed.try_get::("exact_reconcile_triggers")?, 1); + await_run_status(&test, &run, ExecutionRunStatus::WaitingExternal).await?; + assert_parked_has_no_active_compute(&fixture, &pool, &run).await?; + + let terminal_event_id = "task12-terminal-1"; + let terminal_body = controller.callback_body( + start.provider_job_id.clone(), + terminal_event_id, + AsyncToolJobCallbackOutcome::Terminal { + outcome: AsyncToolJobTerminalOutcome::Completed { + output: json!({"provider": "done"}), + }, + }, + ); + assert_eq!( + post_external_job_callback( + &fixture, + start.context.external_job_uid, + job_generation, + terminal_event_id, + &terminal_body, + ) + .await?, + reqwest::StatusCode::NO_CONTENT + ); + let completed = await_run_status(&test, &run, ExecutionRunStatus::Completed).await?; + assert_eq!(completed.output, Some(json!({"external": "complete"}))); + assert_eq!( + post_external_job_callback( + &fixture, + start.context.external_job_uid, + job_generation, + terminal_event_id, + &terminal_body, + ) + .await?, + reqwest::StatusCode::NO_CONTENT + ); + let terminal = sqlx::query( + "SELECT job.state, job.output, job.completed_at, \ + (SELECT COUNT(*) FROM moa.execution_external_job_callback_receipt AS receipt \ + WHERE receipt.external_job_uid = job.external_job_uid \ + AND receipt.provider_event_id = $2) AS receipt_count, \ + (SELECT COUNT(*) FROM moa.execution_capacity_reservation AS capacity \ + WHERE capacity.external_job_uid = job.external_job_uid \ + AND capacity.resource_dimension = 'external_jobs' \ + AND capacity.state = 'released') AS released_external_receipts \ + FROM moa.execution_external_job AS job WHERE job.external_job_uid = $1", + ) + .bind(start.context.external_job_uid) + .bind(terminal_event_id) + .fetch_one(&pool) + .await?; + assert_eq!(terminal.try_get::("state")?, "completed"); + assert_eq!( + terminal.try_get::("output")?, + json!({"provider": "done"}) + ); + assert!( + terminal + .try_get::>, _>("completed_at")? + .is_some() + ); + assert_eq!(terminal.try_get::("receipt_count")?, 1); + assert_eq!(terminal.try_get::("released_external_receipts")?, 1); + assert_eq!(controller.starts().len(), 1); + Ok(()) +} + +#[tokio::test] +#[ignore = "requires Docker for the real Restate/Postgres/Valkey execution fixture"] +async fn terminal_callback_before_attempt_release_defers_then_settles_once_service_e2e() +-> Result<()> { + // Pins: a provider terminal callback can win after durable bind but before TaskAttempt has + // released its capacity. Persistence records DeferredRelease without waking the controller; + // the exact post-bind attempt then releases capacity and consumes that terminal result once. + let fixture = external_job_execution_fixture(Vec::new()).await?; + let test = fixture.isolated().await; + let pool = PgPool::connect(&fixture.postgres_url).await?; + let run = start_plan( + &test, + "external-terminal-before-release", + vec![ + external_job_capability_node("external-job", json!({"value": "early-terminal"})), + output_node(&["external-job"], json!({"deferred_release": "complete"})), + ], + Duration::from_secs(30), + ) + .await?; + let controller = fixture + .fixture_external_job() + .context("external-job fixture omitted provider controller")?; + let starts = controller.wait_for_starts(1, SCENARIO_TIMEOUT).await?; + let start = &starts[0]; + controller.release_starts(1); + let after_bind = controller.wait_for_after_bind(1, SCENARIO_TIMEOUT).await?; + assert_eq!(after_bind[0].context, start.context); + let bound_owner = sqlx::query( + "SELECT job.job_generation, job.state AS job_state, task.status AS task_status, \ + task.attempt_state, task.active_dispatch_uid, \ + capacity.state AS task_capacity_state \ + FROM moa.execution_external_job AS job \ + JOIN moa.execution_task AS task \ + ON task.run_uid = job.run_uid AND task.task_id = job.task_id \ + JOIN moa.execution_capacity_reservation AS capacity \ + ON capacity.run_uid = task.run_uid AND capacity.task_id = task.task_id \ + AND capacity.attempt_generation = task.attempt_generation \ + AND capacity.resource_dimension = 'active_tasks' \ + WHERE job.run_uid = $1 AND job.external_job_uid = $2", + ) + .bind(run.run_uid) + .bind(start.context.external_job_uid) + .fetch_one(&pool) + .await?; + let job_generation = u64::try_from(bound_owner.try_get::("job_generation")?)?; + assert_eq!(bound_owner.try_get::("job_state")?, "running"); + assert_eq!(bound_owner.try_get::("task_status")?, "running"); + assert_eq!( + bound_owner.try_get::("attempt_state")?, + "running" + ); + assert_eq!( + bound_owner.try_get::("task_capacity_state")?, + "reserved" + ); + assert!( + bound_owner + .try_get::, _>("active_dispatch_uid")? + .is_some() + ); + + let event_id = "task12-terminal-before-release"; + let body = controller.callback_body( + start.provider_job_id.clone(), + event_id, + AsyncToolJobCallbackOutcome::Terminal { + outcome: AsyncToolJobTerminalOutcome::Completed { + output: json!({"provider": "early"}), + }, + }, + ); + assert_eq!( + post_external_job_callback( + &fixture, + start.context.external_job_uid, + job_generation, + event_id, + &body, + ) + .await?, + reqwest::StatusCode::NO_CONTENT + ); + let deferred = sqlx::query( + "SELECT job.state AS job_state, task.status AS task_status, task.attempt_state, \ + task.active_dispatch_uid, task.external_job_uid, \ + task_capacity.state AS task_capacity_state, \ + external_capacity.state AS external_capacity_state, \ + (SELECT COUNT(*) FROM moa.execution_dispatch_outbox AS activation \ + WHERE activation.run_uid = job.run_uid \ + AND activation.dispatch_kind = 'run_activation' \ + AND activation.payload ->> 'source' = 'external_job_callback' \ + AND activation.payload ->> 'external_job_uid' = $2) AS callback_activations \ + FROM moa.execution_external_job AS job \ + JOIN moa.execution_task AS task \ + ON task.run_uid = job.run_uid AND task.task_id = job.task_id \ + JOIN moa.execution_capacity_reservation AS task_capacity \ + ON task_capacity.run_uid = task.run_uid AND task_capacity.task_id = task.task_id \ + AND task_capacity.attempt_generation = task.attempt_generation \ + AND task_capacity.resource_dimension = 'active_tasks' \ + JOIN moa.execution_capacity_reservation AS external_capacity \ + ON external_capacity.external_job_uid = job.external_job_uid \ + AND external_capacity.resource_dimension = 'external_jobs' \ + WHERE job.run_uid = $1 AND job.external_job_uid = $3", + ) + .bind(run.run_uid) + .bind(start.context.external_job_uid.to_string()) + .bind(start.context.external_job_uid) + .fetch_one(&pool) + .await?; + assert_eq!(deferred.try_get::("job_state")?, "completed"); + assert_eq!(deferred.try_get::("task_status")?, "running"); + assert_eq!(deferred.try_get::("attempt_state")?, "running"); + assert_eq!( + deferred.try_get::, _>("external_job_uid")?, + None, + "callback must not forge the pre-release checkpoint" + ); + assert!( + deferred + .try_get::, _>("active_dispatch_uid")? + .is_some() + ); + assert_eq!( + deferred.try_get::("task_capacity_state")?, + "reserved" + ); + assert_eq!( + deferred.try_get::("external_capacity_state")?, + "released" + ); + assert_eq!(deferred.try_get::("callback_activations")?, 0); + + controller.release_after_bind(1); + let completed = await_run_status(&test, &run, ExecutionRunStatus::Completed).await?; + assert_eq!( + completed.output, + Some(json!({"deferred_release": "complete"})) + ); + let settled = sqlx::query( + "SELECT task.status, task.attempt_state, task.active_dispatch_uid, \ + task.external_job_uid, capacity.state AS capacity_state, \ + (SELECT COUNT(*) FROM moa.execution_external_job_callback_receipt \ + WHERE external_job_uid = $2 AND provider_event_id = $3) AS receipt_count \ + FROM moa.execution_task AS task \ + JOIN moa.execution_capacity_reservation AS capacity \ + ON capacity.run_uid = task.run_uid AND capacity.task_id = task.task_id \ + AND capacity.attempt_generation = task.attempt_generation \ + AND capacity.resource_dimension = 'active_tasks' \ + WHERE task.run_uid = $1 AND task.node_id = 'external-job'", + ) + .bind(run.run_uid) + .bind(start.context.external_job_uid) + .bind(event_id) + .fetch_one(&pool) + .await?; + assert_eq!(settled.try_get::("status")?, "completed"); + assert_eq!(settled.try_get::("attempt_state")?, "terminal"); + assert_eq!( + settled.try_get::, _>("active_dispatch_uid")?, + None + ); + assert_eq!( + settled.try_get::, _>("external_job_uid")?, + Some(start.context.external_job_uid) + ); + assert_eq!(settled.try_get::("capacity_state")?, "released"); + assert_eq!(settled.try_get::("receipt_count")?, 1); + assert_eq!(controller.starts().len(), 1); + assert_eq!(controller.after_bind().len(), 1); + Ok(()) +} + +#[tokio::test] +#[ignore = "requires Docker for the real Restate/Postgres/Valkey execution fixture"] +async fn paused_external_reconcile_settles_storage_then_resume_activates_once_service_e2e() +-> Result<()> { + // Pins: provider time continues while a run is Paused. A due sparse reconciliation persists + // the terminal job/task outcome without controller compute, and resume emits one activation + // that observes the already-settled dependency instead of reissuing provider work. + let fixture = external_job_execution_fixture(Vec::new()).await?; + let test = fixture.isolated().await; + let pool = PgPool::connect(&fixture.postgres_url).await?; + let run = start_plan( + &test, + "paused-external-reconcile", + vec![ + external_job_capability_node("external-job", json!({"value": "pause"})), + output_node(&["external-job"], json!({"external_pause": "complete"})), + ], + Duration::from_secs(30), + ) + .await?; + let controller = fixture + .fixture_external_job() + .context("external-job fixture omitted provider controller")?; + let starts = controller.wait_for_starts(1, SCENARIO_TIMEOUT).await?; + let start = &starts[0]; + controller.release_starts(1); + let after_bind = controller.wait_for_after_bind(1, SCENARIO_TIMEOUT).await?; + assert_eq!(after_bind[0].context, start.context); + controller.release_after_bind(1); + await_run_status(&test, &run, ExecutionRunStatus::WaitingExternal).await?; + let waiting = await_task_status( + &test, + &run, + "external-job", + ExecutionTaskStatus::WaitingExternal, + ) + .await?; + let before_pause = sqlx::query( + "SELECT run.controller_generation, job.job_generation, job.next_reconcile_at, \ + trigger.trigger_uid, trigger.due_at, trigger.state AS trigger_state \ + FROM moa.execution_run AS run \ + JOIN moa.execution_external_job AS job ON job.run_uid = run.run_uid \ + JOIN moa.execution_trigger AS trigger \ + ON trigger.payload->>'external_job_uid' = job.external_job_uid::TEXT \ + AND (trigger.payload->>'job_generation')::BIGINT = job.job_generation \ + AND trigger.trigger_kind = 'external_reconcile' \ + WHERE run.run_uid = $1 AND job.task_id = $2", + ) + .bind(run.run_uid) + .bind(task_id(&waiting).as_uuid()) + .fetch_one(&pool) + .await?; + let initial_generation = + u64::try_from(before_pause.try_get::("controller_generation")?)?; + let job_generation = u64::try_from(before_pause.try_get::("job_generation")?)?; + let reconcile_trigger_uid: Uuid = before_pause.try_get("trigger_uid")?; + let due_at: DateTime = before_pause.try_get("due_at")?; + assert_eq!( + before_pause.try_get::, _>("next_reconcile_at")?, + due_at + ); + assert_eq!( + before_pause.try_get::("trigger_state")?, + "pending" + ); + + let pause_request = ExecutionRunControlRequest { + run: run.request.clone(), + expected_controller_generation: initial_generation, + }; + let pause: ExecutionRunControlResponse = test + .client() + .post_call("/Execution/pause", &pause_request) + .await?; + let paused_generation = match pause { + ExecutionRunControlResponse::Applied { + run: summary, + controller_generation, + .. + } => { + assert_eq!(summary.status, ExecutionRunStatus::Paused); + assert_eq!(controller_generation, initial_generation + 1); + controller_generation + } + other => bail!("external waiting run did not pause: {other:?}"), + }; + assert_parked_has_no_active_compute(&fixture, &pool, &run).await?; + let paused_external_receipts: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM moa.execution_capacity_reservation \ + WHERE run_uid = $1 AND external_job_uid = $2 \ + AND resource_dimension = 'external_jobs' AND state <> 'released'", + ) + .bind(run.run_uid) + .bind(start.context.external_job_uid) + .fetch_one(&pool) + .await?; + assert_eq!(paused_external_receipts, 1); + controller.queue_reconcile_outcomes([AsyncToolJobCallbackOutcome::Terminal { + outcome: AsyncToolJobTerminalOutcome::Completed { + output: json!({"provider": "reconciled"}), + }, + }]); + let reconciliations = controller + .wait_for_reconciliations(1, SCENARIO_TIMEOUT) + .await?; + assert_eq!(reconciliations.len(), 1); + assert_eq!( + reconciliations[0].request.external_job_uid, + start.context.external_job_uid + ); + assert_eq!(reconciliations[0].request.job_generation, job_generation); + assert_eq!( + reconciliations[0].request.trigger_uid, + reconcile_trigger_uid + ); + assert_eq!( + reconciliations[0].request.idempotency_key, + start.context.idempotency_key + ); + + let deadline = Instant::now() + SCENARIO_TIMEOUT; + let settled = loop { + let row = sqlx::query( + "SELECT run.status AS run_status, run.activation_state, run.controller_generation, \ + job.state AS job_state, job.output, task.status AS task_status, \ + trigger.state AS trigger_state, dispatch.state AS dispatch_state, \ + (SELECT COUNT(*) FROM moa.execution_capacity_reservation AS capacity \ + WHERE capacity.external_job_uid = job.external_job_uid \ + AND capacity.resource_dimension = 'external_jobs' \ + AND capacity.state = 'released') AS released_external_receipts, \ + (SELECT COUNT(*) FROM moa.execution_dispatch_outbox AS activation \ + WHERE activation.run_uid = run.run_uid \ + AND activation.dispatch_kind = 'run_activation' \ + AND activation.controller_generation = $3) AS paused_activations \ + FROM moa.execution_run AS run \ + JOIN moa.execution_external_job AS job ON job.run_uid = run.run_uid \ + JOIN moa.execution_task AS task \ + ON task.run_uid = job.run_uid AND task.task_id = job.task_id \ + JOIN moa.execution_trigger AS trigger ON trigger.trigger_uid = $2 \ + JOIN moa.execution_dispatch_outbox AS dispatch \ + ON dispatch.trigger_uid = trigger.trigger_uid \ + AND dispatch.dispatch_kind = 'trigger_delivery' \ + WHERE run.run_uid = $1", + ) + .bind(run.run_uid) + .bind(reconcile_trigger_uid) + .bind(i64::try_from(paused_generation)?) + .fetch_one(&pool) + .await?; + if row.try_get::("job_state")? == "completed" + && row.try_get::("task_status")? == "completed" + && row.try_get::("trigger_state")? == "superseded" + && row.try_get::("dispatch_state")? == "cancelled" + { + break row; + } + if Instant::now() >= deadline { + bail!( + "paused external reconciliation did not settle: job={}, task={}, trigger={}", + row.try_get::("job_state")?, + row.try_get::("task_status")?, + row.try_get::("trigger_state")?, + ); + } + tokio::time::sleep(POLL_INTERVAL).await; + }; + assert_eq!(settled.try_get::("run_status")?, "paused"); + assert_eq!(settled.try_get::("activation_state")?, "paused"); + assert_eq!( + u64::try_from(settled.try_get::("controller_generation")?)?, + paused_generation + ); + assert_eq!(settled.try_get::("trigger_state")?, "superseded"); + assert_eq!(settled.try_get::("dispatch_state")?, "cancelled"); + assert_eq!(settled.try_get::("released_external_receipts")?, 1); + assert_eq!( + settled.try_get::("output")?, + json!({"provider": "reconciled"}) + ); + assert_eq!(settled.try_get::("paused_activations")?, 0); + assert_parked_has_no_active_compute(&fixture, &pool, &run).await?; + + let resume_request = ExecutionRunControlRequest { + run: run.request.clone(), + expected_controller_generation: paused_generation, + }; + let resume: ExecutionRunControlResponse = test + .client() + .post_call("/Execution/resume", &resume_request) + .await?; + let (resumed_generation, resumed_wake_epoch) = match resume { + ExecutionRunControlResponse::Applied { + controller_generation, + wake_epoch, + .. + } => { + assert_eq!(controller_generation, paused_generation + 1); + (controller_generation, wake_epoch) + } + other => bail!("settled external run did not resume: {other:?}"), + }; + let completed = await_run_status(&test, &run, ExecutionRunStatus::Completed).await?; + assert_eq!( + completed.output, + Some(json!({"external_pause": "complete"})) + ); + let resumed_activations: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM moa.execution_dispatch_outbox \ + WHERE run_uid = $1 AND dispatch_kind = 'run_activation' \ + AND controller_generation = $2 AND wake_epoch = $3", + ) + .bind(run.run_uid) + .bind(i64::try_from(resumed_generation)?) + .bind(i64::try_from(resumed_wake_epoch)?) + .fetch_one(&pool) + .await?; + assert_eq!(resumed_activations, 1); + assert_eq!(controller.starts().len(), 1); + assert_eq!(controller.reconciliations().len(), 1); + Ok(()) +} + +#[tokio::test] +#[ignore = "requires Docker for the real Restate/Postgres/Valkey execution fixture"] +async fn agent_action_review_parks_then_resumes_persisted_continuation_service_e2e() -> Result<()> { + // Pins: an Agent's governed effect is checkpointed before WaitingReview, + // owns no active capacity while parked, and a tenant-admin decision resumes + // the persisted invocation in a new bounded slice instead of a live promise. + let tool_name = "long_horizon_agent_review_probe"; + let registered_tool_name = moa_hands::mcp_tool_reference("fixture-capability", tool_name); + let completed = serde_json::to_string(&ExecutionTaskResult::Completed { + output: json!({"review": "continued"}), + citations: Vec::::new(), + })?; + let fixture = execution_fixture_with_script_and_tools( + json!({ + "default": { + "completion": { + "content": "unexpected agent review continuation path", + "tool_calls": [] + } + }, + "keyed": [ + { + "match": "reviewed_effect", + "completion": {"content": completed, "tool_calls": []} + }, + { + "match": "agent-review-operation", + "completion": { + "content": "Requesting the reviewed effect.", + "tool_calls": [{ + "name": registered_tool_name, + "id": "agent-review-effect", + "input": {"case": "agent-review"} + }] + } + } + ] + }), + vec![FixtureCapabilityTool { + name: tool_name.to_string(), + description: "Deterministic Task 12 Agent review continuation".to_string(), + input_schema: json!({ + "type": "object", + "additionalProperties": false, + "required": ["case"], + "properties": {"case": {"type": "string"}} + }), + item_key_pointer: None, + idempotent: true, + outcomes: vec![FixtureCapabilityOutcome::Success { + output: json!({"reviewed_effect": "applied"}), + }], + }], + Vec::new(), + ) + .await?; + let test = fixture.isolated().await; + let pool = PgPool::connect(&fixture.postgres_url).await?; + let run = start_plan_with_capability_policy( + &test, + "agent-action-review", + vec![ + node( + "reviewing-agent", + &[], + ExecutionOperation::Agent { + instructions: "agent-review-operation".to_string(), + skill_refs: Vec::new(), + capability_refs: vec![CapabilityReference { + name: registered_tool_name.clone(), + version: FIXTURE_CAPABILITY_VERSION.to_string(), + }], + max_turns: 3, + }, + json!({"type": "object"}), + ), + output_node(&["reviewing-agent"], json!({"review": "complete"})), + ], + Duration::from_secs(30), + Some(ActionPolicyEffect::AdminReview), + ) + .await?; + let waiting = await_task_status( + &test, + &run, + "reviewing-agent", + ExecutionTaskStatus::WaitingReview, + ) + .await?; + await_run_status(&test, &run, ExecutionRunStatus::WaitingReview).await?; + assert_eq!(waiting.attempt, 1); + assert_parked_has_no_active_compute(&fixture, &pool, &run).await?; + + let review = await_execution_action_review(&test, run.tenant_id, task_id(&waiting)).await?; + let origin = review + .envelope + .owner + .execution_origin() + .context("Agent action review omitted execution origin")?; + assert_eq!(origin.task_uid, task_id(&waiting).as_uuid()); + assert_eq!(origin.generation, waiting.generation); + let checkpoint = sqlx::query( + "SELECT checkpoint_kind, task_generation, attempt_generation, payload, \ + workspace_release_receipt, \ + (SELECT COUNT(*) FROM moa.execution_capacity_reservation AS capacity \ + WHERE capacity.run_uid = checkpoint.run_uid \ + AND capacity.task_id = checkpoint.task_id \ + AND capacity.attempt_generation = checkpoint.attempt_generation \ + AND capacity.resource_dimension = 'active_tasks' \ + AND capacity.state = 'released') AS released_capacity, \ + (SELECT COUNT(*) FROM moa.execution_dispatch_outbox AS activation \ + WHERE activation.run_uid = checkpoint.run_uid \ + AND activation.dispatch_kind = 'run_activation' \ + AND activation.payload->>'source' = 'task_attempt_review_park' \ + AND activation.payload->>'task_id' = checkpoint.task_id::TEXT \ + AND (activation.payload->>'attempt_generation')::BIGINT = \ + checkpoint.attempt_generation) AS controller_activations \ + FROM moa.execution_task_checkpoint AS checkpoint \ + WHERE checkpoint.run_uid = $1 AND checkpoint.task_id = $2 \ + AND checkpoint.superseded_at IS NULL", + ) + .bind(run.run_uid) + .bind(task_id(&waiting).as_uuid()) + .fetch_one(&pool) + .await?; + assert_eq!( + checkpoint.try_get::("checkpoint_kind")?, + "agent_continuation" + ); + assert_eq!( + u64::try_from(checkpoint.try_get::("task_generation")?)?, + waiting.generation + ); + assert_eq!(checkpoint.try_get::("attempt_generation")?, 1); + assert_eq!(checkpoint.try_get::("released_capacity")?, 1); + assert_eq!(checkpoint.try_get::("controller_activations")?, 1); + let release_receipt: Value = checkpoint + .try_get::, _>("workspace_release_receipt")? + .context("the review checkpoint omitted its exact hand-release proof")?; + assert_eq!(release_receipt.get("workspace_id"), Some(&Value::Null)); + assert_eq!( + release_receipt.get("hand_provisioning_operation_id"), + Some(&Value::Null) + ); + assert_eq!( + release_receipt.get("hand_lease_generation"), + Some(&Value::Null), + "a non-sandbox review must persist verified absence, not invent a hand identity" + ); + let payload: Value = checkpoint.try_get("payload")?; + assert_eq!( + payload.pointer("/state/kind").and_then(Value::as_str), + Some("agent") + ); + let review_id = review.id.to_string(); + assert_eq!( + payload + .pointer("/state/pending_review/review_uid") + .and_then(Value::as_str), + Some(review_id.as_str()) + ); + assert_eq!( + payload + .pointer("/state/pending_review/invocation/id") + .and_then(Value::as_str), + Some("agent-review-effect") + ); + assert_eq!(payload.get("review_resolution"), Some(&Value::Null)); + let controller = fixture + .fixture_capability() + .context("Agent review fixture omitted capability controller")?; + assert!( + controller.calls().is_empty(), + "reviewed effect ran before approval" + ); + + let client = fixture.client.clone(); + let tenant_id = run.tenant_id; + let action_review_uid = review.id; + let decision = tokio::spawn(async move { + client + .post_void( + "/ActionReviews/decide", + &DecideActionReviewRequest { + tenant_id, + review_id: action_review_uid, + decision: ActionReviewDecisionKind::Cleared, + reason: None, + }, + ) + .await + }); + let calls = controller.wait_for_calls(1, SCENARIO_TIMEOUT).await?; + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].capability, tool_name); + assert_eq!(calls[0].input, json!({"case": "agent-review"})); + controller.release(1); + tokio::time::timeout(SCENARIO_TIMEOUT, decision) + .await + .context("Agent action-review decision did not durably settle")???; + + let (attempt_count, last_error) = await_execution_review_delivery(&pool, review.id).await?; + assert_eq!( + attempt_count, 1, + "the admin decision must dispatch its exact execution resolution once; last_error={last_error:?}" + ); + assert!( + last_error.is_none(), + "the exact execution resolution failed before delivery: {last_error:?}" + ); + let completed_task = await_task_status( + &test, + &run, + "reviewing-agent", + ExecutionTaskStatus::Completed, + ) + .await?; + assert_eq!(completed_task.attempt, 1); + assert_eq!(completed_task.generation, waiting.generation); + assert_eq!(controller.calls().len(), 1); + let redispatch = sqlx::query( + "SELECT COUNT(*) AS dispatch_count, COUNT(DISTINCT dispatch_uid) AS distinct_dispatches, \ + MIN(attempt_generation) AS first_generation, \ + MAX(attempt_generation) AS last_generation \ + FROM moa.execution_dispatch_outbox \ + WHERE run_uid = $1 AND task_id = $2 AND dispatch_kind = 'task_attempt'", + ) + .bind(run.run_uid) + .bind(task_id(&completed_task).as_uuid()) + .fetch_one(&pool) + .await?; + assert_eq!(redispatch.try_get::("dispatch_count")?, 2); + assert_eq!(redispatch.try_get::("distinct_dispatches")?, 2); + assert_eq!( + redispatch.try_get::, _>("first_generation")?, + Some(1) + ); + assert_eq!( + redispatch.try_get::, _>("last_generation")?, + Some(2) + ); + let terminal = await_run_status(&test, &run, ExecutionRunStatus::Completed).await?; + assert_eq!(terminal.output, Some(json!({"review": "complete"}))); + Ok(()) +} + +#[tokio::test] +#[ignore = "requires Docker for the real Restate/Postgres/Valkey/sandbox-workspace fixture"] +async fn sandbox_hand_releases_during_signal_wait_and_reacquires_after_resume_service_e2e() +-> Result<()> { + // Pins: a real sandbox-required execution capability owns one ActiveHands + // receipt while running, releases it before a storage-only signal wait, + // and a downstream sandbox task acquires and releases its own receipt. + let fixture = sandbox_execution_fixture().await?; + let test = fixture.isolated().await; + let pool = PgPool::connect(&fixture.postgres_url).await?; + let run = start_plan( + &test, + "sandbox-hand-park-resume", + vec![ + hand_capability_node( + "sandbox-before-wait", + &[], + "bash", + json!({ + "cmd": "sleep 3", + "timeout_secs": 10 + }), + ), + node( + "sandbox-signal-wait", + &["sandbox-before-wait"], + ExecutionOperation::WaitSignal { + signal_name: "resume-sandbox".to_string(), + wait_policy: ExecutionWaitPolicy { + expiry: after_logical_days(7), + on_expiry: ExecutionWaitExpiryAction::FailRun, + }, + }, + json!({"type": "object"}), + ), + hand_capability_node( + "sandbox-after-wait", + &["sandbox-signal-wait"], + "bash", + json!({ + "cmd": "sleep 3", + "timeout_secs": 10 + }), + ), + output_node( + &["sandbox-after-wait"], + json!({"sandbox": "released-and-reacquired"}), + ), + ], + Duration::from_secs(35), + ) + .await?; + + let first_task = await_task_status( + &test, + &run, + "sandbox-before-wait", + ExecutionTaskStatus::Running, + ) + .await?; + let first_workspace = + await_active_execution_task_hand(&pool, &run, task_id(&first_task)).await?; + let waiting = await_task_status( + &test, + &run, + "sandbox-signal-wait", + ExecutionTaskStatus::WaitingSignal, + ) + .await?; + await_run_status(&test, &run, ExecutionRunStatus::WaitingSignal).await?; + assert_released_execution_task_hand(&pool, &run, task_id(&first_task), first_workspace).await?; + assert_parked_has_no_active_compute(&fixture, &pool, &run).await?; + + let signal: ExecutionMutationResponse = test + .client() + .post_call( + "/Execution/deliver_signal", + &ExecutionSignalRequest { + tenant_id: run.tenant_id, + contact_id: None, + run_uid: run.run_uid, + task_id: task_id(&waiting), + expected_generation: waiting.generation, + signal_name: "resume-sandbox".to_string(), + payload: json!({"resume": true}), + }, + ) + .await?; + assert!(matches!(signal, ExecutionMutationResponse::Applied { .. })); + let second_task = await_task_status( + &test, + &run, + "sandbox-after-wait", + ExecutionTaskStatus::Running, + ) + .await?; + let second_workspace = + await_active_execution_task_hand(&pool, &run, task_id(&second_task)).await?; + assert_ne!( + second_workspace, first_workspace, + "execution-task workspace ownership must remain scoped to the exact logical task" + ); + let terminal = await_run_status(&test, &run, ExecutionRunStatus::Completed).await?; + assert_eq!( + terminal.output, + Some(json!({"sandbox": "released-and-reacquired"})) + ); + assert_released_execution_task_hand(&pool, &run, task_id(&second_task), second_workspace) + .await?; + fixture.cleanup_sandbox_workspace_namespace().await?; + Ok(()) +} + +#[tokio::test] +#[ignore = "requires Docker for the real Restate/Postgres/Valkey execution fixture"] +async fn paused_timer_settles_without_activation_then_resume_advances_once_service_e2e() +-> Result<()> { + // Pins: wall time continues while a storage-only timer run is Paused; the + // due trigger and exact task settle durably without controller activation, + // then one generation-fenced resume activation advances the settled graph. + let fixture = execution_fixture(vec![( + "MOA_EXECUTION_TRIGGER_RECONCILIATION_CADENCE_SECONDS".to_string(), + "1".to_string(), + )]) + .await?; + let test = fixture.isolated().await; + let pool = PgPool::connect(&fixture.postgres_url).await?; + let run = start_plan( + &test, + "pause-due-timer", + vec![ + node( + "paused-timer", + &[], + ExecutionOperation::WaitUntil { + wake: after_logical_days(5), + result: json!({"timer": "elapsed"}), + }, + json!({"type": "object"}), + ), + output_node(&["paused-timer"], json!({"pause_timer": "complete"})), + ], + Duration::from_secs(30), + ) + .await?; + await_run_status(&test, &run, ExecutionRunStatus::WaitingTimer).await?; + let waiting = await_task_status( + &test, + &run, + "paused-timer", + ExecutionTaskStatus::WaitingTimer, + ) + .await?; + let timer_row = sqlx::query( + "SELECT trigger.trigger_uid, trigger.due_at, run.controller_generation, run.wake_epoch \ + FROM moa.execution_trigger AS trigger \ + JOIN moa.execution_run AS run ON run.run_uid = trigger.run_uid \ + WHERE trigger.run_uid = $1 AND trigger.task_id = $2 \ + AND trigger.trigger_kind = 'task_timer'", + ) + .bind(run.run_uid) + .bind(task_id(&waiting).as_uuid()) + .fetch_one(&pool) + .await?; + let trigger_uid: Uuid = timer_row.try_get("trigger_uid")?; + let due_at: DateTime = timer_row.try_get("due_at")?; + let initial_generation = u64::try_from(timer_row.try_get::("controller_generation")?)?; + let initial_wake_epoch = u64::try_from(timer_row.try_get::("wake_epoch")?)?; + assert!( + due_at > Utc::now() + TimeDelta::seconds(2), + "setup failed to pause the timer safely before its persisted due time" + ); + + let pause_request = ExecutionRunControlRequest { + run: run.request.clone(), + expected_controller_generation: initial_generation, + }; + let pause: ExecutionRunControlResponse = test + .client() + .post_call("/Execution/pause", &pause_request) + .await?; + let (paused_generation, paused_wake_epoch) = match pause { + ExecutionRunControlResponse::Applied { + run: summary, + controller_generation, + wake_epoch, + } => { + assert_eq!(summary.status, ExecutionRunStatus::Paused); + assert_eq!(controller_generation, initial_generation + 1); + assert!(wake_epoch >= initial_wake_epoch); + (controller_generation, wake_epoch) + } + other => bail!("waiting-timer pause was not applied directly: {other:?}"), + }; + assert_parked_has_no_active_compute(&fixture, &pool, &run).await?; + + let deadline = Instant::now() + SCENARIO_TIMEOUT; + let settled = loop { + let row = sqlx::query( + "SELECT trigger.state AS trigger_state, trigger.delivered_at, \ + trigger.controller_generation AS trigger_controller_generation, \ + dispatch.state AS dispatch_state, dispatch.delivered_at AS dispatch_delivered_at, \ + task.status AS task_status, task.outcome_audit, task.generation_history, \ + run.status AS run_status, run.activation_state, \ + run.controller_generation, run.wake_epoch, \ + (SELECT capacity.state FROM moa.execution_capacity_reservation AS capacity \ + WHERE capacity.trigger_uid = trigger.trigger_uid \ + AND capacity.resource_dimension = 'scheduled_triggers') \ + AS scheduled_trigger_capacity_state, \ + (SELECT COUNT(*) FROM moa.execution_dispatch_outbox AS activation \ + WHERE activation.run_uid = run.run_uid \ + AND activation.dispatch_kind = 'run_activation' \ + AND activation.controller_generation = $3) AS paused_activations \ + FROM moa.execution_trigger AS trigger \ + JOIN moa.execution_dispatch_outbox AS dispatch \ + ON dispatch.trigger_uid = trigger.trigger_uid \ + AND dispatch.dispatch_kind = 'trigger_delivery' \ + JOIN moa.execution_task AS task \ + ON task.run_uid = trigger.run_uid AND task.task_id = trigger.task_id \ + JOIN moa.execution_run AS run ON run.run_uid = trigger.run_uid \ + WHERE trigger.run_uid = $1 AND trigger.trigger_uid = $2", + ) + .bind(run.run_uid) + .bind(trigger_uid) + .bind(i64::try_from(paused_generation)?) + .fetch_one(&pool) + .await?; + if row.try_get::("trigger_state")? == "delivered" + && row.try_get::("task_status")? == "completed" + { + break row; + } + if Instant::now() >= deadline { + bail!( + "paused timer {trigger_uid} did not settle after persisted due_at {due_at}; \ + trigger={}, task={}", + row.try_get::("trigger_state")?, + row.try_get::("task_status")?, + ); + } + tokio::time::sleep(POLL_INTERVAL).await; + }; + assert_eq!(settled.try_get::("dispatch_state")?, "delivered"); + let trigger_delivered_at = settled + .try_get::>, _>("delivered_at")? + .context("delivered timer omitted delivered_at")?; + let dispatch_delivered_at = settled + .try_get::>, _>("dispatch_delivered_at")? + .context("delivered timer outbox omitted delivered_at")?; + assert!(trigger_delivered_at >= due_at); + assert!(dispatch_delivered_at >= due_at); + assert_eq!( + u64::try_from(settled.try_get::("trigger_controller_generation")?)?, + initial_generation, + "pause must not rearm the immutable timer under the new controller generation" + ); + assert_eq!( + settled.try_get::("scheduled_trigger_capacity_state")?, + "released" + ); + assert_eq!(settled.try_get::("run_status")?, "paused"); + assert_eq!(settled.try_get::("activation_state")?, "paused"); + assert_eq!( + u64::try_from(settled.try_get::("controller_generation")?)?, + paused_generation + ); + assert!( + u64::try_from(settled.try_get::("wake_epoch")?)? >= paused_wake_epoch, + "paused settlement must not move the run wake epoch backwards" + ); + assert_eq!(settled.try_get::("paused_activations")?, 0); + let outcome_audit: Value = settled.try_get("outcome_audit")?; + assert_eq!( + outcome_audit + .as_array() + .context("timer outcome audit was not an array")? + .iter() + .filter(|entry| entry.get("accepted").and_then(Value::as_bool) == Some(true)) + .count(), + 1, + "the due timer outcome must be accepted exactly once while paused" + ); + let generation_history: Value = settled.try_get("generation_history")?; + assert_eq!( + generation_history + .as_array() + .context("timer generation history was not an array")? + .iter() + .filter(|entry| { + entry.get("kind").and_then(Value::as_str) == Some("storage_wait_settlement") + }) + .count(), + 1, + "the due timer must record exactly one storage-wait settlement" + ); + await_run_status(&test, &run, ExecutionRunStatus::Paused).await?; + await_task_status(&test, &run, "paused-timer", ExecutionTaskStatus::Completed).await?; + assert_parked_has_no_active_compute(&fixture, &pool, &run).await?; + + let resume_request = ExecutionRunControlRequest { + run: run.request.clone(), + expected_controller_generation: paused_generation, + }; + let resume: ExecutionRunControlResponse = test + .client() + .post_call("/Execution/resume", &resume_request) + .await?; + let (resumed_generation, resumed_wake_epoch) = match resume { + ExecutionRunControlResponse::Applied { + run: summary, + controller_generation, + wake_epoch, + } => { + assert_eq!(summary.status, ExecutionRunStatus::Queued); + assert_eq!(controller_generation, paused_generation + 1); + (controller_generation, wake_epoch) + } + other => bail!("paused settled timer did not resume: {other:?}"), + }; + let terminal = await_run_status(&test, &run, ExecutionRunStatus::Completed).await?; + assert_eq!(terminal.output, Some(json!({"pause_timer": "complete"}))); + let resumed_activation_count: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM moa.execution_dispatch_outbox \ + WHERE run_uid = $1 AND dispatch_kind = 'run_activation' \ + AND controller_generation = $2 AND wake_epoch = $3", + ) + .bind(run.run_uid) + .bind(i64::try_from(resumed_generation)?) + .bind(i64::try_from(resumed_wake_epoch)?) + .fetch_one(&pool) + .await?; + assert_eq!( + resumed_activation_count, 1, + "resume must enqueue exactly one activation for its returned generation and wake epoch" + ); + let final_counts = sqlx::query( + "SELECT \ + (SELECT COUNT(*) FROM moa.execution_trigger \ + WHERE run_uid = $1 AND task_id = $2 AND trigger_kind = 'task_timer' \ + AND state = 'delivered') AS delivered_timers, \ + (SELECT COUNT(*) FROM moa.execution_dispatch_outbox \ + WHERE run_uid = $1 AND dispatch_kind = 'run_activation' \ + AND controller_generation = $3 AND wake_epoch = $5) AS resumed_activations, \ + (SELECT COUNT(*) FROM moa.execution_capacity_reservation \ + WHERE run_uid = $1 AND resource_dimension = 'parked_runs' \ + AND controller_generation = $4 AND state = 'released') \ + AS released_paused_receipts", + ) + .bind(run.run_uid) + .bind(task_id(&waiting).as_uuid()) + .bind(i64::try_from(resumed_generation)?) + .bind(i64::try_from(paused_generation)?) + .bind(i64::try_from(resumed_wake_epoch)?) + .fetch_one(&pool) + .await?; + assert_eq!(final_counts.try_get::("delivered_timers")?, 1); + assert_eq!(final_counts.try_get::("resumed_activations")?, 1); + assert_eq!( + final_counts.try_get::("released_paused_receipts")?, + 1, + "resume must release the exact paused-generation ParkedRuns receipt" + ); + Ok(()) +} + +#[tokio::test] +#[ignore = "requires Docker for the real Restate/Postgres/Valkey execution fixture"] +async fn active_attempt_pause_drains_then_generation_fenced_resume_completes_service_e2e() +-> Result<()> { + // Pins: pause fences a running bounded attempt, drains all compute before + // Paused, replays the exact control request, and resume advances one generation. + let tool_name = "long_horizon_pause_probe"; + let fixture = execution_fixture_with_tools( + vec![FixtureCapabilityTool { + name: tool_name.to_string(), + description: "Deterministic Task 12 pause barrier".to_string(), + input_schema: json!({ + "type": "object", + "additionalProperties": false, + "required": ["case"], + "properties": {"case": {"type": "string"}} + }), + item_key_pointer: None, + idempotent: true, + outcomes: vec![FixtureCapabilityOutcome::Success { + output: json!({"result": "resumed"}), + }], + }], + Vec::new(), + ) + .await?; + let test = fixture.isolated().await; + let pool = PgPool::connect(&fixture.postgres_url).await?; + let run = start_plan( + &test, + "pause-resume", + vec![ + fixture_capability_node("pausable-agent", tool_name, json!({"case": "pause-resume"})), + output_node(&["pausable-agent"], json!({"pause": "resumed"})), + ], + Duration::from_secs(30), + ) + .await?; + let controller = fixture + .fixture_capability() + .context("pause fixture omitted capability controller")?; + let first = controller.wait_for_calls(1, SCENARIO_TIMEOUT).await?; + assert_eq!(first.len(), 1); + assert_eq!(first[0].capability, tool_name); + let running = + await_task_status(&test, &run, "pausable-agent", ExecutionTaskStatus::Running).await?; + let active_before_pause = sqlx::query( + "SELECT generation, attempt_generation, active_dispatch_uid \ + FROM moa.execution_task WHERE run_uid = $1 AND task_id = $2", + ) + .bind(run.run_uid) + .bind(task_id(&running).as_uuid()) + .fetch_one(&pool) + .await?; + let task_generation = u64::try_from(active_before_pause.try_get::("generation")?)?; + let attempt_generation = + u64::try_from(active_before_pause.try_get::("attempt_generation")?)?; + let active_dispatch_uid: Uuid = active_before_pause.try_get("active_dispatch_uid")?; + let initial_generation: i64 = sqlx::query_scalar( + "SELECT controller_generation FROM moa.execution_run WHERE run_uid = $1", + ) + .bind(run.run_uid) + .fetch_one(&pool) + .await?; + let initial_generation = u64::try_from(initial_generation)?; + let pause_request = ExecutionRunControlRequest { + run: run.request.clone(), + expected_controller_generation: initial_generation, + }; + let pause: ExecutionRunControlResponse = test + .client() + .post_call("/Execution/pause", &pause_request) + .await?; + let paused_generation = match pause { + ExecutionRunControlResponse::Applied { + run: summary, + controller_generation, + .. + } => { + assert!(matches!( + summary.status, + ExecutionRunStatus::Pausing | ExecutionRunStatus::Paused + )); + assert_eq!(controller_generation, initial_generation + 1); + controller_generation + } + other => bail!("pause was not applied: {other:?}"), + }; + let cancelling: String = sqlx::query_scalar( + "SELECT attempt_state FROM moa.execution_task \ + WHERE run_uid = $1 AND node_id = 'pausable-agent'", + ) + .bind(run.run_uid) + .fetch_one(&pool) + .await?; + assert_eq!(cancelling, "cancelling"); + let cancel = sqlx::query( + "SELECT controller_generation, attempt_generation, payload \ + FROM moa.execution_dispatch_outbox \ + WHERE run_uid = $1 AND task_id = $2 \ + AND dispatch_kind = 'task_attempt_cancel'", + ) + .bind(run.run_uid) + .bind(task_id(&running).as_uuid()) + .fetch_one(&pool) + .await?; + assert_eq!( + u64::try_from(cancel.try_get::("controller_generation")?)?, + paused_generation + ); + assert_eq!( + u64::try_from(cancel.try_get::("attempt_generation")?)?, + attempt_generation + ); + let cancel_payload: Value = cancel.try_get("payload")?; + assert_eq!( + cancel_payload + .get("controller_generation") + .and_then(Value::as_u64), + Some(paused_generation), + "pause cancellation payload used the pre-pause generation" + ); + assert_eq!( + cancel_payload + .get("task_generation") + .and_then(Value::as_u64), + Some(task_generation) + ); + assert_eq!( + cancel_payload + .get("attempt_generation") + .and_then(Value::as_u64), + Some(attempt_generation) + ); + let active_dispatch_uid_string = active_dispatch_uid.to_string(); + assert_eq!( + cancel_payload + .get("active_dispatch_uid") + .and_then(Value::as_str), + Some(active_dispatch_uid_string.as_str()) + ); + controller.release(1); + await_run_status(&test, &run, ExecutionRunStatus::Paused).await?; + assert_parked_has_no_active_compute(&fixture, &pool, &run).await?; + let drained = sqlx::query( + "SELECT task.attempt_state, task.active_dispatch_uid, dispatch.state AS cancel_state, \ + capacity.state AS capacity_state, watchdog.state AS watchdog_state \ + FROM moa.execution_task AS task \ + JOIN moa.execution_dispatch_outbox AS dispatch \ + ON dispatch.run_uid = task.run_uid AND dispatch.task_id = task.task_id \ + AND dispatch.dispatch_kind = 'task_attempt_cancel' \ + JOIN moa.execution_capacity_reservation AS capacity \ + ON capacity.run_uid = task.run_uid AND capacity.task_id = task.task_id \ + AND capacity.attempt_generation = $3 \ + AND capacity.resource_dimension = 'active_tasks' \ + JOIN moa.execution_trigger AS watchdog \ + ON watchdog.run_uid = task.run_uid AND watchdog.task_id = task.task_id \ + AND watchdog.attempt_generation = $3 AND watchdog.trigger_kind = 'task_watchdog' \ + WHERE task.run_uid = $1 AND task.task_id = $2", + ) + .bind(run.run_uid) + .bind(task_id(&running).as_uuid()) + .bind(i64::try_from(attempt_generation)?) + .fetch_one(&pool) + .await?; + assert_eq!(drained.try_get::("attempt_state")?, "idle"); + assert_eq!( + drained.try_get::, _>("active_dispatch_uid")?, + None + ); + assert_eq!(drained.try_get::("cancel_state")?, "delivered"); + assert_eq!(drained.try_get::("capacity_state")?, "released"); + assert_eq!( + drained.try_get::("watchdog_state")?, + "superseded" + ); + let pause_replay: ExecutionRunControlResponse = test + .client() + .post_call("/Execution/pause", &pause_request) + .await?; + assert!(matches!( + pause_replay, + ExecutionRunControlResponse::Replayed { + controller_generation, + .. + } if controller_generation == paused_generation + )); + + let stale_resume: ExecutionRunControlResponse = test + .client() + .post_call("/Execution/resume", &pause_request) + .await?; + assert_eq!( + stale_resume, + ExecutionRunControlResponse::Conflict { + reason: ExecutionConflictReason::GenerationMismatch, + } + ); + let resume_request = ExecutionRunControlRequest { + run: run.request.clone(), + expected_controller_generation: paused_generation, + }; + let resume: ExecutionRunControlResponse = test + .client() + .post_call("/Execution/resume", &resume_request) + .await?; + assert!(matches!( + resume, + ExecutionRunControlResponse::Applied { + controller_generation, + .. + } if controller_generation == paused_generation + 1 + )); + let second = controller.wait_for_calls(2, SCENARIO_TIMEOUT).await?; + assert_eq!(second[0].input, second[1].input); + assert_ne!(second[0].invocation_id, second[1].invocation_id); + controller.release(1); + let terminal = match await_run_status(&test, &run, ExecutionRunStatus::Completed).await { + Ok(terminal) => terminal, + Err(error) => { + match active_pause_timeout_diagnostic(&pool, run.run_uid, task_id(&running)).await { + Ok(diagnostic) => bail!("{error:#}; active_pause_diagnostic={diagnostic}"), + Err(diagnostic_error) => bail!( + "{error:#}; active-pause diagnostic query also failed: {diagnostic_error:#}" + ), + } + } + }; + assert_eq!(terminal.output, Some(json!({"pause": "resumed"}))); + Ok(()) +} + +async fn active_pause_timeout_diagnostic( + pool: &PgPool, + run_uid: Uuid, + task_id: ExecutionTaskId, +) -> Result { + sqlx::query_scalar( + "SELECT jsonb_build_object( \ + 'run', (SELECT jsonb_build_object( \ + 'status', status, 'activation_state', activation_state, \ + 'controller_generation', controller_generation, 'wake_epoch', wake_epoch, \ + 'processed_wake_epoch', processed_wake_epoch, \ + 'ready_task_count', ready_task_count, 'active_task_count', active_task_count, \ + 'waiting_task_count', waiting_task_count) \ + FROM moa.execution_run WHERE run_uid=$1), \ + 'task', (SELECT jsonb_build_object( \ + 'status', status, 'attempt_state', attempt_state, 'generation', generation, \ + 'attempt_generation', attempt_generation, \ + 'active_dispatch_uid', active_dispatch_uid) \ + FROM moa.execution_task WHERE run_uid=$1 AND task_id=$2), \ + 'capacity', COALESCE((SELECT jsonb_agg(jsonb_build_object( \ + 'reservation_uid', reservation_uid, 'state', state, \ + 'attempt_generation', attempt_generation, \ + 'resource_dimension', resource_dimension) ORDER BY created_at) \ + FROM moa.execution_capacity_reservation \ + WHERE run_uid=$1 AND task_id=$2), '[]'::JSONB), \ + 'watchdogs', COALESCE((SELECT jsonb_agg(jsonb_build_object( \ + 'trigger_uid', trigger_uid, 'state', state, \ + 'controller_generation', controller_generation, \ + 'attempt_generation', attempt_generation) ORDER BY created_at) \ + FROM moa.execution_trigger \ + WHERE run_uid=$1 AND task_id=$2 AND trigger_kind='task_watchdog'), '[]'::JSONB), \ + 'activations', COALESCE((SELECT jsonb_agg(recent.value) FROM ( \ + SELECT jsonb_build_object( \ + 'dispatch_uid', dispatch_uid, 'state', state, \ + 'controller_generation', controller_generation, 'wake_epoch', wake_epoch, \ + 'last_error', last_error) AS value \ + FROM moa.execution_dispatch_outbox \ + WHERE run_uid=$1 AND dispatch_kind='run_activation' \ + ORDER BY created_at DESC LIMIT 5) AS recent), '[]'::JSONB))", + ) + .bind(run_uid) + .bind(task_id.as_uuid()) + .fetch_one(pool) + .await + .context("load active-pause timeout diagnostic") +} + +#[tokio::test] +#[ignore = "requires Docker for the real Restate/Postgres/Valkey execution fixture"] +async fn parked_signal_rejects_wrong_generation_and_replays_duplicate_after_valkey_loss_service_e2e() +-> Result<()> { + // Pins: a storage-only externally resumed wait owns no active compute, + // rejects a wrong generation, and applies/replays the exact callback once + // even when Valkey state is replaced before delivery. + let fixture = execution_fixture(Vec::new()).await?; + let test = fixture.isolated().await; + let pool = PgPool::connect(&fixture.postgres_url).await?; + let run = start_plan( + &test, + "external-signal-fence", + vec![ + node( + "external-signal", + &[], + ExecutionOperation::WaitSignal { + signal_name: "provider-complete".to_string(), + wait_policy: ExecutionWaitPolicy { + expiry: after_logical_days(5), + on_expiry: ExecutionWaitExpiryAction::FailRun, + }, + }, + json!({"type": "object"}), + ), + output_node(&["external-signal"], json!({"external": "settled"})), + ], + Duration::from_secs(15), + ) + .await?; + await_run_status(&test, &run, ExecutionRunStatus::WaitingSignal).await?; + let waiting = await_task_status( + &test, + &run, + "external-signal", + ExecutionTaskStatus::WaitingSignal, + ) + .await?; + assert_parked_has_no_active_compute(&fixture, &pool, &run).await?; + fixture.recreate_valkey_after_loss().await?; + + let stale: ExecutionMutationResponse = test + .client() + .post_call( + "/Execution/deliver_signal", + &ExecutionSignalRequest { + tenant_id: run.tenant_id, + contact_id: None, + run_uid: run.run_uid, + task_id: task_id(&waiting), + expected_generation: waiting.generation + 1, + signal_name: "provider-complete".to_string(), + payload: json!({"callback": "late"}), + }, + ) + .await?; + assert_eq!( + stale, + ExecutionMutationResponse::Conflict { + reason: ExecutionConflictReason::GenerationMismatch, + } + ); + + let request = ExecutionSignalRequest { + tenant_id: run.tenant_id, + contact_id: None, + run_uid: run.run_uid, + task_id: task_id(&waiting), + expected_generation: waiting.generation, + signal_name: "provider-complete".to_string(), + payload: json!({"callback": "current"}), + }; + let applied: ExecutionMutationResponse = test + .client() + .post_call("/Execution/deliver_signal", &request) + .await?; + assert!(matches!(applied, ExecutionMutationResponse::Applied { .. })); + let replay: ExecutionMutationResponse = test + .client() + .post_call("/Execution/deliver_signal", &request) + .await?; + assert!(matches!(replay, ExecutionMutationResponse::Replayed { .. })); + + let terminal = await_run_status(&test, &run, ExecutionRunStatus::Completed).await?; + assert_eq!(terminal.output, Some(json!({"external": "settled"}))); + Ok(()) +} + +#[tokio::test] +#[ignore = "requires Docker for the real Restate/Postgres/Valkey execution fixture"] +async fn input_wait_releases_compute_and_resumes_same_attempt_under_new_generation_service_e2e() +-> Result<()> { + // Pins: a model-authored user-input wait parks without active compute, then + // resumes the same logical attempt under a new generation and replays input once. + let needs_input = serde_json::to_string(&ExecutionTaskResult::NeedsInput { + question: "Which source should be used?".to_string(), + audience: InputAudience::User, + })?; + let completed = serde_json::to_string(&json!({"result": "used analyst notes"}))?; + let fixture = execution_fixture_with_script( + json!({ + "default": {"content": needs_input, "tool_calls": []}, + "keyed": [{ + "match": "analyst-notes", + "completion": {"content": completed, "tool_calls": []} + }] + }), + Vec::new(), + ) + .await?; + let test = fixture.isolated().await; + let pool = PgPool::connect(&fixture.postgres_url).await?; + let run = start_plan( + &test, + "input-wait", + vec![ + node( + "input-agent", + &[], + ExecutionOperation::Agent { + instructions: "Return a typed execution result.".to_string(), + skill_refs: Vec::new(), + capability_refs: Vec::new(), + max_turns: 2, + }, + json!({"type": "object"}), + ), + output_node(&["input-agent"], json!({"input": "settled"})), + ], + Duration::from_secs(15), + ) + .await?; + await_run_status(&test, &run, ExecutionRunStatus::WaitingInput).await?; + let waiting = await_task_status( + &test, + &run, + "input-agent", + ExecutionTaskStatus::WaitingInput, + ) + .await?; + assert_eq!(waiting.attempt, 1); + assert_parked_has_no_active_compute(&fixture, &pool, &run).await?; + + let request = ExecutionInputRequest { + tenant_id: run.tenant_id, + contact_id: None, + session_id: Some(run.request.session_id), + run_uid: run.run_uid, + task_id: task_id(&waiting), + expected_generation: waiting.generation, + audience: InputAudience::User, + input: json!({"source": "analyst-notes"}), + }; + let applied: ExecutionMutationResponse = test + .client() + .post_call("/Execution/deliver_input", &request) + .await?; + assert!(matches!(applied, ExecutionMutationResponse::Applied { .. })); + let replay: ExecutionMutationResponse = test + .client() + .post_call("/Execution/deliver_input", &request) + .await?; + assert!(matches!(replay, ExecutionMutationResponse::Replayed { .. })); + + let completed_task = + await_task_status(&test, &run, "input-agent", ExecutionTaskStatus::Completed).await?; + assert_eq!(completed_task.attempt, 1); + assert_eq!(completed_task.generation, waiting.generation + 1); + let terminal = await_run_status(&test, &run, ExecutionRunStatus::Completed).await?; + assert_eq!(terminal.output, Some(json!({"input": "settled"}))); + Ok(()) +} + +async fn await_active_execution_task_hand( + pool: &PgPool, + run: &StartedRun, + expected_task_id: ExecutionTaskId, +) -> Result { + let deadline = Instant::now() + SCENARIO_TIMEOUT; + loop { + let rows: Vec = sqlx::query_scalar( + "SELECT workspace.workspace_id \ + FROM moa.sandbox_workspaces AS workspace \ + JOIN moa.sandbox_capacity_reservations AS reservation \ + ON reservation.tenant_id = workspace.tenant_id \ + AND reservation.workspace_id = workspace.workspace_id \ + WHERE workspace.scope_kind = 'execution_task' \ + AND workspace.scope_run_id = $1 AND workspace.scope_task_id = $2 \ + AND reservation.resource_dimension = 'active_hands' \ + AND reservation.reservation_state <> 'released'", + ) + .bind(run.run_uid) + .bind(expected_task_id.as_uuid()) + .fetch_all(pool) + .await?; + match rows.as_slice() { + [workspace_id] => return Ok(*workspace_id), + [] if Instant::now() < deadline => tokio::time::sleep(POLL_INTERVAL).await, + [] => { + bail!("execution task {expected_task_id} never acquired an ActiveHands reservation") + } + _ => bail!( + "execution task {expected_task_id} acquired multiple live ActiveHands reservations: {rows:?}" + ), + } + } +} + +async fn await_execution_action_review( + test: &IsolatedTest<'_>, + tenant_id: TenantId, + expected_task_id: ExecutionTaskId, +) -> Result { + let deadline = Instant::now() + SCENARIO_TIMEOUT; + loop { + let reviews: Vec = test + .client() + .post_call( + "/ActionReviews/list_pending", + &ListActionReviewsRequest { tenant_id }, + ) + .await?; + if let Some(review) = reviews.into_iter().find(|review| { + review + .envelope + .owner + .execution_origin() + .is_some_and(|origin| origin.task_uid == expected_task_id.as_uuid()) + }) { + assert_eq!(review.status, ActionReviewStatus::Pending); + return Ok(review); + } + if Instant::now() >= deadline { + bail!("task {expected_task_id} did not publish an action review") + } + tokio::time::sleep(POLL_INTERVAL).await; + } +} + +async fn await_execution_review_delivery( + pool: &PgPool, + review_uid: Uuid, +) -> Result<(i32, Option)> { + let deadline = Instant::now() + SCENARIO_TIMEOUT; + loop { + let delivery: Option<(i32, Option>, Option)> = sqlx::query_as( + "SELECT attempt_count, delivered_at, last_error \ + FROM moa.execution_action_review_outbox WHERE review_uid=$1", + ) + .bind(review_uid) + .fetch_optional(pool) + .await?; + if let Some((attempt_count, Some(_), last_error)) = &delivery { + return Ok((*attempt_count, last_error.clone())); + } + if Instant::now() >= deadline { + bail!( + "execution review {review_uid} was not delivered within {SCENARIO_TIMEOUT:?}; delivery={delivery:?}" + ) + } + tokio::time::sleep(POLL_INTERVAL).await; + } +} + +async fn assert_released_execution_task_hand( + pool: &PgPool, + run: &StartedRun, + expected_task_id: ExecutionTaskId, + expected_workspace_id: Uuid, +) -> Result<()> { + let states: Vec = sqlx::query_scalar( + "SELECT reservation.reservation_state \ + FROM moa.sandbox_workspaces AS workspace \ + JOIN moa.sandbox_capacity_reservations AS reservation \ + ON reservation.tenant_id = workspace.tenant_id \ + AND reservation.workspace_id = workspace.workspace_id \ + WHERE workspace.scope_kind = 'execution_task' \ + AND workspace.scope_run_id = $1 AND workspace.scope_task_id = $2 \ + AND workspace.workspace_id = $3 \ + AND reservation.resource_dimension = 'active_hands'", + ) + .bind(run.run_uid) + .bind(expected_task_id.as_uuid()) + .bind(expected_workspace_id) + .fetch_all(pool) + .await?; + assert_eq!( + states, + vec!["released".to_string()], + "the exact execution-task ActiveHands receipt must be released while compute is parked" + ); + Ok(()) +} diff --git a/crates/moa-orchestrator/tests/orchestrator_db.rs b/crates/moa-orchestrator/tests/orchestrator_db.rs index 3ee3906d9..61047e5fc 100644 --- a/crates/moa-orchestrator/tests/orchestrator_db.rs +++ b/crates/moa-orchestrator/tests/orchestrator_db.rs @@ -18,6 +18,10 @@ mod authz_admin_db; mod authz_challenges_db; #[path = "orchestrator_db/contacts_db.rs"] mod contacts_db; +#[path = "orchestrator_db/execution_dispatch_reconciliation_db.rs"] +mod execution_dispatch_reconciliation_db; +#[path = "orchestrator_db/execution_schedule_db.rs"] +mod execution_schedule_db; #[path = "orchestrator_db/execution_service_db.rs"] mod execution_service_db; #[path = "orchestrator_db/fga_mock.rs"] diff --git a/crates/moa-orchestrator/tests/orchestrator_db/action_reviews_reaper_db.rs b/crates/moa-orchestrator/tests/orchestrator_db/action_reviews_reaper_db.rs index 959872c28..c0dfe35e5 100644 --- a/crates/moa-orchestrator/tests/orchestrator_db/action_reviews_reaper_db.rs +++ b/crates/moa-orchestrator/tests/orchestrator_db/action_reviews_reaper_db.rs @@ -24,7 +24,8 @@ use moa_orchestrator::services::{ ExecutionActionReviewSettlement, SettleExecutionActionReviewRequest, settle_execution_action_review, }, - action_reviews_reaper::ActionReviewReaper, + action_reviews_reaper::{ActionReviewReaper, ActionReviewTimeoutDelivery}, + durable_timeout::ActionReviewTimeout, }; use moa_test_support::postgres::TestDb; use sqlx::PgPool; @@ -224,6 +225,92 @@ async fn unexpired_pending_review_survives_sweep_db() { assert_eq!(status, "pending", "unexpired review stays pending"); } +#[tokio::test] +async fn durable_timeout_requires_the_exact_action_review_generation_db() { + // Pins: a delayed timeout from an older owner generation is a successful + // no-op; only the exact persisted owner incarnation may fail the review closed. + let test_db = test_pool().await; + let pool = test_db.store().pool().clone(); + let review_id = insert_review(&pool, "command_execution", "high", ReviewClock::Expired).await; + let envelope: ActionEnvelope = serde_json::from_value( + sqlx::query_scalar("SELECT envelope FROM tenant_action_reviews WHERE id = $1") + .bind(review_id) + .fetch_one(&pool) + .await + .expect("review envelope should load"), + ) + .expect("review envelope should decode"); + let ActionReviewOwner::Coordinator { + session_id, + turn_id, + generation, + } = envelope.owner.clone() + else { + panic!("fixture should create a coordinator review"); + }; + let stale = ActionReviewTimeout { + tenant_id: envelope.tenant_id, + review_id, + owner: ActionReviewOwner::Coordinator { + session_id, + turn_id: turn_id.clone(), + generation: generation + 1, + }, + }; + let reaper = ActionReviewReaper::new(pool.clone()); + + assert_eq!( + reaper + .apply_timeout(&stale) + .await + .expect("stale timeout should be a no-op"), + moa_orchestrator::services::action_reviews_reaper::ActionReviewTimeoutDelivery::Stale + ); + let status: String = + sqlx::query_scalar("SELECT status FROM tenant_action_reviews WHERE id = $1") + .bind(review_id) + .fetch_one(&pool) + .await + .expect("review status should load"); + assert_eq!(status, "pending", "stale generation must not fail closed"); + + let expected_owner = envelope.owner.clone(); + let exact = ActionReviewTimeout { + tenant_id: envelope.tenant_id, + review_id, + owner: envelope.owner, + }; + let delivery = reaper + .apply_timeout(&exact) + .await + .expect("exact timeout should apply"); + let ActionReviewTimeoutDelivery::Conversational { + timed_out_at, + release, + } = delivery + else { + panic!("exact conversational timeout should return its owner release"); + }; + assert_eq!(release.review_id, review_id); + assert_eq!(release.owner, expected_owner); + assert!( + !release.resume_queued, + "the exact delayed delivery releases only its current owner hold" + ); + let (status, decided_at): (String, Option>) = + sqlx::query_as("SELECT status, decided_at FROM tenant_action_reviews WHERE id = $1") + .bind(review_id) + .fetch_one(&pool) + .await + .expect("review status should load"); + assert_eq!(status, "timeout", "exact generation must fail closed"); + assert_eq!( + decided_at, + Some(timed_out_at), + "the delivery must expose the timestamp committed by the timeout transition" + ); +} + #[tokio::test] async fn expired_review_claimed_by_durable_execution_is_not_timed_out_db() { // Pins: once the keyed action-review service has durably claimed a clear, @@ -819,6 +906,12 @@ async fn insert_execution_task(pool: &PgPool, tenant_id: TenantId) -> ExecutionT let plan = serde_json::to_value(CanonicalExecutionPlan { definition: ExecutionPlanDefinition { cancel_policy: ExecutionCancelPolicy::RetainEffects, + input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { + expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::At { + at: chrono::Utc::now() + chrono::TimeDelta::hours(1), + }, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + }, input_schema: serde_json::json!({ "type": "object" }), output_schema: serde_json::json!({ "type": "object" }), nodes: Vec::new(), @@ -901,6 +994,7 @@ async fn insert_execution_task(pool: &PgPool, tenant_id: TenantId) -> ExecutionT run_uid, task_uid, generation: 1, + attempt_generation: 1, } } @@ -935,6 +1029,7 @@ async fn insert_execution_compensation( run_uid: task_origin.run_uid, compensation_id, generation: 1, + attempt_generation: 1, } } diff --git a/crates/moa-orchestrator/tests/orchestrator_db/analytics_export_db.rs b/crates/moa-orchestrator/tests/orchestrator_db/analytics_export_db.rs index 0f1de62a8..b9bd5ea08 100644 --- a/crates/moa-orchestrator/tests/orchestrator_db/analytics_export_db.rs +++ b/crates/moa-orchestrator/tests/orchestrator_db/analytics_export_db.rs @@ -226,6 +226,12 @@ async fn seed_execution_analytics_fixture( let plan = serde_json::to_value(CanonicalExecutionPlan { definition: ExecutionPlanDefinition { cancel_policy: ExecutionCancelPolicy::RetainEffects, + input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { + expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::At { + at: chrono::Utc::now() + chrono::TimeDelta::hours(1), + }, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + }, input_schema: json!({ "type": "object" }), output_schema: json!({ "type": "object" }), nodes: Vec::new(), diff --git a/crates/moa-orchestrator/tests/orchestrator_db/authz_challenges_db.rs b/crates/moa-orchestrator/tests/orchestrator_db/authz_challenges_db.rs index 44de52241..9bcff764b 100644 --- a/crates/moa-orchestrator/tests/orchestrator_db/authz_challenges_db.rs +++ b/crates/moa-orchestrator/tests/orchestrator_db/authz_challenges_db.rs @@ -3,7 +3,10 @@ use async_trait::async_trait; use chrono::{DateTime, Utc}; use moa_authz::{AwakeableResolveError, AwakeableResolver}; -use moa_orchestrator::services::authz_challenges_reaper::AuthzChallengeReaper; +use moa_orchestrator::services::{ + authz_challenges_reaper::{AuthzChallengeReaper, AuthzChallengeTimeoutDelivery}, + durable_timeout::AuthzChallengeTimeout, +}; use moa_test_support::fixtures::quote_identifier; use serde_json::json; use sqlx::{PgPool, postgres::PgPoolOptions}; @@ -114,6 +117,73 @@ async fn concurrent_reapers_resolve_terminal_challenge_once_db() { ); } +#[tokio::test] +async fn durable_timeout_requires_the_exact_authz_awakeable_incarnation_db() { + // Pins: a delayed timeout cannot resolve or terminalize a replacement + // awakeable; the exact id/awakeable incarnation is claimed only once. + let pool = test_pool().await; + let challenge_id = insert_expired_pending_challenge(&pool, "awakeable-current").await; + let reaper = AuthzChallengeReaper::new(pool.clone()); + + let stale = reaper + .apply_timeout(&AuthzChallengeTimeout { + challenge_id, + awakeable_id: "awakeable-older-incarnation".to_string(), + }) + .await + .expect("stale timeout should be a no-op"); + assert_eq!(stale, AuthzChallengeTimeoutDelivery::Stale); + let status: String = + sqlx::query_scalar("SELECT status FROM builtin_pending_approvals WHERE id = $1") + .bind(challenge_id) + .fetch_one(&pool) + .await + .expect("challenge status should load"); + assert_eq!(status, "pending", "stale awakeable must not fail closed"); + + let exact = AuthzChallengeTimeout { + challenge_id, + awakeable_id: "awakeable-current".to_string(), + }; + let delivery = reaper + .apply_timeout(&exact) + .await + .expect("exact timeout should apply"); + let AuthzChallengeTimeoutDelivery::Resolve { + challenge_id: delivered_challenge_id, + awakeable_id, + resolve_claim_token, + newly_timed_out, + } = delivery + else { + panic!("exact timeout should return its claimed awakeable delivery"); + }; + assert_eq!(delivered_challenge_id, challenge_id); + assert_eq!(awakeable_id, "awakeable-current"); + assert!(newly_timed_out, "the first exact timeout changes the row"); + let (status, stored_claim): (String, Option) = sqlx::query_as( + "SELECT status, resolve_claim_token FROM builtin_pending_approvals WHERE id = $1", + ) + .bind(challenge_id) + .fetch_one(&pool) + .await + .expect("claimed challenge state should load"); + assert_eq!(status, "timeout"); + assert_eq!( + stored_claim, + Some(resolve_claim_token), + "the returned delivery token must be the transaction's durable claim" + ); + assert_eq!( + reaper + .apply_timeout(&exact) + .await + .expect("claimed timeout replay should be a no-op"), + AuthzChallengeTimeoutDelivery::AlreadyDelivered, + "the exact resolution claim prevents duplicate delivery" + ); +} + async fn test_pool() -> PgPool { let database_url = std::env::var("MOA_DATABASE_URL") .unwrap_or_else(|_| "postgres://moa_owner:dev@localhost:10040/moa".to_string()); @@ -206,6 +276,29 @@ async fn insert_terminal_challenge( challenge_id } +async fn insert_expired_pending_challenge(pool: &PgPool, awakeable_id: &str) -> Uuid { + let challenge_id = Uuid::new_v4(); + sqlx::query( + r#" + INSERT INTO builtin_pending_approvals + (id, session_id, deciding_user_id, tenant_id, awakeable_id, + action_summary, action_details, status, expires_at) + VALUES + ($1, $2, $3, $4, $5, 'approve deploy', '{}'::jsonb, + 'pending', NOW() - INTERVAL '1 minute') + "#, + ) + .bind(challenge_id) + .bind(Uuid::new_v4()) + .bind(Uuid::new_v4()) + .bind(Uuid::new_v4()) + .bind(awakeable_id) + .execute(pool) + .await + .expect("expired pending challenge should insert"); + challenge_id +} + #[derive(Default)] struct MissingAwakeableResolver { calls: AtomicUsize, diff --git a/crates/moa-orchestrator/tests/orchestrator_db/execution_dispatch_reconciliation_db.rs b/crates/moa-orchestrator/tests/orchestrator_db/execution_dispatch_reconciliation_db.rs new file mode 100644 index 000000000..1e87d5c73 --- /dev/null +++ b/crates/moa-orchestrator/tests/orchestrator_db/execution_dispatch_reconciliation_db.rs @@ -0,0 +1,318 @@ +//! DB-backed execution dispatch reconciliation and exact trigger delivery contracts. + +use std::time::Duration as StdDuration; + +use chrono::{Duration, Utc}; +use moa_config::ExecutionConfig; +use moa_core::{ + traits::{Identity, IdentityType}, + types::identifiers::TenantId, +}; +use moa_execution::repository::{ + ExecutionRepository, ExecutionScope, + outbox::{ + ExecutionDispatchFailureOutcome, ExecutionDispatchRetryPolicy, ExecutionMaintenanceJobKind, + ExecutionMaintenanceSettlementOutcome, + }, + trigger::{ + ExecutionTriggerFireOutcome, ExecutionTriggerKind, ExecutionTriggerNoOp, + NewExecutionTrigger, + }, +}; +use serde_json::json; +use uuid::Uuid; + +type TestResult = Result<(), Box>; + +#[tokio::test] +async fn reconciliation_checkpoint_rejects_superseded_completion_and_records_failure_db() +-> TestResult { + // Pins: overlapping infrastructure invocations cannot let an older pass + // overwrite the durable completion receipt for a newer reconciliation. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let repository = ExecutionRepository::new(test_db.store().pool().clone()); + let scope = ExecutionScope::ControlPlane; + let kind = ExecutionMaintenanceJobKind::DispatchReconciliation; + let first = repository.begin_execution_maintenance(scope, kind).await?; + let second = repository.begin_execution_maintenance(scope, kind).await?; + assert_eq!(second.generation, first.generation + 1); + assert_eq!( + repository + .complete_execution_maintenance(scope, kind, first.generation) + .await?, + ExecutionMaintenanceSettlementOutcome::StaleOrMissing + ); + let failed = repository + .fail_execution_maintenance( + scope, + kind, + second.generation, + "injected bounded reconciliation failure", + ) + .await?; + let ExecutionMaintenanceSettlementOutcome::Applied(failed) = failed else { + return Err("current checkpoint generation must accept failure".into()); + }; + assert_eq!( + failed.last_error.as_deref(), + Some("injected bounded reconciliation failure") + ); + assert!(failed.last_failure_at.is_some()); + assert!(failed.last_succeeded_at.is_none()); + Ok(()) +} + +#[tokio::test] +async fn reconciliation_repairs_only_one_bounded_indexed_window_db() -> TestResult { + // Pins: the infrastructure reconciliation target never becomes an unbounded + // poller; each invocation repairs at most its configured SKIP LOCKED window. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let schedule_uid = insert_schedule(&pool, tenant_id).await?; + let scope = ExecutionScope::ControlPlane; + + for occurrence_sequence in 1..=3 { + let write = repository + .create_trigger( + scope, + &ExecutionConfig::default(), + schedule_trigger( + tenant_id, + schedule_uid, + occurrence_sequence, + Utc::now() - Duration::minutes(1), + ), + ) + .await?; + sqlx::query("DELETE FROM moa.execution_dispatch_outbox WHERE dispatch_uid = $1") + .bind(write.dispatch.dispatch_uid) + .execute(&pool) + .await?; + } + + assert_eq!( + repository + .reconcile_due_trigger_dispatches(scope, 2) + .await? + .len(), + 2 + ); + assert_eq!( + repository + .reconcile_due_trigger_dispatches(scope, 2) + .await? + .len(), + 1 + ); + assert!( + repository + .reconcile_due_trigger_dispatches(scope, 2) + .await? + .is_empty() + ); + Ok(()) +} + +#[tokio::test] +async fn claimed_dispatches_ack_retry_and_dead_letter_under_exact_owner_fences_db() -> TestResult { + // Pins: a dispatcher ACKs only after accepted delivery, abandoned owners + // cannot settle another claim, and bounded failures eventually dead-letter. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let schedule_uid = insert_schedule(&pool, tenant_id).await?; + let scope = ExecutionScope::ControlPlane; + for occurrence_sequence in 1..=3 { + repository + .create_trigger( + scope, + &ExecutionConfig::default(), + schedule_trigger( + tenant_id, + schedule_uid, + occurrence_sequence, + Utc::now() - Duration::minutes(1), + ), + ) + .await?; + } + + let claimed = repository + .claim_due_dispatches(scope, "dispatcher-a", 3, StdDuration::from_secs(30)) + .await?; + assert_eq!(claimed.len(), 3); + assert!( + repository + .claim_due_dispatches(scope, "dispatcher-b", 3, StdDuration::from_secs(30)) + .await? + .is_empty() + ); + + assert_eq!( + repository + .mark_dispatches_delivered(scope, &[claimed[0].dispatch_uid], "dispatcher-b") + .await?, + Vec::::new() + ); + assert_eq!( + repository + .mark_dispatches_delivered(scope, &[claimed[0].dispatch_uid], "dispatcher-a") + .await?, + vec![claimed[0].dispatch_uid] + ); + + let retry = ExecutionDispatchRetryPolicy { + max_attempts: 2, + base_delay: StdDuration::from_secs(1), + maximum_delay: StdDuration::from_secs(2), + }; + assert!(matches!( + repository + .record_dispatch_failure( + scope, + claimed[1].dispatch_uid, + "dispatcher-a", + "injected acceptance failure", + retry, + ) + .await?, + ExecutionDispatchFailureOutcome::RetryScheduled { .. } + )); + let dead_letter = ExecutionDispatchRetryPolicy { + max_attempts: 1, + base_delay: StdDuration::from_secs(1), + maximum_delay: StdDuration::from_secs(1), + }; + assert_eq!( + repository + .record_dispatch_failure( + scope, + claimed[2].dispatch_uid, + "dispatcher-a", + "injected permanent acceptance failure", + dead_letter, + ) + .await?, + ExecutionDispatchFailureOutcome::DeadLettered + ); + Ok(()) +} + +#[tokio::test] +async fn trigger_delivery_rechecks_due_time_and_settles_fallback_outbox_db() -> TestResult { + // Pins: send_after cannot fire a trigger early, while a current due delivery + // atomically settles both canonical trigger state and its recovery outbox row. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let schedule_uid = insert_schedule(&pool, tenant_id).await?; + let scope = ExecutionScope::ControlPlane; + let future = repository + .create_trigger( + scope, + &ExecutionConfig::default(), + schedule_trigger(tenant_id, schedule_uid, 2, Utc::now() + Duration::hours(1)), + ) + .await?; + assert_eq!( + repository + .fire_trigger(scope, future.trigger.trigger_uid,) + .await?, + ExecutionTriggerFireOutcome::NoOp(ExecutionTriggerNoOp::NotDue) + ); + + let due = repository + .create_trigger( + scope, + &ExecutionConfig::default(), + schedule_trigger( + tenant_id, + schedule_uid, + 1, + Utc::now() - Duration::seconds(1), + ), + ) + .await?; + assert_eq!( + repository + .fire_trigger(scope, due.trigger.trigger_uid) + .await?, + ExecutionTriggerFireOutcome::Delivered { activation: None } + ); + let fallback_state: String = sqlx::query_scalar( + "SELECT state FROM moa.execution_dispatch_outbox WHERE dispatch_uid = $1", + ) + .bind(due.dispatch.dispatch_uid) + .fetch_one(&pool) + .await?; + assert_eq!(fallback_state, "delivered"); + Ok(()) +} + +fn schedule_trigger( + tenant_id: TenantId, + schedule_uid: Uuid, + occurrence_sequence: u64, + due_at: chrono::DateTime, +) -> NewExecutionTrigger { + NewExecutionTrigger { + trigger_uid: Uuid::now_v7(), + tenant_id, + run_uid: None, + task_id: None, + compensation_id: None, + schedule_uid: Some(schedule_uid), + schedule_incarnation: Some(1), + kind: ExecutionTriggerKind::ScheduleOccurrence, + controller_generation: None, + attempt_generation: None, + compensation_generation: None, + compensation_attempt_generation: None, + occurrence_sequence: Some(occurrence_sequence), + due_at, + payload: json!({ "occurrence_sequence": occurrence_sequence }), + } +} + +async fn insert_schedule(pool: &sqlx::PgPool, tenant_id: TenantId) -> Result { + let schedule_uid = Uuid::now_v7(); + let identity = Identity { + identity_type: IdentityType::Service, + id: Uuid::now_v7(), + tenant_id, + api_key_id: None, + acting_on_behalf_of: None, + }; + let identity = serde_json::to_value(identity).expect("fixture identity must serialize"); + sqlx::query( + r#" + INSERT INTO moa.execution_schedule ( + schedule_uid, tenant_id, owner_user_id, name, timezone, + calendar_expression, template_revision_uid, template_snapshot, + template_hash, run_as_identity, creation_origin, missed_fire_policy, + overlap_policy, dst_policy, occurrence_budget, start_at + ) VALUES ( + $1, $2, 'scheduler', $3, 'UTC', '0 * * * *', $4, '{}'::JSONB, + $5, $6, jsonb_build_object( + 'request_uid', $7::TEXT, + 'created_by', $6::JSONB, + 'source', jsonb_build_object('kind', 'tenant_api') + ), 'skip', 'skip', 'earliest', '{}'::JSONB, now() + ) + "#, + ) + .bind(schedule_uid) + .bind(tenant_id.0) + .bind(format!("reconcile-{schedule_uid}")) + .bind(Uuid::now_v7()) + .bind("0".repeat(64)) + .bind(identity) + .bind(Uuid::now_v7()) + .execute(pool) + .await?; + Ok(schedule_uid) +} diff --git a/crates/moa-orchestrator/tests/orchestrator_db/execution_schedule_db.rs b/crates/moa-orchestrator/tests/orchestrator_db/execution_schedule_db.rs new file mode 100644 index 000000000..981f5a56b --- /dev/null +++ b/crates/moa-orchestrator/tests/orchestrator_db/execution_schedule_db.rs @@ -0,0 +1,1035 @@ +//! Database contract coverage for recurring durable execution schedules. + +use chrono::{Duration, Utc}; +use moa_artifacts::execution_plan::{ + ExecutionBudgetLimit, ExecutionCancelPolicy, ExecutionGoalContract, ExecutionNode, + ExecutionOperation, ExecutionPlanDefinition, ExecutionTemporalTarget, + ExecutionWaitExpiryAction, ExecutionWaitPolicy, RetryPolicy, +}; +use moa_config::ExecutionConfig; +use moa_core::{ + traits::{Identity, IdentityType}, + types::{ + execution_planning::{ + ExecutionScheduleCreateRequest, ExecutionScheduleDstPolicy, + ExecutionScheduleMissedFirePolicy, ExecutionScheduleOrigin, + ExecutionScheduleOriginSource, ExecutionScheduleOverlapPolicy, ExecutionSchedulePolicy, + ExecutionScheduleStatus, ExecutionScheduleTemplate, ExecutionScheduleUpdateRequest, + ExecutionSourceProvenance, execution_schedule_template_hash, + }, + identifiers::{SessionId, TenantId}, + }, +}; +use moa_execution::repository::{ + ExecutionRepository, ExecutionScope, + schedule::{ + ExecutionScheduleCreateOutcome, ExecutionScheduleMutationOutcome, + ExecutionScheduleOccurrence, ExecutionScheduleRunAdmission, + ExecutionScheduleRunAdmissionOutcome, ExecutionScheduleRunBlueprint, + execution_schedule_run_blueprint, + }, +}; +use moa_execution::{ + capability::{ + ExecutionAuthorizationEnvelope, ExecutionCapabilityCatalog, ExecutionEstimate, + ExecutionHash, + }, + compiler::{CanonicalExecutionPlan, ExecutionValidationReport}, +}; +use uuid::Uuid; + +type TestResult = Result<(), Box>; + +#[tokio::test] +async fn schedule_create_pause_resume_is_tenant_scoped_and_generation_fenced_db() -> TestResult { + // Pins: one committed schedule carries exact immutable identity/provenance, an occurrence + // trigger and outbox row commit together, and pause/resume invalidate the old incarnation. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let other_tenant_id = TenantId::new(); + let schedule_uid = Uuid::now_v7(); + let scope = ExecutionScope::Tenant { tenant_id }; + let now = moa_test_support::fixtures::pg_now(); + let first = ExecutionScheduleOccurrence { + at: now + Duration::minutes(5), + local: (now + Duration::minutes(5)).naive_utc(), + }; + let mut request = schedule_request(tenant_id, schedule_uid, now); + request.policy.start_at = now - Duration::minutes(1); + + let created = repository + .create_schedule( + scope, + &moa_config::ExecutionConfig::default(), + request.clone(), + Some(first), + ) + .await?; + let ExecutionScheduleCreateOutcome::Created { schedule, trigger } = created else { + panic!("fresh schedule must be created"); + }; + assert_eq!(schedule.template, request.template); + assert_eq!(schedule.run_as_identity, request.run_as_identity); + assert_eq!(schedule.origin, request.origin); + assert_eq!(schedule.schedule_incarnation, 1); + let blueprint = execution_schedule_run_blueprint(&schedule)?; + sqlx::query( + "INSERT INTO moa.execution_planning_context (\ + planning_context_uid, tenant_id, session_id, originating_user_sequence_num, \ + originating_user_event_hash, owner_user_id, planning_context_hash, snapshot\ + ) VALUES ($1, $2, $3, $4, $5, $6, $5, '{}'::JSONB)", + ) + .bind(blueprint.planning_context_uid) + .bind(tenant_id.0) + .bind(blueprint.session_id.0) + .bind(i64::try_from(blueprint.originating_user_sequence_num)?) + .bind(blueprint.planning_context_hash.to_string()) + .bind(schedule.run_as_identity.id.to_string()) + .execute(&pool) + .await?; + let trigger = trigger.expect("active schedule must atomically arm a trigger"); + assert_eq!(trigger.trigger.schedule_incarnation, Some(1)); + assert_eq!(trigger.trigger.occurrence_sequence, Some(1)); + assert_eq!( + trigger.dispatch.trigger_uid, + Some(trigger.trigger.trigger_uid) + ); + assert!( + repository + .load_schedule( + ExecutionScope::Tenant { + tenant_id: other_tenant_id, + }, + other_tenant_id, + schedule_uid, + ) + .await? + .is_none(), + "forced tenant RLS must hide the schedule from another tenant" + ); + + let paused = repository + .pause_schedule( + scope, + &moa_config::ExecutionConfig::default(), + tenant_id, + schedule_uid, + ) + .await?; + let ExecutionScheduleMutationOutcome::Updated { + schedule: paused, .. + } = paused + else { + panic!("active schedule must pause"); + }; + assert_eq!(paused.status, ExecutionScheduleStatus::Paused); + assert_eq!(paused.schedule_incarnation, 2); + assert_eq!(paused.next_occurrence_at, None); + + let next = ExecutionScheduleOccurrence { + at: now - Duration::seconds(1), + local: (now - Duration::seconds(1)).naive_utc(), + }; + let resumed = repository + .resume_schedule( + scope, + &moa_config::ExecutionConfig::default(), + tenant_id, + schedule_uid, + Some(next), + ) + .await?; + let ExecutionScheduleMutationOutcome::Updated { + schedule: resumed, + trigger: Some(resumed_trigger), + } = resumed + else { + panic!("paused schedule must resume and arm a new trigger"); + }; + assert_eq!(resumed.status, ExecutionScheduleStatus::Active); + assert_eq!(resumed.schedule_incarnation, 3); + assert_eq!(resumed_trigger.trigger.schedule_incarnation, Some(3)); + assert_ne!( + resumed_trigger.trigger.trigger_uid, + trigger.trigger.trigger_uid + ); + let occurrence_run = blueprint.instantiate(&resumed, next, 1, 30 * 24 * 60 * 60)?; + assert_eq!( + occurrence_run.approved_budget.deadline_at, + Some(next.at + Duration::hours(2)), + "a resumed recurrence must receive a fresh occurrence-relative deadline" + ); + + let old_state: String = + sqlx::query_scalar("SELECT state FROM moa.execution_trigger WHERE trigger_uid=$1") + .bind(trigger.trigger.trigger_uid) + .fetch_one(&pool) + .await?; + assert_eq!(old_state, "superseded"); + + let admission = repository + .admit_schedule_occurrence( + scope, + &moa_config::ExecutionConfig::default(), + ExecutionScheduleRunAdmission { + tenant_id, + schedule_uid, + schedule_incarnation: resumed.schedule_incarnation, + occurrence_sequence: 1, + trigger_uid: resumed_trigger.trigger.trigger_uid, + trigger_dispatch_uid: resumed_trigger.dispatch.dispatch_uid, + occurrence: next, + run: occurrence_run.clone(), + next_occurrence: None, + }, + ) + .await?; + let ExecutionScheduleRunAdmissionOutcome::Admitted { + run, activation, .. + } = admission + else { + panic!("due resumed occurrence must admit one fresh run"); + }; + assert_eq!(run.controller_generation, 1); + assert_eq!(run.wake_epoch, 1); + assert_eq!(run.processed_wake_epoch, 0); + assert_eq!(activation.controller_generation, Some(1)); + assert_eq!(activation.wake_epoch, Some(1)); + let seeded_node: (String, i64, String) = sqlx::query_as( + "SELECT node_id, remaining_dependency_count, node_status \ + FROM moa.execution_node_state WHERE run_uid=$1", + ) + .bind(run.run_uid) + .fetch_one(&pool) + .await?; + assert_eq!( + seeded_node, + ("output".to_string(), 0, "pending".to_string()) + ); + let deadline_shape: (String, chrono::DateTime, String, String) = sqlx::query_as( + "SELECT trigger.state, trigger.due_at, dispatch.state, capacity.state \ + FROM moa.execution_trigger AS trigger \ + JOIN moa.execution_dispatch_outbox AS dispatch \ + ON dispatch.trigger_uid=trigger.trigger_uid \ + AND dispatch.dispatch_kind='trigger_delivery' \ + JOIN moa.execution_capacity_reservation AS capacity \ + ON capacity.trigger_uid=trigger.trigger_uid \ + AND capacity.resource_dimension='scheduled_triggers' \ + WHERE trigger.run_uid=$1 AND trigger.trigger_kind='run_deadline'", + ) + .bind(run.run_uid) + .fetch_one(&pool) + .await?; + assert_eq!(deadline_shape.0, "pending"); + assert_eq!(deadline_shape.1, next.at + Duration::hours(2)); + assert_eq!(deadline_shape.2, "pending"); + assert_eq!(deadline_shape.3, "reserved"); + + let replay = repository + .admit_schedule_occurrence( + scope, + &moa_config::ExecutionConfig::default(), + ExecutionScheduleRunAdmission { + tenant_id, + schedule_uid, + schedule_incarnation: resumed.schedule_incarnation, + occurrence_sequence: 1, + trigger_uid: resumed_trigger.trigger.trigger_uid, + trigger_dispatch_uid: resumed_trigger.dispatch.dispatch_uid, + occurrence: next, + run: occurrence_run, + next_occurrence: None, + }, + ) + .await?; + assert_eq!( + replay, + ExecutionScheduleRunAdmissionOutcome::Replayed { + run_uid: Some(run.run_uid), + activation_dispatch_uid: Some(activation.dispatch_uid), + }, + "replay after completion clears mutable next fields but remains an accepted no-op" + ); + Ok(()) +} + +#[tokio::test] +async fn schedule_update_binds_every_mutable_field_to_its_named_column_db() -> TestResult { + // Pins: schedule policy replacement must bind each SQL parameter exactly once so changes to + // the name cannot shift timezone, calendar, policy, concurrency, budget, or time boundaries. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let repository = ExecutionRepository::new(test_db.store().pool().clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let now = moa_test_support::fixtures::pg_now(); + let original_occurrence = ExecutionScheduleOccurrence { + at: now + Duration::minutes(5), + local: (now + Duration::minutes(5)).naive_utc(), + }; + let create = schedule_request(tenant_id, Uuid::now_v7(), now); + let ExecutionScheduleCreateOutcome::Created { + schedule, + trigger: Some(original_trigger), + } = repository + .create_schedule( + scope, + &ExecutionConfig::default(), + create, + Some(original_occurrence), + ) + .await? + else { + panic!("update fixture must create one active schedule"); + }; + let next = ExecutionScheduleOccurrence { + at: now + Duration::hours(3), + local: (now + Duration::hours(3)).naive_utc(), + }; + let updated_budget = ExecutionBudgetLimit { + max_cost_microusd: Some(2_001), + max_tokens: Some(3_002), + max_tasks: Some(4), + max_tool_calls: Some(5), + max_retrieved_bytes: Some(6_003), + deadline_at: None, + }; + let update = ExecutionScheduleUpdateRequest { + tenant_id, + schedule_uid: schedule.schedule_uid, + expected_incarnation: schedule.schedule_incarnation, + name: "monthly close report".to_string(), + policy: ExecutionSchedulePolicy { + timezone: "America/New_York".to_string(), + calendar_expression: "0 30 17 1 * *".to_string(), + start_at: now + Duration::hours(1), + end_at: Some(now + Duration::days(90)), + missed_fire_policy: ExecutionScheduleMissedFirePolicy::Skip, + overlap_policy: ExecutionScheduleOverlapPolicy::Allow, + dst_policy: ExecutionScheduleDstPolicy::Latest, + maximum_concurrent_runs: 7, + occurrence_budget: serde_json::to_value(&updated_budget)?, + }, + }; + + let ExecutionScheduleMutationOutcome::Updated { + schedule: updated, + trigger: Some(updated_trigger), + } = repository + .update_schedule( + scope, + &ExecutionConfig::default(), + update.clone(), + Some(next), + ) + .await? + else { + panic!("exact current incarnation must accept the policy replacement"); + }; + assert_eq!(updated.name, update.name); + assert_eq!(updated.policy, update.policy); + assert_eq!( + updated.schedule_incarnation, + schedule.schedule_incarnation + 1 + ); + assert_eq!(updated.next_occurrence_at, Some(next.at)); + assert_eq!(updated.next_occurrence_local, Some(next.local)); + assert_eq!(updated.template, schedule.template); + assert_eq!(updated.run_as_identity, schedule.run_as_identity); + assert_eq!(updated.origin, schedule.origin); + assert_eq!( + updated_trigger.trigger.schedule_incarnation, + Some(updated.schedule_incarnation) + ); + let original_state: String = + sqlx::query_scalar("SELECT state FROM moa.execution_trigger WHERE trigger_uid=$1") + .bind(original_trigger.trigger.trigger_uid) + .fetch_one(test_db.store().pool()) + .await?; + assert_eq!(original_state, "cancelled"); + Ok(()) +} + +#[tokio::test] +async fn concurrent_schedule_update_and_occurrence_fire_share_capacity_first_lock_order_db() +-> TestResult { + // Pins: schedule CRUD and occurrence delivery must acquire ScheduledTriggers capacity before + // the schedule row, so either mutation wins atomically without a schedule/trigger deadlock. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let repository = ExecutionRepository::new(test_db.store().pool().clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let config = ExecutionConfig::default(); + let due_at = moa_test_support::fixtures::pg_now() - Duration::seconds(1); + let due = ExecutionScheduleOccurrence { + at: due_at, + local: due_at.naive_utc(), + }; + let create = schedule_request(tenant_id, Uuid::now_v7(), due_at); + let ExecutionScheduleCreateOutcome::Created { + schedule, + trigger: Some(trigger), + } = repository + .create_schedule(scope, &config, create, Some(due)) + .await? + else { + panic!("concurrency fixture must create one due schedule"); + }; + let run = execution_schedule_run_blueprint(&schedule)?.instantiate( + &schedule, + due, + 1, + config.maximum_horizon_seconds, + )?; + let next = ExecutionScheduleOccurrence { + at: due_at + Duration::hours(1), + local: (due_at + Duration::hours(1)).naive_utc(), + }; + let update = ExecutionScheduleUpdateRequest { + tenant_id, + schedule_uid: schedule.schedule_uid, + expected_incarnation: schedule.schedule_incarnation, + name: "concurrent replacement".to_string(), + policy: ExecutionSchedulePolicy { + end_at: Some(due_at + Duration::days(60)), + ..schedule.policy.clone() + }, + }; + let admission = ExecutionScheduleRunAdmission { + tenant_id, + schedule_uid: schedule.schedule_uid, + schedule_incarnation: schedule.schedule_incarnation, + occurrence_sequence: 1, + trigger_uid: trigger.trigger.trigger_uid, + trigger_dispatch_uid: trigger.dispatch.dispatch_uid, + occurrence: due, + run, + next_occurrence: None, + }; + + let (update_result, fire_result) = + tokio::time::timeout(std::time::Duration::from_secs(5), async { + tokio::join!( + repository.update_schedule(scope, &config, update, Some(next)), + repository.admit_schedule_occurrence(scope, &config, admission), + ) + }) + .await + .expect("capacity-first lock order must complete both contenders without deadlock"); + let update_result = update_result?; + let fire_result = fire_result?; + assert!( + matches!( + (&update_result, &fire_result), + ( + ExecutionScheduleMutationOutcome::Updated { .. }, + ExecutionScheduleRunAdmissionOutcome::Stale + ) | ( + ExecutionScheduleMutationOutcome::Stale, + ExecutionScheduleRunAdmissionOutcome::Admitted { .. } + ) + ), + "exactly one schedule-row mutation must win: update={update_result:?}, fire={fire_result:?}" + ); + Ok(()) +} + +#[tokio::test] +async fn schedule_occurrence_respects_joint_active_and_parked_resident_ceiling_db() -> TestResult { + // Pins: schedule-owned admission cannot bypass the joint resident-run ceiling merely because + // ActiveRuns compute capacity remains available; saturation consumes the occurrence once and + // leaves no run, activation, or ActiveRuns receipt to replay. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let config = moa_config::ExecutionConfig { + max_tenant_active_runs: 10, + max_fleet_active_runs: 10, + max_tenant_parked_runs: 1, + max_fleet_parked_runs: 1, + ..moa_config::ExecutionConfig::default() + }; + config.validate()?; + let due_at = moa_test_support::fixtures::pg_now() - Duration::seconds(1); + let due = ExecutionScheduleOccurrence { + at: due_at, + local: due_at.naive_utc(), + }; + + let first_request = schedule_request(tenant_id, Uuid::now_v7(), due.at); + let ExecutionScheduleCreateOutcome::Created { + schedule: first_schedule, + trigger: Some(first_trigger), + } = repository + .create_schedule(scope, &config, first_request, Some(due)) + .await? + else { + panic!("first resident schedule must arm its occurrence"); + }; + let first_run = execution_schedule_run_blueprint(&first_schedule)?.instantiate( + &first_schedule, + due, + 1, + config.maximum_horizon_seconds, + )?; + let ExecutionScheduleRunAdmissionOutcome::Admitted { + run: admitted_run, .. + } = repository + .admit_schedule_occurrence( + scope, + &config, + ExecutionScheduleRunAdmission { + tenant_id, + schedule_uid: first_schedule.schedule_uid, + schedule_incarnation: first_schedule.schedule_incarnation, + occurrence_sequence: 1, + trigger_uid: first_trigger.trigger.trigger_uid, + trigger_dispatch_uid: first_trigger.dispatch.dispatch_uid, + occurrence: due, + run: first_run, + next_occurrence: None, + }, + ) + .await? + else { + panic!("first occurrence must consume the sole resident entitlement"); + }; + + let second_request = schedule_request(tenant_id, Uuid::now_v7(), due.at); + let ExecutionScheduleCreateOutcome::Created { + schedule: second_schedule, + trigger: Some(second_trigger), + } = repository + .create_schedule(scope, &config, second_request, Some(due)) + .await? + else { + panic!("second schedule must arm before its resident admission check"); + }; + let second_run = execution_schedule_run_blueprint(&second_schedule)?.instantiate( + &second_schedule, + due, + 1, + config.maximum_horizon_seconds, + )?; + let replay_run = second_run.clone(); + let saturated_request = ExecutionScheduleRunAdmission { + tenant_id, + schedule_uid: second_schedule.schedule_uid, + schedule_incarnation: second_schedule.schedule_incarnation, + occurrence_sequence: 1, + trigger_uid: second_trigger.trigger.trigger_uid, + trigger_dispatch_uid: second_trigger.dispatch.dispatch_uid, + occurrence: due, + run: second_run, + next_occurrence: None, + }; + let saturated = repository + .admit_schedule_occurrence(scope, &config, saturated_request) + .await?; + assert!( + matches!( + saturated, + ExecutionScheduleRunAdmissionOutcome::Skipped { .. } + ), + "joint resident saturation must consume and skip the occurrence" + ); + + let second_run_count: i64 = + sqlx::query_scalar("SELECT count(*) FROM moa.execution_run WHERE schedule_uid=$1") + .bind(second_schedule.schedule_uid) + .fetch_one(&pool) + .await?; + assert_eq!(second_run_count, 0); + let resident_receipts: Vec<(String, i64)> = sqlx::query_as( + "SELECT resource_dimension, count(*) \ + FROM moa.execution_capacity_reservation \ + WHERE tenant_id=$1 AND state IN ('reserved','reconciling') \ + AND resource_dimension IN ('active_runs','parked_runs') \ + GROUP BY resource_dimension ORDER BY resource_dimension", + ) + .bind(tenant_id.0) + .fetch_all(&pool) + .await?; + assert_eq!(resident_receipts, vec![("active_runs".to_string(), 1)]); + let second_trigger_state: (String, String) = sqlx::query_as( + "SELECT trigger.state, capacity.state \ + FROM moa.execution_trigger AS trigger \ + JOIN moa.execution_capacity_reservation AS capacity \ + ON capacity.trigger_uid=trigger.trigger_uid \ + AND capacity.resource_dimension='scheduled_triggers' \ + WHERE trigger.trigger_uid=$1", + ) + .bind(second_trigger.trigger.trigger_uid) + .fetch_one(&pool) + .await?; + assert_eq!( + second_trigger_state, + ("delivered".to_string(), "released".to_string()) + ); + + assert_eq!( + repository + .admit_schedule_occurrence( + scope, + &config, + ExecutionScheduleRunAdmission { + tenant_id, + schedule_uid: second_schedule.schedule_uid, + schedule_incarnation: second_schedule.schedule_incarnation, + occurrence_sequence: 1, + trigger_uid: second_trigger.trigger.trigger_uid, + trigger_dispatch_uid: second_trigger.dispatch.dispatch_uid, + occurrence: due, + run: replay_run, + next_occurrence: None, + }, + ) + .await?, + ExecutionScheduleRunAdmissionOutcome::Replayed { + run_uid: None, + activation_dispatch_uid: None, + } + ); + let first_run_still_present: bool = + sqlx::query_scalar("SELECT EXISTS (SELECT 1 FROM moa.execution_run WHERE run_uid=$1)") + .bind(admitted_run.run_uid) + .fetch_one(&pool) + .await?; + assert!(first_run_still_present); + Ok(()) +} + +#[tokio::test] +async fn schedule_overlap_probes_are_indexed_and_bounded_across_large_terminal_history_db() +-> TestResult { + // Pins: terminal occurrence history never expands an overlap decision. Skip and QueueOne + // stop at one indexed row, while Allow reads no more than maximum_concurrent_runs even when + // more live rows and thousands of terminal rows exist for the same schedule. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let config = ExecutionConfig::default(); + let now = moa_test_support::fixtures::pg_now(); + + for (ordinal, overlap_policy, maximum_concurrent_runs, extra_queued) in [ + (1_i64, ExecutionScheduleOverlapPolicy::Skip, 1_u64, 0_i64), + (2, ExecutionScheduleOverlapPolicy::QueueOne, 1, 0), + (3, ExecutionScheduleOverlapPolicy::Allow, 4, 8), + ] { + let schedule_uid = Uuid::now_v7(); + let occurrence = ExecutionScheduleOccurrence { + at: now - Duration::seconds(20 - ordinal), + local: (now - Duration::seconds(20 - ordinal)).naive_utc(), + }; + let next_occurrence = ExecutionScheduleOccurrence { + at: now - Duration::seconds(1), + local: (now - Duration::seconds(1)).naive_utc(), + }; + let mut request = schedule_request(tenant_id, schedule_uid, occurrence.at); + request.policy.start_at = occurrence.at - Duration::minutes(1); + request.policy.overlap_policy = overlap_policy; + request.policy.maximum_concurrent_runs = maximum_concurrent_runs; + let ExecutionScheduleCreateOutcome::Created { + schedule, + trigger: Some(trigger), + } = repository + .create_schedule(scope, &config, request, Some(occurrence)) + .await? + else { + panic!("overlap fixture must create one due schedule"); + }; + let blueprint = execution_schedule_run_blueprint(&schedule)?; + insert_schedule_planning_context(&pool, tenant_id, schedule.run_as_identity.id, &blueprint) + .await?; + let first_run = + blueprint.instantiate(&schedule, occurrence, 1, config.maximum_horizon_seconds)?; + let ExecutionScheduleRunAdmissionOutcome::Admitted { + run, + next_trigger: Some(next_trigger), + .. + } = repository + .admit_schedule_occurrence( + scope, + &config, + ExecutionScheduleRunAdmission { + tenant_id, + schedule_uid, + schedule_incarnation: schedule.schedule_incarnation, + occurrence_sequence: 1, + trigger_uid: trigger.trigger.trigger_uid, + trigger_dispatch_uid: trigger.dispatch.dispatch_uid, + occurrence, + run: first_run, + next_occurrence: Some(next_occurrence), + }, + ) + .await? + else { + panic!("first overlap fixture occurrence must be admitted"); + }; + + seed_schedule_overlap_history(&pool, run.run_uid, 2_501, extra_queued).await?; + sqlx::query("ANALYZE moa.execution_run") + .execute(&pool) + .await?; + assert_schedule_overlap_probe( + &pool, + tenant_id, + schedule_uid, + overlap_policy, + maximum_concurrent_runs, + ) + .await?; + + let current = repository + .load_schedule(scope, tenant_id, schedule_uid) + .await? + .expect("advanced overlap fixture schedule remains visible"); + let second_run = + blueprint.instantiate(¤t, next_occurrence, 2, config.maximum_horizon_seconds)?; + assert!(matches!( + repository + .admit_schedule_occurrence( + scope, + &config, + ExecutionScheduleRunAdmission { + tenant_id, + schedule_uid, + schedule_incarnation: current.schedule_incarnation, + occurrence_sequence: 2, + trigger_uid: next_trigger.trigger.trigger_uid, + trigger_dispatch_uid: next_trigger.dispatch.dispatch_uid, + occurrence: next_occurrence, + run: second_run, + next_occurrence: None, + }, + ) + .await?, + ExecutionScheduleRunAdmissionOutcome::Skipped { .. } + )); + } + Ok(()) +} + +async fn insert_schedule_planning_context( + pool: &sqlx::PgPool, + tenant_id: TenantId, + owner_id: Uuid, + blueprint: &ExecutionScheduleRunBlueprint, +) -> TestResult { + sqlx::query( + "INSERT INTO moa.execution_planning_context (\ + planning_context_uid, tenant_id, session_id, originating_user_sequence_num, \ + originating_user_event_hash, owner_user_id, planning_context_hash, snapshot\ + ) VALUES ($1, $2, $3, $4, $5, $6, $5, '{}'::JSONB)", + ) + .bind(blueprint.planning_context_uid) + .bind(tenant_id.0) + .bind(blueprint.session_id.0) + .bind(i64::try_from(blueprint.originating_user_sequence_num)?) + .bind(blueprint.planning_context_hash.to_string()) + .bind(owner_id.to_string()) + .execute(pool) + .await?; + Ok(()) +} + +async fn seed_schedule_overlap_history( + pool: &sqlx::PgPool, + template_run_uid: Uuid, + terminal_count: i64, + extra_queued_count: i64, +) -> TestResult { + let mut transaction = pool.begin().await?; + let insert_columns: Vec = sqlx::query_scalar( + "SELECT quote_ident(attname::TEXT) FROM pg_catalog.pg_attribute \ + WHERE attrelid='moa.execution_run'::REGCLASS AND attnum>0 \ + AND NOT attisdropped AND attgenerated='' ORDER BY attnum", + ) + .fetch_all(transaction.as_mut()) + .await?; + let column_list = insert_columns.join(", "); + let selected_columns = insert_columns + .iter() + .map(|column| format!("populated.{column}")) + .collect::>() + .join(", "); + sqlx::query("ALTER TABLE moa.execution_run DISABLE TRIGGER USER") + .execute(transaction.as_mut()) + .await?; + let terminal_sql = format!( + "INSERT INTO moa.execution_run ({column_list}) \ + SELECT {selected_columns} \ + FROM moa.execution_run AS template_run \ + CROSS JOIN generate_series(1, $2::BIGINT) AS series(ordinal) \ + CROSS JOIN LATERAL jsonb_populate_record(\ + NULL::moa.execution_run, \ + to_jsonb(template_run) || jsonb_build_object(\ + 'run_uid', gen_random_uuid(), \ + 'idempotency_key', NULL, \ + 'schedule_occurrence_sequence', 10000 + series.ordinal, \ + 'status', 'failed', \ + 'activation_state', 'terminal', \ + 'terminal_cause', jsonb_build_object('kind', 'internal_failure'), \ + 'terminal_satisfied_requirement_count', 0, \ + 'terminal_requirement_count', 0, \ + 'terminal_reason', 'internal_failure', \ + 'completed_at', now(), \ + 'processed_wake_epoch', template_run.wake_epoch, \ + 'next_wake_at', NULL\ + )\ + ) AS populated \ + WHERE template_run.run_uid=$1" + ); + let terminal = sqlx::query(&terminal_sql) + .bind(template_run_uid) + .bind(terminal_count) + .execute(transaction.as_mut()) + .await?; + assert_eq!(terminal.rows_affected(), u64::try_from(terminal_count)?); + let queued_sql = format!( + "INSERT INTO moa.execution_run ({column_list}) \ + SELECT {selected_columns} \ + FROM moa.execution_run AS template_run \ + CROSS JOIN generate_series(1, $2::BIGINT) AS series(ordinal) \ + CROSS JOIN LATERAL jsonb_populate_record(\ + NULL::moa.execution_run, \ + to_jsonb(template_run) || jsonb_build_object(\ + 'run_uid', gen_random_uuid(), \ + 'idempotency_key', NULL, \ + 'schedule_occurrence_sequence', 20000 + series.ordinal\ + )\ + ) AS populated \ + WHERE template_run.run_uid=$1" + ); + let queued = sqlx::query(&queued_sql) + .bind(template_run_uid) + .bind(extra_queued_count) + .execute(transaction.as_mut()) + .await?; + assert_eq!(queued.rows_affected(), u64::try_from(extra_queued_count)?); + sqlx::query("ALTER TABLE moa.execution_run ENABLE TRIGGER USER") + .execute(transaction.as_mut()) + .await?; + transaction.commit().await?; + Ok(()) +} + +async fn assert_schedule_overlap_probe( + pool: &sqlx::PgPool, + tenant_id: TenantId, + schedule_uid: Uuid, + overlap_policy: ExecutionScheduleOverlapPolicy, + maximum_concurrent_runs: u64, +) -> TestResult { + let mut transaction = pool.begin().await?; + sqlx::query("SET LOCAL enable_seqscan=off") + .execute(transaction.as_mut()) + .await?; + let (explain, expected_index, expected_rows) = match overlap_policy { + ExecutionScheduleOverlapPolicy::Skip => ( + sqlx::query_scalar( + "EXPLAIN (ANALYZE, COSTS OFF, FORMAT JSON) \ + SELECT EXISTS (SELECT 1 FROM moa.execution_run \ + WHERE tenant_id=$1 AND schedule_uid=$2 \ + AND status NOT IN \ + ('completed','partial','blocked','unsupported','failed','cancelled'))", + ) + .bind(tenant_id.0) + .bind(schedule_uid) + .fetch_one(transaction.as_mut()) + .await?, + "execution_run_schedule_nonterminal_idx", + 1, + ), + ExecutionScheduleOverlapPolicy::QueueOne => ( + sqlx::query_scalar( + "EXPLAIN (ANALYZE, COSTS OFF, FORMAT JSON) \ + SELECT EXISTS (SELECT 1 FROM moa.execution_run \ + WHERE tenant_id=$1 AND schedule_uid=$2 AND status='queued')", + ) + .bind(tenant_id.0) + .bind(schedule_uid) + .fetch_one(transaction.as_mut()) + .await?, + "execution_run_schedule_queued_idx", + 1, + ), + ExecutionScheduleOverlapPolicy::Allow => ( + sqlx::query_scalar( + "EXPLAIN (ANALYZE, COSTS OFF, FORMAT JSON) \ + SELECT count(*) FROM (SELECT 1 FROM moa.execution_run \ + WHERE tenant_id=$1 AND schedule_uid=$2 \ + AND status NOT IN \ + ('completed','partial','blocked','unsupported','failed','cancelled') \ + LIMIT $3) AS bounded_nonterminal_runs", + ) + .bind(tenant_id.0) + .bind(schedule_uid) + .bind(i64::try_from(maximum_concurrent_runs)?) + .fetch_one(transaction.as_mut()) + .await?, + "execution_run_schedule_nonterminal_idx", + maximum_concurrent_runs, + ), + }; + let scan = explain_index_scan(&explain, expected_index) + .expect("overlap probe must use its policy-specific partial index"); + assert_eq!( + scan.get("Actual Rows").and_then(serde_json::Value::as_u64), + Some(expected_rows), + "overlap probe must stop at its semantic bound" + ); + transaction.rollback().await?; + Ok(()) +} + +fn explain_index_scan<'a>( + value: &'a serde_json::Value, + expected_index: &str, +) -> Option<&'a serde_json::Map> { + match value { + serde_json::Value::Object(object) => { + if object.get("Index Name").and_then(serde_json::Value::as_str) == Some(expected_index) + { + return Some(object); + } + object + .values() + .find_map(|child| explain_index_scan(child, expected_index)) + } + serde_json::Value::Array(values) => values + .iter() + .find_map(|child| explain_index_scan(child, expected_index)), + _ => None, + } +} + +fn schedule_request( + tenant_id: TenantId, + schedule_uid: Uuid, + now: chrono::DateTime, +) -> ExecutionScheduleCreateRequest { + let identity = Identity { + identity_type: IdentityType::Operator, + id: Uuid::now_v7(), + tenant_id, + api_key_id: None, + acting_on_behalf_of: None, + }; + let revision_uid = Uuid::now_v7(); + let approved_budget = ExecutionBudgetLimit { + max_cost_microusd: Some(1_000), + max_tokens: Some(1_000), + max_tasks: Some(10), + max_tool_calls: Some(100), + max_retrieved_bytes: Some(10_000), + deadline_at: None, + }; + let catalog = + ExecutionCapabilityCatalog::build(Vec::new()).expect("empty capability catalog is valid"); + let blueprint = ExecutionScheduleRunBlueprint { + session_id: SessionId::new(), + originating_user_sequence_num: 1, + planning_context_uid: Uuid::now_v7(), + planning_context_hash: ExecutionHash::from_bytes([9; 32]), + goal: ExecutionGoalContract { + objective: "produce the recurring weekday report".to_string(), + requirements: Vec::new(), + deliverables: Vec::new(), + coverage: Vec::new(), + constraints: Vec::new(), + completion_checks: Vec::new(), + }, + plan: CanonicalExecutionPlan { + definition: ExecutionPlanDefinition { + cancel_policy: ExecutionCancelPolicy::RetainEffects, + input_wait_policy: ExecutionWaitPolicy { + expiry: ExecutionTemporalTarget::After { + delay_seconds: 3_600, + }, + on_expiry: ExecutionWaitExpiryAction::FailTask, + }, + input_schema: serde_json::json!({"type":"object"}), + output_schema: serde_json::json!({"type":"object"}), + nodes: vec![ExecutionNode { + id: "output".to_string(), + requirement_ids: Vec::new(), + depends_on: Vec::new(), + when: None, + input: serde_json::json!({}), + output_schema: serde_json::json!({"type":"object"}), + operation: ExecutionOperation::Output { + value: serde_json::json!({"report":"ready"}), + }, + compensation: None, + retry: RetryPolicy { + max_attempts: 1, + initial_backoff_ms: 1, + max_backoff_ms: 1, + }, + budget: None, + }], + }, + plan_hash: ExecutionHash::from_bytes([10; 32]), + catalog_hash: catalog.catalog_hash, + estimate: ExecutionEstimate { + cost_microusd: 1, + tokens: 1, + tasks: 1, + tool_calls: 1, + retrieved_bytes: 1, + }, + report: ExecutionValidationReport::default(), + }, + catalog, + authorization: ExecutionAuthorizationEnvelope { + capability_refs: Vec::new(), + skill_refs: Vec::new(), + }, + pinned_instruction_skills: Vec::new(), + source_provenance: ExecutionSourceProvenance::SkillTemplate { + skill_template_ref: "skill://weekday-report".to_string(), + skill_template_revision_uid: revision_uid, + }, + input: serde_json::json!({"report":"weekday"}), + approved_budget: approved_budget.clone(), + deadline_offset_seconds: Some(2 * 60 * 60), + }; + let template_snapshot = + serde_json::to_value(blueprint).expect("schedule blueprint fixture must serialize"); + ExecutionScheduleCreateRequest { + tenant_id, + schedule_uid, + name: "weekday report".to_string(), + template: ExecutionScheduleTemplate { + revision_uid, + template_hash: execution_schedule_template_hash(&template_snapshot) + .expect("template fixture must canonicalize"), + snapshot: template_snapshot, + }, + run_as_identity: identity.clone(), + origin: ExecutionScheduleOrigin { + request_uid: Uuid::now_v7(), + created_by: identity, + source: ExecutionScheduleOriginSource::TenantApi, + }, + policy: ExecutionSchedulePolicy { + timezone: "UTC".to_string(), + calendar_expression: "0 0 9 * * 1-5".to_string(), + start_at: now, + end_at: Some(now + Duration::days(30)), + missed_fire_policy: ExecutionScheduleMissedFirePolicy::FireOnce, + overlap_policy: ExecutionScheduleOverlapPolicy::QueueOne, + dst_policy: ExecutionScheduleDstPolicy::Earliest, + maximum_concurrent_runs: 1, + occurrence_budget: serde_json::to_value(approved_budget) + .expect("budget fixture must serialize"), + }, + } +} diff --git a/crates/moa-orchestrator/tests/orchestrator_db/execution_service_db.rs b/crates/moa-orchestrator/tests/orchestrator_db/execution_service_db.rs index a7377d847..0629f1deb 100644 --- a/crates/moa-orchestrator/tests/orchestrator_db/execution_service_db.rs +++ b/crates/moa-orchestrator/tests/orchestrator_db/execution_service_db.rs @@ -5,6 +5,7 @@ use moa_artifacts::execution_plan::{ ExecutionBudgetLimit, ExecutionCitation, ExecutionGoalContract, ExecutionPlanDefinition, ExecutionTaskOutcome, ExecutionTaskResult, ExecutionUsage, RetryPolicy, }; +use moa_config::ExecutionConfig; use moa_core::{ events::ExecutionTaskResultsRef, types::{ @@ -23,9 +24,14 @@ use moa_execution::{ terminal_evidence_from_evaluation, }, repository::{ - ExecutionRepository, ExecutionScope, FinalizationOutcome, NewExecutionPlanningContext, - NewExecutionRun, PlanningContextWriteOutcome, ReservationOutcome, RunFinalizationRequest, - TaskOutcomeWrite, TransitionOutcome, + ExecutionRepository, ExecutionScope, NewExecutionRun, ReservationOutcome, + RunControllerClaimOutcome, TaskOutcomeWrite, TransitionOutcome, + audit::{NewExecutionPlanningContext, PlanningContextWriteOutcome}, + run::RunAdmissionOutcome, + terminal::{ + FinalizationOutcome, RunFinalizationRequest, RunTriggerDrainOutcome, + RunTriggerDrainRequest, + }, }, state::{ ExecutionRunStatus, ExecutionTaskId, ExecutionTaskStatus, ExecutionTerminalCause, @@ -218,6 +224,12 @@ async fn execution_task_citation_lineage_survives_reload_and_terminal_summary_db let plan = CanonicalExecutionPlan { definition: ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, + input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { + expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::At { + at: chrono::Utc::now() + chrono::TimeDelta::hours(1), + }, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + }, input_schema: json!({"type": "object"}), output_schema: json!({"type": "object"}), nodes: Vec::new(), @@ -233,9 +245,10 @@ async fn execution_task_citation_lineage_survives_reload_and_terminal_summary_db }, report: ExecutionValidationReport::default(), }; - let run = repository + let RunAdmissionOutcome::Admitted(run) = repository .create_run( scope, + &ExecutionConfig::default(), NewExecutionRun { tenant_id, contact_id: None, @@ -244,6 +257,13 @@ async fn execution_task_citation_lineage_survives_reload_and_terminal_summary_db planning_context_uid: planning_context.planning_context_uid, planning_context_hash: planning_hash, owner_user_id, + admitted_identity: moa_core::traits::Identity { + identity_type: moa_core::traits::IdentityType::Service, + id: Uuid::new_v4(), + tenant_id, + api_key_id: None, + acting_on_behalf_of: None, + }, goal: ExecutionGoalContract { objective: "preserve exact execution citation lineage".to_string(), requirements: Vec::new(), @@ -266,17 +286,9 @@ async fn execution_task_citation_lineage_survives_reload_and_terminal_summary_db idempotency_key: Some("execution-lineage".to_string()), }, ) - .await?; - let TransitionOutcome::RunApplied(running) = repository - .transition_run_wait( - scope, - run.run_uid, - ExecutionRunStatus::Queued, - ExecutionRunStatus::Running, - ) .await? else { - panic!("execution lineage fixture must transition its run to running"); + panic!("execution lineage fixture must be admitted"); }; let task_id = ExecutionTaskId::derive(run.run_uid, "collect", "primary")?; let task = LogicalTask { @@ -284,7 +296,7 @@ async fn execution_task_citation_lineage_survives_reload_and_terminal_summary_db node_id: "collect".to_string(), item_key: "primary".to_string(), requirement_ids: vec!["lineage".to_string()], - plan_revision: running.plan_revision, + plan_revision: run.plan_revision, generation: 1, input: json!({"source_set": "primary"}), kind: LogicalTaskKind::Output { @@ -305,7 +317,7 @@ async fn execution_task_citation_lineage_survives_reload_and_terminal_summary_db }, }; let tasks = repository - .materialize_tasks(scope, run.run_uid, running.plan_revision, vec![task]) + .materialize_tasks(scope, run.run_uid, run.plan_revision, vec![task]) .await?; assert_eq!(tasks.len(), 1, "fixture must materialize exactly one task"); assert_eq!(tasks[0].task_id, task_id); @@ -394,6 +406,39 @@ async fn execution_task_citation_lineage_survives_reload_and_terminal_summary_db "another tenant must not load the owning execution run" ); + let current = repository + .load_run(scope, run.run_uid) + .await? + .expect("execution lineage run must remain visible"); + let claimed = match repository + .claim_controller_wake( + scope, + current.run_uid, + current.controller_generation, + current.wake_epoch, + ) + .await? + { + RunControllerClaimOutcome::Claimed(claimed) => claimed, + outcome => panic!("terminal controller wake must be claimable: {outcome:?}"), + }; + let RunTriggerDrainOutcome::ReadyToFinalize { run: drained, .. } = repository + .drain_run_triggers_page( + scope, + &ExecutionConfig::default(), + RunTriggerDrainRequest { + run_uid: claimed.run_uid, + controller_generation: claimed.controller_generation, + wake_epoch: claimed.wake_epoch, + page_limit: 2, + now: Utc::now(), + }, + ) + .await? + else { + panic!("bounded trigger drain must make the fixture ready to finalize"); + }; + let completion = CompletionEvaluation { status: CompletionStatus::Completed, limit_stop: None, @@ -408,8 +453,8 @@ async fn execution_task_citation_lineage_survives_reload_and_terminal_summary_db }; let finalization = RunFinalizationRequest { run_uid: run.run_uid, - expected_revision: replayed_run.plan_revision, - expected_wake_epoch: replayed_run.wake_epoch, + expected_revision: drained.plan_revision, + expected_wake_epoch: drained.wake_epoch, terminal_projection: terminal.clone(), completion_evaluation: completion.clone(), terminal_evidence: terminal_evidence_from_evaluation(cause.clone(), &completion)?, @@ -429,9 +474,9 @@ async fn execution_task_citation_lineage_survives_reload_and_terminal_summary_db assert_eq!(replayed_finalized_run, finalized_run); let delivery = repository - .load_terminal_delivery(scope, run.run_uid) + .load_bounded_terminal_delivery(scope, run.run_uid) .await? - .expect("completed execution run must expose terminal delivery"); + .expect("completed execution run must expose bounded terminal delivery"); let expected_source_ids = (0..EXECUTION_TERMINAL_MAX_CITATION_IDS) .map(|index| format!("source-{index:03}")) .collect::>(); @@ -447,7 +492,7 @@ async fn execution_task_citation_lineage_survives_reload_and_terminal_summary_db ); assert_eq!( repository - .load_terminal_delivery(other_scope, run.run_uid) + .load_bounded_terminal_delivery(other_scope, run.run_uid) .await?, None, "another tenant must not derive the execution terminal summary" @@ -471,6 +516,12 @@ async fn execution_service_rows_require_parent_session_and_keep_authorization_im let plan = serde_json::to_value(CanonicalExecutionPlan { definition: ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, + input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { + expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::At { + at: chrono::Utc::now() + chrono::TimeDelta::hours(1), + }, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + }, input_schema: json!({ "type": "object" }), output_schema: json!({ "type": "object" }), nodes: Vec::new(), diff --git a/crates/moa-orchestrator/tests/orchestrator_offline/session_vo.rs b/crates/moa-orchestrator/tests/orchestrator_offline/session_vo.rs index 833e15075..e5e43786d 100644 --- a/crates/moa-orchestrator/tests/orchestrator_offline/session_vo.rs +++ b/crates/moa-orchestrator/tests/orchestrator_offline/session_vo.rs @@ -132,11 +132,29 @@ fn active_execution_run(run_uid: Uuid, origin: u64) -> ActiveExecutionRunState { } fn execution_progress(run_uid: Uuid) -> moa_core::events::ExecutionProgress { + let now = moa_test_support::fixtures::pg_now(); moa_core::events::ExecutionProgress { run_uid, originating_user_sequence_num: 7, plan_revision: 1, status: "running".to_string(), + phase: moa_core::events::ExecutionProgressPhase::Running, + waiting_since: None, + next_wake_at: None, + last_progress_at: now, + external_job_uid: None, + ready_tasks: 3, + active_tasks: 1, + parked_tasks: 0, + blocker_audience: None, + remaining_budget: moa_core::events::ExecutionRemainingBudget { + cost_microusd: Some(70), + tokens: Some(700), + tasks: Some(6), + tool_calls: Some(12), + retrieved_bytes: Some(7_000), + deadline_at: Some(now + chrono::Duration::hours(2)), + }, total: 8, completed: 2, failed: 1, @@ -155,6 +173,23 @@ fn session_progress_projects_exact_persisted_active_execution_values() { originating_user_sequence_num: 41, plan_revision: 5, status: "waiting_input".to_string(), + phase: moa_core::events::ExecutionProgressPhase::WaitingInput, + waiting_since: Some(chrono::Utc::now()), + next_wake_at: None, + last_progress_at: chrono::Utc::now(), + external_job_uid: None, + ready_tasks: 1, + active_tasks: 0, + parked_tasks: 1, + blocker_audience: Some(moa_core::events::ExecutionBlockerAudience::User), + remaining_budget: moa_core::events::ExecutionRemainingBudget { + cost_microusd: Some(40), + tokens: Some(400), + tasks: Some(5), + tool_calls: Some(10), + retrieved_bytes: Some(4_000), + deadline_at: None, + }, total: 13, completed: 8, failed: 3, @@ -174,9 +209,9 @@ fn session_progress_projects_exact_persisted_active_execution_values() { } #[test] -fn execution_progress_requires_cadence_and_changed_exact_aggregate_tuple() { - // Pins: every tuple member participates in delta detection, while an early changed tuple - // and a due identical tuple are both suppressed. +fn execution_progress_requires_cadence_and_changed_exact_public_projection() { + // Pins: every public progress member participates in delta detection, while an early changed + // projection and a due identical projection are both suppressed. let run_uid = Uuid::from_u128(70); let start = moa_test_support::fixtures::pg_now(); let baseline = execution_progress(run_uid); @@ -215,6 +250,27 @@ fn execution_progress_requires_cadence_and_changed_exact_aggregate_tuple() { plan_revision_changed.plan_revision += 1; let mut status_changed = baseline.clone(); status_changed.status = "waiting_input".to_string(); + let mut phase_changed = baseline.clone(); + phase_changed.phase = moa_core::events::ExecutionProgressPhase::WaitingInput; + let mut waiting_since_changed = baseline.clone(); + waiting_since_changed.waiting_since = Some(start); + let mut next_wake_at_changed = baseline.clone(); + next_wake_at_changed.next_wake_at = Some(start + chrono::Duration::hours(1)); + let mut last_progress_at_changed = baseline.clone(); + last_progress_at_changed.last_progress_at += chrono::Duration::seconds(1); + let mut external_job_uid_changed = baseline.clone(); + external_job_uid_changed.external_job_uid = Some(Uuid::from_u128(701)); + let mut ready_tasks_changed = baseline.clone(); + ready_tasks_changed.ready_tasks += 1; + let mut active_tasks_changed = baseline.clone(); + active_tasks_changed.active_tasks += 1; + let mut parked_tasks_changed = baseline.clone(); + parked_tasks_changed.parked_tasks += 1; + let mut blocker_audience_changed = baseline.clone(); + blocker_audience_changed.blocker_audience = + Some(moa_core::events::ExecutionBlockerAudience::External); + let mut remaining_budget_changed = baseline.clone(); + remaining_budget_changed.remaining_budget.tasks = Some(5); let mut total_changed = baseline.clone(); total_changed.total += 1; let mut completed_changed = baseline.clone(); @@ -226,6 +282,16 @@ fn execution_progress_requires_cadence_and_changed_exact_aggregate_tuple() { let changed_members = [ ("plan_revision", plan_revision_changed), ("status", status_changed), + ("phase", phase_changed), + ("waiting_since", waiting_since_changed), + ("next_wake_at", next_wake_at_changed), + ("last_progress_at", last_progress_at_changed), + ("external_job_uid", external_job_uid_changed), + ("ready_tasks", ready_tasks_changed), + ("active_tasks", active_tasks_changed), + ("parked_tasks", parked_tasks_changed), + ("blocker_audience", blocker_audience_changed), + ("remaining_budget", remaining_budget_changed), ("total", total_changed), ("completed", completed_changed), ("failed", failed_changed), @@ -254,6 +320,113 @@ fn execution_progress_requires_cadence_and_changed_exact_aggregate_tuple() { } } +#[test] +fn execution_progress_emits_parked_transitions_inside_cadence_and_throttles_counters() { + // Pins: counter-only churn updates the hot projection without emitting inside cadence, while + // Pausing->Paused and WaitingExternal transitions publish immediately with exact wait data. + let run_uid = Uuid::from_u128(702); + let start = moa_test_support::fixtures::pg_now(); + let mut state = SessionVoState::default(); + state + .active_execution_runs + .push(active_execution_run(run_uid, 7)); + + let mut pausing = execution_progress(run_uid); + pausing.status = "pausing".to_string(); + pausing.phase = moa_core::events::ExecutionProgressPhase::Pausing; + assert!( + state + .apply_execution_progress(pausing.clone(), start, 1_000) + .expect("initial pausing projection emits") + ); + + let mut counter_only = pausing.clone(); + counter_only.completed += 1; + counter_only.ready_tasks += 1; + assert!( + !state + .apply_execution_progress( + counter_only.clone(), + start + chrono::Duration::milliseconds(1), + 1_000, + ) + .expect("counter-only projection is cadence-throttled") + ); + assert_eq!( + state.active_execution_runs[0].progress.as_ref(), + Some(&counter_only), + "throttling the event must not leave Session/progress stale" + ); + + let mut parked_changed = counter_only.clone(); + parked_changed.parked_tasks += 1; + assert!( + state + .apply_execution_progress( + parked_changed.clone(), + start + chrono::Duration::milliseconds(2), + 1_000, + ) + .expect("parked-count changes are immediate wait semantics") + ); + + let mut blocker_changed = parked_changed; + blocker_changed.blocker_audience = + Some(moa_core::events::ExecutionBlockerAudience::TenantReviewer); + assert!( + state + .apply_execution_progress( + blocker_changed.clone(), + start + chrono::Duration::milliseconds(3), + 1_000, + ) + .expect("blocker audience is immediate wait semantics") + ); + + let mut paused = blocker_changed; + paused.status = "paused".to_string(); + paused.phase = moa_core::events::ExecutionProgressPhase::Paused; + paused.waiting_since = Some(start + chrono::Duration::milliseconds(4)); + assert!( + state + .apply_execution_progress( + paused.clone(), + start + chrono::Duration::milliseconds(4), + 1_000, + ) + .expect("paused transition emits inside cadence") + ); + + let mut waiting_external = paused; + waiting_external.status = "waiting_external".to_string(); + waiting_external.phase = moa_core::events::ExecutionProgressPhase::WaitingExternal; + waiting_external.waiting_since = Some(start + chrono::Duration::milliseconds(5)); + waiting_external.next_wake_at = Some(start + chrono::Duration::hours(1)); + waiting_external.external_job_uid = Some(Uuid::from_u128(703)); + assert!( + state + .apply_execution_progress( + waiting_external.clone(), + start + chrono::Duration::milliseconds(5), + 1_000, + ) + .expect("external wait transition emits inside cadence") + ); + assert_eq!( + state.active_execution_runs[0].progress, + Some(waiting_external.clone()) + ); + assert!( + !state + .apply_execution_progress( + waiting_external, + start + chrono::Duration::milliseconds(6), + 1_000, + ) + .expect("identical external wait is delta-suppressed") + ); +} + #[test] fn terminal_synthesis_dispatch_clears_active_state_once_and_replays_stable_marker() { // Pins: run state remains active before dispatch, then one stable run+origin marker clears diff --git a/crates/moa-orchestrator/tests/orchestrator_offline/tool_executor.rs b/crates/moa-orchestrator/tests/orchestrator_offline/tool_executor.rs index e488a716f..8f14f6d86 100644 --- a/crates/moa-orchestrator/tests/orchestrator_offline/tool_executor.rs +++ b/crates/moa-orchestrator/tests/orchestrator_offline/tool_executor.rs @@ -122,6 +122,7 @@ fn tool_definition( schema: json!({"type": "object"}), policy, idempotency_class, + async_mode: moa_core::types::tools::ToolAsyncMode::SynchronousOnly, rollback: None, max_output_tokens: 8_000, } diff --git a/crates/moa-orchestrator/tests/recovery_matrix_sandbox_workspace_service_e2e.rs b/crates/moa-orchestrator/tests/recovery_matrix_sandbox_workspace_service_e2e.rs index 269e8302a..24e982744 100644 --- a/crates/moa-orchestrator/tests/recovery_matrix_sandbox_workspace_service_e2e.rs +++ b/crates/moa-orchestrator/tests/recovery_matrix_sandbox_workspace_service_e2e.rs @@ -445,7 +445,7 @@ async fn seed_ambiguous_absent_operation(pool: &sqlx::PgPool) -> Result Result String { self.create_calls.fetch_add(1, Ordering::SeqCst); - let provider_reference = format!("soak-storage/{}", candidate.request.workspace_id); - let resource_fingerprint = - format!("sha256:soak-resource-{}", candidate.request.workspace_id); + let provider_reference = format!("soak-storage/{}", candidate.workspace_id); + let resource_fingerprint = format!("sha256:soak-resource-{}", candidate.workspace_id); self.resources.lock().await.push(ProviderInventoryResource { kind: ProviderInventoryResourceKind::MutableFilesystem, provider_reference: provider_reference.clone(), resource_fingerprint, - evidence_digest: format!("sha256:soak-evidence-{}", candidate.request.workspace_id), + evidence_digest: format!("sha256:soak-evidence-{}", candidate.workspace_id), verified_owner: Some(ProviderInventoryOwner { tenant_id: candidate.tenant_id, - workspace_id: candidate.request.workspace_id, + workspace_id: candidate.workspace_id, provisioning_operation_id: None, writer_epoch: Some(0), instance_generation: Some(0), @@ -173,22 +172,20 @@ async fn seed_candidate( operations: &PostgresWorkspaceOperationRepository, pool: &sqlx::PgPool, account_id: ProviderAccountId, -) -> Result { - let tenant_id = TenantId::new(); - let workspace_id = SandboxWorkspaceId::new(); - let operation_id = WorkspaceOperationId::new(); + candidate: &Candidate, +) -> Result<()> { sqlx::query( "INSERT INTO moa.sandbox_tenant_capacity_limits (tenant_id, configured_limits) \ VALUES ($1, '{\"workspaces\": 1}'::jsonb)", ) - .bind(tenant_id) + .bind(candidate.tenant_id) .execute(pool) .await .context("seed one-tenant workspace quota")?; workspaces .create(&CreateWorkspaceRequest { - workspace_id, - tenant_id, + workspace_id: candidate.workspace_id, + tenant_id: candidate.tenant_id, scope: SandboxWorkspaceScope::ExecutionTask { run_id: ExecutionRunScopeId::new(), task_id: ExecutionTaskScopeId::new(), @@ -204,13 +201,13 @@ async fn seed_candidate( let now = Utc::now(); operations .persist_intent(&WorkspaceOperationIntent { - operation_id, - tenant_id, - workspace_id, + operation_id: candidate.operation_id, + tenant_id: candidate.tenant_id, + workspace_id: candidate.workspace_id, provider_account_id: account_id, provider_account_generation: 1, kind: WorkspaceOperationKind::Create, - request_hash: format!("sha256:soak-{operation_id}"), + request_hash: format!("sha256:soak-{}", candidate.operation_id), expected_writer_epoch: 0, expected_instance_generation: 0, expected_checkpoint_generation: 0, @@ -219,22 +216,7 @@ async fn seed_candidate( }) .await .context("persist soak create intent through production repository")?; - Ok(Candidate { - tenant_id, - request: CapacityReservationRequest { - tenant_id, - workspace_id, - operation_id, - provider_account_id: account_id, - provider_account_generation: 1, - expected_writer_epoch: 0, - expected_instance_generation: 0, - quantities: vec![CapacityQuantity { - dimension: WorkspaceCapacityDimension::Workspaces, - quantity: 1, - }], - }, - }) + Ok(()) } #[tokio::test] @@ -273,7 +255,6 @@ async fn sandbox_workspace_1000_tenant_soak_exact_capacity_and_zero_drift_servic let workspaces = PostgresWorkspaceRepository::new(pool.clone()); let operations = PostgresWorkspaceOperationRepository::new(pool.clone()); - let capacity = PostgresWorkspaceCapacityRepository::new(pool.clone()); let storage_resources = PostgresWorkspaceStorageResourceRepository::new(pool.clone()); let scripted_provider = Arc::new(ScriptedSoakStorageProvider::new()); let local_root = @@ -298,26 +279,21 @@ async fn sandbox_workspace_1000_tenant_soak_exact_capacity_and_zero_drift_servic )?; let mut admitted = Vec::with_capacity(TENANT_COUNT); for _ in 0..TENANT_COUNT { - let candidate = seed_candidate(&workspaces, &operations, &pool, account_id).await?; - let reservation = capacity - .reserve(&candidate.request) - .await - .context("reserve one exact tenant/provider workspace slot")?; - assert_eq!( - reservation.len(), - 1, - "each tenant consumes exactly one slot" - ); + let candidate = Candidate { + tenant_id: TenantId::new(), + workspace_id: SandboxWorkspaceId::new(), + operation_id: WorkspaceOperationId::new(), + }; + seed_candidate(&workspaces, &operations, &pool, account_id, &candidate).await?; let storage_resource_id = Uuid::now_v7(); - let deterministic_name = format!("soak-storage-{}", candidate.request.workspace_id); - let verified_owner_fingerprint = - format!("sha256:soak-owner-{}", candidate.request.workspace_id); + let deterministic_name = format!("soak-storage-{}", candidate.workspace_id); + let verified_owner_fingerprint = format!("sha256:soak-owner-{}", candidate.workspace_id); storage_resources .persist_create_intent(&StorageResourceCreateIntent { storage_resource_id, tenant_id: candidate.tenant_id, - workspace_id: candidate.request.workspace_id, - create_operation_id: candidate.request.operation_id, + workspace_id: candidate.workspace_id, + create_operation_id: candidate.operation_id, provider_account_id: account_id, provider_account_generation: 1, security_class: "scheduled-soak".to_string(), @@ -333,7 +309,7 @@ async fn sandbox_workspace_1000_tenant_soak_exact_capacity_and_zero_drift_servic candidate.tenant_id, storage_resource_id, 1, - candidate.request.operation_id, + candidate.operation_id, &provider_reference, ) .await @@ -346,11 +322,14 @@ async fn sandbox_workspace_1000_tenant_soak_exact_capacity_and_zero_drift_servic TENANT_COUNT ); - let rejected = seed_candidate(&workspaces, &operations, &pool, account_id).await?; - let error = capacity - .reserve(&rejected.request) + let rejected = Candidate { + tenant_id: TenantId::new(), + workspace_id: SandboxWorkspaceId::new(), + operation_id: WorkspaceOperationId::new(), + }; + let error = seed_candidate(&workspaces, &operations, &pool, account_id, &rejected) .await - .expect_err("provider ceiling plus one must fail before provider I/O"); + .expect_err("provider ceiling plus one must fail during workspace creation"); assert_eq!( scripted_provider.create_calls.load(Ordering::SeqCst), TENANT_COUNT, @@ -358,19 +337,33 @@ async fn sandbox_workspace_1000_tenant_soak_exact_capacity_and_zero_drift_servic ); assert!( matches!( - error, - MoaError::ValidationError(ref detail) - if detail == "provider account workspaces capacity exceeded: 1000 + 1 > 1000" + error.downcast_ref::(), + Some(MoaError::StorageError(detail)) + if detail.contains("provider account workspaces capacity exceeded: 1000 + 1 > 1000") ), - "capacity rejection must identify the exact provider-account boundary: {error}" + "atomic create rejection must identify the exact provider-account boundary: {error}" ); - let row: (i64, i64, i64, i64) = sqlx::query_as( + let rejected_state: (i64, i64) = sqlx::query_as( + r#" + SELECT (SELECT count(*) FROM moa.sandbox_workspaces WHERE workspace_id = $1)::BIGINT, + (SELECT count(*) FROM moa.sandbox_capacity_reservations + WHERE workspace_id = $1 AND resource_dimension = 'workspaces')::BIGINT + "#, + ) + .bind(rejected.workspace_id) + .fetch_one(&pool) + .await + .context("verify atomic create admission rollback")?; + assert_eq!(rejected_state, (0, 0)); + + let row: (i64, i64, i64, i64, i64) = sqlx::query_as( r#" SELECT count(*)::BIGINT, count(DISTINCT tenant_id)::BIGINT, count(DISTINCT workspace_id)::BIGINT, - count(DISTINCT operation_id)::BIGINT + count(*) FILTER (WHERE reservation_state = 'committed')::BIGINT, + count(operation_id)::BIGINT FROM moa.sandbox_capacity_reservations WHERE provider_account_id = $1 AND resource_dimension = 'workspaces' @@ -381,7 +374,7 @@ async fn sandbox_workspace_1000_tenant_soak_exact_capacity_and_zero_drift_servic .fetch_one(&pool) .await .context("measure exact non-double-counted capacity")?; - assert_eq!(row, (1_000, 1_000, 1_000, 1_000)); + assert_eq!(row, (1_000, 1_000, 1_000, 1_000, 0)); let workspace_fences: (i64, i64, i64, i64, i64, i64, i64, i64) = sqlx::query_as( r#" @@ -401,7 +394,7 @@ async fn sandbox_workspace_1000_tenant_soak_exact_capacity_and_zero_drift_servic .fetch_one(&pool) .await .context("measure workspace ownership and monotonic head fences")?; - assert_eq!(workspace_fences, (1_001, 1_001, 0, 0, 0, 0, 0, 0)); + assert_eq!(workspace_fences, (1_000, 1_000, 0, 0, 0, 0, 0, 0)); let durable_storage: (i64, i64, i64) = sqlx::query_as( r#" @@ -420,9 +413,11 @@ async fn sandbox_workspace_1000_tenant_soak_exact_capacity_and_zero_drift_servic assert_eq!(durable_storage, (1_000, 1_000, 1_000)); let inventory = maintenance - .reconcile_provider_inventory_once() + .reconcile_claimed_provider_inventory_once(1) .await - .context("run production inventory reconciliation after soak admission")?; + .context( + "claim and reconcile the production provider-account shard after soak admission", + )?; assert_eq!(inventory.accounts, 1); assert_eq!(inventory.resources, TENANT_COUNT as u64); assert_eq!(inventory.unresolved_findings, 0); diff --git a/crates/moa-session/src/store/dashboard.rs b/crates/moa-session/src/store/dashboard.rs index 124e9b31a..2b6ba19a8 100644 --- a/crates/moa-session/src/store/dashboard.rs +++ b/crates/moa-session/src/store/dashboard.rs @@ -416,8 +416,15 @@ fn redacted_event_summary(event: &Event) -> String { format!("execution run {} started", started.run_uid) } Event::ExecutionProgress(progress) => format!( - "execution run {} progress {}/{} status={}", - progress.run_uid, progress.completed, progress.total, progress.status + "execution run {} progress {}/{} ready={} active={} parked={} blocker={:?} status={}", + progress.run_uid, + progress.completed, + progress.total, + progress.ready_tasks, + progress.active_tasks, + progress.parked_tasks, + progress.blocker_audience, + progress.status ), Event::ExecutionInputRequired(required) => { format!("execution run {} requires user input", required.run_uid) diff --git a/crates/moa-session/tests/session_db/execution_events_db.rs b/crates/moa-session/tests/session_db/execution_events_db.rs index 602252364..3b796e146 100644 --- a/crates/moa-session/tests/session_db/execution_events_db.rs +++ b/crates/moa-session/tests/session_db/execution_events_db.rs @@ -52,6 +52,23 @@ async fn execution_events_db_round_trip_compact_payloads_without_task_output_cop originating_user_sequence_num: 4, plan_revision: 2, status: "running".to_string(), + phase: moa_core::events::ExecutionProgressPhase::Running, + waiting_since: None, + next_wake_at: None, + last_progress_at: chrono::Utc::now(), + external_job_uid: None, + ready_tasks: 2, + active_tasks: 1, + parked_tasks: 0, + blocker_audience: None, + remaining_budget: moa_core::events::ExecutionRemainingBudget { + cost_microusd: Some(100), + tokens: Some(1_000), + tasks: Some(3), + tool_calls: Some(6), + retrieved_bytes: Some(10_000), + deadline_at: None, + }, total: 6, completed: 3, failed: 1, diff --git a/crates/moa-session/tests/session_db/learning_candidate_planning_audit_db.rs b/crates/moa-session/tests/session_db/learning_candidate_planning_audit_db.rs index eb9f00051..fa4a8c634 100644 --- a/crates/moa-session/tests/session_db/learning_candidate_planning_audit_db.rs +++ b/crates/moa-session/tests/session_db/learning_candidate_planning_audit_db.rs @@ -16,7 +16,9 @@ use moa_core::types::{ session::SessionMeta, }; use moa_core::{canonical_json::canonical_json_bytes, traits::SessionStore}; -use moa_execution::repository::{CompileAuditWriteOutcome, ExecutionRepository, ExecutionScope}; +use moa_execution::repository::{ + ExecutionRepository, ExecutionScope, audit::CompileAuditWriteOutcome, +}; use moa_test_support::postgres::{TestDb, bootstrap_test_db}; use serde_json::{Value, json}; use uuid::Uuid; diff --git a/crates/moa-test-support/src/fixture_capability.rs b/crates/moa-test-support/src/fixture_capability.rs index 7a0d763d3..445188c58 100644 --- a/crates/moa-test-support/src/fixture_capability.rs +++ b/crates/moa-test-support/src/fixture_capability.rs @@ -77,6 +77,7 @@ pub fn reversible_fixture_tool_definitions() -> ( schema: input_schema.clone(), policy: policy.clone(), idempotency_class: IdempotencyClass::NonIdempotent, + async_mode: moa_core::types::tools::ToolAsyncMode::SynchronousOnly, rollback: Some(ToolRollbackDefinition { compensator_tool_name: REVERSIBLE_FIXTURE_COMPENSATOR_TOOL.to_string(), input_mapping: ToolRollbackInputMapping { @@ -97,6 +98,7 @@ pub fn reversible_fixture_tool_definitions() -> ( schema: input_schema, policy, idempotency_class: IdempotencyClass::Idempotent, + async_mode: moa_core::types::tools::ToolAsyncMode::SynchronousOnly, rollback: None, max_output_tokens: 128, }; diff --git a/crates/moa-test-support/src/lib.rs b/crates/moa-test-support/src/lib.rs index cbc409a28..6b41797ee 100644 --- a/crates/moa-test-support/src/lib.rs +++ b/crates/moa-test-support/src/lib.rs @@ -26,10 +26,12 @@ mod orchestrator_fixture; #[cfg(feature = "orchestrator-fixture")] pub use orchestrator_fixture::{ - ConversationOptions, FixtureCapabilityAttempt, FixtureCapabilityCall, - FixtureCapabilityController, FixtureCapabilityOptions, FixtureCapabilityOutcome, - FixtureCapabilityTool, IsolatedTest, OrchestratorTestFixture, RustFsFixture, - SandboxWorkspaceCrashBarrier, SandboxWorkspaceCrashControl, SandboxWorkspaceFixture, - TestApiClient, TestSessionHandle, WorkspaceRestartProbe, drive_conversation, - provision_workspace_maintenance_login, + ConversationOptions, FIXTURE_EXTERNAL_JOB_CALLBACK_TOKEN, FIXTURE_EXTERNAL_JOB_PROVIDER, + FixtureCapabilityAttempt, FixtureCapabilityCall, FixtureCapabilityController, + FixtureCapabilityOptions, FixtureCapabilityOutcome, FixtureCapabilityTool, + FixtureExternalJobAfterBind, FixtureExternalJobController, FixtureExternalJobReconciliation, + FixtureExternalJobRecovery, FixtureExternalJobStart, FixtureHandlerRevision, IsolatedTest, + OrchestratorTestFixture, RustFsFixture, SandboxWorkspaceCrashBarrier, + SandboxWorkspaceCrashControl, SandboxWorkspaceFixture, TestApiClient, TestSessionHandle, + WorkspaceRestartProbe, drive_conversation, provision_workspace_maintenance_login, }; diff --git a/crates/moa-test-support/src/orchestrator_fixture.rs b/crates/moa-test-support/src/orchestrator_fixture.rs index efc7ac85c..bca97d716 100644 --- a/crates/moa-test-support/src/orchestrator_fixture.rs +++ b/crates/moa-test-support/src/orchestrator_fixture.rs @@ -56,6 +56,7 @@ const STARTUP_TIMEOUT: Duration = Duration::from_secs(60); mod client; mod conversation; +mod external_job; mod openfga; mod otlp_capture; mod postgres; @@ -72,6 +73,11 @@ pub use crate::fixture_capability::{ }; pub use client::{TestApiClient, TestSessionHandle}; pub use conversation::{ConversationOptions, drive_conversation}; +pub use external_job::{ + FIXTURE_EXTERNAL_JOB_CALLBACK_TOKEN, FIXTURE_EXTERNAL_JOB_PROVIDER, + FixtureExternalJobAfterBind, FixtureExternalJobController, FixtureExternalJobReconciliation, + FixtureExternalJobRecovery, FixtureExternalJobStart, +}; pub use otlp_capture::OtlpCapture; pub use rustfs::RustFsFixture; pub use sandbox_workspace::{ @@ -79,6 +85,17 @@ pub use sandbox_workspace::{ WorkspaceRestartProbe, }; +/// One concrete Restate handler deployment owned by a restartable fixture. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct FixtureHandlerRevision { + /// Restate-assigned immutable deployment identity. + pub deployment_id: String, + /// Handler endpoint registered for this revision. + pub deployment_uri: String, + /// Health endpoint port for the corresponding orchestrator child. + pub health_port: u16, +} + use openfga::{ bootstrap_openfga, external_fga_client, fixture_fga_endpoint_from_env, start_openfga_container, wait_for_openfga, @@ -90,9 +107,10 @@ use process::{ locate_orchestrator_binary, read_child_logs, repo_root, reserve_orchestrator_ports, spawn_orchestrator, terminate_child, wait_for_orchestrator_health, }; -use redis::{start_redis_container, wait_for_redis}; +use redis::{start_redis_container, start_redis_container_on_port, wait_for_redis}; use restate::{ - derive_admin_url, register_deployment, start_restate_container, trim_url, + delete_deployment, derive_admin_url, find_deployment, pinned_invocation_count, + register_deployment, start_restate_container, start_restate_container_on_ports, trim_url, wait_for_registered_services, wait_for_restate_admin, }; use scripted_provider::default_script; @@ -113,13 +131,16 @@ pub struct OrchestratorTestFixture { pub test_prefix: String, _script_dir: Option, _postgres: Option>, - _restate: Option>, + restate: Mutex>>, _openfga: Option>, - _redis: Option>, + redis: Mutex>>, orchestrator: Mutex>, + maintenance: Mutex>, + handler_revisions: Mutex>, _orchestrator_binary_snapshot: Option, restart_config: Option, fixture_capability: Option, + fixture_external_job: Option, otlp_capture: Option, sandbox_workspace: Option, } @@ -153,7 +174,7 @@ impl OrchestratorTestFixture { if let Ok(ingress_url) = std::env::var("MOA_RESTATE_INGRESS_URL") { return Self::external(ingress_url); } - Self::internal(None, Vec::new(), None, true, false).await + Self::internal(None, Vec::new(), None, true, false, false).await } /// Starts a dedicated fixture with a scripted provider fixture loaded at startup. @@ -161,7 +182,7 @@ impl OrchestratorTestFixture { if std::env::var("MOA_RESTATE_INGRESS_URL").is_ok() { bail!("dedicated scripted fixtures cannot use an external orchestrator"); } - Self::internal(Some(script), Vec::new(), None, true, false).await + Self::internal(Some(script), Vec::new(), None, true, false, false).await } /// Starts a dedicated scripted fixture with extra orchestrator process environment. @@ -172,7 +193,7 @@ impl OrchestratorTestFixture { if std::env::var("MOA_RESTATE_INGRESS_URL").is_ok() { bail!("dedicated scripted fixtures cannot use an external orchestrator"); } - Self::internal(Some(script), extra_env, None, true, false).await + Self::internal(Some(script), extra_env, None, true, false, false).await } /// Starts a restartable dedicated fixture with a scripted provider and fake MCP capabilities. @@ -185,7 +206,19 @@ impl OrchestratorTestFixture { } validate_execution_fixture_env(&options.orchestrator_env)?; let extra_env = options.orchestrator_env.clone(); - Self::internal(Some(script), extra_env, Some(options), true, false).await + Self::internal(Some(script), extra_env, Some(options), true, false, false).await + } + + /// Starts a restartable execution fixture with the deterministic asynchronous provider. + pub async fn with_external_job_execution_fixture( + script: serde_json::Value, + extra_env: Vec<(String, String)>, + ) -> Result { + if std::env::var("MOA_RESTATE_INGRESS_URL").is_ok() { + bail!("dedicated external-job fixtures cannot use an external orchestrator"); + } + validate_execution_fixture_env(&extra_env)?; + Self::internal(Some(script), extra_env, None, true, false, true).await } /// Starts a dedicated scripted fixture with durable workspace owners. @@ -197,7 +230,7 @@ impl OrchestratorTestFixture { bail!("sandbox-workspace fixtures cannot use an external orchestrator"); } validate_execution_fixture_env(&extra_env)?; - Self::internal(Some(script), extra_env, None, true, true).await + Self::internal(Some(script), extra_env, None, true, true, false).await } /// Starts a durable-workspace fixture with the fake MCP capability server. @@ -210,7 +243,7 @@ impl OrchestratorTestFixture { } validate_execution_fixture_env(&options.orchestrator_env)?; let extra_env = options.orchestrator_env.clone(); - Self::internal(Some(script), extra_env, Some(options), true, true).await + Self::internal(Some(script), extra_env, Some(options), true, true, false).await } /// Starts a dedicated fixture backed by a configured real LLM provider. @@ -237,7 +270,7 @@ impl OrchestratorTestFixture { "MOA_RUN_LIVE_EXECUTION_EVALS=1 requires MOA_ANTHROPIC_API_KEY, MOA_OPENAI_API_KEY, or MOA_GOOGLE_API_KEY" ); } - Self::internal(None, Vec::new(), None, false, false).await + Self::internal(None, Vec::new(), None, false, false, false).await } fn external(raw_ingress_url: String) -> Result { @@ -262,13 +295,16 @@ impl OrchestratorTestFixture { test_prefix: format!("external-{}", Uuid::now_v7().simple()), _script_dir: None, _postgres: None, - _restate: None, + restate: Mutex::new(None), _openfga: None, - _redis: None, + redis: Mutex::new(None), orchestrator: Mutex::new(None), + maintenance: Mutex::new(None), + handler_revisions: Mutex::new(HashMap::new()), _orchestrator_binary_snapshot: None, restart_config: None, fixture_capability: None, + fixture_external_job: None, otlp_capture: None, sandbox_workspace: None, }) @@ -280,6 +316,7 @@ impl OrchestratorTestFixture { capability_options: Option, use_provider_override: bool, use_sandbox_workspace: bool, + use_external_job_fixture: bool, ) -> Result { if capability_options.is_some() && !use_provider_override { bail!("fixture capabilities require the scripted-provider override"); @@ -444,6 +481,24 @@ impl OrchestratorTestFixture { } None => None, }; + let fixture_external_job = if use_external_job_fixture { + if extra_env + .iter() + .any(|(key, _)| key == "MOA_FIXTURE_EXTERNAL_JOB_ADAPTER_URL") + { + bail!( + "external-job fixture owns reserved environment key `MOA_FIXTURE_EXTERNAL_JOB_ADAPTER_URL`" + ); + } + let runtime = external_job::FixtureExternalJobRuntime::start().await?; + extra_env.push(( + "MOA_FIXTURE_EXTERNAL_JOB_ADAPTER_URL".to_string(), + runtime.endpoint().to_string(), + )); + Some(runtime) + } else { + None + }; let client = TestApiClient::new(&ingress_url) .context("construct test client")? .with_identity(default_test_identity()); @@ -524,13 +579,16 @@ impl OrchestratorTestFixture { test_prefix: format!("fixture-{}", Uuid::now_v7().simple()), _script_dir: script_dir, _postgres: postgres, - _restate: Some(restate), + restate: Mutex::new(Some(restate)), _openfga: openfga_container, - _redis: redis_container, + redis: Mutex::new(redis_container), orchestrator: Mutex::new(Some(orchestrator)), + maintenance: Mutex::new(None), + handler_revisions: Mutex::new(HashMap::new()), _orchestrator_binary_snapshot: Some(orchestrator_binary_snapshot), restart_config, fixture_capability, + fixture_external_job, otlp_capture: Some(otlp_capture), sandbox_workspace, }) @@ -572,6 +630,134 @@ impl OrchestratorTestFixture { self.replace_orchestrator(Vec::new(), true).await } + /// Stops the fixture-owned Restate node while retaining its local durable state. + pub async fn stop_restate(&self) -> Result<()> { + let restate = self.restate.lock().await; + let container = restate + .as_ref() + .context("external orchestrator fixture does not own Restate")?; + container.stop().await.context("stop fixture Restate") + } + + /// Starts a previously stopped Restate node and waits for its registered deployment. + pub async fn restart_restate(&self) -> Result<()> { + let restate = self.restate.lock().await; + let container = restate + .as_ref() + .context("external orchestrator fixture does not own Restate")?; + container.start().await.context("restart fixture Restate")?; + wait_for_restate_admin(&self.admin_url).await?; + wait_for_registered_services(&self.admin_url).await + } + + /// Replaces Restate with an empty node on the same endpoints to model journal loss. + /// + /// PostgreSQL execution state and its dispatch outbox remain intact. The replacement + /// is therefore a proof that MOA can rediscover committed work without relying on the + /// lost Restate journal. This intentionally does not pretend to restore Restate state. + pub async fn recreate_restate_after_loss(&self) -> Result<()> { + let config = self.restart_config.as_ref().context( + "external orchestrator fixture cannot replace a Restate node it does not own", + )?; + let ingress_port = url_port(&config.ingress_url, "Restate ingress")?; + let admin_port = url_port(&config.admin_url, "Restate admin")?; + let mut restate = self.restate.lock().await; + let previous = restate + .take() + .context("fixture-owned Restate container is unavailable")?; + previous + .rm() + .await + .context("remove lost fixture Restate node")?; + let (replacement, mapped_ingress, mapped_admin) = + start_restate_container_on_ports(Some((ingress_port, admin_port))).await?; + if mapped_ingress != ingress_port || mapped_admin != admin_port { + bail!( + "replacement Restate remapped endpoints: expected {ingress_port}/{admin_port}, got {mapped_ingress}/{mapped_admin}" + ); + } + *restate = Some(replacement); + drop(restate); + wait_for_restate_admin(&config.admin_url).await?; + register_deployment(&config.admin_url, &config.deployment_uri()).await?; + wait_for_registered_services(&config.admin_url).await?; + self.restart_execution_maintenance_owner().await + } + + async fn restart_execution_maintenance_owner(&self) -> Result<()> { + let config = self.restart_config.as_ref().context( + "external orchestrator fixture cannot restart an execution maintenance owner", + )?; + let health_listener = std::net::TcpListener::bind("0.0.0.0:0") + .context("reserve fixture maintenance health port")?; + let health_port = health_listener + .local_addr() + .context("read fixture maintenance health port")? + .port(); + let mut maintenance = self.maintenance.lock().await; + if let Some(child) = maintenance.take() { + terminate_child(child); + } + drop(health_listener); + let mut child_guard = config.spawn_maintenance(health_port)?; + wait_for_orchestrator_health( + health_port, + child_guard + .child_mut() + .context("maintenance child guard is unexpectedly disarmed")?, + ) + .await + .context("restart fixture execution maintenance owner")?; + *maintenance = Some( + child_guard + .disarm() + .context("healthy maintenance child guard is unexpectedly disarmed")?, + ); + Ok(()) + } + + /// Stops the fixture-owned Valkey process while retaining its container identity. + pub async fn stop_valkey(&self) -> Result<()> { + let redis = self.redis.lock().await; + let container = redis + .as_ref() + .context("external orchestrator fixture does not own Valkey")?; + container.stop().await.context("stop fixture Valkey") + } + + /// Starts a previously stopped Valkey process and waits for TCP readiness. + pub async fn restart_valkey(&self) -> Result<()> { + let config = self.restart_config.as_ref().context( + "external orchestrator fixture cannot restart a Valkey node it does not own", + )?; + let redis = self.redis.lock().await; + let container = redis + .as_ref() + .context("fixture-owned Valkey container is unavailable")?; + container.start().await.context("restart fixture Valkey")?; + wait_for_redis(&config.redis_url).await + } + + /// Replaces Valkey on the same endpoint to prove runtime-cache loss is recoverable. + pub async fn recreate_valkey_after_loss(&self) -> Result<()> { + let config = self.restart_config.as_ref().context( + "external orchestrator fixture cannot replace a Valkey node it does not own", + )?; + let port = url_port(&config.redis_url, "Valkey")?; + let mut redis = self.redis.lock().await; + let previous = redis + .take() + .context("fixture-owned Valkey container is unavailable")?; + previous + .rm() + .await + .context("remove lost fixture Valkey node")?; + let replacement = start_redis_container_on_port(Some(port)).await?; + *redis = Some(replacement); + drop(redis); + wait_for_redis(&config.redis_url).await + } + /// Restarts the dedicated orchestrator once with additional child-process environment. /// /// The additional environment is not retained by later calls to @@ -584,6 +770,170 @@ impl OrchestratorTestFixture { self.replace_orchestrator(extra_env, false).await } + /// Returns the fixture's initially registered handler deployment. + pub async fn current_handler_revision(&self) -> Result { + let config = self + .restart_config + .as_ref() + .context("external orchestrator fixture does not expose an owned handler deployment")?; + let deployment_uri = config.deployment_uri(); + let (deployment_id, registered_uri) = + find_deployment(&config.admin_url, &deployment_uri).await?; + Ok(FixtureHandlerRevision { + deployment_id, + deployment_uri: registered_uri, + health_port: config.health_port, + }) + } + + /// Starts and registers one additional real handler revision on fresh ports. + /// + /// Restate routes new invocations to the newest registered deployment while + /// already-pinned invocations remain observable on their previous deployment. + pub async fn start_handler_revision( + &self, + revision_label: &str, + ) -> Result { + if revision_label.trim().is_empty() { + bail!("fixture handler revision label must not be empty"); + } + let config = self + .restart_config + .as_ref() + .context("external orchestrator fixture cannot start an owned handler revision")?; + let ports = reserve_orchestrator_ports()?.release(); + let mut revision_env = config.extra_env.clone(); + revision_env.push(( + "MOA_FIXTURE_HANDLER_REVISION".to_string(), + revision_label.to_string(), + )); + let mut child_guard = spawn_orchestrator(OrchestratorSpawnConfig { + binary: &config.binary, + port: ports.restate, + health_port: ports.health, + scim_port: ports.scim, + credential_port: ports.credential, + postgres_url: &config.postgres_url, + ingress_url: &config.ingress_url, + redis_url: &config.redis_url, + script_path: config.script_path.as_deref(), + journal_path: config.journal_path.as_deref(), + fga_config: &config.fga_config, + extra_env: &revision_env, + otlp_endpoint: &config.otlp_endpoint, + observability_service_name: revision_label, + })?; + wait_for_orchestrator_health( + ports.health, + child_guard + .child_mut() + .context("new handler revision child guard is unexpectedly disarmed")?, + ) + .await + .with_context(|| format!("start fixture handler revision `{revision_label}`"))?; + let deployment_uri = format!("http://host.docker.internal:{}", ports.restate); + register_deployment(&config.admin_url, &deployment_uri).await?; + let (deployment_id, registered_uri) = + find_deployment(&config.admin_url, &deployment_uri).await?; + let child = child_guard + .disarm() + .context("healthy handler revision child guard is unexpectedly disarmed")?; + let previous = self + .handler_revisions + .lock() + .await + .insert(deployment_id.clone(), child); + if let Some(previous) = previous { + terminate_child(previous); + bail!("Restate reused deployment id `{deployment_id}` for a distinct handler URI"); + } + Ok(FixtureHandlerRevision { + deployment_id, + deployment_uri: registered_uri, + health_port: ports.health, + }) + } + + /// Reads the exact count of nonterminal invocations pinned to a handler revision. + pub async fn handler_revision_pinned_invocations( + &self, + revision: &FixtureHandlerRevision, + ) -> Result { + pinned_invocation_count(&self.admin_url, &revision.deployment_id).await + } + + /// Waits for a handler revision to own zero nonterminal pinned invocations. + pub async fn wait_for_handler_revision_drained( + &self, + revision: &FixtureHandlerRevision, + timeout: Duration, + ) -> Result<()> { + let deadline = Instant::now() + timeout; + loop { + let pinned = self.handler_revision_pinned_invocations(revision).await?; + if pinned == 0 { + return Ok(()); + } + if Instant::now() >= deadline { + bail!( + "handler deployment {} did not drain within {timeout:?}; {pinned} invocations remain pinned", + revision.deployment_id + ); + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + } + + /// Deregisters and stops an owned handler revision after it is fully drained. + pub async fn stop_drained_handler_revision( + &self, + revision: &FixtureHandlerRevision, + ) -> Result<()> { + let pinned = self.handler_revision_pinned_invocations(revision).await?; + if pinned != 0 { + bail!( + "refusing to stop handler deployment {} with {pinned} pinned invocations", + revision.deployment_id + ); + } + let is_additional = self + .handler_revisions + .lock() + .await + .contains_key(&revision.deployment_id); + let config = self + .restart_config + .as_ref() + .context("external orchestrator fixture cannot stop an owned handler revision")?; + let initial_uri = config.deployment_uri(); + let is_initial = + revision.deployment_uri.trim_end_matches('/') == initial_uri.trim_end_matches('/'); + if !is_additional && !is_initial { + bail!( + "handler deployment {} is not owned by this fixture", + revision.deployment_id + ); + } + delete_deployment(&self.admin_url, &revision.deployment_id).await?; + if let Some(child) = self + .handler_revisions + .lock() + .await + .remove(&revision.deployment_id) + { + terminate_child(child); + return Ok(()); + } + let child = self + .orchestrator + .lock() + .await + .take() + .context("fixture initial orchestrator is already stopped")?; + terminate_child(child); + Ok(()) + } + async fn replace_orchestrator( &self, extra_env: Vec<(String, String)>, @@ -740,6 +1090,38 @@ impl OrchestratorTestFixture { .map(crate::fixture_capability::FixtureCapabilityRuntime::controller) } + /// Returns the deterministic asynchronous-provider controller for opt-in fixtures. + #[must_use] + pub fn fixture_external_job(&self) -> Option<&FixtureExternalJobController> { + self.fixture_external_job + .as_ref() + .map(external_job::FixtureExternalJobRuntime::controller) + } + + /// Builds the private callback-ingress URL for one exact fixture job generation and event. + pub fn external_job_callback_url( + &self, + external_job_uid: Uuid, + job_generation: u64, + provider_event_id: &str, + ) -> Result { + if provider_event_id.trim().is_empty() + || !provider_event_id + .chars() + .all(|character| character.is_ascii_alphanumeric() || "-_.:".contains(character)) + { + bail!("fixture provider event id must be non-empty and path-segment safe"); + } + let config = self + .restart_config + .as_ref() + .context("external-job callback ingress requires a dedicated orchestrator fixture")?; + Ok(format!( + "http://127.0.0.1:{}/internal/v1/execution/external-jobs/{external_job_uid}/generations/{job_generation}/callbacks/{provider_event_id}", + config.credential_port + )) + } + /// Grants the provided identity tenant-operator access. pub async fn grant_tenant_operator_identity( &self, @@ -838,6 +1220,13 @@ impl OrchestratorTestFixture { } } +fn url_port(raw_url: &str, dependency: &str) -> Result { + url::Url::parse(raw_url) + .with_context(|| format!("parse fixture {dependency} URL"))? + .port_or_known_default() + .with_context(|| format!("fixture {dependency} URL has no port")) +} + async fn fixture_host_port_ipv4( container: &ContainerAsync, label: &'static str, @@ -884,7 +1273,10 @@ fn validate_execution_fixture_env(extra_env: &[(String, String)]) -> Result<()> } if matches!( key.as_str(), - "MOA_MCP_SERVERS_JSON" | "MOA_SCRIPTED_PROVIDER_REQUEST_LOG" | "MOA_PROVIDERS_OVERRIDE" + "MOA_FIXTURE_EXTERNAL_JOB_ADAPTER_URL" + | "MOA_MCP_SERVERS_JSON" + | "MOA_SCRIPTED_PROVIDER_REQUEST_LOG" + | "MOA_PROVIDERS_OVERRIDE" ) { bail!("execution fixture owns reserved environment key `{key}`"); } @@ -916,9 +1308,18 @@ impl Drop for OrchestratorTestFixture { if let Some(child) = self.orchestrator.get_mut().take() { terminate_child(child); } + if let Some(child) = self.maintenance.get_mut().take() { + terminate_child(child); + } + for (_, child) in self.handler_revisions.get_mut().drain() { + terminate_child(child); + } if let Some(runtime) = self.fixture_capability.as_mut() { runtime.stop(); } + if let Some(runtime) = self.fixture_external_job.as_mut() { + runtime.stop(); + } if let Some(capture) = self.otlp_capture.as_mut() { capture.stop(); } @@ -1106,13 +1507,16 @@ mod tests { test_prefix: "inert".to_string(), _script_dir: None, _postgres: None, - _restate: None, + restate: Mutex::new(None), _openfga: None, - _redis: None, + redis: Mutex::new(None), orchestrator: Mutex::new(None), + maintenance: Mutex::new(None), + handler_revisions: Mutex::new(HashMap::new()), _orchestrator_binary_snapshot: None, restart_config: None, fixture_capability: None, + fixture_external_job: None, otlp_capture: None, sandbox_workspace: None, } diff --git a/crates/moa-test-support/src/orchestrator_fixture/external_job.rs b/crates/moa-test-support/src/orchestrator_fixture/external_job.rs new file mode 100644 index 000000000..10b1b2b8a --- /dev/null +++ b/crates/moa-test-support/src/orchestrator_fixture/external_job.rs @@ -0,0 +1,786 @@ +//! Deterministic loopback provider for durable asynchronous execution-tool tests. + +use super::*; + +use std::collections::{HashMap, VecDeque}; +use std::sync::{Arc, Mutex as StdMutex, MutexGuard}; + +use axum::extract::State; +use axum::routing::post; +use axum::{Json, Router}; +use moa_core::types::tools::{ + AsyncToolJob, AsyncToolJobCallbackOutcome, AsyncToolJobCancelOutcome, ExternalJobStartContext, +}; +use moa_execution::wire::{ + ExecutionExternalJobCancelRequest, ExecutionExternalJobReconcileRequest, +}; +use serde::{Deserialize, Serialize}; +use tokio::sync::{Notify, oneshot}; +use tokio::task::JoinHandle; + +/// Stable provider key registered by the provider-override orchestrator runtime. +pub const FIXTURE_EXTERNAL_JOB_PROVIDER: &str = "fixture-external-job"; +/// Stable callback credential accepted only by the deterministic fixture adapter. +pub const FIXTURE_EXTERNAL_JOB_CALLBACK_TOKEN: &str = "fixture-callback-token"; + +/// One provider start observed after MOA durably reserved its external-job identity. +#[derive(Clone, Debug, PartialEq)] +pub struct FixtureExternalJobStart { + /// Reserved identity, adapter key, and deterministic provider idempotency key. + pub context: ExternalJobStartContext, + /// Governed tool-call payload received by the provider adapter. + pub call: serde_json::Value, + /// Stable provider job identity committed for this reservation. + pub provider_job_id: String, + /// One-based order among unique provider starts. + pub arrival_order: u64, +} + +/// One provider start-recovery lookup observed after an unbound intent became due. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct FixtureExternalJobRecovery { + /// Exact reserved context reused by recovery. + pub context: ExternalJobStartContext, + /// One-based order among recovery requests. + pub arrival_order: u64, +} + +/// One post-bind barrier reached before the owning attempt releases active compute. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct FixtureExternalJobAfterBind { + /// Exact reserved provider context whose job has already been bound in PostgreSQL. + pub context: ExternalJobStartContext, + /// One-based order among unique post-bind barriers. + pub arrival_order: u64, +} + +/// One bounded sparse-reconciliation request received by the provider fixture. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct FixtureExternalJobReconciliation { + /// Exact generation-fenced durable request. + pub request: ExecutionExternalJobReconcileRequest, + /// One-based order among reconciliation requests. + pub arrival_order: u64, +} + +/// One generation-fenced provider cancellation request received by the fixture. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct FixtureExternalJobCancellation { + /// Exact generation-fenced durable request. + pub request: ExecutionExternalJobCancelRequest, + /// One-based order among cancellation requests. + pub arrival_order: u64, +} + +/// Controller for observing and deterministically releasing asynchronous provider operations. +#[derive(Clone)] +pub struct FixtureExternalJobController { + state: Arc, +} + +impl FixtureExternalJobController { + /// Waits for at least `count` unique provider starts. + pub async fn wait_for_starts( + &self, + count: usize, + timeout: Duration, + ) -> Result> { + wait_for_observations( + &self.state.start_notify, + timeout, + count, + || self.starts(), + "provider starts", + ) + .await + } + + /// Releases exactly `count` pending starts in unique-arrival order. + pub fn release_starts(&self, count: usize) { + if count == 0 { + return; + } + let pending = { + let observations = lock_unpoisoned(&self.state.observations); + observations + .starts + .iter() + .filter_map(|start| { + observations + .start_gates + .get(&start.context.external_job_uid) + }) + .filter(|gate| !gate.is_released()) + .take(count) + .cloned() + .collect::>() + }; + assert_eq!( + pending.len(), + count, + "release_starts({count}) requires {count} pending fixture starts" + ); + for gate in pending { + gate.release(); + } + } + + /// Returns unique provider starts in deterministic arrival order. + #[must_use] + pub fn starts(&self) -> Vec { + lock_unpoisoned(&self.state.observations).starts.clone() + } + + /// Waits for at least `count` provider start-recovery requests. + pub async fn wait_for_recoveries( + &self, + count: usize, + timeout: Duration, + ) -> Result> { + wait_for_observations( + &self.state.recovery_notify, + timeout, + count, + || self.recoveries(), + "provider start recoveries", + ) + .await + } + + /// Returns all provider start-recovery lookups in arrival order. + #[must_use] + pub fn recoveries(&self) -> Vec { + lock_unpoisoned(&self.state.observations).recoveries.clone() + } + + /// Waits for at least `count` post-bind, pre-release barriers. + pub async fn wait_for_after_bind( + &self, + count: usize, + timeout: Duration, + ) -> Result> { + wait_for_observations( + &self.state.after_bind_notify, + timeout, + count, + || self.after_bind(), + "post-bind barriers", + ) + .await + } + + /// Returns unique post-bind barriers in arrival order. + #[must_use] + pub fn after_bind(&self) -> Vec { + lock_unpoisoned(&self.state.observations).after_bind.clone() + } + + /// Releases exactly `count` pending post-bind barriers in arrival order. + pub fn release_after_bind(&self, count: usize) { + if count == 0 { + return; + } + let pending = { + let observations = lock_unpoisoned(&self.state.observations); + observations + .after_bind + .iter() + .filter_map(|barrier| { + observations + .after_bind_gates + .get(&barrier.context.external_job_uid) + }) + .filter(|gate| !gate.is_released()) + .take(count) + .cloned() + .collect::>() + }; + assert_eq!( + pending.len(), + count, + "release_after_bind({count}) requires {count} pending fixture barriers" + ); + for gate in pending { + gate.release(); + } + } + + /// Waits for at least `count` sparse reconciliation observations. + pub async fn wait_for_reconciliations( + &self, + count: usize, + timeout: Duration, + ) -> Result> { + wait_for_observations( + &self.state.reconcile_notify, + timeout, + count, + || self.reconciliations(), + "provider reconciliations", + ) + .await + } + + /// Returns all sparse reconciliation observations in arrival order. + #[must_use] + pub fn reconciliations(&self) -> Vec { + lock_unpoisoned(&self.state.observations) + .reconciliations + .clone() + } + + /// Queues exact outcomes consumed by subsequent sparse reconciliations. + pub fn queue_reconcile_outcomes( + &self, + outcomes: impl IntoIterator, + ) { + lock_unpoisoned(&self.state.observations) + .reconcile_outcomes + .extend(outcomes); + } + + /// Waits for at least `count` provider cancellation requests. + pub async fn wait_for_cancellations( + &self, + count: usize, + timeout: Duration, + ) -> Result> { + wait_for_observations( + &self.state.cancel_notify, + timeout, + count, + || self.cancellations(), + "provider cancellations", + ) + .await + } + + /// Returns all provider cancellation observations in arrival order. + #[must_use] + pub fn cancellations(&self) -> Vec { + lock_unpoisoned(&self.state.observations) + .cancellations + .clone() + } + + /// Queues exact outcomes consumed by subsequent cancellation requests. + pub fn queue_cancel_outcomes( + &self, + outcomes: impl IntoIterator, + ) { + lock_unpoisoned(&self.state.observations) + .cancel_outcomes + .extend(outcomes); + } + + /// Creates the raw callback envelope parsed by the fixture adapter. + #[must_use] + pub fn callback_body( + &self, + provider_job_id: impl Into, + provider_event_id: impl Into, + outcome: AsyncToolJobCallbackOutcome, + ) -> serde_json::Value { + serde_json::json!({ + "provider_job_id": provider_job_id.into(), + "provider_event_id": provider_event_id.into(), + "outcome": outcome, + }) + } +} + +/// Running deterministic asynchronous-provider fixture. +pub struct FixtureExternalJobRuntime { + controller: FixtureExternalJobController, + endpoint: String, + shutdown: Option>, + task: Option>, +} + +impl FixtureExternalJobRuntime { + /// Starts the fixture server on one ephemeral loopback port. + pub async fn start() -> Result { + let state = Arc::new(FixtureExternalJobState::default()); + let controller = FixtureExternalJobController { + state: Arc::clone(&state), + }; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .context("bind external-job fixture listener")?; + let address = listener + .local_addr() + .context("read external-job fixture listener address")?; + let endpoint = format!("http://{address}"); + let router = Router::new() + .route("/start", post(start)) + .route("/recover_start", post(recover_start)) + .route("/after_bind", post(after_bind)) + .route("/cancel", post(cancel)) + .route("/reconcile", post(reconcile)) + .with_state(state); + let (shutdown_tx, shutdown_rx) = oneshot::channel(); + let task = tokio::spawn(async move { + let server = axum::serve(listener, router).with_graceful_shutdown(async move { + let _ = shutdown_rx.await; + }); + if let Err(error) = server.await { + tracing::warn!(%error, "external-job fixture server stopped unexpectedly"); + } + }); + Ok(Self { + controller, + endpoint, + shutdown: Some(shutdown_tx), + task: Some(task), + }) + } + + /// Returns the base URL configured on the provider-override adapter. + #[must_use] + pub fn endpoint(&self) -> &str { + &self.endpoint + } + + /// Returns the observation and release controller. + #[must_use] + pub fn controller(&self) -> &FixtureExternalJobController { + &self.controller + } + + /// Stops the listener and aborts its accept task. + pub fn stop(&mut self) { + if let Some(shutdown) = self.shutdown.take() { + let _ = shutdown.send(()); + } + if let Some(task) = self.task.take() { + task.abort(); + } + } +} + +impl Drop for FixtureExternalJobRuntime { + fn drop(&mut self) { + self.stop(); + } +} + +#[derive(Default)] +struct FixtureExternalJobState { + observations: StdMutex, + start_notify: Notify, + recovery_notify: Notify, + after_bind_notify: Notify, + reconcile_notify: Notify, + cancel_notify: Notify, +} + +#[derive(Default)] +struct FixtureExternalJobObservations { + starts: Vec, + start_gates: HashMap>, + jobs: HashMap, + recoveries: Vec, + after_bind: Vec, + after_bind_gates: HashMap>, + reconciliations: Vec, + reconcile_outcomes: VecDeque, + cancellations: Vec, + cancel_outcomes: VecDeque, +} + +#[derive(Default)] +struct ReleaseGate { + released: StdMutex, + notify: Notify, +} + +impl ReleaseGate { + fn is_released(&self) -> bool { + *lock_unpoisoned(&self.released) + } + + fn release(&self) { + let mut released = lock_unpoisoned(&self.released); + if !*released { + *released = true; + drop(released); + self.notify.notify_waiters(); + } + } + + async fn wait(&self) { + loop { + let notified = self.notify.notified(); + if self.is_released() { + return; + } + notified.await; + } + } +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct FixtureStartRequest { + context: ExternalJobStartContext, + call: serde_json::Value, +} + +#[derive(Serialize)] +#[serde(tag = "outcome", rename_all = "snake_case", deny_unknown_fields)] +enum FixtureStartOutcome { + ExternalJob(AsyncToolJob), +} + +#[derive(Serialize)] +#[serde(tag = "outcome", rename_all = "snake_case", deny_unknown_fields)] +enum FixtureStartRecovery { + Started(AsyncToolJob), +} + +async fn start( + State(state): State>, + Json(request): Json, +) -> Result, axum::http::StatusCode> { + if request.context.provider != FIXTURE_EXTERNAL_JOB_PROVIDER { + return Err(axum::http::StatusCode::BAD_REQUEST); + } + let (job, gate, is_new) = { + let mut observations = lock_unpoisoned(&state.observations); + if let Some(job) = observations + .jobs + .get(&request.context.external_job_uid) + .cloned() + { + let gate = observations + .start_gates + .get(&request.context.external_job_uid) + .cloned() + .ok_or(axum::http::StatusCode::INTERNAL_SERVER_ERROR)?; + (job, gate, false) + } else { + let job = fixture_job(&request.context); + let gate = Arc::new(ReleaseGate::default()); + let start = FixtureExternalJobStart { + context: request.context.clone(), + call: request.call, + provider_job_id: job.provider_job_id.clone(), + arrival_order: observations.starts.len() as u64 + 1, + }; + observations.starts.push(start); + observations + .start_gates + .insert(request.context.external_job_uid, Arc::clone(&gate)); + observations + .jobs + .insert(request.context.external_job_uid, job.clone()); + (job, gate, true) + } + }; + if is_new { + state.start_notify.notify_waiters(); + } + gate.wait().await; + Ok(Json(FixtureStartOutcome::ExternalJob(job))) +} + +async fn recover_start( + State(state): State>, + Json(context): Json, +) -> Result, axum::http::StatusCode> { + let job = { + let mut observations = lock_unpoisoned(&state.observations); + let job = observations + .jobs + .get(&context.external_job_uid) + .cloned() + .ok_or(axum::http::StatusCode::NOT_FOUND)?; + let arrival_order = observations.recoveries.len() as u64 + 1; + observations.recoveries.push(FixtureExternalJobRecovery { + context, + arrival_order, + }); + job + }; + state.recovery_notify.notify_waiters(); + Ok(Json(FixtureStartRecovery::Started(job))) +} + +async fn after_bind( + State(state): State>, + Json(context): Json, +) -> Result, axum::http::StatusCode> { + let (gate, is_new) = { + let mut observations = lock_unpoisoned(&state.observations); + if !observations.jobs.contains_key(&context.external_job_uid) { + return Err(axum::http::StatusCode::NOT_FOUND); + } + if let Some(gate) = observations + .after_bind_gates + .get(&context.external_job_uid) + .cloned() + { + (gate, false) + } else { + let gate = Arc::new(ReleaseGate::default()); + let arrival_order = observations.after_bind.len() as u64 + 1; + observations.after_bind.push(FixtureExternalJobAfterBind { + context: context.clone(), + arrival_order, + }); + observations + .after_bind_gates + .insert(context.external_job_uid, Arc::clone(&gate)); + (gate, true) + } + }; + if is_new { + state.after_bind_notify.notify_waiters(); + } + gate.wait().await; + Ok(Json(())) +} + +async fn cancel( + State(state): State>, + Json(request): Json, +) -> Json { + let outcome = { + let mut observations = lock_unpoisoned(&state.observations); + let arrival_order = observations.cancellations.len() as u64 + 1; + observations + .cancellations + .push(FixtureExternalJobCancellation { + request, + arrival_order, + }); + observations + .cancel_outcomes + .pop_front() + .unwrap_or(AsyncToolJobCancelOutcome::Unsupported) + }; + state.cancel_notify.notify_waiters(); + Json(outcome) +} + +async fn reconcile( + State(state): State>, + Json(request): Json, +) -> Result, axum::http::StatusCode> { + let outcome = { + let mut observations = lock_unpoisoned(&state.observations); + let arrival_order = observations.reconciliations.len() as u64 + 1; + observations + .reconciliations + .push(FixtureExternalJobReconciliation { + request, + arrival_order, + }); + observations.reconcile_outcomes.pop_front() + }; + state.reconcile_notify.notify_waiters(); + outcome + .map(Json) + .ok_or(axum::http::StatusCode::SERVICE_UNAVAILABLE) +} + +fn fixture_job(context: &ExternalJobStartContext) -> AsyncToolJob { + AsyncToolJob { + provider: context.provider.clone(), + provider_job_id: format!("fixture-job-{}", context.external_job_uid), + idempotency_key: context.idempotency_key.clone(), + callback_auth_reference: FIXTURE_EXTERNAL_JOB_CALLBACK_TOKEN.to_string(), + progress_phase: "queued".to_string(), + cancel_supported: true, + next_reconcile_at: Utc::now() + chrono::Duration::seconds(5), + } +} + +async fn wait_for_observations( + notify: &Notify, + timeout: Duration, + count: usize, + snapshot: impl Fn() -> Vec, + label: &str, +) -> Result> { + let deadline = tokio::time::Instant::now() + timeout; + loop { + let notified = notify.notified(); + let observations = snapshot(); + if observations.len() >= count { + return Ok(observations); + } + tokio::time::timeout_at(deadline, notified) + .await + .with_context(|| { + format!( + "external-job fixture observed {} of {count} {label} within {timeout:?}", + observations.len() + ) + })?; + } +} + +fn lock_unpoisoned(mutex: &StdMutex) -> MutexGuard<'_, T> { + mutex + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) +} + +#[cfg(test)] +mod tests { + use super::*; + + // Pins: provider work is committed under the reserved identity before the start response is + // released, and crash recovery resolves the same provider job without a second start. + #[tokio::test] + async fn external_job_fixture_recovers_committed_blocked_start_without_replay_offline() + -> Result<()> { + let mut runtime = FixtureExternalJobRuntime::start().await?; + let controller = runtime.controller().clone(); + let endpoint = runtime.endpoint().to_string(); + let context = ExternalJobStartContext { + external_job_uid: Uuid::now_v7(), + provider: FIXTURE_EXTERNAL_JOB_PROVIDER.to_string(), + idempotency_key: "fixture-start-key".to_string(), + }; + let client = reqwest::Client::new(); + let start_context = context.clone(); + let start = tokio::spawn(async move { + client + .post(format!("{endpoint}/start")) + .json(&serde_json::json!({ + "context": start_context, + "call": {"tool_call_id": "opaque-to-provider-fixture"}, + })) + .send() + .await? + .error_for_status()? + .json::() + .await + }); + + let starts = controller + .wait_for_starts(1, Duration::from_secs(2)) + .await?; + assert_eq!(starts[0].context, context); + assert_eq!(starts[0].arrival_order, 1); + assert_eq!(controller.starts().len(), 1); + + let recovery = reqwest::Client::new() + .post(format!("{}/recover_start", runtime.endpoint())) + .json(&context) + .send() + .await? + .error_for_status()? + .json::() + .await?; + assert_eq!(recovery["outcome"], "started"); + assert_eq!( + recovery["provider_job_id"], + starts[0].provider_job_id.as_str() + ); + assert_eq!(controller.recoveries().len(), 1); + assert_eq!(controller.starts().len(), 1); + + controller.release_starts(1); + let started = start.await.context("join blocked fixture start")??; + assert_eq!(started["outcome"], "external_job"); + assert_eq!( + started["provider_job_id"], + starts[0].provider_job_id.as_str() + ); + assert_eq!(started["idempotency_key"], context.idempotency_key); + let after_bind_endpoint = runtime.endpoint().to_string(); + let after_bind_context = context.clone(); + let after_bind = tokio::spawn(async move { + reqwest::Client::new() + .post(format!("{after_bind_endpoint}/after_bind")) + .json(&after_bind_context) + .send() + .await? + .error_for_status()? + .json::() + .await + }); + let barriers = controller + .wait_for_after_bind(1, Duration::from_secs(2)) + .await?; + assert_eq!(barriers[0].context, context); + controller.release_after_bind(1); + assert_eq!( + after_bind.await.context("join post-bind barrier")??, + serde_json::Value::Null + ); + runtime.stop(); + Ok(()) + } + + // Pins: sparse reconcile and cancellation scripts are consumed once in request order, and + // the controller observes the exact generation-fenced request sent by the production adapter. + #[tokio::test] + async fn external_job_fixture_routes_generation_fenced_provider_operations_offline() + -> Result<()> { + let mut runtime = FixtureExternalJobRuntime::start().await?; + let controller = runtime.controller().clone(); + let tenant_id = TenantId::from(Uuid::now_v7()); + let external_job_uid = Uuid::now_v7(); + let reconcile = ExecutionExternalJobReconcileRequest { + tenant_id, + external_job_uid, + trigger_uid: Uuid::now_v7(), + job_generation: 3, + provider: FIXTURE_EXTERNAL_JOB_PROVIDER.to_string(), + provider_job_id: "fixture-provider-job".to_string(), + idempotency_key: "fixture-provider-key".to_string(), + }; + let next_reconcile_at = Utc::now() + chrono::Duration::minutes(1); + let progress = AsyncToolJobCallbackOutcome::Progress { + progress_phase: "working".to_string(), + next_reconcile_at, + }; + controller.queue_reconcile_outcomes([progress.clone()]); + let response = reqwest::Client::new() + .post(format!("{}/reconcile", runtime.endpoint())) + .json(&reconcile) + .send() + .await? + .error_for_status()? + .json::() + .await?; + assert_eq!(response, progress); + let reconciliations = controller + .wait_for_reconciliations(1, Duration::from_secs(2)) + .await?; + assert_eq!(reconciliations[0].request, reconcile); + + let cancel = ExecutionExternalJobCancelRequest { + tenant_id, + external_job_uid, + job_generation: 3, + provider: FIXTURE_EXTERNAL_JOB_PROVIDER.to_string(), + provider_job_id: "fixture-provider-job".to_string(), + idempotency_key: "fixture-provider-key".to_string(), + }; + let accepted = AsyncToolJobCancelOutcome::Accepted { + next_reconcile_at, + progress_phase: "cancelling".to_string(), + }; + controller.queue_cancel_outcomes([accepted.clone()]); + let response = reqwest::Client::new() + .post(format!("{}/cancel", runtime.endpoint())) + .json(&cancel) + .send() + .await? + .error_for_status()? + .json::() + .await?; + assert_eq!(response, accepted); + let cancellations = controller + .wait_for_cancellations(1, Duration::from_secs(2)) + .await?; + assert_eq!(cancellations[0].request, cancel); + runtime.stop(); + Ok(()) + } +} diff --git a/crates/moa-test-support/src/orchestrator_fixture/postgres.rs b/crates/moa-test-support/src/orchestrator_fixture/postgres.rs index b44f4c378..b46c7ae64 100644 --- a/crates/moa-test-support/src/orchestrator_fixture/postgres.rs +++ b/crates/moa-test-support/src/orchestrator_fixture/postgres.rs @@ -1,6 +1,7 @@ //! Postgres container bootstrap for orchestrator service fixtures. use super::*; +use sqlx::Connection as _; pub(super) async fn start_postgres_container() -> Result> { GenericImage::new(POSTGRES_IMAGE, POSTGRES_TAG) @@ -62,20 +63,32 @@ pub(super) async fn ensure_postgres_image(repo_root: &Path) -> Result<()> { pub(super) async fn wait_for_postgres(postgres_url: &str) -> Result<()> { let deadline = Instant::now() + STARTUP_TIMEOUT; loop { - match PgPoolOptions::new() - .max_connections(1) - .connect(postgres_url) - .await - { - Ok(pool) => { - pool.close().await; + let probe = tokio::time::timeout( + Duration::from_secs(1), + sqlx::PgConnection::connect(postgres_url), + ) + .await; + match probe { + Ok(Ok(connection)) => { + connection + .close() + .await + .context("close Postgres readiness connection")?; return Ok(()); } - Err(error) if Instant::now() < deadline => { + Ok(Err(error)) if Instant::now() < deadline => { tracing::debug!(%error, "waiting for Postgres testcontainer"); tokio::time::sleep(Duration::from_millis(250)).await; } - Err(error) => return Err(error).context("Postgres testcontainer did not become ready"), + Ok(Err(error)) => { + return Err(error).context("Postgres testcontainer did not become ready"); + } + Err(_) if Instant::now() < deadline => { + tracing::debug!("Postgres testcontainer readiness probe timed out"); + } + Err(error) => { + return Err(error).context("Postgres testcontainer readiness probe timed out"); + } } } } diff --git a/crates/moa-test-support/src/orchestrator_fixture/process.rs b/crates/moa-test-support/src/orchestrator_fixture/process.rs index b870cabf0..78f0ddb1f 100644 --- a/crates/moa-test-support/src/orchestrator_fixture/process.rs +++ b/crates/moa-test-support/src/orchestrator_fixture/process.rs @@ -180,6 +180,29 @@ impl OrchestratorRestartConfig { pub(super) fn deployment_uri(&self) -> String { format!("http://host.docker.internal:{}", self.port) } + + /// Spawns the maintenance owner against the same durable fixture dependencies. + pub(super) fn spawn_maintenance(&self, health_port: u16) -> Result { + spawn_maintenance( + OrchestratorSpawnConfig { + binary: &self.binary, + port: self.port, + health_port, + scim_port: self.scim_port, + credential_port: self.credential_port, + postgres_url: &self.postgres_url, + ingress_url: &self.ingress_url, + redis_url: &self.redis_url, + script_path: self.script_path.as_deref(), + journal_path: self.journal_path.as_deref(), + fga_config: &self.fga_config, + extra_env: &self.extra_env, + otlp_endpoint: &self.otlp_endpoint, + observability_service_name: &self.observability_service_name, + }, + health_port, + ) + } } /// Four distinct TCP ports used by one orchestrator child. @@ -266,7 +289,23 @@ pub(super) fn hard_kill_child(mut child: Child) -> Result<()> { Ok(()) } +/// Spawns one normal Restate handler runtime for the fixture. pub(super) fn spawn_orchestrator(config: OrchestratorSpawnConfig<'_>) -> Result { + spawn_orchestrator_process(config, None) +} + +/// Spawns the real singleton maintenance role on a dedicated health port. +pub(super) fn spawn_maintenance( + config: OrchestratorSpawnConfig<'_>, + health_port: u16, +) -> Result { + spawn_orchestrator_process(config, Some(health_port)) +} + +fn spawn_orchestrator_process( + config: OrchestratorSpawnConfig<'_>, + maintenance_health_port: Option, +) -> Result { let mut command = Command::new(config.binary); command .env_remove("MOA_MCP_SERVERS_JSON") @@ -287,14 +326,6 @@ pub(super) fn spawn_orchestrator(config: OrchestratorSpawnConfig<'_>) -> Result< // hermetic child depend on external availability and load. .env_remove("MOA_PII_SERVICE_URL") .env_remove("OTEL_METRIC_EXPORT_INTERVAL") - .arg("--port") - .arg(config.port.to_string()) - .arg("--health-port") - .arg(config.health_port.to_string()) - .arg("--scim-port") - .arg(config.scim_port.to_string()) - .arg("--credential-port") - .arg(config.credential_port.to_string()) .env("MOA_DATABASE_URL", config.postgres_url) .env("MOA_RESTATE_INGRESS_URL", config.ingress_url) .env("MOA_RUNTIME_CACHE_BACKEND", "redis") @@ -319,6 +350,25 @@ pub(super) fn spawn_orchestrator(config: OrchestratorSpawnConfig<'_>) -> Result< // the quiet default keeps ordinary runs readable. std::env::var("MOA_FIXTURE_RUST_LOG").unwrap_or_else(|_| "warn".to_string()), ); + match maintenance_health_port { + Some(health_port) => { + command + .arg("--health-port") + .arg(health_port.to_string()) + .arg("maintenance"); + } + None => { + command + .arg("--port") + .arg(config.port.to_string()) + .arg("--health-port") + .arg(config.health_port.to_string()) + .arg("--scim-port") + .arg(config.scim_port.to_string()) + .arg("--credential-port") + .arg(config.credential_port.to_string()); + } + } if let Some(script_path) = config.script_path { command.env( "MOA_PROVIDERS_OVERRIDE", @@ -361,7 +411,17 @@ pub(super) fn spawn_orchestrator(config: OrchestratorSpawnConfig<'_>) -> Result< .stderr(Stdio::inherit()) .spawn() .map(ChildGuard::new) - .with_context(|| format!("spawn orchestrator binary {}", config.binary.display())) + .with_context(|| { + let role = if maintenance_health_port.is_some() { + "maintenance" + } else { + "runtime" + }; + format!( + "spawn orchestrator {role} binary {}", + config.binary.display() + ) + }) } pub(super) async fn wait_for_orchestrator_health( diff --git a/crates/moa-test-support/src/orchestrator_fixture/redis.rs b/crates/moa-test-support/src/orchestrator_fixture/redis.rs index 008633335..c11078ea7 100644 --- a/crates/moa-test-support/src/orchestrator_fixture/redis.rs +++ b/crates/moa-test-support/src/orchestrator_fixture/redis.rs @@ -8,12 +8,20 @@ use super::*; pub(super) async fn start_redis_container() -> Result> { - GenericImage::new(REDIS_IMAGE, REDIS_TAG) + start_redis_container_on_port(None).await +} + +pub(super) async fn start_redis_container_on_port( + host_port: Option, +) -> Result> { + let image = GenericImage::new(REDIS_IMAGE, REDIS_TAG) .with_exposed_port(6379.tcp()) - .with_wait_for(WaitFor::message_on_stdout("Ready to accept connections")) - .start() - .await - .context("start Valkey testcontainer") + .with_wait_for(WaitFor::message_on_stdout("Ready to accept connections")); + let image = match host_port { + Some(host_port) => image.with_mapped_port(host_port, 6379.tcp()), + None => image.into(), + }; + image.start().await.context("start Valkey testcontainer") } pub(super) async fn wait_for_redis(redis_url: &str) -> Result<()> { diff --git a/crates/moa-test-support/src/orchestrator_fixture/restate.rs b/crates/moa-test-support/src/orchestrator_fixture/restate.rs index 5ade97314..70149df47 100644 --- a/crates/moa-test-support/src/orchestrator_fixture/restate.rs +++ b/crates/moa-test-support/src/orchestrator_fixture/restate.rs @@ -3,9 +3,15 @@ use super::*; pub(super) async fn start_restate_container() -> Result<(ContainerAsync, u16, u16)> { + start_restate_container_on_ports(None).await +} + +pub(super) async fn start_restate_container_on_ports( + host_ports: Option<(u16, u16)>, +) -> Result<(ContainerAsync, u16, u16)> { let mut failures = Vec::new(); for attempt in 1..=3 { - let container = match GenericImage::new(RESTATE_IMAGE, RESTATE_TAG) + let image = GenericImage::new(RESTATE_IMAGE, RESTATE_TAG) .with_exposed_port(8080.tcp()) .with_exposed_port(9070.tcp()) .with_wait_for(WaitFor::seconds(1)) @@ -17,10 +23,14 @@ pub(super) async fn start_restate_container() -> Result<(ContainerAsync image + .with_mapped_port(ingress, 8080.tcp()) + .with_mapped_port(admin, 9070.tcp()), + None => image, + }; + let container = match image.start().await { Ok(container) => container, Err(error) => { failures.push(format!("attempt {attempt} failed to start: {error}")); @@ -117,6 +127,118 @@ pub(super) async fn register_deployment(admin_url: &str, deployment_uri: &str) - } } +pub(super) async fn find_deployment( + admin_url: &str, + deployment_uri: &str, +) -> Result<(String, String)> { + let client = reqwest::Client::new(); + let expected = deployment_uri.trim_end_matches('/'); + let deadline = Instant::now() + STARTUP_TIMEOUT; + loop { + let payload = client + .get(format!("{admin_url}/deployments")) + .send() + .await + .context("list Restate fixture deployments")? + .error_for_status() + .context("Restate fixture deployment list failed")? + .json::() + .await + .context("decode Restate fixture deployment list")?; + if let Some(deployment) = payload.deployments.into_iter().find(|deployment| { + deployment + .uri + .as_deref() + .is_some_and(|uri| uri.trim_end_matches('/') == expected) + }) { + return Ok(( + deployment.id, + deployment.uri.unwrap_or_else(|| deployment_uri.to_string()), + )); + } + if Instant::now() >= deadline { + bail!("Restate did not expose registered deployment URI `{deployment_uri}`"); + } + tokio::time::sleep(Duration::from_millis(250)).await; + } +} + +pub(super) async fn pinned_invocation_count(admin_url: &str, deployment_id: &str) -> Result { + #[derive(Deserialize)] + struct CountRow { + pinned_count: u64, + } + #[derive(Deserialize)] + struct QueryResponse { + rows: Vec, + } + + let escaped = deployment_id.replace('\'', "''"); + let query = format!( + "SELECT COUNT(*) AS pinned_count FROM sys_invocation \ + WHERE (pinned_deployment_id = '{escaped}' OR last_attempt_deployment_id = '{escaped}') \ + AND status NOT IN ('completed', 'killed')" + ); + let response = reqwest::Client::new() + .post(format!("{admin_url}/query")) + .header("content-type", "application/json") + .header("accept", "application/json") + .json(&serde_json::json!({ "query": query })) + .send() + .await + .context("query Restate pinned fixture invocations")? + .error_for_status() + .context("Restate pinned-invocation query failed")? + .json::() + .await + .context("decode Restate pinned-invocation query")?; + response + .rows + .first() + .map(|row| row.pinned_count) + .context("Restate pinned-invocation query returned no aggregate row") +} + +pub(super) async fn delete_deployment(admin_url: &str, deployment_id: &str) -> Result<()> { + let client = reqwest::Client::new(); + let response = client + .delete(format!( + "{admin_url}/deployments/{deployment_id}?force=true" + )) + .send() + .await + .context("delete drained Restate fixture deployment")?; + if response.status() != StatusCode::ACCEPTED { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + bail!("delete Restate deployment {deployment_id} returned {status}: {body}"); + } + let deadline = Instant::now() + STARTUP_TIMEOUT; + loop { + let payload = client + .get(format!("{admin_url}/deployments")) + .send() + .await + .context("list Restate deployments after delete")? + .error_for_status() + .context("Restate deployment list failed after delete")? + .json::() + .await + .context("decode Restate deployments after delete")?; + if payload + .deployments + .iter() + .all(|deployment| deployment.id != deployment_id) + { + return Ok(()); + } + if Instant::now() >= deadline { + bail!("Restate deployment {deployment_id} remained registered after delete"); + } + tokio::time::sleep(Duration::from_millis(100)).await; + } +} + pub(super) async fn wait_for_registered_services(admin_url: &str) -> Result<()> { let client = reqwest::Client::new(); let deadline = Instant::now() + STARTUP_TIMEOUT; @@ -154,7 +276,9 @@ struct DeploymentsResponse { #[derive(Deserialize)] struct Deployment { + id: String, services: Vec, + uri: Option, } #[derive(Deserialize)] @@ -163,12 +287,16 @@ struct RegisteredService { } fn deployment_is_routable(deployment: &Deployment) -> bool { - const REQUIRED_SERVICES: [&str; 5] = [ + const REQUIRED_SERVICES: [&str; 9] = [ "Session", "ActionReviewDispatcher", "Execution", - "ExecutionRun", - "ExecutionTask", + "ExecutionDispatcher", + "ExecutionTrigger", + "ExecutionRunController", + "ExecutionTaskAttempt", + "LLMGateway", + "ToolExecutor", ]; REQUIRED_SERVICES.iter().all(|required| { @@ -210,15 +338,21 @@ mod tests { name: name.to_string(), }; let mut deployment = Deployment { + id: "fixture-deployment".to_string(), services: vec![service("Session")], + uri: Some("http://127.0.0.1:8080".to_string()), }; assert!(!deployment_is_routable(&deployment)); deployment.services.extend([ service("ActionReviewDispatcher"), service("Execution"), - service("ExecutionRun"), - service("ExecutionTask"), + service("ExecutionDispatcher"), + service("ExecutionTrigger"), + service("ExecutionRunController"), + service("ExecutionTaskAttempt"), + service("LLMGateway"), + service("ToolExecutor"), ]); assert!(deployment_is_routable(&deployment)); } diff --git a/crates/moa-wire/src/turn.rs b/crates/moa-wire/src/turn.rs index 98626b618..8f1fbb7f6 100644 --- a/crates/moa-wire/src/turn.rs +++ b/crates/moa-wire/src/turn.rs @@ -876,6 +876,23 @@ mod tests { originating_user_sequence_num: 17, plan_revision: 3, status: "waiting_input".to_string(), + phase: moa_core::events::ExecutionProgressPhase::WaitingInput, + waiting_since: Some(chrono::Utc::now()), + next_wake_at: None, + last_progress_at: chrono::Utc::now(), + external_job_uid: None, + ready_tasks: 1, + active_tasks: 0, + parked_tasks: 1, + blocker_audience: Some(moa_core::events::ExecutionBlockerAudience::User), + remaining_budget: moa_core::events::ExecutionRemainingBudget { + cost_microusd: Some(100), + tokens: Some(1_000), + tasks: Some(4), + tool_calls: Some(8), + retrieved_bytes: Some(10_000), + deadline_at: None, + }, total: 11, completed: 7, failed: 2, diff --git a/crates/xtask/src/check_architecture_boundaries/budgets.rs b/crates/xtask/src/check_architecture_boundaries/budgets.rs index 8b67e40c8..551c3d7a1 100644 --- a/crates/xtask/src/check_architecture_boundaries/budgets.rs +++ b/crates/xtask/src/check_architecture_boundaries/budgets.rs @@ -29,8 +29,8 @@ const LOC_BUDGETS: &[LocBudget] = &[ label: "moa-core Rust source", path: "crates/moa-core/src", scope: LocScope::RustTree, - max_lines: 25_836, - reason: "Unified Execute routing adds shared Respond/Execute/NeedsInput decisions, Inline/Durable strategies, classifier provenance and configuration, normalized planning audits, session events, and observability DTOs without rebuilding the moa-core root facade", + max_lines: 26_291, + reason: "long-horizon execution adds shared durable execution identities and events, typed asynchronous tool ownership, and provider-neutral sandbox checkpoint and release contracts without rebuilding the moa-core root facade", }, LocBudget { label: "public edge route ladder", @@ -127,8 +127,8 @@ const LOC_BUDGETS: &[LocBudget] = &[ label: "execution service shell", path: "crates/moa-orchestrator/src/services/execution.rs", scope: LocScope::File, - max_lines: 250, - reason: "the execution Restate surface delegates to focused handlers, capability catalog, planning context, and support modules", + max_lines: 263, + reason: "the execution Restate trait and DTO shell adds the authorized pause/resume public contract while delegating behavior to focused handlers, capability catalog, planning context, and support modules", }, LocBudget { label: "execution service handlers", @@ -218,8 +218,8 @@ const LOC_BUDGETS: &[LocBudget] = &[ label: "execution lifecycle DB behavior module", path: "crates/moa-execution/tests/execution_db/scope_and_lifecycle_db.rs", scope: LocScope::File, - max_lines: 1_500, - reason: "execution scope and lifecycle scenarios remain in their concrete owner instead of regrowing the shared execution DB harness", + max_lines: 1_514, + reason: "execution scope and lifecycle scenarios add exact durable-admission and session-fence Pins in their concrete behavior owner instead of regrowing the shared execution DB harness", }, LocBudget { label: "connector Restate service shell", diff --git a/crates/xtask/src/execution_trace_manifest.rs b/crates/xtask/src/execution_trace_manifest.rs index dd05a590b..cef1fbb58 100644 --- a/crates/xtask/src/execution_trace_manifest.rs +++ b/crates/xtask/src/execution_trace_manifest.rs @@ -62,6 +62,12 @@ struct ReceiverManifestEntry { receiver: ReceiverKind, } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct ReceiverHandlerManifestEntry { + client: &'static str, + operation: &'static str, +} + macro_rules! sender { ($path:literal, $symbol:literal, $helper:expr, $client:literal, $operation:literal) => { SenderManifestEntry { @@ -179,6 +185,27 @@ const SENDERS: &[SenderManifestEntry] = &[ "WorkerClient", "status" ), + sender!( + "crates/moa-orchestrator/src/objects/execution_run_controller/progress.rs", + "deliver", + IDENTITY_TRACE_HELPER, + "SessionClient", + "execution_progress" + ), + sender!( + "crates/moa-orchestrator/src/objects/execution_run_controller/progress.rs", + "deliver", + IDENTITY_TRACE_HELPER, + "SessionClient", + "execution_terminal" + ), + sender!( + "crates/moa-orchestrator/src/objects/cron_job.rs", + "schedule_next_tick", + TRACE_HELPER, + "CronJobClient", + "tick" + ), sender!( "crates/moa-orchestrator/src/objects/session/execution_runs.rs", "accept_execution_run_started", @@ -298,13 +325,6 @@ const SENDERS: &[SenderManifestEntry] = &[ "TurnExecutionClient", "request_cancel" ), - sender!( - "crates/moa-orchestrator/src/objects/session/execution_runs.rs", - "dispatch_execution_run", - TRACE_HELPER, - "ExecutionRunClient", - "run" - ), sender!( "crates/moa-orchestrator/src/objects/session/mod.rs", "append_session_event_deduped", @@ -418,18 +438,11 @@ const SENDERS: &[SenderManifestEntry] = &[ "run" ), sender!( - "crates/moa-orchestrator/src/services/action_review_dispatcher.rs", - "deliver_one", - TRACE_HELPER, - "ExecutionCompensationClient", - "resolve_action_review" - ), - sender!( - "crates/moa-orchestrator/src/services/action_review_dispatcher.rs", - "deliver_one", + "crates/moa-orchestrator/src/services/action_reviews.rs", + "decide", TRACE_HELPER, - "ExecutionTaskClient", - "resolve_action_review" + "ActionReviewDispatcherClient", + "dispatch" ), sender!( "crates/moa-orchestrator/src/services/action_reviews.rs", @@ -438,6 +451,13 @@ const SENDERS: &[SenderManifestEntry] = &[ "RestateSessionStoreClient", "append_event" ), + sender!( + "crates/moa-orchestrator/src/services/action_review_dispatcher.rs", + "deliver_one", + TRACE_HELPER, + "ExecutionDispatcherClient", + "dispatch" + ), sender!( "crates/moa-orchestrator/src/services/action_reviews.rs", "decide", @@ -488,6 +508,20 @@ const SENDERS: &[SenderManifestEntry] = &[ "SessionClient", "release_action_review" ), + sender!( + "crates/moa-orchestrator/src/services/action_reviews.rs", + "release_timed_out_conversational_review", + TRACE_HELPER, + "SessionClient", + "release_action_review" + ), + sender!( + "crates/moa-orchestrator/src/services/action_reviews.rs", + "release_timed_out_conversational_review", + TRACE_HELPER, + "WorkerClient", + "release_action_review" + ), sender!( "crates/moa-orchestrator/src/services/action_reviews.rs", "release_conversational_review", @@ -538,60 +572,208 @@ const SENDERS: &[SenderManifestEntry] = &[ "start_turn" ), sender!( - "crates/moa-orchestrator/src/services/execution/handlers.rs", - "apply_amendment", + "crates/moa-orchestrator/src/services/durable_timeout.rs", + "deliver_action_review_release", TRACE_HELPER, - "ExecutionTaskClient", - "cancel" + "RestateSessionStoreClient", + "append_event" + ), + sender!( + "crates/moa-orchestrator/src/services/durable_timeout.rs", + "deliver_action_review_timeout", + TRACE_HELPER, + "ActionReviewDispatcherClient", + "dispatch" + ), + sender!( + "crates/moa-orchestrator/src/services/durable_timeout.rs", + "schedule_durable_timeout", + TRACE_HELPER, + "DurableTimeoutClient", + "expire" + ), + sender!( + "crates/moa-orchestrator/src/services/execution/handlers.rs", + "start", + IDENTITY_TRACE_HELPER, + "SessionClient", + "execution_run_started" ), sender!( "crates/moa-orchestrator/src/services/execution/handlers.rs", - "apply_planned_amendment", + "kick_execution_dispatcher", TRACE_HELPER, - "ExecutionTaskClient", + "ExecutionDispatcherClient", + "dispatch" + ), + sender!( + "crates/moa-orchestrator/src/services/execution_dispatcher.rs", + "accept_target", + TRACE_HELPER, + "ExecutionCompensationAttemptClient", "cancel" ), sender!( - "crates/moa-orchestrator/src/services/execution/handlers.rs", - "decide_review", + "crates/moa-orchestrator/src/services/execution_dispatcher.rs", + "accept_target", TRACE_HELPER, - "ExecutionTaskClient", - "review_decided" + "ExecutionCompensationAttemptClient", + "run" ), sender!( - "crates/moa-orchestrator/src/services/execution/handlers.rs", - "deliver_input", + "crates/moa-orchestrator/src/services/execution_dispatcher.rs", + "accept_batch", TRACE_HELPER, - "ExecutionTaskClient", + "ExecutionRunControllerClient", + "advance" + ), + sender!( + "crates/moa-orchestrator/src/services/execution_dispatcher.rs", + "accept_target", + TRACE_HELPER, + "ExecutionTaskAttemptClient", "cancel" ), sender!( - "crates/moa-orchestrator/src/services/execution/handlers.rs", - "deliver_input", + "crates/moa-orchestrator/src/services/execution_dispatcher.rs", + "accept_target", TRACE_HELPER, - "ExecutionTaskClient", - "input_delivered" + "ExecutionTaskAttemptClient", + "run" ), sender!( - "crates/moa-orchestrator/src/services/execution/handlers.rs", - "deliver_signal", + "crates/moa-orchestrator/src/services/execution_dispatcher.rs", + "accept_batch", TRACE_HELPER, - "ExecutionTaskClient", - "signal_delivered" + "ExecutionTriggerClient", + "fire" ), sender!( - "crates/moa-orchestrator/src/services/execution/handlers.rs", - "start", - IDENTITY_TRACE_HELPER, - "SessionClient", - "execution_run_started" + "crates/moa-orchestrator/src/services/execution_dispatcher.rs", + "accept_target", + TRACE_HELPER, + "ToolExecutorClient", + "cancel_external_job" + ), + sender!( + "crates/moa-orchestrator/src/services/execution_dispatcher.rs", + "reconcile_bounded", + TRACE_HELPER, + "ExecutionDispatchDrainClient", + "drain" ), sender!( - "crates/moa-orchestrator/src/services/execution/support.rs", - "call_run_wake", + "crates/moa-orchestrator/src/services/execution_dispatcher.rs", + "dispatch", TRACE_HELPER, - "ExecutionRunClient", - "wake" + "ExecutionDispatchDrainClient", + "drain" + ), + sender!( + "crates/moa-orchestrator/src/services/execution_dispatcher.rs", + "drain", + TRACE_HELPER, + "ExecutionDispatchDrainClient", + "drain" + ), + sender!( + "crates/moa-orchestrator/src/services/execution_retention.rs", + "send_next", + TRACE_HELPER, + "ExecutionRetentionClient", + "run" + ), + sender!( + "crates/moa-orchestrator/src/services/execution_schedule.rs", + "kick_new_schedule_trigger", + TRACE_HELPER, + "ExecutionDispatcherClient", + "dispatch" + ), + sender!( + "crates/moa-orchestrator/src/services/execution_trigger.rs", + "fire", + TRACE_HELPER, + "ExecutionScheduleClient", + "fire_occurrence" + ), + sender!( + "crates/moa-orchestrator/src/services/execution_trigger.rs", + "fire", + TRACE_HELPER, + "ExecutionCompensationAttemptClient", + "watchdog" + ), + sender!( + "crates/moa-orchestrator/src/services/execution_trigger.rs", + "fire", + TRACE_HELPER, + "ExecutionTaskAttemptClient", + "watchdog" + ), + sender!( + "crates/moa-orchestrator/src/services/execution_trigger.rs", + "fire", + TRACE_HELPER, + "ToolExecutorClient", + "reconcile_external_job" + ), + sender!( + "crates/moa-orchestrator/src/services/execution_trigger.rs", + "fire", + TRACE_HELPER, + "ToolExecutorClient", + "recover_external_job_start" + ), + sender!( + "crates/moa-orchestrator/src/services/tool_executor.rs", + "append_tool_call_event", + TRACE_HELPER, + "RestateSessionStoreClient", + "append_event" + ), + sender!( + "crates/moa-orchestrator/src/services/tool_executor.rs", + "append_tool_canary_block_events", + TRACE_HELPER, + "RestateSessionStoreClient", + "append_event", + 2 + ), + sender!( + "crates/moa-orchestrator/src/services/tool_executor.rs", + "append_tool_dispatch_denied_event", + TRACE_HELPER, + "RestateSessionStoreClient", + "append_event" + ), + sender!( + "crates/moa-orchestrator/src/services/tool_executor.rs", + "append_tool_error_event", + TRACE_HELPER, + "RestateSessionStoreClient", + "append_event" + ), + sender!( + "crates/moa-orchestrator/src/services/tool_executor.rs", + "append_tool_result_event", + TRACE_HELPER, + "RestateSessionStoreClient", + "append_event" + ), + sender!( + "crates/moa-orchestrator/src/services/tool_executor.rs", + "finalize_recovered_compensation_external_start", + TRACE_HELPER, + "ToolExecutorClient", + "checkpoint_and_release_execution_hands" + ), + sender!( + "crates/moa-orchestrator/src/services/tool_executor.rs", + "release_session_hands", + TRACE_HELPER, + "ToolExecutorClient", + "release_session_hands" ), sender!( "crates/moa-orchestrator/src/services/artifact_release.rs", @@ -649,20 +831,6 @@ const SENDERS: &[SenderManifestEntry] = &[ "LLMGatewayClient", "cancel_owner" ), - sender!( - "crates/moa-orchestrator/src/services/llm_gateway.rs", - "cancel_completion_owner_from_service", - TRACE_HELPER, - "LLMGatewayClient", - "cancel_owner" - ), - sender!( - "crates/moa-orchestrator/src/services/llm_gateway.rs", - "cancel_completion_owner_from_workflow", - TRACE_HELPER, - "LLMGatewayClient", - "cancel_owner" - ), sender!( "crates/moa-orchestrator/src/services/llm_gateway.rs", "record_completion", @@ -754,6 +922,13 @@ const SENDERS: &[SenderManifestEntry] = &[ "SessionStatusMigratorClient", "migrate_status_idle" ), + sender!( + "crates/moa-orchestrator/src/services/tool_executor.rs", + "recover_external_job_start", + TRACE_HELPER, + "ExecutionDispatcherClient", + "dispatch" + ), sender!( "crates/moa-orchestrator/src/tool_invocation/governed.rs", "append_session_event", @@ -819,109 +994,88 @@ const SENDERS: &[SenderManifestEntry] = &[ "mark_consolidation_started" ), sender!( - "crates/moa-orchestrator/src/workflows/execution_compensation.rs", - "cleanup_compensation_hands", + "crates/moa-orchestrator/src/workflows/execution_compensation_attempt.rs", + "kick_dispatcher", TRACE_HELPER, - "ToolExecutorClient", - "release_execution_compensation_hands" + "ExecutionDispatcherClient", + "dispatch" ), sender!( - "crates/moa-orchestrator/src/workflows/execution_run.rs", - "complete", + "crates/moa-orchestrator/src/workflows/execution_compensation_attempt.rs", + "kick_dispatcher_shared", TRACE_HELPER, - "LLMGatewayClient", - "complete" + "ExecutionDispatcherClient", + "dispatch" ), sender!( - "crates/moa-orchestrator/src/workflows/execution_run.rs", - "deliver_session_projection", + "crates/moa-orchestrator/src/workflows/execution_compensation_attempt/external.rs", + "yield_external_job", TRACE_HELPER, - "SessionClient", - "execution_input_required" + "ToolExecutorClient", + "checkpoint_and_release_execution_hands" ), sender!( - "crates/moa-orchestrator/src/workflows/execution_run.rs", - "deliver_session_projection", + "crates/moa-orchestrator/src/workflows/execution_compensation_attempt/yielding.rs", + "park_compensation_review", TRACE_HELPER, - "SessionClient", - "execution_progress" + "ActionReviewsClient", + "acknowledge_execution_owner_review" ), sender!( - "crates/moa-orchestrator/src/workflows/execution_run.rs", - "deliver_session_projection", + "crates/moa-orchestrator/src/workflows/execution_compensation_attempt/yielding.rs", + "release_and_settle_compensation_shared", TRACE_HELPER, - "SessionClient", - "execution_terminal" + "ToolExecutorClient", + "checkpoint_and_release_execution_hands" ), sender!( - "crates/moa-orchestrator/src/workflows/execution_run.rs", - "plan_and_apply_waiting_replan", + "crates/moa-orchestrator/src/workflows/execution_compensation_attempt/yielding.rs", + "release_compensation_hands_workflow", TRACE_HELPER, - "ExecutionClient", - "apply_planned_amendment" + "ToolExecutorClient", + "checkpoint_and_release_execution_hands" ), sender!( - "crates/moa-orchestrator/src/workflows/execution_run.rs", - "run", + "crates/moa-orchestrator/src/workflows/execution_task_attempt.rs", + "kick_dispatcher", TRACE_HELPER, - "ExecutionCompensationClient", - "run" + "ExecutionDispatcherClient", + "dispatch" ), sender!( - "crates/moa-orchestrator/src/workflows/execution_run.rs", - "run", + "crates/moa-orchestrator/src/workflows/execution_task_attempt.rs", + "kick_dispatcher_shared", TRACE_HELPER, - "ExecutionTaskClient", - "run" + "ExecutionDispatcherClient", + "dispatch" ), sender!( - "crates/moa-orchestrator/src/workflows/execution_run.rs", - "signal_task_cancellation", + "crates/moa-orchestrator/src/workflows/execution_task_attempt/active.rs", + "execute_agent_turn", TRACE_HELPER, - "ExecutionTaskClient", - "cancel" + "LLMGatewayClient", + "complete_bounded" ), sender!( - "crates/moa-orchestrator/src/workflows/execution_task.rs", - "cleanup_task_hands", + "crates/moa-orchestrator/src/workflows/execution_task_attempt/yielding.rs", + "checkpoint_task_hands_shared", TRACE_HELPER, "ToolExecutorClient", - "release_execution_task_hands" + "checkpoint_and_release_execution_hands" ), sender!( - "crates/moa-orchestrator/src/workflows/execution_task.rs", - "execute_agent", + "crates/moa-orchestrator/src/workflows/execution_task_attempt/yielding.rs", + "checkpoint_task_hands_workflow", TRACE_HELPER, - "LLMGatewayClient", - "complete" + "ToolExecutorClient", + "checkpoint_and_release_execution_hands" ), sender!( - "crates/moa-orchestrator/src/workflows/execution_task.rs", - "invoke_capability_tool", + "crates/moa-orchestrator/src/workflows/execution_task_attempt/yielding.rs", + "park_review", TRACE_HELPER, "ActionReviewsClient", - "settle_execution_owner_review" - ), - sender!( - "crates/moa-orchestrator/src/workflows/execution_task.rs", - "record_execution_task_transition", - TRACE_HELPER, - "SecurityEventsClient", - "record_circuit_transition" - ), - sender!( - "crates/moa-orchestrator/src/workflows/execution_task.rs", - "record_execution_task_transition", - TRACE_HELPER, - "RestateSessionStoreClient", - "append_event" - ), - sender!( - "crates/moa-orchestrator/src/workflows/execution_task.rs", - "send_run_wake", - TRACE_HELPER, - "ExecutionRunClient", - "wake" + "acknowledge_execution_owner_review" ), sender!( "crates/moa-orchestrator/src/workflows/experiment_cancel.rs", @@ -1418,6 +1572,14 @@ const SENDERS: &[SenderManifestEntry] = &[ ]; const RECEIVERS: &[ReceiverManifestEntry] = &[ + ReceiverManifestEntry { + client: "ActionReviewDispatcherClient", + receiver: ReceiverKind::MoaHandler { + path: "crates/moa-orchestrator/src/services/action_review_dispatcher.rs", + symbol: "*", + adoption_symbol: "crate::ctx::adopt_incoming_trace_parent", + }, + }, ReceiverManifestEntry { client: "ActionPolicyClient", receiver: ReceiverKind::MoaHandler { @@ -1467,25 +1629,73 @@ const RECEIVERS: &[ReceiverManifestEntry] = &[ }, }, ReceiverManifestEntry { - client: "ExecutionCompensationClient", + client: "ExecutionCompensationAttemptClient", + receiver: ReceiverKind::MoaHandler { + path: "crates/moa-orchestrator/src/workflows/execution_compensation_attempt.rs", + symbol: "*", + adoption_symbol: "crate::ctx::adopt_incoming_trace_parent", + }, + }, + ReceiverManifestEntry { + client: "ExecutionDispatcherClient", + receiver: ReceiverKind::MoaHandler { + path: "crates/moa-orchestrator/src/services/execution_dispatcher.rs", + symbol: "*", + adoption_symbol: "crate::ctx::adopt_incoming_trace_parent", + }, + }, + ReceiverManifestEntry { + client: "ExecutionDispatchDrainClient", + receiver: ReceiverKind::MoaHandler { + path: "crates/moa-orchestrator/src/services/execution_dispatcher.rs", + symbol: "*", + adoption_symbol: "crate::ctx::adopt_incoming_trace_parent", + }, + }, + ReceiverManifestEntry { + client: "ExecutionDispatchReconcilerClient", + receiver: ReceiverKind::MoaHandler { + path: "crates/moa-orchestrator/src/services/execution_dispatcher.rs", + symbol: "*", + adoption_symbol: "crate::ctx::adopt_incoming_trace_parent", + }, + }, + ReceiverManifestEntry { + client: "ExecutionRetentionClient", + receiver: ReceiverKind::MoaHandler { + path: "crates/moa-orchestrator/src/services/execution_retention.rs", + symbol: "*", + adoption_symbol: "crate::ctx::adopt_incoming_trace_parent", + }, + }, + ReceiverManifestEntry { + client: "ExecutionRunControllerClient", receiver: ReceiverKind::MoaHandler { - path: "crates/moa-orchestrator/src/workflows/execution_compensation.rs", + path: "crates/moa-orchestrator/src/objects/execution_run_controller.rs", symbol: "*", adoption_symbol: "crate::ctx::adopt_incoming_trace_parent", }, }, ReceiverManifestEntry { - client: "ExecutionRunClient", + client: "ExecutionScheduleClient", + receiver: ReceiverKind::MoaHandler { + path: "crates/moa-orchestrator/src/services/execution_schedule.rs", + symbol: "*", + adoption_symbol: "prepare_handler", + }, + }, + ReceiverManifestEntry { + client: "ExecutionTaskAttemptClient", receiver: ReceiverKind::MoaHandler { - path: "crates/moa-orchestrator/src/workflows/execution_run.rs", + path: "crates/moa-orchestrator/src/workflows/execution_task_attempt.rs", symbol: "*", adoption_symbol: "crate::ctx::adopt_incoming_trace_parent", }, }, ReceiverManifestEntry { - client: "ExecutionTaskClient", + client: "ExecutionTriggerClient", receiver: ReceiverKind::MoaHandler { - path: "crates/moa-orchestrator/src/workflows/execution_task.rs", + path: "crates/moa-orchestrator/src/services/execution_trigger.rs", symbol: "*", adoption_symbol: "crate::ctx::adopt_incoming_trace_parent", }, @@ -1538,6 +1748,14 @@ const RECEIVERS: &[ReceiverManifestEntry] = &[ adoption_symbol: "crate::ctx::adopt_incoming_trace_parent", }, }, + ReceiverManifestEntry { + client: "DurableTimeoutClient", + receiver: ReceiverKind::MoaHandler { + path: "crates/moa-orchestrator/src/services/durable_timeout.rs", + symbol: "*", + adoption_symbol: "crate::ctx::adopt_incoming_trace_parent", + }, + }, ReceiverManifestEntry { client: "RestateSessionStoreClient", receiver: ReceiverKind::MoaHandler { @@ -1640,6 +1858,99 @@ const RECEIVERS: &[ReceiverManifestEntry] = &[ }, ]; +// Receiver-only handlers are explicit because shared workflow methods and +// externally invoked operational services can have no in-process producer. +const REQUIRED_RECEIVER_HANDLERS: &[ReceiverHandlerManifestEntry] = &[ + ReceiverHandlerManifestEntry { + client: "DurableTimeoutClient", + operation: "expire", + }, + ReceiverHandlerManifestEntry { + client: "ExecutionCompensationAttemptClient", + operation: "cancel", + }, + ReceiverHandlerManifestEntry { + client: "ExecutionCompensationAttemptClient", + operation: "run", + }, + ReceiverHandlerManifestEntry { + client: "ExecutionCompensationAttemptClient", + operation: "watchdog", + }, + ReceiverHandlerManifestEntry { + client: "ExecutionDispatcherClient", + operation: "dispatch", + }, + ReceiverHandlerManifestEntry { + client: "ExecutionDispatchDrainClient", + operation: "drain", + }, + ReceiverHandlerManifestEntry { + client: "ExecutionDispatchReconcilerClient", + operation: "reconcile", + }, + ReceiverHandlerManifestEntry { + client: "ExecutionRunControllerClient", + operation: "advance", + }, + ReceiverHandlerManifestEntry { + client: "ExecutionRetentionClient", + operation: "run", + }, + ReceiverHandlerManifestEntry { + client: "ExecutionScheduleClient", + operation: "cancel", + }, + ReceiverHandlerManifestEntry { + client: "ExecutionScheduleClient", + operation: "create", + }, + ReceiverHandlerManifestEntry { + client: "ExecutionScheduleClient", + operation: "fire_occurrence", + }, + ReceiverHandlerManifestEntry { + client: "ExecutionScheduleClient", + operation: "list", + }, + ReceiverHandlerManifestEntry { + client: "ExecutionScheduleClient", + operation: "pause", + }, + ReceiverHandlerManifestEntry { + client: "ExecutionScheduleClient", + operation: "resume", + }, + ReceiverHandlerManifestEntry { + client: "ExecutionScheduleClient", + operation: "status", + }, + ReceiverHandlerManifestEntry { + client: "ExecutionScheduleClient", + operation: "update", + }, + ReceiverHandlerManifestEntry { + client: "ExecutionTaskAttemptClient", + operation: "cancel", + }, + ReceiverHandlerManifestEntry { + client: "ExecutionTaskAttemptClient", + operation: "run", + }, + ReceiverHandlerManifestEntry { + client: "ExecutionTaskAttemptClient", + operation: "watchdog", + }, + ReceiverHandlerManifestEntry { + client: "ExecutionTriggerClient", + operation: "fire", + }, + ReceiverHandlerManifestEntry { + client: "SessionRetentionClient", + operation: "run", + }, +]; + const RAW_SENDERS: &[SenderManifestEntry] = &[ sender!( "crates/moa-edge/src/proxy.rs", @@ -1761,7 +2072,13 @@ pub(crate) fn audit(root: &Path) -> Result> { fs::read_to_string(&path).with_context(|| format!("read {}", path.display()))?; sources.insert(relative, source); } - Ok(audit_sources(&sources, SENDERS, RAW_SENDERS, RECEIVERS)) + Ok(audit_sources( + &sources, + SENDERS, + RAW_SENDERS, + RECEIVERS, + REQUIRED_RECEIVER_HANDLERS, + )) } fn audit_sources( @@ -1769,6 +2086,7 @@ fn audit_sources( manifest: &[SenderManifestEntry], raw_manifest: &[SenderManifestEntry], receivers: &[ReceiverManifestEntry], + required_receiver_handlers: &[ReceiverHandlerManifestEntry], ) -> Vec { let mut diagnostics = Vec::new(); let mut discovered = Vec::new(); @@ -1779,7 +2097,14 @@ fn audit_sources( compare_manifest(&discovered, manifest, &mut diagnostics); audit_discovered_raw_senders(sources, raw_manifest, &mut diagnostics); audit_raw_manifest(sources, raw_manifest, &mut diagnostics); - audit_receiver_manifest(sources, manifest, raw_manifest, receivers, &mut diagnostics); + audit_receiver_manifest( + sources, + manifest, + raw_manifest, + receivers, + required_receiver_handlers, + &mut diagnostics, + ); audit_identity_helper_delegation(sources, &mut diagnostics); diagnostics .sort_by(|left, right| (&left.path, &left.detail).cmp(&(&right.path, &right.detail))); @@ -1932,12 +2257,14 @@ fn audit_receiver_manifest( manifest: &[SenderManifestEntry], raw_manifest: &[SenderManifestEntry], receivers: &[ReceiverManifestEntry], + required_receiver_handlers: &[ReceiverHandlerManifestEntry], diagnostics: &mut Vec, ) { let used_clients = manifest .iter() .chain(raw_manifest) .map(|entry| entry.client) + .chain(required_receiver_handlers.iter().map(|entry| entry.client)) .collect::>(); for client in used_clients { let Some(receiver) = receivers.iter().find(|receiver| receiver.client == client) else { @@ -1979,6 +2306,12 @@ fn audit_receiver_manifest( .chain(raw_manifest) .filter(|entry| entry.client == client) .map(|entry| entry.operation) + .chain( + required_receiver_handlers + .iter() + .filter(|entry| entry.client == client) + .map(|entry| entry.operation), + ) .collect::>(); for operation in operations { let receiver_symbol = if symbol == "*" { operation } else { symbol }; @@ -2332,12 +2665,18 @@ struct Function<'a> { } fn functions(source: &str) -> Vec> { - let production_end = source.find("\n#[cfg(test)]").unwrap_or(source.len()); - let source = &source[..production_end]; + let test_modules = cfg_test_module_ranges(source); let mut functions = Vec::new(); let mut cursor = 0usize; while let Some(fn_offset) = find_fn_token(&source[cursor..]) { let fn_start = cursor + fn_offset; + if let Some((_, end)) = test_modules + .iter() + .find(|(start, end)| *start <= fn_start && fn_start < *end) + { + cursor = *end; + continue; + } let name_start = fn_start + 3; let name_end = source[name_start..] .find(|character: char| !character.is_ascii_alphanumeric() && character != '_') @@ -2363,6 +2702,35 @@ fn functions(source: &str) -> Vec> { functions } +fn cfg_test_module_ranges(source: &str) -> Vec<(usize, usize)> { + let mut ranges = Vec::new(); + let mut cursor = 0usize; + while let Some(relative) = source[cursor..].find("#[cfg(test)]") { + let start = cursor + relative; + let after_attribute = start + "#[cfg(test)]".len(); + let whitespace = source[after_attribute..] + .find(|character: char| !character.is_whitespace()) + .unwrap_or(source.len() - after_attribute); + let mod_start = after_attribute + whitespace; + if !source[mod_start..].starts_with("mod ") { + cursor = after_attribute; + continue; + } + let Some(terminator_relative) = source[mod_start..].find(['{', ';']) else { + break; + }; + let terminator = mod_start + terminator_relative; + let end = if source.as_bytes()[terminator] == b'{' { + matching_delimiter(source, terminator, '{', '}').map_or(source.len(), |close| close + 1) + } else { + terminator + 1 + }; + ranges.push((start, end)); + cursor = end; + } + ranges +} + fn find_fn_token(source: &str) -> Option { let mut cursor = 0usize; while let Some(offset) = source[cursor..].find("fn ") { @@ -2464,10 +2832,10 @@ mod tests { } const FIXTURE_RECEIVER: ReceiverManifestEntry = ReceiverManifestEntry { - client: "ExecutionRunClient", + client: "ExecutionRunControllerClient", receiver: ReceiverKind::MoaHandler { path: "receiver.rs", - symbol: "run", + symbol: "advance", adoption_symbol: "adopt_incoming_trace_parent", }, }; @@ -2480,28 +2848,29 @@ mod tests { "sender.rs", "dispatch", TRACE_HELPER, - "ExecutionRunClient", - "run" + "ExecutionRunControllerClient", + "advance" )]; let diagnostics = audit_sources( &sources(&[ ( "sender.rs", - "async fn dispatch(ctx: &Context<'_>) { ctx.workflow_client::(\"run\").run(()).send(); }", + "async fn dispatch(ctx: &Context<'_>) { ctx.object_client::(\"run\").advance(()).send(); }", ), ( "receiver.rs", - "async fn run(ctx: WorkflowContext<'_>) { adopt_incoming_trace_parent(&ctx); annotate_restate_handler_span(\"ExecutionRun\", \"run\"); }", + "async fn advance(ctx: ObjectContext<'_>) { adopt_incoming_trace_parent(&ctx); annotate_restate_handler_span(\"ExecutionRunController\", \"advance\"); }", ), ]), &manifest, &[], &[FIXTURE_RECEIVER], + &[], ); assert!(diagnostics.iter().any(|diagnostic| { diagnostic.detail() - == "sender.rs::dispatch: generated Restate sender must use `replay_safe_request` for client=ExecutionRunClient operation=run; found no approved trace wrapper" + == "sender.rs::dispatch: generated Restate sender must use `replay_safe_request` for client=ExecutionRunControllerClient operation=advance; found no approved trace wrapper" }), "unexpected diagnostics: {diagnostics:#?}"); } @@ -2512,22 +2881,23 @@ mod tests { "sender.rs", "dispatch", TRACE_HELPER, - "ExecutionRunClient", - "run" + "ExecutionRunControllerClient", + "advance" )]; let diagnostics = audit_sources( &sources(&[( "sender.rs", - "async fn dispatch(ctx: &Context<'_>) { replay_safe_request(ctx.workflow_client::(\"run\").run(())).send(); }", + "async fn dispatch(ctx: &Context<'_>) { replay_safe_request(ctx.object_client::(\"run\").advance(())).send(); }", )]), &manifest, &[], &[FIXTURE_RECEIVER], + &[], ); assert!(diagnostics.iter().any(|diagnostic| { diagnostic.detail() - == "receiver.rs::run: receiver for client=ExecutionRunClient is missing" + == "receiver.rs::advance: receiver for client=ExecutionRunControllerClient is missing" })); } @@ -2539,31 +2909,379 @@ mod tests { "sender.rs", "dispatch", TRACE_HELPER, - "ExecutionRunClient", - "run" + "ExecutionRunControllerClient", + "advance" )]; let diagnostics = audit_sources( &sources(&[ ( "sender.rs", - "async fn dispatch(ctx: &Context<'_>) { replay_safe_request(ctx.workflow_client::(\"run\").run(())).send(); }", + "async fn dispatch(ctx: &Context<'_>) { replay_safe_request(ctx.object_client::(\"run\").advance(())).send(); }", ), ( "receiver.rs", - "async fn run(ctx: WorkflowContext<'_>) { let _span = tracing::info_span!(\"run\"); adopt_incoming_trace_parent(&ctx); }", + "async fn advance(ctx: ObjectContext<'_>) { let _span = tracing::info_span!(\"advance\"); adopt_incoming_trace_parent(&ctx); }", ), ]), &manifest, &[], &[FIXTURE_RECEIVER], + &[], ); assert!(diagnostics.iter().any(|diagnostic| { diagnostic.detail() - == "receiver.rs::run: receiver creates its handler span before `adopt_incoming_trace_parent` for client=ExecutionRunClient operation=run" + == "receiver.rs::advance: receiver creates its handler span before `adopt_incoming_trace_parent` for client=ExecutionRunControllerClient operation=advance" + })); + } + + #[test] + fn generated_sender_discovery_keeps_production_after_an_early_test_module() { + // Pins: placing an out-of-line test module near the top of a production + // owner cannot hide later generated-client dispatches from the audit. + let discovered = discover_generated_senders( + "controller.rs", + r#" + #[cfg(test)] + mod inline_tests { + async fn hidden(ctx: &Context<'_>) { + ctx.service_client::() + .fire(()) + .send(); + } + } + + #[cfg(test)] + mod tests; + + async fn advance(ctx: &Context<'_>) { + replay_safe_request( + ctx.object_client::("run") + .advance(()), + ) + .call() + .await; + } + "#, + ); + + assert_eq!(discovered.len(), 1, "discovered senders: {discovered:#?}"); + let sender = &discovered[0]; + assert_eq!(sender.path, "controller.rs"); + assert_eq!(sender.symbol, "advance"); + assert_eq!(sender.helper, Some(TRACE_HELPER)); + assert_eq!(sender.client, "ExecutionRunControllerClient"); + assert_eq!(sender.operation, "advance"); + } + + #[test] + fn checked_in_manifest_pins_long_horizon_producer_identities() { + // Pins: every bounded activation, durable trigger, dispatch, cancellation, + // and timeout producer remains attributable to its exact owning function. + for (path, symbol, helper, client, operation) in [ + ( + "crates/moa-orchestrator/src/objects/execution_run_controller/progress.rs", + "deliver", + IDENTITY_TRACE_HELPER, + "SessionClient", + "execution_progress", + ), + ( + "crates/moa-orchestrator/src/services/durable_timeout.rs", + "schedule_durable_timeout", + TRACE_HELPER, + "DurableTimeoutClient", + "expire", + ), + ( + "crates/moa-orchestrator/src/services/execution/handlers.rs", + "kick_execution_dispatcher", + TRACE_HELPER, + "ExecutionDispatcherClient", + "dispatch", + ), + ( + "crates/moa-orchestrator/src/services/execution_dispatcher.rs", + "accept_batch", + TRACE_HELPER, + "ExecutionRunControllerClient", + "advance", + ), + ( + "crates/moa-orchestrator/src/services/execution_dispatcher.rs", + "accept_target", + TRACE_HELPER, + "ExecutionTaskAttemptClient", + "run", + ), + ( + "crates/moa-orchestrator/src/services/execution_dispatcher.rs", + "accept_target", + TRACE_HELPER, + "ExecutionTaskAttemptClient", + "cancel", + ), + ( + "crates/moa-orchestrator/src/services/execution_dispatcher.rs", + "accept_target", + TRACE_HELPER, + "ExecutionCompensationAttemptClient", + "run", + ), + ( + "crates/moa-orchestrator/src/services/execution_dispatcher.rs", + "accept_target", + TRACE_HELPER, + "ExecutionCompensationAttemptClient", + "cancel", + ), + ( + "crates/moa-orchestrator/src/services/execution_dispatcher.rs", + "reconcile_bounded", + TRACE_HELPER, + "ExecutionDispatchDrainClient", + "drain", + ), + ( + "crates/moa-orchestrator/src/services/execution_retention.rs", + "send_next", + TRACE_HELPER, + "ExecutionRetentionClient", + "run", + ), + ( + "crates/moa-orchestrator/src/services/execution_schedule.rs", + "kick_new_schedule_trigger", + TRACE_HELPER, + "ExecutionDispatcherClient", + "dispatch", + ), + ( + "crates/moa-orchestrator/src/services/execution_trigger.rs", + "fire", + TRACE_HELPER, + "ExecutionScheduleClient", + "fire_occurrence", + ), + ] { + assert!( + SENDERS.iter().any(|entry| { + entry.path == path + && entry.symbol == symbol + && entry.helper == helper + && entry.client == client + && entry.operation == operation + && entry.expected_count == 1 + }), + "missing producer {path}::{symbol} -> {client}/{operation} via {helper}" + ); + } + } + + #[test] + fn checked_in_manifest_pins_bounded_session_hand_release_self_continuation() { + // Pins: incomplete session teardown self-schedules through the exact ToolExecutor + // handler with replay-safe trace propagation; it cannot disappear into an untraced + // raw send or drift to a different receiver implementation. + let matching = SENDERS + .iter() + .filter(|entry| { + entry.path == "crates/moa-orchestrator/src/services/tool_executor.rs" + && entry.symbol == "release_session_hands" + && entry.client == "ToolExecutorClient" + && entry.operation == "release_session_hands" + }) + .collect::>(); + assert_eq!(matching.len(), 1); + assert_eq!(matching[0].helper, TRACE_HELPER); + assert_eq!(matching[0].expected_count, 1); + assert!(RECEIVERS.iter().any(|entry| { + entry.client == "ToolExecutorClient" + && entry.receiver + == ReceiverKind::MoaHandler { + path: "crates/moa-orchestrator/src/services/tool_executor.rs", + symbol: "*", + adoption_symbol: "crate::ctx::adopt_incoming_trace_parent", + } })); } + #[test] + fn checked_in_manifest_rejects_retired_execution_workflow_clients() { + // Pins: the hard cutover cannot silently restore lifetime-spanning run, task, or + // compensation workflows as trace-manifest senders or receivers. + const RETIRED_CLIENTS: [&str; 3] = [ + "ExecutionRunClient", + "ExecutionTaskClient", + "ExecutionCompensationClient", + ]; + assert!( + SENDERS + .iter() + .all(|entry| !RETIRED_CLIENTS.contains(&entry.client)), + "sender manifest restored a retired execution workflow client" + ); + assert!( + RECEIVERS + .iter() + .all(|entry| !RETIRED_CLIENTS.contains(&entry.client)), + "receiver manifest restored a retired execution workflow client" + ); + } + + #[test] + fn checked_in_manifest_pins_long_horizon_receivers_and_shared_handlers() { + // Pins: receiver-only operational and shared workflow handlers remain in + // the trace contract even when no Rust producer currently calls them. + assert_eq!( + REQUIRED_RECEIVER_HANDLERS, + &[ + ReceiverHandlerManifestEntry { + client: "DurableTimeoutClient", + operation: "expire", + }, + ReceiverHandlerManifestEntry { + client: "ExecutionCompensationAttemptClient", + operation: "cancel", + }, + ReceiverHandlerManifestEntry { + client: "ExecutionCompensationAttemptClient", + operation: "run", + }, + ReceiverHandlerManifestEntry { + client: "ExecutionCompensationAttemptClient", + operation: "watchdog", + }, + ReceiverHandlerManifestEntry { + client: "ExecutionDispatcherClient", + operation: "dispatch", + }, + ReceiverHandlerManifestEntry { + client: "ExecutionDispatchDrainClient", + operation: "drain", + }, + ReceiverHandlerManifestEntry { + client: "ExecutionDispatchReconcilerClient", + operation: "reconcile", + }, + ReceiverHandlerManifestEntry { + client: "ExecutionRunControllerClient", + operation: "advance", + }, + ReceiverHandlerManifestEntry { + client: "ExecutionRetentionClient", + operation: "run", + }, + ReceiverHandlerManifestEntry { + client: "ExecutionScheduleClient", + operation: "cancel", + }, + ReceiverHandlerManifestEntry { + client: "ExecutionScheduleClient", + operation: "create", + }, + ReceiverHandlerManifestEntry { + client: "ExecutionScheduleClient", + operation: "fire_occurrence", + }, + ReceiverHandlerManifestEntry { + client: "ExecutionScheduleClient", + operation: "list", + }, + ReceiverHandlerManifestEntry { + client: "ExecutionScheduleClient", + operation: "pause", + }, + ReceiverHandlerManifestEntry { + client: "ExecutionScheduleClient", + operation: "resume", + }, + ReceiverHandlerManifestEntry { + client: "ExecutionScheduleClient", + operation: "status", + }, + ReceiverHandlerManifestEntry { + client: "ExecutionScheduleClient", + operation: "update", + }, + ReceiverHandlerManifestEntry { + client: "ExecutionTaskAttemptClient", + operation: "cancel", + }, + ReceiverHandlerManifestEntry { + client: "ExecutionTaskAttemptClient", + operation: "run", + }, + ReceiverHandlerManifestEntry { + client: "ExecutionTaskAttemptClient", + operation: "watchdog", + }, + ReceiverHandlerManifestEntry { + client: "ExecutionTriggerClient", + operation: "fire", + }, + ReceiverHandlerManifestEntry { + client: "SessionRetentionClient", + operation: "run", + }, + ] + ); + + for (client, path) in [ + ( + "DurableTimeoutClient", + "crates/moa-orchestrator/src/services/durable_timeout.rs", + ), + ( + "ExecutionCompensationAttemptClient", + "crates/moa-orchestrator/src/workflows/execution_compensation_attempt.rs", + ), + ( + "ExecutionDispatcherClient", + "crates/moa-orchestrator/src/services/execution_dispatcher.rs", + ), + ( + "ExecutionDispatchReconcilerClient", + "crates/moa-orchestrator/src/services/execution_dispatcher.rs", + ), + ( + "ExecutionRunControllerClient", + "crates/moa-orchestrator/src/objects/execution_run_controller.rs", + ), + ( + "ExecutionRetentionClient", + "crates/moa-orchestrator/src/services/execution_retention.rs", + ), + ( + "ExecutionScheduleClient", + "crates/moa-orchestrator/src/services/execution_schedule.rs", + ), + ( + "ExecutionTaskAttemptClient", + "crates/moa-orchestrator/src/workflows/execution_task_attempt.rs", + ), + ( + "ExecutionTriggerClient", + "crates/moa-orchestrator/src/services/execution_trigger.rs", + ), + ( + "SessionRetentionClient", + "crates/moa-orchestrator/src/workflows/session_retention.rs", + ), + ] { + assert!( + RECEIVERS.iter().any(|entry| { + entry.client == client + && matches!( + entry.receiver, + ReceiverKind::MoaHandler { path: actual, .. } if actual == path + ) + }), + "missing receiver mapping for {client} at {path}" + ); + } + } + #[test] fn checked_in_manifest_pins_named_cross_boundary_mappings() { // Pins: the Task 11 manifest retains its explicitly required memory, diff --git a/docker-compose.yml b/docker-compose.yml index 3b4b19696..6a0764481 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -365,7 +365,7 @@ services: environment: MOA_EDGE_BIND: 0.0.0.0:8080 MOA_EDGE_UPSTREAM: http://restate:8080 - MOA_EDGE_CONNECTOR_CREDENTIAL_UPSTREAM: http://moa-orchestrator:10023 + MOA_EDGE_INTERNAL_INGRESS_UPSTREAM: http://moa-orchestrator:10023 # Local development opts into the staged connector-management surface. MOA_EDGE_CONNECTOR_MANAGEMENT_ENABLED: "true" MOA_DATABASE_URL: postgres://moa_owner:${POSTGRES_PASSWORD:-dev}@postgres:5432/moa?sslmode=disable diff --git a/docs/00-direction.md b/docs/00-direction.md index f6ddc1bff..f349739a8 100644 --- a/docs/00-direction.md +++ b/docs/00-direction.md @@ -23,7 +23,12 @@ MOA is not a personal assistant or chat wrapper. It is an execution platform wit ## What MOA Provides -- **Durable work:** sessions, conversational workers, and execution runs survive process restarts because Restate owns orchestration and Postgres owns product data. Sandbox compute remains ephemeral; tenant filesystem work survives only through the independently retained `SandboxWorkspace` and its verified portable checkpoints, as defined in [Sandbox Workspaces](25-sandbox-workspaces.md). +- **Durable work at every horizon:** sessions and conversational workers use + Restate directly; execution runs persist their complete product state in + Postgres and advance through bounded Restate controller, task-attempt, + compensation-attempt, and trigger activations. A run waiting for input, + review, signal, time, external completion, or operator resume retains no live + handler or sandbox compute. - **Task segmentation:** conversations are split into discrete task segments so one long session can contain many independently tracked outcomes. - **Outcome assessment:** MOA records whether each task segment resolved, partially resolved, failed, was abandoned, or remains unknown without requiring explicit user feedback. - **Per-tenant learning:** task outcomes become experience records, attributions, candidates, skill changes, and memory updates at tenant scope without requiring a fixed session intent taxonomy. @@ -53,7 +58,11 @@ MOA is not a personal assistant or chat wrapper. It is an execution platform wit MOA's differentiators are architectural, not cosmetic: - **Restate-native agents:** sessions and workers map to virtual objects with single-writer semantics and durable waits. -- **Reliable bulk execution:** `ExecutionRun` and `ExecutionTask` durably execute validated plans with atomic budgets, exact logical coverage, a positive bounded window for live task invocations, storage-only pending rows, and compact terminal delivery to the owning session. +- **Reliable long-horizon execution:** `ExecutionRunController`, + `ExecutionTaskAttempt`, and `ExecutionTrigger` execute validated plans through + bounded, generation-fenced activations. Postgres owns plan, identity, wait, + task, budget, schedule, external-job, and outbox truth; fleet/tenant admission + bounds active compute separately from storage-only parked work. - **Experience-level analytics:** learning is derived from assessed task segments, not whole-session guesses. - **Resolution-weighted improvement:** skills and future retrieval decisions can use measured success rates. - **Candidate-gated adaptation:** reusable skills, memory proposals, policy proposals, and eval proposals start as learning candidates before promotion. @@ -66,5 +75,8 @@ MOA's differentiators are architectural, not cosmetic: - MOA does not require a durable session intent taxonomy for routing or learning. The agent loop and skills decide dynamically from context. - MOA does not keep durable product state only in Restate. Restate is orchestration state; Postgres is the product record. - MOA does not retain sandbox process memory by default or treat a live sandbox, mutable volume, paused instance, or provider snapshot as the committed filesystem revision. The filesystem-only workspace contract requires a verified portable checkpoint. +- MOA does not represent a day- or week-scale execution as one lifetime-spanning + workflow invocation. Every activation returns after bounded progress; every + yield checkpoints required filesystem state and destroys active compute. - MOA does not bind agent work to a single front door. REST/gateway, API automation, and messaging adapters are peers over the same runtime model. - MOA does not optimize for a single-user personal desktop workflow. Local mode is a development and operator path over the same enterprise runtime model. diff --git a/docs/01-architecture-overview.md b/docs/01-architecture-overview.md index 8634fc6ac..e330f9dd1 100644 --- a/docs/01-architecture-overview.md +++ b/docs/01-architecture-overview.md @@ -18,7 +18,7 @@ Brain and execution Context pipeline -> provider router -> LLM Tool router -> built-ins / hands / operator MCP / tenant connector actions Execute Inline delegation -> Restate Worker virtual objects - Execute Durable planning/compiler -> Restate ExecutionRun / ExecutionTask workflows + Execute Durable planning/compiler -> bounded Restate execution activations | v Product data in Postgres / Neon @@ -29,7 +29,9 @@ Product data in Postgres / Neon graph nodes, graph edges, sidecar indexes, configured vector records connector connections, HTTP action bindings, invocation ledgers knowledge projections, sync runs, document versions, chunks - execution runs, execution tasks, plan history, budgets, completion checks + execution runs/nodes/tasks/attempts, compensation, waits, triggers, schedules, + external jobs, dispatch outbox, admission capacity, plan history, budgets, + completion checks learning_log analytics.turn_lineage, analytics.score_run, analytics.scores, moa.experiment_run, compliance audit tables @@ -82,9 +84,10 @@ node in Durable Execute; no skill is required to define a plan. Inline Execute is the bounded root model/tool loop, including repeat and tool-call limits, visible skills, and conversational `Worker` delegation. -Durable Execute instantiates or compiles an immutable plan, starts a detached -`ExecutionRun`, publishes compact progress, and synthesizes its terminal result -into the owning session automatically. An initial root Inline turn may make one +Durable Execute instantiates or compiles an immutable plan, admits a detached +run in Postgres, dispatches its first bounded `ExecutionRunController` +activation, publishes compact progress, and synthesizes its terminal result into +the owning session automatically. An initial root Inline turn may make one evidence-preserving upgrade to Durable; it cannot downgrade or classify again. The workflow exposes `request_durable_execution` only to that eligible turn, requires it to be the sole tool call in the model response, and validates its @@ -127,8 +130,11 @@ authoring and import/export format. Optional `ui` metadata is non-semantic. | `ExecutionCompiler` | `moa-execution` | Validates, canonicalizes, estimates, and hashes initial plans and amendments against the capability catalog and remaining budget. | | `ExecutionProjection` | `moa-execution` | Supplies ordered node/task state to the pure scheduler; it contains no repository or provider handle. | | `ExecutionRepository` | `moa-execution` | Owns scoped run/task persistence, idempotent materialization, atomic budget accounting, generation-fenced outcomes, amendment history, and cancellation. It depends only on shared database/core types, never on Restate or runtime owners. | -| `ExecutionRun` | `moa-orchestrator` Restate workflow | Sole DAG-advancement owner. Drives one durable plan from persisted state, keeps at most `execution.max_in_flight_tasks` attached task calls live, and parks on persisted wake epochs or owned task completion. | -| `ExecutionTask` | `moa-orchestrator` Restate workflow | Executes one stable logical node or map-item instance and records one typed outcome. | +| `ExecutionRunController` | `moa-orchestrator` Restate virtual object | Serializes by `run_uid`, runs at most `maximum_activation_steps`, dispatches at most `dispatch_batch_size` ready rows, commits the next product state, and returns. It never spans a product wait. | +| `ExecutionTaskAttempt` | `moa-orchestrator` Restate workflow | Executes one generation-fenced attempt for a stable logical node or map-item, bounded by the active-attempt timeout and task deadline. | +| `ExecutionTrigger` | `moa-orchestrator` Restate service | Delivers one immutable, generation-fenced deadline, timer, wait-expiry, watchdog, external-reconcile, or schedule occurrence into the transactional dispatch outbox. | +| `ExecutionCompensationAttempt` | `moa-orchestrator` ingress-private Restate workflow | Reverses one committed compensatable effect under a stable compensation identity; ambiguous results become explicit repair state. | +| `ExecutionRetention` | `moa-orchestrator` ingress-private Restate service | Archives or deletes one bounded terminal-detail page, persists the next generation, and self-schedules; a coarse CronJob only repairs a missing delayed invocation. | | Sandbox workspace | `moa-hands` domain/repositories/adapters, composed by `moa-orchestrator` | Owns one tenant-scoped, worker-owned or execution-task-owned filesystem lifecycle; Postgres stores ownership/fences and provider/object storage owns bytes. | | Connector definition | `moa-artifacts` | Owns immutable reviewed HTTP transport, schema, data-class, credential-slot, and action contracts; the platform supplies the fixed external-write/high-risk/admin-review floor. | | Connector connection | `moa-connectors` | Owns tenant lifecycle/health/generation, HTTP action bindings, and durable send outcomes. | @@ -146,11 +152,11 @@ deliverables can produce `partial`, `blocked`, or `unsupported`, never a false outputs, citations, and explicit gaps. An `ExecutionPlanDefinition` carries an explicit `cancel_policy`, -`input_schema`, `output_schema`, and `nodes`. Each node carries +`input_schema`, `output_schema`, required `input_wait_policy`, and `nodes`. Each node carries `id`, `depends_on`, optional `when`, `input`, `output_schema`, one `operation`, an explicit optional compensation contract, retry policy, optional budget, and the goal requirement IDs it serves. The dependency graph is acyclic, and -its operation enum has exactly seven variants: +its operation enum has exactly eight variants: 1. `Capability { reference }` invokes one registered governed capability. 2. `Agent { instructions, skill_refs, capability_refs, max_turns }` runs one @@ -161,9 +167,18 @@ its operation enum has exactly seven variants: 4. `Reduce { items, max_items, reducer, batch_size }` reduces bounded structured results through a deterministic capability or a bounded hierarchical agent reducer. -5. `Review { prompt }` waits for a tenant review decision. -6. `WaitSignal { signal_name }` waits for one external or user signal. -7. `Output { value }` resolves and validates the terminal output. +5. `Review { prompt, wait_policy }` waits for a tenant review decision. +6. `WaitSignal { signal_name, wait_policy }` waits for one external or user signal. +7. `WaitUntil { wake, result }` produces its declared result at an exact durable time. +8. `Output { value }` resolves and validates the terminal output. + +Every wait expiry uses `ExecutionWaitPolicy { expiry, on_expiry }`, where +`on_expiry` is `FailTask`, `FailRun`, or `ContinueWith { output }`. +`ExecutionTemporalTarget::At { at }` is an exact UTC instant and is allowed in +one-off compiled plans. `After { delay_seconds }` is nonzero and resolves from +the instant the task actually enters its wait, not from planning or run +admission. Reusable skill templates accept only `After`; this preserves their +meaning whenever earlier dependencies take a different amount of time. Dependencies provide parallelism and joins; there are no implicit start, parallel, join, worker, tool, action, skill-action, or memory node kinds. Dynamic @@ -245,13 +260,13 @@ compiled worst-case estimate above the unattended threshold is persisted as Ready map items are materialized as stable tasks keyed by `(run_uid, node_id, item_key)`. Materialization is deterministic and complete, -but pending rows are storage-only: they have no live `ExecutionTask` invocation -until the run admits them into its positive physical window. The default -`execution.max_in_flight_tasks` is 64 and is independent of logical `max_tasks` -and provider-specific concurrency. `ExecutionRun` starts the first stable -undispatched tasks that fit the window, acknowledges the processed wake epoch, -and suspends on the epoch promise plus its attached task-call handles. Each -completed handle opens a slot for the next stable pending row. +but pending and waiting rows are storage-only. The controller admits ready rows +through tenant/fleet active-attempt buckets, dispatches at most +`execution.dispatch_batch_size`, applies at most +`execution.maximum_activation_steps`, commits its processed wake generation, +and returns. `execution.max_in_flight_tasks` remains a per-run ceiling within +the wider tenant/fleet admission and is independent of logical `max_tasks` and +provider-specific concurrency. `ExecutionTaskId` is UUIDv5 over length-framed run UUID, node ID, and item key. Ordinary tasks use item key `""`, map tasks use the typed canonical extracted @@ -486,17 +501,21 @@ binding without changing handler contracts. Core production bindings: -- Virtual objects: `Session`, `Worker`, `Tenant`, `CronJob`, `IngestionVO`. +- Virtual objects: `Session`, `Worker`, `Tenant`, `CronJob`, `IngestionVO`, + `ExecutionRunController`, and the fleet-keyed `ExecutionDispatchDrain` behind the stateless + head-coalescing `ExecutionDispatcher` router. `Session` and `Worker` additionally own the generation fence and the derived scheduling index for the action reviews their own turns raise. - Services: `ActionReviews`, `AgentDefinitions`, `Agents`, `AdminMaintenance`, `ApiKeys`, `Artifacts`, `Authz`, `AuthzChallenges`, - `ConnectorConnections`, `Contacts`, `Execution`, `Experiments`, + `ConnectorConnections`, `Contacts`, `DurableTimeout`, `Execution`, + `ExecutionDispatcher`, `ExecutionDispatchReconciler`, `ExecutionRetention`, `ExecutionSchedule`, + `ExecutionTrigger`, `Experiments`, `GraphMemoryMaint`, `Knowledge`, `LearningReview`, `LLMGateway`, `Memory`, `NeonMaint`, `Privacy`, `SessionStore`, `Skills`, `Tenants`, `ToolExecutor`, `ActionPolicy` -- Workflows: `ExecutionRun`, `ExecutionTask`, `KnowledgeSyncIngestion`, - `Consolidate`, `ExperimentRun`, `ExperimentTrialRun`, +- Workflows: `KnowledgeSyncIngestion`, `Consolidate`, `ExecutionTaskAttempt`, + `ExecutionCompensationAttempt`, `ExperimentRun`, `ExperimentTrialRun`, `SkillLearning`, `TenantPurge`, `TurnExecution`, `WorkerTurnExecution` @@ -542,10 +561,16 @@ child turn and reports turn-scoped mutations back to the VO. Segment assessment happens at turn, segment, idle, cancellation, and timeout boundaries as an auditable learning artifact, not as a live-loop control signal. -`ExecutionRun` and `ExecutionTask` are the separate durable bulk-execution -family. Their full state and aggregate counters come from execution persistence, -not the `Session` VO. A run links to its owning session only for compact -progress, exact input requests, and one deduplicated terminal synthesis turn. +Long-horizon execution is a separate bounded-activation family. Postgres owns +its complete run/task/compensation state and counters; Restate owns only short +`ExecutionRunController`, `ExecutionTaskAttempt`, `ExecutionCompensationAttempt`, +`ExecutionTrigger`, `ExecutionDispatcher`, `ExecutionDispatchDrain`, and +`ExecutionDispatchReconciler` +activations. `ExecutionSchedule` and `DurableTimeout` provide bounded schedule +and timeout mutations, while `ExecutionRetention` archives or deletes one +terminal-detail page per activation. A run links to its owning session only for compact progress, exact +input requests, and one deduplicated terminal synthesis turn. Parked work owns +no handler invocation, attempt reservation, or hand. Coordinator turns can return while detached workers keep running across non-sticky replicas. Cadence-limited turn progress is delivered directly from @@ -604,7 +629,7 @@ User message -> TurnExecution selects Respond, Execute, or NeedsInput Respond: one model response, no tools or planning call Execute/Inline: bounded model/tool loop; optional Worker delegation - Execute/Durable: instantiate/compile, persist, and detach ExecutionRun + Execute/Durable: instantiate/compile, persist, and dispatch controller activation NeedsInput: deterministic bounded clarification -> Query rewrite may mark `is_new_task` -> SegmentTracker opens or rolls a task segment @@ -785,7 +810,7 @@ separate surfaces: gateway. The typed decision and policy binding are terminal evidence. Execution targets invoke a serving skill revision's exact pinned `execution_plan` through the same origin-bound planning/admission path, start the common - `ExecutionRun`, and link its `execution_run_uid`. The `moa.experiment_run` row is the experiment ledger and + execution run, and link its `execution_run_uid`. The `moa.experiment_run` row is the experiment ledger and links to the session, execution run, pinned artifact revisions, and `analytics.score_run`. `ExperimentTrialRun` owns per-trial simulator execution. The public edge diff --git a/docs/02-brain-orchestration.md b/docs/02-brain-orchestration.md index 220a36aa4..458a4706c 100644 --- a/docs/02-brain-orchestration.md +++ b/docs/02-brain-orchestration.md @@ -13,7 +13,7 @@ _Restate orchestration, hosted API runtime mode, turn execution, and workers._ - Worker VO: `crates/moa-orchestrator/src/objects/worker/` - Turn workflows: `crates/moa-orchestrator/src/workflows/turn_execution/mod.rs` and `crates/moa-orchestrator/src/workflows/worker_turn_execution.rs` - Execution domain: `crates/moa-execution/` -- Execution workflows: `crates/moa-orchestrator/src/workflows/execution_run.rs` and `crates/moa-orchestrator/src/workflows/execution_task.rs` +- Execution persistence/activation contracts: `crates/moa-execution/src/repository/` and `crates/moa-orchestrator/src/runtime/endpoint.rs` - CronJob VO: `crates/moa-orchestrator/src/objects/cron_job.rs` - Pipeline assembly: `crates/moa-brain/src/pipeline/mod.rs` - Sandbox workspace contract: `docs/25-sandbox-workspaces.md` @@ -54,9 +54,9 @@ Core production Restate bindings: | Restate primitive | Handlers | |---|---| -| Virtual Object | `Session`, `Worker`, `Tenant`, `CronJob`, `IngestionVO` | -| Service | `ActionReviews`, `AgentDefinitions`, `Agents`, `AdminMaintenance`, `ApiKeys`, `Artifacts`, `Authz`, `AuthzChallenges`, `Contacts`, `Execution`, `Experiments`, `GraphMemoryMaint`, `Knowledge`, `LearningReview`, `LLMGateway`, `Memory`, `NeonMaint`, `Privacy`, `SessionStore`, `Skills`, `Tenants`, `ToolExecutor`, `ActionPolicy` | -| Workflow | `ExecutionRun`, `ExecutionTask`, `ExecutionCompensation`, `KnowledgeSyncIngestion`, `Consolidate`, `SkillLearning`, `TurnExecution`, `WorkerTurnExecution`, `ExperimentRun`, `ExperimentTrialRun` | +| Virtual Object | `Session`, `Worker`, `Tenant`, `CronJob`, `IngestionVO`, `ExecutionRunController`, fleet-keyed `ExecutionDispatchDrain` | +| Service | `ActionReviews`, `AgentDefinitions`, `Agents`, `AdminMaintenance`, `ApiKeys`, `Artifacts`, `Authz`, `AuthzChallenges`, `Contacts`, `DurableTimeout`, `Execution`, `ExecutionDispatcher`, `ExecutionDispatchReconciler`, `ExecutionRetention`, `ExecutionSchedule`, `ExecutionTrigger`, `Experiments`, `GraphMemoryMaint`, `Knowledge`, `LearningReview`, `LLMGateway`, `Memory`, `NeonMaint`, `Privacy`, `SessionStore`, `Skills`, `Tenants`, `ToolExecutor`, `ActionPolicy` | +| Workflow | `KnowledgeSyncIngestion`, `Consolidate`, `ExecutionTaskAttempt`, `ExecutionCompensationAttempt`, `SkillLearning`, `TurnExecution`, `WorkerTurnExecution`, `ExperimentRun`, `ExperimentTrialRun` | Internal application boundaries for action reviews, builtin async-authz challenges, learning review, experiments, privacy, provider routing, and memory @@ -80,10 +80,15 @@ verified portable checkpoint; see [Sandbox Workspaces](25-sandbox-workspaces.md) `Artifacts` owns import, export, listing, validation, and publish for canonical skills, connectors, actions, and agents. `moa-execution` owns execution-plan compilation, pure scheduling, budgets, completion, and run/task persistence. -The `Execution` service exposes start, status, list, cancel, review, signal, and -bounded task-result operations. `ExecutionRun` and `ExecutionTask` own durable -graph execution. The open-ended agent loop remains in `Session` and -`TurnExecution`. +The `Execution` service exposes start, status, list, cancel, pause/resume, +review, signal, callback, and bounded result operations. Postgres owns durable +graph execution; `ExecutionRunController`, `ExecutionTaskAttempt`, +`ExecutionCompensationAttempt`, `ExecutionTrigger`, the head-coalescing `ExecutionDispatcher`, +`ExecutionDispatchDrain`, and +`ExecutionDispatchReconciler` are bounded activations over that state. +`ExecutionSchedule` and `DurableTimeout` own bounded schedule/timeout mutations; +`ExecutionRetention` owns bounded terminal-detail archival and deletion. The +open-ended agent loop remains in `Session` and `TurnExecution`. ## Session Flow @@ -143,7 +148,7 @@ result selects Execute/Inline without retry or planner fallback. - Execute carries exactly one explicit strategy. Inline runs the bounded root model/tool loop and may use conversational workers. Durable instantiates a pinned skill template or compiles a strict generated plan, persists it, - starts `ExecutionRun` detached, and returns acceptance without polling it + admits a run and dispatches `ExecutionRunController` detached, then returns acceptance without polling it from the root model. - NeedsInput appends one deterministic clarification carrying bounded missing fields. @@ -273,8 +278,8 @@ The owner then runs one continuation turn: no durable upgrade, one bounded `Respond` call, at most one visible answer. - Worker: one no-tools synthesis turn that updates local history and result, then normal parent-result and cleanup ownership resumes. -- `ExecutionTask`: no conversational callback at all; it stays on the durable - run/task outbox and ack path. +- Execution task owner: no conversational callback at all; it stays on the + durable task-generation outbox and acknowledgement path. Review timeout remains fail-closed and produces no conversational resume. The reaper durably releases the Session or Worker lifecycle hold through a @@ -303,22 +308,23 @@ internal atomic apply handler that performs the whole read-score-write and returns the exact transition, so two tool results landing in the same turn cannot interleave into a lost update. -An execution task instead scores against a circuit held by its own task -workflow. Do not "simplify" this into the Session VO alongside the other two. +An execution task instead scores against circuit state held by its own persisted +task generation and loaded by the bounded attempt. Do not "simplify" this into +the Session VO alongside the other two. A single shared circuit alternates owners as work moves between the coordinator and detached tasks, and adopting a new owner generation clears the capability map — so an attacker who has tripped the coordinator's circuit could reset it just by causing any detached task to run. Per-task circuits under per-task owners remove that move entirely. The cheaper arguments point the same way: a -task turn is a single sequential writer, so there is no interleaving to defend -against, and the journal replays its state deterministically without a VO -round-trip per scored output. The owner is +task attempt is a single sequential writer, so there is no interleaving to +defend against, and generation-fenced Postgres state survives retries without a +VO round-trip per scored output. The owner is generation-fenced — `Coordinator { turn_id, generation }`, `Worker { worker_id, turn_id, generation }`, or `ExecutionTask { run_uid, task_uid, generation }` — and state resets only for a genuinely new owner generation, never for a new input fingerprint, new tool -arguments, a fallback Hand provider, or a workflow replay. A delayed -action-review continuation runs under a new workflow id but keeps the original +arguments, a fallback Hand provider, or an activation replay. A delayed +action-review continuation runs under a new dispatch id but keeps the original logical owner, which is why the owner travels in the request rather than being inferred from the caller. @@ -374,7 +380,8 @@ relevant skill instructions; optional controls bound tools, tokens, and turns. Workers support interactive, steerable delegation inside Execute/Inline. They are not plan nodes, map items, reducers, or the bulk DAG substrate. Work that needs an explicit dependency graph, durable joins, scalable map materialization, review -waits, or exact coverage uses `ExecutionRun` and stable `ExecutionTask` rows. +waits, or exact coverage uses stable execution run and task rows plus bounded +controller/attempt activations. Conversational worker fan-out limits are separate from the `execution.max_in_flight_tasks` physical window that bounds live DAG tasks. @@ -456,7 +463,7 @@ state, reschedules once for the exact latest-heartbeat deadline when still fresh, or sends one joined `HeartbeatStale` signal to Session and stops when genuinely stale. Session never polls Worker state to discover staleness. -## Workflows And Execution Runs +## Workflows And Bounded Execution Activations Restate workflows run internal durable jobs: @@ -464,33 +471,35 @@ Restate workflows run internal durable jobs: - `KnowledgeSyncIngestion`: one tenant knowledge sync ingestion pass. - `TurnExecution`: one durable session turn keyed by `turn_id`; runs the top-level session brain loop and calls back to `Session` on completion, cancellation, or failure. - `WorkerTurnExecution`: one admitted worker turn keyed by `turn_id`; runs child-local LLM/tool loops and calls back to `Worker` with turn-scoped mutations. -- `ExecutionRun`: one immutable goal contract and active plan keyed by `run_uid`. -- `ExecutionTask`: one stable logical node or map item keyed by its task identity. - -These are workflow-shaped because rerunning the same logical job should be -explicit and observable. - -`ExecutionRun` loads the persisted canonical plan and asks the pure interpreter -for ready logical work. It materializes every stable row keyed by -`(run_uid, node_id, item_key)`, but pending rows remain storage-only. The -positive `execution.max_in_flight_tasks` setting (64 by default) is the physical -window for live attached `ExecutionTask` calls and is independent of logical -`max_tasks` and provider concurrency. - -On each dispatch step the run fills only open window slots with the first stable -undispatched rows, acknowledges the processed Postgres wake epoch, and suspends -on the matching Restate promise plus the attached calls it owns. A completed -handle opens one slot, which the run refills before parking again. The only -advancement sources are an already-persisted run/task transition named by a wake -epoch or completion of an attached task call. Pending rows do not own a task -runtime, and elapsed time never causes a scan or dispatch. - -`ExecutionTask` atomically reserves its worst-case integer cost, token, task, -tool-call, retrieved-byte, and deadline allowance before dispatch. A failed -reservation starts no work. It resolves only compiler-approved references, -executes one governed capability or bounded agent task, reconciles actual usage, -persists citations and output, and completes through a generation fence so a -stale attempt cannot overwrite newer work. + +Long-horizon execution is deliberately not workflow-shaped. A run may remain +nonterminal for seconds through weeks, but no Restate invocation spans that +lifetime. Postgres stores the immutable admitted `Identity`, plan, scheduler +projection, run and task generations, attempt leases, compensation stack, +waits, triggers, external jobs, capacity reservations, and dispatch outbox. + +`ExecutionRunController/advance` serializes activations by `run_uid`, claims one +persisted wake epoch, performs at most `maximum_activation_steps`, dispatches at +most `dispatch_batch_size` ready tasks, commits progress, and returns. +`ExecutionTaskAttempt/run` and the `ExecutionCompensationAttempt` slice each +execute one immutable dispatch generation and return after outcome, retry, +review, signal, timer, pause, external-job start, or watchdog classification. +Every stale attempt loses its generation fence. + +Input, review, signal, timer, pause, and external-job waits are storage-only. +Entering a wait resolves an `After { delay_seconds }` target from that exact +wait-entry instant or retains an explicit UTC `At { at }`, persists `due_at`, +releases active-attempt and hand capacity, and enqueues an immutable trigger. +Reusable templates accept only nonzero `After`; generated one-off plans may use +`At`. `ExecutionTrigger/fire` and the reconciliation owner redeliver the same +generation-fenced transition through the outbox, so no polling workflow is +needed. + +Task dispatch atomically reserves tenant and fleet active-attempt capacity plus +the task's worst-case integer cost, token, task, tool-call, retrieved-byte, and +deadline allowance. Parked-run, scheduled-trigger, and external-job ceilings +are independent durable capacity classes. Weighted tenant dispatch prevents a +single tenant from consuming the fleet queue. A sandbox-using worker or execution task is also the only valid owner of a `SandboxWorkspace`; a coordinator/bare session is not. Mutating sandbox tools @@ -500,7 +509,7 @@ Sandbox dispatch without a typed worker or execution-task workspace scope is rejected before workspace reads or provider I/O. The plan is an acyclic graph with exactly `Capability`, `Agent`, `Map`, -`Reduce`, `Review`, `WaitSignal`, and `Output`. A map task is only a capability +`Reduce`, `Review`, `WaitSignal`, `WaitUntil`, and `Output`. A map task is only a capability or agent and cannot nest another map. Agent tasks can use declared instruction-only skills and capabilities with bounded turns and budgets. They cannot mutate the graph. Unexpected conditions return typed `NeedsInput` or @@ -510,7 +519,7 @@ Repeated hashes, recurring failure fingerprints, no progress, deadline, or resource exhaustion terminate with exact partial/blocked coverage instead of an infinite loop. -Cancellation first fences new reservations and cancels and joins active tasks. +Cancellation first fences new reservations and cancels active attempts. The plan's explicit policy then either retains already committed effects or enters `Compensating` and invokes atomically registered compensators in reverse commit order. Compensation uses stable identities and generation fencing, so a @@ -523,12 +532,11 @@ output, citations, failures, and gaps to the owning session. The session starts at most one deduplicated synthesis turn for the originating user sequence; it does not ingest every raw map output or poll the run through the root model. -Public `confirm`, `cancel`, `deliver_input`, `decide_review`, -`deliver_signal`, and `apply_amendment` requests acknowledge success only after -the repository mutation commits, any exact task-specific shared handler accepts -it, and `ExecutionRun::wake` accepts the committed epoch. Task terminal outcomes -may send their run wake detached because outcome and epoch are already committed -and the run retains the attached child call as the second recovery path. +Public `confirm`, `cancel`, `pause`, `resume`, `deliver_input`, `decide_review`, +`deliver_signal`, external callback, and `apply_amendment` requests acknowledge +success only after the repository mutation and generation-fenced dispatch row +commit. If immediate delivery fails, the singleton maintenance owner reclaims +the outbox row; replay cannot repeat the logical transition. Behavior-lab execution uses the same reserve-before-dispatch rule. An `ExperimentRun` requires one exact immutable `experiment_plan` revision and @@ -553,9 +561,19 @@ the same persisted projection, task rows, planning audits, and bounded session event evidence as runtime inspection; they do not reconstruct success from a prose transcript. -Reusable scheduled work is anchored by the `CronJob` virtual object. Each job -key stores its cron expression, timezone, target service handler, and a version -counter that invalidates stale delayed ticks after reconfiguration. +Reusable product execution schedules live in `moa.execution_schedule`. Each +occurrence has a stable schedule/occurrence identity, admitted owner `Identity`, +timezone-aware policy, misfire policy, concurrency policy, next occurrence, and +generation. The maintenance owner incrementally materializes immutable +`schedule_occurrence` triggers; start-run admission remains tenant/fleet +capacity-gated and idempotent. Stopping or reconfiguring a schedule advances a +monotonic incarnation tombstone, and every tick from an older incarnation is a +no-op. + +Platform maintenance schedules remain anchored by the `CronJob` virtual object. +Each job key stores its cron expression, timezone, target service handler, and +the same monotonic incarnation rule; stop/reconfigure never resets the counter +or lets a late tick regain authority. ### Background Maintenance Jobs diff --git a/docs/05-session-event-log.md b/docs/05-session-event-log.md index b7aa4ff8c..915e4fd76 100644 --- a/docs/05-session-event-log.md +++ b/docs/05-session-event-log.md @@ -18,7 +18,9 @@ Postgres stores: - task segments - learning log entries - live behavior experiment run metadata -- execution run/task state, immutable plan snapshots, and completion results +- execution run/node/task/attempt/compensation state, immutable plan snapshots, + admitted identities, waits, triggers, schedules, external jobs, capacity, + dispatch outbox, and completion results - normalized execution route, planner-call, and compiler audit records - graph changelog outbox rows and per-tenant changelog versions - large event payload claim-check blobs @@ -334,24 +336,41 @@ the originating user sequence and run ID. durable typed DAG work. They are separate from session events and protected by the same tenant/contact/admin scope rules. -An execution-run row stores its immutable `ExecutionGoalContract`, canonical +An execution-run row stores its immutable admitted `Identity`, +`ExecutionGoalContract`, canonical initial plan, active plan, plan revision and append-only amendment history, plan hashes, skill-template or compiled-plan provenance, input/output, completion-check evidence, terminal gaps, status, integer budget and usage, -aggregate counters, owning session/tenant/user scope, idempotency key, and -timestamps. It cannot be marked `completed` while a required deliverable, +aggregate counters, controller generation/activation state, next wake, +owning session/tenant/user scope, idempotency key, and timestamps. The full +identity tuple (`identity_type`, ID, tenant ID, API-key ID, and +`acting_on_behalf_of`) is persisted at admission and never reconstructed from a +later session, contact, or request. A run cannot be marked `completed` while a required deliverable, coverage item, schema check, citation requirement, or budget/deadline check is unsatisfied. An execution-task row stores one logical node or map-item instance, unique by `(run_uid, node_id, item_key)`. It records requirement IDs, plan revision, -status, attempt, generation fence, input/output/error, reserved and actual -usage, citations, and timestamps. Atomic SQL reserves every worst-case budget -dimension before dispatch. Generation-fenced completion prevents a stale retry -from overwriting current state; cancellation prevents new reservations while -leaving completed results queryable. Deterministic materialization may create -every pending row, but pending rows remain storage-only until `ExecutionRun` -admits them into its positive `execution.max_in_flight_tasks` window. +status, active attempt/dispatch identity, generation fence, wait state and +`due_at`, input/output/error, reserved and actual usage, citations, and +timestamps. `moa.execution_node_state` stores aggregate node counters; +`moa.execution_compensation` stores strict reverse-order undo state. + +Atomic SQL reserves every worst-case budget dimension and both tenant/fleet +active-attempt capacity before dispatch. Generation-fenced completion prevents +a stale retry from overwriting current state; cancellation prevents new +reservations while leaving completed results queryable. Deterministic +materialization may create every pending row, but pending and waiting rows are +storage-only until a bounded controller activation dispatches an attempt. + +`moa.execution_trigger` and `moa.execution_dispatch_outbox` form the recovery +bridge to Restate. Exact run deadlines, task timers, wait expiry, watchdogs, +external reconciliation, and schedule occurrences are immutable trigger rows; +the same transaction that changes product state inserts an idempotent dispatch +row. `moa.execution_external_job` records asynchronous provider identity, +generation, callback disposition, and sparse reconciliation. Capacity bucket, +tenant dispatch, reservation, and schedule rows keep fairness and admission +durable without using Valkey as an authority. ## Idempotent Append @@ -477,6 +496,14 @@ The orchestrator publishes live runtime events during turn execution. Visible history is recoverable from the durable event log; hot turn/worker progress is queryable through Restate where a durable execution primitive owns it. +Long-horizon execution replay is product-state-first, not a reconstruction from +session events or a lifetime journal. The controller reloads the admitted +identity and current Postgres projection, claims its generation/wake epoch, +and applies the pure scheduler. A restored or rebuilt Restate cluster receives +the same activation through the transactional outbox and due-trigger scan. +External effects are resumed only from their persisted idempotency and +unknown-outcome ledgers. + Replay uses persisted session contact metadata; clients cannot provide a new contact per message to change historical attribution. Tool-call records only need the session id because the session store can recover the contact binding. diff --git a/docs/06-hands-and-mcp.md b/docs/06-hands-and-mcp.md index 9d2ee114c..cf9ef8e79 100644 --- a/docs/06-hands-and-mcp.md +++ b/docs/06-hands-and-mcp.md @@ -361,14 +361,20 @@ fences provider compute. Siblings never share a writable workspace or hand: only when the checkpoint commit barrier published them. This Worker model remains for conversational delegation in `act`; Worker is not -an execution-plan node or bulk DAG primitive. A sandbox-using `ExecutionTask` -gets the same isolation and generation-fenced recovery under its task identity. -Dynamic map execution materializes every stable logical item deterministically -after atomic budget reservation, -but only the positive `execution.max_in_flight_tasks` window owns live attached -task calls and therefore live task sandboxes. Pending rows remain storage-only; -provider pacing and governed hand capacity apply independent limits within the -run-owned window. +an execution-plan node or bulk DAG primitive. A sandbox-using bounded +`ExecutionTaskAttempt` gets the same isolation under its persisted logical task +and generation. Dynamic map execution materializes stable logical items +deterministically, but only attempts holding tenant/fleet active-attempt and +`active_hands` reservations may own live compute. Pending, input/review/signal, +timer, external-job, and paused rows remain storage-only. + +Every attempt yield is a compute-release boundary. If the attempt may need its +filesystem again, it quiesces the writer, publishes and verifies the portable +checkpoint, releases the exact hand/capacity generation, and destroys provider +compute before recording the wait or returning. Resume provisions fresh +compute, restores the committed checkpoint, and reacquires current policy and +capacity. A parked task retaining an active hand is a correctness and cost +incident, not an optimization choice. Before the LLM call for a turn, the context pipeline selects relevant skills. The selected trusted sandbox file references are copied into `ToolCallRequest`. @@ -526,6 +532,7 @@ connection lifecycle, credential vault, management API, and rollout contract. - Use parsed command normalization for shell action-policy patterns. - Keep generated-code compute ephemeral; persist only the filesystem-only mutable root through the governed `SandboxWorkspace` commit barrier. -- Destroy hands when their worker/execution-task scope - stops so stale credentials and processes do not linger; retain or delete the +- Destroy hands when their worker scope stops or an execution attempt reaches + any wait, retry, pause, external-job, compensation, or terminal yield so + stale credentials and processes do not linger; retain or delete the workspace only through its independent policy and purge lifecycle. diff --git a/docs/10-technology-stack.md b/docs/10-technology-stack.md index 558c3e973..814b20fb0 100644 --- a/docs/10-technology-stack.md +++ b/docs/10-technology-stack.md @@ -66,13 +66,15 @@ those instances explicitly rather than installing process globals. | Containers/tools | Docker integration, Daytona/E2B HTTP clients, MCP revision `2026-07-28` Streamable HTTP client/server | | Lineage and audit | OTel/OpenInference bridge, Parquet/Arrow cold export, Object Lock audit storage | -`moa-migrations` owns the fresh-install-only, contiguous 57-file PostgreSQL -chain and the central table-ownership manifest. The current 143 table -families span 148 `CREATE TABLE` declarations and map one-to-one to 143 -ownership entries. `cargo run -p xtask --locked -- check-migrations` enforces -this contract. V57 adds provider-visible hand operation identities, absolute +`moa-migrations` owns the fresh-install-only, contiguous V1-V60 PostgreSQL +chain and the central table-ownership manifest. Every final logical table +family must map one-to-one to an ownership entry; the repository's +`xtask check-migrations` command enforces this contract. V57 adds +provider-visible hand operation identities, absolute create deadlines, delayed reconciliation, and a database generation-rotation -guard that rejects pre-V57 writers before provider I/O. +guard that rejects pre-V57 writers before provider I/O. V59 installs bounded +execution activations, triggers, schedules, external jobs, admission, and the +dispatch outbox; V60 installs exact active-compute capacity reservations. ## External Services @@ -94,7 +96,7 @@ the Compose `pii` profile when `MOA_PII_SERVICE_URL` is configured. | Service | Purpose | |---|---| | Restate | Durable orchestration engine | -| Postgres/Neon | Product data store, relational graph storage, and pgvector transactional vector source | +| Postgres/Neon | Product data store, long-horizon execution/wait/schedule/outbox authority, relational graph storage, and pgvector transactional vector source | | Redis or Valkey | Shared runtime cache for orchestrator replicas | | AWS S3 or GCS | Session attachment byte storage in cloud | | Turbopuffer | Cloud vector backend for storage partitions configured away from local pgvector | @@ -279,7 +281,15 @@ and deployment setup. Key groups: Implemented architectural pillars: - Restate cloud orchestration with session, worker, tenant, service, and workflow handlers. -- Dynamic `respond`/`act`/`run` routing with `ExecutionRun` and `ExecutionTask` as the only durable typed-DAG runtime. +- Dynamic `respond`/`act`/`run` routing with Postgres-backed execution runs and + the bounded `ExecutionRunController`, `ExecutionTaskAttempt`, + `ExecutionCompensationAttempt`, `ExecutionTrigger`, `ExecutionDispatcher`, + fleet-keyed `ExecutionDispatchDrain`, and `ExecutionDispatchReconciler` + activation family as the durable typed-DAG runtime; the drain serializes + bounded outbox delivery and admission against fleet-global capacity. + `ExecutionSchedule` and `DurableTimeout` own bounded schedule and timeout + delivery, while `ExecutionRetention` owns bounded terminal-detail archival + and deletion. - `moa-execution` ownership of canonical plan compilation, bindings, pure scheduling, pure integer budget transitions, completion checks, and replan-stop evaluation; these core APIs have no I/O, provider, Restate, or persistence dependencies. - One `moa-orchestrator` production binary for local development and cloud execution, with domain logic kept behind in-process application and repository boundaries. - Constructor-based runtime composition: `RuntimeDeps::build` constructs the @@ -333,6 +343,15 @@ MOA_KMS_ROOT_KEY_DIR=/var/run/secrets/moa-kms/root-keys MOA_KMS_REQUIRED_GENERATION=primary ``` +Production runs two roles from the same immutable orchestrator image. The +versioned `RestateDeployment` serves handler, SCIM, channel, and credential +ingress. A stable one-replica `moa-maintenance` Deployment runs +`moa-orchestrator maintenance`, exposes health/metrics only, and owns trigger +and dispatch-outbox reconciliation, retention, approval/authz reconciliation, +workspace/hand reaping, and provider inventory. Serving revisions never start +duplicate correctness scanners. Draining Restate revisions retain at least one +recovery replica and autoscale down as their bounded invocations finish. + Production Kubernetes provisions `moa-kms-root-keys` externally and mounts it read-only into orchestrator pods only. The edge never receives root-key material. The local Kustomize overlay and Docker Compose use a fixed public @@ -441,15 +460,13 @@ and Postgres, never in process memory or Redis: RestateDeployment readiness, rather than an Admin API call from each replica, gates registered service traffic. -Durable execution runs use a separate topology. `ExecutionRun` and -`ExecutionTask` workflows recover from Postgres execution rows plus Restate -journals; the `Session` VO stores only compact linkage and terminal-synthesis -dedupe state. Ready map items have stable logical task identities and are fully -materialized, but pending rows are storage-only. Each run owns at most -`execution.max_in_flight_tasks` attached `ExecutionTask` calls (64 by default), -a positive physical window separate from logical `max_tasks` and provider -concurrency. The run fills open slots in stable order, acknowledges its -processed wake epoch, and suspends on the epoch promise plus owned task handles; -only a persisted transition or attached task completion advances it. Public -execution mutations return accepted responses only after the committed task/run -wake is acknowledged. +Durable execution runs use a separate topology. Postgres owns run, node, task, +attempt, compensation, wait, trigger, schedule, external-job, capacity, and +dispatch-outbox recovery; the `Session` VO stores only compact linkage and +terminal-synthesis dedupe state. Ready map items have stable logical task +identities, but pending and waiting rows are storage-only. Controller +activations advance and dispatch bounded batches, task and compensation +activations run one generation, and every activation returns. The singleton +maintenance role repairs due triggers and undelivered outbox rows. No process +memory, Valkey entry, or lifetime Restate invocation is required to keep a +day- or week-scale run alive. diff --git a/docs/12-restate-architecture.md b/docs/12-restate-architecture.md index da1d6be25..ecb96e94f 100644 --- a/docs/12-restate-architecture.md +++ b/docs/12-restate-architecture.md @@ -6,8 +6,10 @@ _Durable execution on Restate services, virtual objects, and workflows._ Restate is MOA's durable execution engine. Postgres remains the product record for sessions, events, memory, analytics, learning, lineage, and audit. Restate -owns orchestration state: queues, workflow progress, awakeables, retries, and -handler journals. +owns orchestration state: queues, bounded activation delivery, awakeables, +retries, and handler journals. Postgres is the recovery authority for +long-horizon execution; a run never depends on one invocation surviving for its +product lifetime. Restate does not own arbitrary sandbox filesystem bytes and cannot make them durable merely because a session or workflow is durable. Sandbox compute is @@ -23,9 +25,9 @@ what must stay out of Restate state. | Restate primitive | Use in MOA | Reason | |---|---|---| -| Service | Durable stateless calls such as `ActionReviews`, `AuthzChallenges`, `Execution`, `LearningReview`, `ToolExecutor`, `LLMGateway`, `SecurityEvents`, `SessionStore`, `Authz`, `Memory`, `Skills`, `Tenants` | Durable RPC with retries, no keyed state. | -| Virtual Object | `Session`, `Worker`, `Tenant`, `CronJob`, `IngestionVO` | Single-writer-per-key semantics and small hot state. | -| Workflow | `TurnExecution`, `WorkerTurnExecution`, `ExecutionRun`, `ExecutionTask`, `ExecutionCompensation`, `KnowledgeSyncIngestion`, `Consolidate`, `ExperimentRun`, `ExperimentTrialRun` | One logical run, task, or compensation per ID with explicit progress and completion. | +| Service | Durable calls such as `ActionReviews`, `AuthzChallenges`, `DurableTimeout`, `Execution`, `ExecutionDispatcher`, `ExecutionDispatchReconciler`, `ExecutionRetention`, `ExecutionSchedule`, `ExecutionTrigger`, `LearningReview`, `ToolExecutor`, `LLMGateway`, `SecurityEvents`, `SessionStore`, `Authz`, `Memory`, `Skills`, `Tenants` | Durable bounded RPC. | +| Virtual Object | `Session`, `Worker`, `Tenant`, `CronJob`, `IngestionVO`, `ExecutionRunController`, fleet-keyed `ExecutionDispatchDrain` | Single-writer-per-key semantics and small hot state; run controllers serialize per-run activation while the drain serializes admission against fleet-global capacity. Producer kicks enter the stateless `ExecutionDispatcher`, which coalesces them by the exact indexed outbox head. | +| Workflow | `TurnExecution`, `WorkerTurnExecution`, `ExecutionTaskAttempt`, `ExecutionCompensationAttempt`, `KnowledgeSyncIngestion`, `Consolidate`, `ExperimentRun`, `ExperimentTrialRun` | One bounded logical job per ID with explicit progress and completion. | Use the weakest primitive that gives the needed correctness property. Do not use a workflow for conversational actors; do not use virtual-object state as a @@ -39,9 +41,10 @@ product database. | Top-level turn | Workflow | `turn_id` | | Worker | Virtual Object | `worker_id` | | Worker turn | Workflow | `turn_id` | -| Execution run | Workflow plus `moa.execution_run` | `run_uid` | -| Execution task | Workflow plus `moa.execution_task` | stable hash of `(run_uid, node_id, item_key)` | -| Execution compensation | Workflow plus `moa.execution_compensation` | stable compensation registration ID | +| Execution run | Postgres aggregate plus keyed `ExecutionRunController` activations | `run_uid` | +| Execution task | Postgres row plus bounded `ExecutionTaskAttempt` activation | stable hash of `(run_uid, node_id, item_key)` and generation | +| Execution compensation | Postgres stack plus bounded compensation-attempt slice | stable compensation registration and dispatch IDs | +| Execution timer/deadline/schedule | Immutable Postgres trigger plus `ExecutionTrigger` delivery | `trigger_uid` and generation | | Sandbox workspace lifecycle | `moa-hands` application/repository boundary called from Restate handlers/workflows, plus Postgres rows | `workspace_id` with writer and instance generations | | Tool execution | Service | none | | LLM call | Service | none | @@ -54,9 +57,9 @@ product database. Sessions and workers are virtual objects because they receive multiple messages over time. `TurnExecution` and `WorkerTurnExecution` are workflows -because one admitted turn should have one observable durable run. `ExecutionRun` -and `ExecutionTask` are workflows because typed graph work has stable run/task -identities, durable waits, recovery, and explicit terminal outcomes. Tenant +because one admitted turn should have one observable bounded run. Typed graph +work instead uses Postgres state plus short controller, attempt, and trigger +activations so waits do not pin handlers or deployment revisions. Tenant knowledge sync ingestion and consolidation are workflows for the same reason. Knowledge index rebuild/rechunk and the hosted tenant `Eval` service are not runtime surfaces. Regression evals run through the platform-only @@ -97,7 +100,7 @@ Respond, Execute, or NeedsInput. Respond makes one no-tool model call. NeedsInput emits one bounded deterministic clarification. Execute carries one explicit internal strategy: Inline retains the bounded root tool loop and optional conversational Worker delegation; Durable persists an -immutable goal contract and canonical plan, starts `ExecutionRun` detached, and +immutable goal contract and canonical plan, dispatches `ExecutionRunController` detached, and returns without making the root model poll status. A bounded free-form classifier rationale may accompany the active turn, but it is never a workflow control input and is not persisted in route audits, runs, or analytics. @@ -159,9 +162,9 @@ Core production bindings: | Primitive | Handlers | |---|---| -| Virtual Object | `Session`, `Worker`, `Tenant`, `CronJob`, `IngestionVO` | -| Workflow | `TurnExecution`, `WorkerTurnExecution`, `ExecutionRun`, `ExecutionTask`, `ExecutionCompensation`, `KnowledgeSyncIngestion`, `Consolidate`, `ExperimentRun`, `ExperimentTrialRun` | -| Service | `ActionReviews`, `ActionReviewDispatcher`, `AgentDefinitions`, `Agents`, `AdminMaintenance`, `ApiKeys`, `Artifacts`, `Authz`, `AuthzChallenges`, `Contacts`, `Execution`, `Experiments`, `GraphMemoryMaint`, `Knowledge`, `LearningReview`, `LLMGateway`, `Memory`, `NeonMaint`, `Privacy`, `SecurityEvents`, `SessionStore`, `Skills`, `Tenants`, `ToolExecutor`, `ActionPolicy` | +| Virtual Object | `Session`, `Worker`, `Tenant`, `CronJob`, `IngestionVO`, `ExecutionRunController`, fleet-keyed `ExecutionDispatchDrain` | +| Workflow | `TurnExecution`, `WorkerTurnExecution`, `ExecutionTaskAttempt`, `ExecutionCompensationAttempt`, `KnowledgeSyncIngestion`, `Consolidate`, `ExperimentRun`, `ExperimentTrialRun` | +| Service | `ActionReviews`, `ActionReviewDispatcher`, `AgentDefinitions`, `Agents`, `AdminMaintenance`, `ApiKeys`, `Artifacts`, `Authz`, `AuthzChallenges`, `Contacts`, `DurableTimeout`, `Execution`, `ExecutionDispatcher`, `ExecutionDispatchReconciler`, `ExecutionRetention`, `ExecutionSchedule`, `ExecutionTrigger`, `Experiments`, `GraphMemoryMaint`, `Knowledge`, `LearningReview`, `LLMGateway`, `Memory`, `NeonMaint`, `Privacy`, `SecurityEvents`, `SessionStore`, `Skills`, `Tenants`, `ToolExecutor`, `ActionPolicy` | Feature-gated bindings: @@ -192,15 +195,16 @@ Restate state should be small, replay-safe, and useful only for orchestration. | Pending message queue | `Session` VO | | Current session turn progress | `TurnExecution` workflow | | Current worker turn progress | `WorkerTurnExecution` workflow | -| Execution goal, plans, provenance, budget, completion, aggregate counters | Postgres `moa.execution_run` | -| Execution task state, generations, reservations, usage, citations, outputs | Postgres `moa.execution_task` | +| Execution admitted identity, goal, plans, provenance, budget, completion, activation state, aggregate counters | Postgres `moa.execution_run` | +| Execution node/task/attempt/compensation state, generations, waits, reservations, usage, citations, outputs | Postgres `moa.execution_node_state`, `moa.execution_task`, and `moa.execution_compensation` | +| Execution deadlines, timers, wait expiry, watchdogs, schedules, external jobs, capacity, and activation delivery | Postgres trigger/schedule/external-job/capacity/dispatch-outbox tables | | Pending tenant action reviews | Postgres `tenant_action_reviews` rows | | Detached worker result waiters | `Worker` VO, resolved by child terminal delivery | | Child heartbeat, one outstanding liveness deadline, last turn summary, pending input requests | `Worker` VO state (`last_heartbeat_at`, liveness generation/outstanding state, `last_turn_summary`, `pending_input_requests`, `cleanup_generation`) | | Unread child signals, current fan-in generation/settlement, resume budget, pending resume | `Session` VO state (`unread_child_signals`, registered child generations, settled generation, `resume_budget`, `pending_parent_resume_signal`, `resume_turn`) | | Prompt-injection circuit for the coordinator turn | `Session` VO state (`security_circuit`) | | Prompt-injection circuit for worker turns | `Worker` VO state (`security_circuit`) | -| Prompt-injection circuit for one execution task turn | `ExecutionTask` workflow, journaled. Deliberately not merged into the Session VO: a shared circuit alternating owners would let a detached task's generation switch clear a tripped coordinator's score. See `docs/02-brain-orchestration.md`. | +| Prompt-injection circuit for one execution task generation | Postgres task-generation state loaded by its bounded attempt. Deliberately not merged into the Session VO: a shared circuit alternating owners would let a detached task's generation switch clear a tripped coordinator's score. See `docs/02-brain-orchestration.md`. | | Signed prompt-injection Detection Findings | Postgres `security_events`, written synchronously by the `SecurityEvents` service | | Tool result and assistant output | Postgres event log | | Graph memory, vectors, changelog | Postgres | @@ -243,12 +247,13 @@ coordination owners: | `Session` | Worker terminal transition | Worker persists terminal state and resolves result waiters, then awaits the sole Session terminal handler. The handler records exactly one failure consequence or, when the last registered child settles successfully or is cancelled, at most one `FanInSettled` consequence for that generation before acknowledgement. | | `Session` | Execution-run terminal synthesis request | The compact terminal run projection and synthesis dedupe are durable before the guarded synthesis turn is dispatched. | | `Worker` | Accepted follow-up/input, an admitted turn callback, or its own exact liveness/cleanup deadline | Worker state and generation fences select the exact continuation; no Session status read discovers Worker work. | -| `ExecutionRun` | A committed run/task mutation wake epoch or completion of an attached `ExecutionTask` call it owns | External mutations await the exact run wake. Task terminal delivery may send after commit because the owned attached call provides the second recovery path. | +| Execution run | A committed run/task/trigger/callback mutation and its dispatch-outbox row | `ExecutionRunController` claims the exact wake generation. Immediate delivery may fail because maintenance redelivers the persisted row. | -Input timeouts, cancellation deadlines, action-review deadlines, admission lease -heartbeats, and maintenance schedules remain exact safety or lease mechanisms; -they do not discover conversational or DAG work. No owner advances from a -repeating status read, recursive progress call, or elapsed-time task scan. +Input/review expiry, run deadlines, task timers/watchdogs, external reconcile, +and schedule occurrences are immutable execution triggers. Conversation lease +heartbeats and platform maintenance schedules remain separate safety +mechanisms. No owner advances from a repeating status read, recursive progress +call, or elapsed-time task scan. ## Main-Agent/Worker Coordination In Inline Execute @@ -350,51 +355,58 @@ coordinator turn and the whole recursive child tree (today's behavior); `CoordinatorOnly` cancels only the active `TurnExecution` and leaves children running. The dead `Soft`/`Hard` `CancelMode` is removed. -## Execution Run Coordination - -`ExecutionRun` is the only durable graph controller. It loads the persisted -goal contract and active canonical plan, asks the pure `moa-execution` -interpreter for ready logical work, and deterministically materializes all -stable task rows. Pending rows are storage-only: they own no Restate task -invocation until admitted into the positive -`execution.max_in_flight_tasks` window (64 by default). This physical window is -separate from the run's logical `max_tasks` budget and every provider-specific -concurrency limit. - -The run retains attached call handles only for tasks it owns. On a dispatch step -it computes the available slots, starts the first stable undispatched rows that -fit, acknowledges the processed Postgres wake epoch, and suspends on the -epoch-keyed Restate promise plus those attached handles. One completed handle -opens one refill slot. Cancellation fences every undispatched row in Postgres, -cancels only owned calls, and joins those calls before terminal settlement. The -only run advancement sources are an already-persisted task/run transition named -by its wake epoch or completion of an attached task call; elapsed time never -causes task discovery. - -Public `confirm`, `cancel`, `deliver_input`, `decide_review`, -`deliver_signal`, and `apply_amendment` handlers commit their repository -transition first, await the exact task shared handler when applicable, and await -`ExecutionRun::wake` for the committed epoch before returning an accepted -response. A task's terminal-outcome wake remains detached because the outcome -and epoch are committed first and the run-owned attached call is an independent -recovery path. - -The graph is acyclic and has exactly seven operations: `Capability`, `Agent`, -`Map`, `Reduce`, `Review`, `WaitSignal`, and `Output`. A map creates one task for +## Long-Horizon Execution Coordination + +Postgres is the only durable graph controller state. Admission persists the +complete `Identity`, immutable goal/plan, absolute run deadline, controller +generation, and initial dispatch row in one transaction. Every subsequent +controller, task, compensation, trigger, callback, or operator mutation loads +that identity; no activation re-derives authority from ambient request state. +Pending and waiting rows are storage-only. + +`ExecutionRunController/advance` is keyed by `run_uid`. One activation claims a +persisted wake epoch, applies at most `maximum_activation_steps`, dispatches at +most `dispatch_batch_size` stable ready rows, records aggregate progress, and +returns. `ExecutionTaskAttempt/run` executes one task generation within the +active-attempt timeout. Compensation uses one bounded +`ExecutionCompensationAttempt` slice per immutable dispatch identity. No +activation sleeps until a product event or retains an attached child for the +run lifetime. + +Public confirm/cancel/pause/resume/input/review/signal/callback/amendment +handlers commit their generation-fenced transition and transactional outbox row +before returning. Immediate Restate delivery is an optimization; the singleton +maintenance owner reclaims undelivered rows and due triggers. The stable +dispatch/trigger identity makes duplicate delivery a no-op. + +The graph is acyclic and has exactly eight operations: `Capability`, `Agent`, +`Map`, `Reduce`, `Review`, `WaitSignal`, `WaitUntil`, and `Output`. A map creates one task for each stable item key and cannot contain another map. Reduce uses structured batches; an agent reducer is a deterministic hierarchical tree bounded by `batch_size`. -Before dispatch, `ExecutionTask` atomically reserves worst-case microusd, -tokens, tasks, tool calls, retrieved bytes, and deadline allowance. A task that -cannot reserve does not start. On completion it reconciles actual integer usage -and writes output/citations through the current generation fence. Retry and -recovery can therefore neither double-spend nor let stale completion overwrite +`Review`, `WaitSignal`, run input, and `WaitUntil` carry explicit expiry or wake +targets. `At { at }` denotes an exact UTC instant and is valid for a generated +one-off plan. Nonzero `After { delay_seconds }` is resolved from the instant the +task enters the wait. Reusable templates reject `At` and use `After`, so earlier +dependency duration cannot make a template timer stale. Entering any wait +persists `due_at`, releases attempt and hand capacity, and schedules an immutable +trigger; expiry follows `FailTask`, `FailRun`, or `ContinueWith { output }`. + +Before dispatch, the repository atomically reserves worst-case microusd, +tokens, tasks, tool calls, retrieved bytes, deadline allowance, and tenant/fleet +active-attempt capacity. Active runs, active attempts, parked runs, scheduled +triggers, and external jobs have distinct tenant/fleet ceilings. Weighted +tenant dispatch controls fleet fairness. A task that cannot reserve starts no +work, and retry/recovery cannot double-spend or let a stale completion overwrite new work. Task-local agents may use declared instruction-only skills and governed -capabilities for a bounded number of turns. They return only `Completed`, -`NeedsInput`, `NeedsReplan`, or `Failed`. `NeedsInput` parks the exact run/task. +capabilities for a bounded number of turns. They return a terminal result, a +typed wait, a bounded replan request, or an asynchronous external-job start. +External jobs persist provider/job identity, generation, callback disposition, +and sparse reconciliation before the attempt returns; callbacks cannot bypass +the outbox or generation fence. A typed wait parks the exact run/task. `NeedsReplan` asks the planner for a structured amendment using the immutable goal, active plan, completed outputs, evidence, remaining budget, and current catalog. The compiler rejects changes to running/completed work, cycles, @@ -404,30 +416,21 @@ append canonical patch/hash/reason records. Repeated hashes or failure fingerprints, no progress, deadline, or resource exhaustion terminate with exact partial/blocked coverage; there is no arbitrary amendment-count cap. -Cancellation fences new reservations, cancels and joins active tasks, and then -applies the plan's explicit policy. `retain_effects` preserves committed work; -`compensate_committed` moves the run into nonterminal `Compensating` and drives -atomically registered compensators in reverse commit order with stable IDs and -generation fencing. Restarts skip already completed undo. Every root-turn, -worker-turn, and execution-task model call carries an internal typed workflow -owner to `LLMGateway`; execution tasks share their owning run's fence. The -gateway removes that metadata before provider -dispatch and, inside the replay-recorded `ctx.run`, observes a kind-scoped, -hashed cancellation fence in the required shared Valkey runtime cache. Each -workflow cancellation handler durably writes that fence before resolving its -local cancellation promise. This lets the gateway drop in-process provider I/O -even though a Restate cancellation signal cannot preempt an already-running -`ctx.run` closure. The resulting zero-usage cancelled completion becomes the -owner's normal `Cancelled` outcome and cannot create an empty `BrainResponse`; -a provider response already ready at the cancellation boundary retains its -actual billed usage. An ambiguous or failed compensator records -`manual_repair_required` instead of a false clean rollback. Forward and -compensation rows remain queryable. Terminal completion requires every -immutable goal requirement, +Cancellation fences new reservations and advances active attempt generations. +`retain_effects` preserves committed work; `compensate_committed` moves the run +to `Compensating` and dispatches registered compensators in strict reverse +commit order. Each bounded slice either commits an outcome, schedules a retry or +review, pauses, or records an ambiguous result as `manual_repair_required`. +Completed undo is never repeated. + +Every root-turn, worker-turn, and execution-attempt model call carries its typed +owner to `LLMGateway`; the gateway removes that metadata before provider +dispatch and observes the shared hashed cancellation fence around active +provider I/O. Postgres, not Valkey, decides the task generation and terminal +state. Terminal completion requires every immutable goal requirement, deliverable, coverage item, schema/citation check, and budget/deadline check to -pass. The run writes compact aggregate output, citations, failures, and gaps, -emits terminal session events, and requests at most one synthesis turn. Raw map -outputs stay in execution persistence, not session history or Session VO state. +pass. Compact aggregate output enters session history; raw task and +compensation evidence remains in execution persistence. ## Determinism Rules @@ -476,8 +479,9 @@ The continuation fact `ActionReviewContinuationRequested` is deduped on dispatches a second continuation turn. Owner generations and continuation scheduling live in Restate VO state as a derived index; the authoritative facts remain the `tenant_action_reviews` row plus the durable `ActionReviewDecided` and -terminal `ToolResult`/`ToolError` events. An `ExecutionTask` owner is excluded -from this path and keeps its run/task/generation outbox and ack contract. +terminal `ToolResult`/`ToolError` events. An execution-task owner is excluded +from this conversational path and keeps its persisted task/generation outbox +and acknowledgement contract. Timed-out Session and Worker reviews use a separate durable release-only delivery on the review row; it removes the lifecycle hold but cannot schedule a continuation. @@ -500,7 +504,7 @@ MOA supports both: | Restate invocation cancellation | Operator hard-stops a stuck invocation through Restate admin APIs. | `Execution/cancel` is the product cancellation path for a run. It fences new -task reservations and settles active tasks before applying the plan's +task reservations and advances active-attempt generations before applying the plan's explicit `retain_effects` or `compensate_committed` policy. Only after retained effects or reverse-order compensation reaches a durable settled state does the run publish terminal session delivery; all evidence is preserved. @@ -539,26 +543,31 @@ provider adapters remain the only owner of repeated paid HTTP attempts; the reuses its durable result rather than multiplying provider calls. `LLMGateway`, `ToolExecutor`, `TurnExecution`, `WorkerTurnExecution`, -`ExecutionRun`, `ExecutionTask`, and `ExecutionCompensation` are ingress-private. They are reachable -only service-to-service; public traffic enters through edge-owned product -surfaces. Their inactivity timeout is 360 seconds with a 60-second abort cleanup -window. Provider stream configuration is capped at 300 seconds, leaving that -cleanup margin. Durable human waits suspend and therefore do not need a larger +`ExecutionRunController`, `ExecutionTaskAttempt`, `ExecutionCompensationAttempt`, +`ExecutionTrigger`, `ExecutionDispatcher`, fleet-keyed `ExecutionDispatchDrain`, +`ExecutionDispatchReconciler`, `ExecutionRetention`, and `DurableTimeout` are +ingress-private. `ExecutionSchedule` remains the tenant-authorized schedule +mutation surface. The private handlers are reachable only service-to-service; +public traffic enters through edge-owned product surfaces. Their inactivity +timeout is 360 seconds with a 60-second abort cleanup window. Provider stream +configuration is capped at 300 seconds, leaving that cleanup margin. Durable +human waits suspend and therefore do not need a larger inactivity timeout. The normal product endpoint does not contain `Session/migrate_status_idle` or `StatusMigrationDispatcher`. Those two handlers exist only in the pre-runtime migration endpoint; the raw Session handler is ingress-private, and the endpoint intentionally omits `Health` and every product turn handler so it cannot open edge admission. -The process action-review reaper owns timeout discovery and gauge sampling, but -it never invokes private workflows. It only wakes `ActionReviewDispatcher`. -That Restate service journals the outbox claim, invokes -`ExecutionTask/resolve_action_review` service-to-service, and journals the -generation-fenced acknowledgement or retry schedule. +The singleton maintenance process owns low-frequency action-review +reconciliation and gauge sampling. Normal expiry is one generation-fenced +`DurableTimeout` delivery scheduled when the review is created. Execution-task +resolution commits the task-generation transition and execution dispatch row; +no lifetime task workflow is resumed. ## Journal And Retention -Journals are for recovery and recent debugging, not long-term product history. +Journals are for bounded activation recovery and recent debugging, not +long-term product history. All bindings explicitly retain idempotency entries and journals for 24 hours, preserving the current effective behavior without depending on a mutable server default. Every workflow `run` handler likewise declares 24-hour completion @@ -574,6 +583,15 @@ recovery is no longer needed. Production should run Restate as a durable cluster, keep the handler endpoint internal, and expose public traffic through `moa-edge`. +The versioned serving role binds Restate, SCIM, channel, and credential +ingress. The stable singleton `moa-orchestrator maintenance` role binds only +health/metrics and owns trigger/outbox reconciliation, retention, +action-review/authz reconciliation, workspace/hand reaping, and provider +inventory. Serving revisions do not duplicate those loops. Because all product +waits are storage-only, an old serving revision retains only bounded +invocations; the RestateDeployment operator may autoscale a draining revision +to one recovery replica and then zero when its invocation count reaches zero. + Deployment requirements: - Postgres/Neon for product data. @@ -607,9 +625,11 @@ attributes. The useful diagnostic chain is: Execution spans add run ID, task ID, plan hash/revision, requirement IDs, reservation/actual usage, retry generation, capability reference, and terminal -reason as trace fields rather than high-cardinality metric labels. Operators -diagnose a run from `moa.execution_run`/`moa.execution_task`, Restate -invocations, traces, and its compact session events. +reason as trace fields rather than high-cardinality metric labels. Fleet +metrics aggregate phase, oldest-ready/deadline/trigger/outbox/attempt/external +age, capacity/fairness, durable maintenance last-success age, and draining revision cost. +Operators diagnose a run from Postgres execution state, its current bounded +Restate activation, traces, and compact session events. Dashboards should separate Restate health, turn latency, LLM/provider behavior, approval latency, tool execution, and sandbox fleet health. @@ -634,8 +654,15 @@ awakeables plus parent-cached terminal results instead of status polling. 2. Sessions and workers are virtual objects. 3. Top-level turns run in `TurnExecution` workflows keyed by turn ID. 4. Worker turns run in `WorkerTurnExecution` workflows keyed by turn ID. -5. `ExecutionRun` and `ExecutionTask` are the only durable typed-DAG runtime; - `Worker` remains conversational delegation in Inline Execute. +5. Postgres execution aggregates plus bounded `ExecutionRunController`, + `ExecutionTaskAttempt`, `ExecutionCompensationAttempt`, `ExecutionTrigger`, + `ExecutionDispatcher`, fleet-keyed `ExecutionDispatchDrain`, and + `ExecutionDispatchReconciler` activations are the only durable typed-DAG + runtime. The singleton `fleet` drain key serializes bounded outbox delivery + and admission against fleet-global capacity; `ExecutionSchedule` and + `DurableTimeout` provide schedule/timeout delivery, `ExecutionRetention` owns + bounded terminal-detail archival and deletion, while `Worker` remains + conversational delegation in Inline Execute. 6. Tenant action reviews use the `ActionReviews` service plus Postgres rows and events; they do not block turn workflows. 7. Product-visible events, execution state, learning, memory, lineage, and audit stay in diff --git a/docs/17-observability.md b/docs/17-observability.md index 5ed747884..61b5fd221 100644 --- a/docs/17-observability.md +++ b/docs/17-observability.md @@ -97,12 +97,26 @@ in-process helper plumbing, not as the hosted observation source of truth. ## Execution Runs Durable execution is observed through `moa.execution_run` and -`moa.execution_task`, Restate invocation state, compact session events, and -bounded trace attributes. A plan fully materializes its approved logical work, -while `execution.max_in_flight_tasks` bounds the attached calls and live task -runtimes owned by one run; pending rows are storage-only. Provider -concurrency/rate pacing and governed tool or hand capacity add independent -physical limits without changing logical coverage. +its node/task/attempt/compensation/trigger/outbox/external-job projections, +bounded Restate activation state, compact session events, and trace attributes. +Pending and every waiting phase are storage-only; only admitted attempts may +own active capacity or hands. + +Fleet health uses bounded labels only. `moa_execution_runs{phase}` separates +active, input/review/signal/timer/external, pause, and compensation phases. +Oldest-ready age, overdue deadlines, trigger/outbox lag and dead letters, +oldest active-attempt/external-job age, admission utilization, tenant maximum +share, durable reconciliation and retention last-success ages, parked tasks retaining +hands, and old Restate deployment age/replica-hours carry no tenant, run, task, +deployment, or provider account identifier. IDs belong in traces and Postgres drilldown. + +Reconciliation and retention expose separate durable health receipts. Trigger/outbox +repair drives `moa_execution_maintenance_*`; terminal-evidence retention drives +`moa_execution_retention_*`. A missing receipt exports as unready with infinite age. +Retention normally completes a bounded pass at least once per hour, so +`MOAExecutionRetentionStale` warns when the receipt is absent, unready, or older than +two hours. No retention backlog series is exported until the repository can provide a +bounded, authoritative backlog snapshot. ### Replay-Safe Trace Correlation @@ -119,13 +133,12 @@ The operational path remains: ```text session turn -> route / planner / compiler - -> ExecutionRun - -> ExecutionTask + -> ExecutionRunController activation + -> ExecutionTaskAttempt activation -> model call or governed capability/tool call -> ActionPolicy and optional action review - -> action-review resolution outbox retry - -> resumed ExecutionTask - -> ExecutionRun fan-in + -> persisted wait/trigger or action-review dispatch + -> later bounded attempt/controller activation -> terminal synthesis turn ``` @@ -137,8 +150,8 @@ than reconstructed from the current handler attempt. Action reviews preserve two distinct contexts. Review creation stores the original execution-task context as the future link target. Terminal resolution stores the resolver's current context as the retry callback's remote parent. -The reaper reinjects the resolution parent; `ExecutionTask/resolve_action_review` -adopts it and links the separately stored original task context. Replay and +The maintenance delivery reinjects the resolution parent; the bounded task +resolution activation adopts it and links the separately stored original task context. Replay and claim retry preserve both byte-for-byte. Invalid `traceparent` is treated as absent; invalid `tracestate` is dropped while a valid parent remains. Non-empty `tracestate` follows W3C Level 2 limits and MOA's 512-byte cap. diff --git a/docs/19-data-operations.md b/docs/19-data-operations.md index 1fe714176..b7dafdb66 100644 --- a/docs/19-data-operations.md +++ b/docs/19-data-operations.md @@ -62,11 +62,10 @@ runner; operators should not use a hard-coded version probe as a schema check. ### Capacity And Backpressure -Execution plans have no application active-worker, plan-node, map-item, or task -fan-out ceiling below the approved run budget. `max_tasks` bounds logical work; -the other four resource dimensions and the deadline bound what that work may -consume. Do not add an application fan-out constant to mitigate provider or -tool pressure. +`max_tasks` bounds logical work; cost, tokens, tool calls, retrieved bytes, and +the absolute deadline bound what that work may consume. Physical execution is +admitted separately so a valid large plan cannot monopolize compute or durable +queues. Physical backpressure is supplied independently: @@ -88,6 +87,12 @@ Physical backpressure is supplied independently: `ActionPolicy`/`ToolExecutor`/`HandProvider`. Tool, MCP, sandbox, and external service quotas queue, retry, or return typed failures at that boundary; they do not reduce logical map coverage. +- Postgres capacity buckets enforce tenant and fleet ceilings for active runs, + active attempts, parked runs, scheduled triggers, and external jobs. The + controller dispatch batch and activation-step limit bound each activation; + weighted tenant dispatch prevents a hot tenant from consuming all ready + slots. Saturation parks durable work without holding a Restate invocation or + sandbox. ### Budgets And Terminal Semantics @@ -123,26 +128,27 @@ reasons. Inspect sources in this order. Do not skip directly to logs or traces: 1. Inspect the durable run through `Execution/status` and - `moa.execution_run`: `status`, `wake_epoch`, `processed_wake_epoch`, + `moa.execution_run`: `status`, controller generation/activation state, + `next_wake_at`, immutable goal contract, active plan hash/revision, typed terminal reason, completion-check evidence, budget/reservation totals, waiting reasons, and - timestamps. A greater `wake_epoch` means the latest scheduling mutation is - not yet acknowledged. Do not infer the execution path from a constant mode - field; use the persisted typed route source and planning audit. -2. Inspect active `moa.execution_task` rows: state, `task_id`, attempt, - generation fence, reservation/actual values, and + timestamps. Do not infer the execution path from a constant mode field; use + the persisted typed route source and planning audit. +2. Inspect active `moa.execution_task` rows: state, `task_id`, attempt dispatch, + generation/lease fence, reservation/actual values, and `reserved_at`/`started_at`/`completed_at`. A stale generation must never overwrite the current one. -3. Inspect the exact waiting input, review, or signal state in the run's - waiting reasons and the owning task generation. Resolve it through +3. Inspect the exact input, review, signal, timer, external-job, or pause state, + its persisted `due_at`, immutable trigger, and owning task generation. + Resolve user-owned waits through `Execution/deliver_input`, `Execution/decide_review`, or `Execution/deliver_signal`; do not edit the task. -4. For action reviews, inspect `moa.execution_action_review_outbox`: - `attempt_count`, `next_attempt_at`, `claimed_at`, `delivered_at`, and - `last_error`, plus the matching tenant action-review row. -5. Inspect Restate invocation and journal state for the keyed `ExecutionRun` - and `ExecutionTask` workflows, including retries and scoped-concurrency - admission. +4. Inspect `moa.execution_dispatch_outbox` and `moa.execution_trigger` claim, + delivery, retry, dead-letter, generation, and error fields. For external + jobs also inspect callback disposition and last reconciliation. +5. Inspect only the current bounded `ExecutionRunController`, + `ExecutionTaskAttempt`, or `ExecutionTrigger` invocation. A parked run should + have none; if it does, treat that as a resource-leak incident. 6. Query spans by stable session/run/task/action-review attributes, then inspect any persisted W3C parent/link contexts on durable callbacks. Do not infer causality from an attempt-local header embedded in a Restate journal command. @@ -155,10 +161,11 @@ Use the parent-scoped product cancellation mutation: - MCP: `execution_run_cancel`; - internal Restate: `Execution/cancel`. -The terminal cancellation transaction fences new reservations, replaces every -active task outcome with cancellation, releases all five reservation -dimensions, preserves completed task evidence, writes the typed cancellation -cause/reason, and wakes terminal delivery. Confirm the result through +The cancellation transaction fences new reservations, advances active attempt +generations, releases their active-capacity and budget reservations, preserves +completed task evidence, writes the typed cause/reason, and enqueues controller +activation. `compensate_committed` remains nonterminal until bounded reverse +compensation settles. Confirm the result through `Execution/status` or `execution_run_status`. Restate admin cancellation is only a hard stop for a stuck invocation. It is @@ -193,6 +200,36 @@ scores. A superseding case may replace one only when it exercises the same production path and strictly contains the old failure condition; record that relationship in the scenario comment. +### Long-Horizon Maintenance And Retention + +The singleton `moa-orchestrator maintenance` deployment is the only fleet +owner for trigger/outbox repair, execution retention, action-review/authz +reconciliation, workspace/hand reaping, and provider inventory. Serving +Restate revisions do not run these scans. Due trigger delivery remains frequent; +full inventory and retention use separate adaptive cadences, account-sharded +leases, and exponential idle backoff. + +Page on overdue deadlines, oldest-ready age, stuck attempt leases, trigger or +outbox dead letters, stale durable maintenance reconciliation, parked tasks retaining +hands, and old deployment drain age. Capacity saturation is normally a warning: +inspect tenant maximum share and fairness before raising fleet ceilings. + +Retention archives and page-deletes terminal run details, tasks, triggers, +outbox rows, external jobs, and compensation evidence only after tenant +retention/legal-hold policy permits it. Active, waiting, paused, compensating, +or unknown-outcome state is never selected. The maintenance owner must prove a +terminal generation and preserve compact run/session/audit evidence before +deletion. + +The one-time V59/V60 hard cut is executed only with +`scripts/cutover-long-horizon-execution.sh`. It always prints the Postgres +nonterminal-run inventory and exact old Restate deployment invocations before +mutation, requires explicit targets and confirmation, archives terminal +execution tables, applies the repository migration runner, clears only the +three retired execution services, and verifies the bounded service inventory. +Admission remains gated until maintenance readiness and archive durability are +independently confirmed. + ## Sandbox Workspace Operations Postgres is the ownership and lifecycle authority for durable sandbox diff --git a/docs/20-testing.md b/docs/20-testing.md index f6d6957ca..4f69a8f1c 100644 --- a/docs/20-testing.md +++ b/docs/20-testing.md @@ -123,16 +123,25 @@ not elapsed quiet periods: deadline movement, no stale signal while terminal or awaiting input, one joined stale signal after the exact deadline, and no periodic Session calls while a healthy child remains active. -- `execution_run_service_e2e` proves that a 500-item map completes with - `peak_live <= execution.max_in_flight_tasks`, restart after 137 completions is - exactly once, and pending rows have no live task invocation outside the - window. -- Public execution-mutation cases stop at the committed-epoch/pre-wake and - wake-ack/pre-park barriers. Success is valid only after the task/run wake is - accepted; replay must neither increment the epoch nor resume twice. +- Long-horizon execution cases prove that every controller activation stays + within its scheduler-step and dispatch-batch bounds, every task or + compensation activation executes one generation, and restart after partial + completion is exactly once. +- Public execution-mutation cases stop after the Postgres transition but before + immediate dispatch. Recovery must redeliver the same outbox/trigger identity, + and replay must neither advance the generation nor apply the transition twice. +- Input/review/signal/timer/external/pause cases assert that parked rows have no + active attempt reservation, Restate handler invocation, or hand. Resume must + provision fresh compute and restore the verified checkpoint when filesystem + state is required. Use the `restate-recovery-pr` profile for the conversational restart matrix and -the `execution-eval-nightly` profile for the 500-task window/recovery matrix. +the `long-horizon-execution` profile for accelerated timer, burst, deployment +drain, and Postgres/outbox reconstruction scenarios. The latter runs one thread, +uses deterministic time/provider fixtures, and must fail when its selector +matches no tests. Twenty-four-hour and seven-day canaries remain ignored behind +`MOA_RUN_LONG_HORIZON_CANARY=1`; provider-live execution additionally requires +its own opt-in and positive cost budget. Both use deterministic local Postgres, Restate, OpenFGA, Valkey, MCP, and fixture-capability dependencies; neither requires a billed provider. diff --git a/docs/22-load-and-chaos-testing.md b/docs/22-load-and-chaos-testing.md index 94dd4b726..7156c3814 100644 --- a/docs/22-load-and-chaos-testing.md +++ b/docs/22-load-and-chaos-testing.md @@ -89,6 +89,113 @@ lease under valid configuration. | T2 capacity | one strong box, compose | nightly (`make loadtest-capacity`) | max sustainable turns/sec per replica + per-turn resource bill | | T3 scale-out | k8s topology (HPA 2–50) | pre-release / on demand | does capacity scale ≈ linearly with replicas, and what breaks first? | +## Long-horizon execution validation + +The `long-horizon-execution` nextest profile is the deterministic durability +lane for execution runs whose product horizon is measured in days or weeks. It +uses the production Restate handlers, PostgreSQL repositories and outbox, and +Valkey runtime-cache backend with fixture providers only. Provider credentials +are removed by `scripts/run-clean-e2e.sh --live --long-horizon`; this lane must +never enable a live-provider flag or consume a provider budget. + +The suite represents eight logical days with compressed real intervals. One +logical day is two real seconds, so plan-authored `At`/`After` waits still travel +through the production compiler, persisted absolute `due_at`, Restate +`send_after`, database `now()`, and generation-fenced trigger delivery. This is +not a fake clock and tests must assert the persisted due time, delivery order, +and exact generation rather than treating elapsed test time as evidence. + +Every implemented user-visible parked status checks the parked-resource +invariant at each input, review, signal, timer, and pause interval: + +- exactly one durable `parked_runs` receipt for the parked run; +- zero non-released `active_tasks` capacity receipts for the parked run; +- no active dispatch UID or running attempt retained by a parked task; +- no live sandbox hand/workspace operation retained for storage-only waits; +- no continuing Restate invocation pinned merely to wait for wall time. + +`WaitingExternal` deliberately retains one bounded `external_jobs` receipt for +the provider-owned job, but still retains zero `active_tasks`, sandbox hands, or +continuing attempt invocations. Provider progress updates only PostgreSQL and a +sparse due trigger; it does not create a hot controller loop. + +The retry-backoff case separately proves a future persisted `ready_at`, no +active task reservation or dispatch during the backoff, and no generation-two +provider call before that timestamp; it does not mislabel retry readiness as a +user-visible parked run. + +The 1,000-run common-wake burst also observes the production OTLP +`moa_execution_dispatch_batch_size` and +`moa_execution_oldest_ready_age_seconds` instruments from the bounded +dispatcher callsite, alongside exact database capacity and Restate invocation +counts. SQL queue age alone is diagnostic evidence, not a substitute for +proving that the production metric is wired. + +The lane drives `WaitingExternal` through an integration-only catalog tool whose +declared async mode selects a deterministic HTTP provider adapter. That path +asserts pre-start reservation and idempotency-key equality, post-bind terminal +callback deferral until the owning attempt releases capacity, progress callback +deduplication and reconcile rearming, sparse reconciliation while paused, and +unbound-start recovery after total Restate-state loss without replaying the +original task attempt. The built-in tool fails closed if execution bypasses its +declared adapter, and neither the adapter nor its catalog entry exists outside +the provider-override integration lane. Sandbox release coverage also uses a real +sandbox-required hand capability: it observes the exact execution-task +`active_hands` receipt while the task runs, its release during a storage-only +wait, and a downstream task's distinct receipt after resume. The implemented +deterministic matrix covers an accelerated week, deadlines and wait expiry, +pause, an Agent action-review checkpoint and redispatch, idempotent versus +ambiguous watchdog expiry, governed retry, burst admission fairness and +fleet/tenant caps, three real Restate handler deployments draining in order, +and dependency recovery. The Agent review case proves the exact pending effect +is checkpointed after active-task capacity release and that approval creates a +new bounded attempt dispatch; it does not wait on a Restate promise. + +Recovery cases include pausing a storage-only timer before its due time: the +timer and task settle once while the run remains parked with no controller +activation, and a generation-fenced resume creates the only activation that +advances the settled graph. Other cases restart the orchestrator and Valkey +repeatedly, stop/start PostgreSQL without replacing its durable volume, replay +late or duplicate input, signal, review, trigger, and outbox deliveries, and +replace Restate with an empty node on the same endpoints. Production +reconciliation re-drives the exact generation-fenced dispatch identity from +PostgreSQL, so cluster replacement cannot duplicate a logical delivery. A +Running non-idempotent +attempt is deliberately excluded from dispatch re-drive; after empty-state +replacement, only its durable watchdog may classify it as `UnknownOutcome`. +Tests use Restate's +deployment and invocation system tables only to observe routing and drain; +PostgreSQL remains the product state source of truth. + +Run the deterministic lane with: + +```bash +MOA_RUN_LIVE_E2E=1 ./scripts/run-clean-e2e.sh --live --long-horizon +``` + +The separate 24-hour and seven-day deployment canaries are ignored by default. +They require both the explicit `MOA_RUN_LONG_HORIZON_CANARY=1` gate and +`MOA_LONG_HORIZON_CANARY_WINDOW=24h` or `7d`. They refuse to construct a local +fallback stack: `MOA_DATABASE_URL`, `MOA_RESTATE_INGRESS_URL`, and +`RESTATE_ADMIN_URL` must identify the deployed system. They remain unbilled and +fail closed if any `MOA_RUN_LIVE_*=1` integration flag is present. Run them +without provider credentials as defense in depth: + +```bash +env -u MOA_ANTHROPIC_API_KEY -u MOA_OPENAI_API_KEY \ + -u MOA_GOOGLE_API_KEY -u MOA_COHERE_API_KEY \ + -u MOA_ZEROENTROPY_API_KEY -u MOA_FIDELITY_SIMULATOR_API_KEY \ + -u MOA_LLAMAPARSE_API_KEY -u MOA_MERGE_API_KEY -u MOA_NANGO_API_KEY \ + -u MOA_NEON_API_KEY -u MOA_DATABASE_NEON_API_KEY \ + -u MOA_REDUCTO_API_KEY -u MOA_TEST_MCP_DEPLOYMENT_API_KEY \ + -u MOA_TURBOPUFFER_API_KEY -u MOA_UNSTRUCTURED_API_KEY \ + MOA_RUN_LONG_HORIZON_CANARY=1 \ + MOA_LONG_HORIZON_CANARY_WINDOW=24h \ + cargo nextest run -p moa-orchestrator --locked \ + --test long_horizon_execution_canary_live \ + --run-ignored ignored-only --no-tests fail +``` + T3 certifies the 10k+ QPS claim as arithmetic validated by measurement: `replicas_needed = ceil(10_000 / per_replica_rate)` must be ≤ HPA max, and a scale-out run at the computed replica count must sustain the target rate diff --git a/docs/23-environment-variables.md b/docs/23-environment-variables.md index fcba80123..3d4990106 100644 --- a/docs/23-environment-variables.md +++ b/docs/23-environment-variables.md @@ -79,18 +79,33 @@ Grouped by top-level config section. `_unset_`/`_none_` means the field is | Variable | Config path | Default | Description | |---|---|---|---| +| `MOA_EXECUTION_ACTIVE_ATTEMPT_TIMEOUT_SECONDS` | `execution.active_attempt_timeout_seconds` | 600 | Maximum wall-clock duration of one bounded active task or compensation attempt | | `MOA_EXECUTION_AGENT_TURN_COST_MICROUSD` | `execution.agent_turn_cost_microusd` | 100000 | Worst-case integer micro-USD estimate for one agent turn | | `MOA_EXECUTION_AGENT_TURN_RETRIEVED_BYTES` | `execution.agent_turn_retrieved_bytes` | 10000000 | Worst-case retrieved-byte estimate for one agent turn | | `MOA_EXECUTION_AGENT_TURN_TOKENS` | `execution.agent_turn_tokens` | 8000 | Worst-case token estimate for one agent turn | | `MOA_EXECUTION_AGENT_TURN_TOOL_CALLS` | `execution.agent_turn_tool_calls` | 8 | Worst-case governed tool-call estimate for one agent turn | +| `MOA_EXECUTION_DISPATCH_BATCH_SIZE` | `execution.dispatch_batch_size` | 64 | Maximum ready task attempts dispatched by one controller activation | | `MOA_EXECUTION_MAX_COST_MICROUSD` | `execution.max_cost_microusd` | 100000000 | Default run cost limit in integer micro-USD | -| `MOA_EXECUTION_MAX_IN_FLIGHT_TASKS` | `execution.max_in_flight_tasks` | 64 | Positive physical window for live `ExecutionTask` invocations owned by one run; distinct from logical task and provider-concurrency limits | +| `MOA_EXECUTION_MAX_FLEET_ACTIVE_RUNS` | `execution.max_fleet_active_runs` | 1000 | Fleet ceiling for admitted non-parked execution runs | +| `MOA_EXECUTION_MAX_FLEET_ACTIVE_TASKS` | `execution.max_fleet_active_tasks` | 4096 | Fleet ceiling for active task and compensation attempts | +| `MOA_EXECUTION_MAX_FLEET_EXTERNAL_JOBS` | `execution.max_fleet_external_jobs` | 10000 | Fleet ceiling for nonterminal asynchronous external jobs | +| `MOA_EXECUTION_MAX_FLEET_PARKED_RUNS` | `execution.max_fleet_parked_runs` | 100000 | Fleet residency entitlement ceiling shared by active and parked runs; must be at least the active-run ceiling | +| `MOA_EXECUTION_MAX_FLEET_SCHEDULED_TRIGGERS` | `execution.max_fleet_scheduled_triggers` | 500000 | Fleet ceiling for pending durable execution triggers | +| `MOA_EXECUTION_MAX_IN_FLIGHT_TASKS` | `execution.max_in_flight_tasks` | 64 | Per-run ceiling contributing to active-attempt admission; it never keeps a waiting task invocation alive | | `MOA_EXECUTION_MAX_RETRIEVED_BYTES` | `execution.max_retrieved_bytes` | 10000000000 | Default run retrieved-byte limit | | `MOA_EXECUTION_MAX_TASKS` | `execution.max_tasks` | 10000 | Default logical-task limit; this is not an active-worker cap | +| `MOA_EXECUTION_MAX_TENANT_ACTIVE_RUNS` | `execution.max_tenant_active_runs` | 100 | Per-tenant ceiling for admitted non-parked execution runs | +| `MOA_EXECUTION_MAX_TENANT_ACTIVE_TASKS` | `execution.max_tenant_active_tasks` | 256 | Per-tenant ceiling for active task and compensation attempts | +| `MOA_EXECUTION_MAX_TENANT_EXTERNAL_JOBS` | `execution.max_tenant_external_jobs` | 1000 | Per-tenant ceiling for nonterminal asynchronous external jobs | +| `MOA_EXECUTION_MAX_TENANT_PARKED_RUNS` | `execution.max_tenant_parked_runs` | 10000 | Per-tenant residency entitlement ceiling shared by active and parked runs; must be at least the active-run ceiling | +| `MOA_EXECUTION_MAX_TENANT_SCHEDULED_TRIGGERS` | `execution.max_tenant_scheduled_triggers` | 50000 | Per-tenant ceiling for pending durable execution triggers | | `MOA_EXECUTION_MAX_TOKENS` | `execution.max_tokens` | 10000000 | Default run token limit | | `MOA_EXECUTION_MAX_TOOL_CALLS` | `execution.max_tool_calls` | 100000 | Default governed tool-call limit | +| `MOA_EXECUTION_MAXIMUM_ACTIVATION_STEPS` | `execution.maximum_activation_steps` | 128 | Maximum pure scheduler transitions applied by one controller activation | +| `MOA_EXECUTION_MAXIMUM_HORIZON_SECONDS` | `execution.maximum_horizon_seconds` | 2592000 | Maximum admitted durable-run horizon, including storage-only waits | | `MOA_EXECUTION_PLANNER_REPAIR_ATTEMPTS` | `execution.planner_repair_attempts` | 1 | Maximum repair attempts for an invalid initial planner response | | `MOA_EXECUTION_REPEATED_FAILURE_LIMIT` | `execution.repeated_failure_limit` | 3 | Repeated normalized failure count that stops replanning | +| `MOA_EXECUTION_TRIGGER_RECONCILIATION_CADENCE_SECONDS` | `execution.trigger_reconciliation_cadence_seconds` | 60 | Safety repair cadence for due execution-trigger delivery; normal delivery remains event-driven | | `MOA_EXECUTION_UNATTENDED_MAX_COST_MICROUSD` | `execution.unattended_max_cost_microusd` | 5000000 | Cost threshold above which a compiled run requires owning-user confirmation | | `MOA_EXECUTION_VERIFIER_TURN_COST_MICROUSD` | `execution.verifier_turn_cost_microusd` | 200000 | Worst-case integer micro-USD estimate for one completion-verifier turn | | `MOA_EXECUTION_VERIFIER_TURN_RETRIEVED_BYTES` | `execution.verifier_turn_retrieved_bytes` | 1000000 | Worst-case retrieved-byte estimate for one completion-verifier turn | @@ -662,7 +677,7 @@ not trip the unknown-variable audit. They do not affect application config. | `MOA_POSTMARK_*` | Postmark live-email credentials | | `MOA_E2B_*` | E2B sandbox credentials | | `MOA_OPENROUTER_*` | OpenRouter credentials (deploy) | -| `MOA_EDGE_*` | Edge binary bind/upstreams, connector rollout switch, and inbound MCP controls (`MOA_EDGE_BIND`, `MOA_EDGE_UPSTREAM`, `MOA_EDGE_CONNECTOR_CREDENTIAL_UPSTREAM`, `MOA_EDGE_CONNECTOR_MANAGEMENT_ENABLED`, `MOA_EDGE_MCP_ALLOWED_HOSTS`, `MOA_EDGE_MCP_ALLOWED_ORIGINS`, `MOA_EDGE_MCP_TOOL_CALLS_PER_MINUTE`). The tool-call limit defaults to 60, must be greater than zero, and is enforced per authenticated tenant/principal by each edge replica; use an upstream distributed limiter as well when the number must be a fleet-wide quota. Connector management defaults dark: when false, every `/v1/connectors/connections...` route returns 404 before authentication, translation, or proxying. The credential upstream must target the orchestrator's private port 10023, never Restate or a public endpoint. Local Compose explicitly opts in; the Kubernetes base explicitly remains false. | +| `MOA_EDGE_*` | Edge binary bind/upstreams, connector rollout switch, and inbound MCP controls (`MOA_EDGE_BIND`, `MOA_EDGE_UPSTREAM`, `MOA_EDGE_INTERNAL_INGRESS_UPSTREAM`, `MOA_EDGE_CONNECTOR_MANAGEMENT_ENABLED`, `MOA_EDGE_MCP_ALLOWED_HOSTS`, `MOA_EDGE_MCP_ALLOWED_ORIGINS`, `MOA_EDGE_MCP_TOOL_CALLS_PER_MINUTE`). The tool-call limit defaults to 60, must be greater than zero, and is enforced per authenticated tenant/principal by each edge replica; use an upstream distributed limiter as well when the number must be a fleet-wide quota. Connector management defaults dark: when false, every `/v1/connectors/connections...` route returns 404 before authentication, translation, or proxying. The internal ingress upstream must target the orchestrator's private credential-and-provider-callback port 10023, never Restate or a public endpoint. Local Compose explicitly opts in; the Kubernetes base explicitly remains false. | | `MOA_RESTATE_DEPLOYMENT_*` | Restate deploy-registration (`MOA_RESTATE_DEPLOYMENT_HOST`/`_URI`) | ### Approved exact names diff --git a/docs/25-sandbox-workspaces.md b/docs/25-sandbox-workspaces.md index 4210d08f4..f979095a1 100644 --- a/docs/25-sandbox-workspaces.md +++ b/docs/25-sandbox-workspaces.md @@ -91,7 +91,7 @@ The canonical Postgres model is: | `moa.sandbox_workspace_grants` | Desired OpenFGA owner/use grants and generation-fenced inverse tuple intent. | | `moa.sandbox_provider_accounts` | Non-secret deployment/provider/isolation-cell identity and generation, organization/project fingerprint, configured limits, observed inventory, headroom, health. | | `moa.sandbox_storage_resources` | Tenant-owned external storage, such as a Daytona tenant volume, with account ownership, provider reference, generation, deletion intent, and verified ownership metadata. | -| `moa.sandbox_capacity_reservations` | Pending/committed capacity by tenant, provider account, operation, and exact resource kind: `workspaces`, `volumes`, `checkpoints`, or `logical_bytes`. | +| `moa.sandbox_capacity_reservations` | Provider-neutral capacity ownership by tenant, provider account, and exact resource kind. `workspaces` is one committed lifetime reservation per workspace; `active_hands` is tied to one provisioning operation and hand-lease generation; `volumes`, `checkpoints`, and `logical_bytes` remain operation-bound. Every row carries the workspace and provider generations needed for exact release. | | `moa.hand_leases` workspace fields | `workspace_id`, workspace writer epoch, workspace instance generation, and restored checkpoint ID; the lease still owns only compute. | Every relationship that crosses a tenant-owned table includes `tenant_id` in @@ -100,6 +100,24 @@ tenant-first predicates, and immutable tenant ownership. Cross-tenant reconciliation uses a separate, narrow maintenance path unavailable to request handlers. +Workspace creation and its committed `workspaces` reservation are one database +transaction, so a tenant or provider limit rejection leaves neither workspace +metadata nor a partial reservation. That lifetime reservation has no operation +owner and is released only after deletion is finalized at its exact next delete +generation. `active_hands` is distinct from durable `volumes`: it is reserved as +pending before provider compute creation, committed only when the exact +provisioning operation and hand-lease generation activate, and released only +after verified provider destruction or transfer to an exact live durable reaper +claim. Stale workspace, lease, provisioning, delete, or reaper generations +cannot release a current owner. + +Checkpoint capacity is also provider-neutral. After the bounded archive and +manifest are built, the publication path reserves one `checkpoints` unit and +the manifest's exact `logical_bytes` before any adapter uploads bytes. The +generation-fenced publication compare-and-set commits those reservations with +the checkpoint head; a failed or ambiguous publication cannot evade or settle +capacity after the upload. + ## Workspace State Machine ```text @@ -209,6 +227,30 @@ and decompression expansion beyond configured limits. Object keys are opaque, writes are create-only, and restore validates into a fresh root before atomic promotion. +### Long-horizon execution yields + +An execution attempt may hold a hand only while it is active. Before returning +for input, review, signal, timer, external-job, retry, pause, compensation, or +terminal state, it runs the commit barrier when filesystem state must survive, +releases the exact `active_hands` and hand-lease generation, and proves provider +compute destroyed. Only then may the task become storage-only. A failed or +ambiguous destroy remains an active reservation owned by reconciliation; the +task is not reported as cleanly parked. + +`moa.sandbox_execution_hand_release_receipts` persists that yield intent before +any checkpoint or provider-destroy I/O. Its exact task-or-compensation owner, +logical generation, bounded-attempt generation, and any workspace, writer, +instance, provisioning, and hand generations fence rotation while pending. +Only verified checkpoint publication where applicable, provider absence, and +lease/active-hand release settle it. A retry returns the stored receipt instead +of repeating the release boundary. + +Resume never revives process memory. It admits current tenant/fleet and provider +capacity, provisions a fresh hand, restores the verified committed checkpoint, +reinstalls trusted runtime material outside the checkpoint root, and advances +the compute-instance fence. The invariant +`parked execution tasks with active hands = 0` is both alerted and tested. + ## Provider Binding And Admission Provider selection uses an operator-authored provider-account route rather than diff --git a/docs/examples/artifacts/damaged-food-order.skill.yaml b/docs/examples/artifacts/damaged-food-order.skill.yaml index f400c2e0e..44ed3746f 100644 --- a/docs/examples/artifacts/damaged-food-order.skill.yaml +++ b/docs/examples/artifacts/damaged-food-order.skill.yaml @@ -36,6 +36,12 @@ definition: kind: output_schema plan: cancel_policy: retain_effects + input_wait_policy: + expiry: + kind: after + delay_seconds: 86400 + on_expiry: + kind: fail_run input_schema: type: object properties: diff --git a/docs/examples/artifacts/patterns/custom-logic.skill.yaml b/docs/examples/artifacts/patterns/custom-logic.skill.yaml index 2850eab50..069a60378 100644 --- a/docs/examples/artifacts/patterns/custom-logic.skill.yaml +++ b/docs/examples/artifacts/patterns/custom-logic.skill.yaml @@ -36,6 +36,12 @@ definition: kind: output_schema plan: cancel_policy: retain_effects + input_wait_policy: + expiry: + kind: after + delay_seconds: 86400 + on_expiry: + kind: fail_run input_schema: type: object properties: diff --git a/docs/examples/artifacts/patterns/human-approval.skill.yaml b/docs/examples/artifacts/patterns/human-approval.skill.yaml index 735d57b91..6b0d58645 100644 --- a/docs/examples/artifacts/patterns/human-approval.skill.yaml +++ b/docs/examples/artifacts/patterns/human-approval.skill.yaml @@ -29,6 +29,12 @@ definition: kind: output_schema plan: cancel_policy: retain_effects + input_wait_policy: + expiry: + kind: after + delay_seconds: 86400 + on_expiry: + kind: fail_run input_schema: type: object output_schema: @@ -61,6 +67,12 @@ definition: operation: kind: review prompt: Approve or reject the drafted customer-visible action. + wait_policy: + expiry: + kind: after + delay_seconds: 86400 + on_expiry: + kind: fail_run compensation: null retry: max_attempts: 1 diff --git a/docs/examples/artifacts/patterns/parallel-review.skill.yaml b/docs/examples/artifacts/patterns/parallel-review.skill.yaml index 58473a2d1..92f1d2c69 100644 --- a/docs/examples/artifacts/patterns/parallel-review.skill.yaml +++ b/docs/examples/artifacts/patterns/parallel-review.skill.yaml @@ -30,6 +30,12 @@ definition: kind: output_schema plan: cancel_policy: retain_effects + input_wait_policy: + expiry: + kind: after + delay_seconds: 86400 + on_expiry: + kind: fail_run input_schema: type: object output_schema: @@ -82,6 +88,12 @@ definition: operation: kind: review prompt: Review gathered evidence before issuing a refund. + wait_policy: + expiry: + kind: after + delay_seconds: 86400 + on_expiry: + kind: fail_run compensation: null retry: max_attempts: 1 diff --git a/docs/examples/artifacts/patterns/react-agent.skill.yaml b/docs/examples/artifacts/patterns/react-agent.skill.yaml index 336b694c0..f25d724c2 100644 --- a/docs/examples/artifacts/patterns/react-agent.skill.yaml +++ b/docs/examples/artifacts/patterns/react-agent.skill.yaml @@ -30,6 +30,12 @@ definition: kind: output_schema plan: cancel_policy: retain_effects + input_wait_policy: + expiry: + kind: after + delay_seconds: 86400 + on_expiry: + kind: fail_run input_schema: type: object output_schema: diff --git a/docs/examples/artifacts/patterns/sequential.skill.yaml b/docs/examples/artifacts/patterns/sequential.skill.yaml index f7d9f173b..e1e712be3 100644 --- a/docs/examples/artifacts/patterns/sequential.skill.yaml +++ b/docs/examples/artifacts/patterns/sequential.skill.yaml @@ -34,6 +34,12 @@ definition: kind: output_schema plan: cancel_policy: retain_effects + input_wait_policy: + expiry: + kind: after + delay_seconds: 86400 + on_expiry: + kind: fail_run input_schema: type: object properties: diff --git a/docs/operations/edge-network-isolation.md b/docs/operations/edge-network-isolation.md index 0caef344b..e1247c04a 100644 --- a/docs/operations/edge-network-isolation.md +++ b/docs/operations/edge-network-isolation.md @@ -22,9 +22,9 @@ proxying. This switch does not make port 10023 safe to expose. ## Compose Orchestrator handler port 9080 is bound to the compose internal network only. -The connector credential listener at `moa-orchestrator:10023` is likewise +The credential-and-provider-callback ingress at `moa-orchestrator:10023` is likewise internal-only and has no host port binding. `moa-edge` reaches it through -`MOA_EDGE_CONNECTOR_CREDENTIAL_UPSTREAM`. +`MOA_EDGE_INTERNAL_INGRESS_UPSTREAM`. Local Compose explicitly sets `MOA_EDGE_CONNECTOR_MANAGEMENT_ENABLED=true` for development. Set it false in a local override when testing Checkpoint A. The default `docker-compose.yml` is a development stack, not an isolation diff --git a/docs/operations/restate-operations.md b/docs/operations/restate-operations.md index 36fe9b640..d3cf1d0e0 100644 --- a/docs/operations/restate-operations.md +++ b/docs/operations/restate-operations.md @@ -143,6 +143,60 @@ the archived LSN gap, replacement duration, new PVC and node identities, and an exactly-once in-flight invocation result in the change record. The repository does not automate destructive cluster or PVC mutation. +### Long-horizon execution recovery objectives + +MOA has two supported disaster-recovery paths. Both preserve the same product +contract: the RPO for a Postgres-committed execution transition and a +ledgered/idempotent external effect is zero. The database PITR policy therefore +sets the real product RPO; an organization with a nonzero database backup gap +must publish that larger value instead. RTO is measured from incident +declaration until admission reopens after invariant verification. + +| Path | Use | RPO evidence | Drill RTO objective | +|---|---|---|---| +| Restate snapshot/journal restore | Restate storage is recoverable and its snapshot/archive set is internally complete | Restored Postgres recovery point plus zero unexplained `APPLIED - ARCHIVED` gap at the selected Restate restore point | 60 minutes | +| Postgres/outbox reconstruction | Restate state is unavailable or journal compatibility cannot be proved | Restored Postgres execution rows, triggers, outbox, external-effect ledgers, and portable checkpoints; no Restate journal is treated as product authority | 4 hours | + +For a Restate restore drill: + +1. Gate execution admission and capture the exact Postgres recovery point, + Restate cluster identity, snapshot pointers, archived LSNs, and deployment + inventory. +2. Restore the complete Restate snapshot/journal set and Postgres to the chosen + coordinated point. Never combine independently timed member volumes. +3. Register the immutable bounded-activation deployment and start the singleton + maintenance role. +4. Verify run/task generations, due triggers, dispatch outbox, external-job + unknown outcomes, active capacity reservations, and the parked-zero-hands + invariant before reopening admission. + +For a Postgres/outbox reconstruction drill: + +1. Gate admission and restore Postgres plus portable checkpoint storage; start + a new empty Restate cluster rather than importing an unproved journal. +2. Register the full bounded runtime inventory from the immutable deployment: + `ExecutionRunController`, `ExecutionTaskAttempt`, + `ExecutionCompensationAttempt`, `ExecutionTrigger`, `ExecutionDispatcher`, + fleet-keyed `ExecutionDispatchDrain`, `ExecutionDispatchReconciler`, + `ExecutionRetention`, `ExecutionSchedule`, and `DurableTimeout`. The + `ExecutionDispatchDrain` virtual object must be registered, and recovery + dispatch must address its canonical `fleet` key: it is the sole serialized + owner that drains reconstructed Postgres outbox heads and admits ready work + against fleet-global capacity. +3. Start maintenance. It reclaims pending outbox rows, due triggers, stale + attempts, external callbacks/reconciliation, and nonterminal run activations + from Postgres. Every delivery retains its original idempotency identity and + generation. +4. Reconcile every transmitting or unknown external effect against its durable + connector/tool/provider ledger. Never replay an effect merely because its + old Restate journal is absent. +5. Require zero overdue unclaimed triggers, zero outbox dead letters, zero + stale active attempts, zero parked tasks with hands, bounded capacity, and a + completed representative run before reopening admission. + +Record observed RPO and RTO, not only the objective. Missing external-effect +evidence, an unbounded archive gap, or a generation mismatch fails the drill. + ## Shutdown and placement Production Restate uses a ten-minute server shutdown timeout and a 660-second @@ -297,6 +351,40 @@ the identical journal prefix. Otherwise restore the original pinned endpoint or cancel through the product path. Forcing deployment removal or killing the invocation discards the safest recovery route. +## Destructive long-horizon execution cutover + +The V59/V60 execution hard cut does not translate live workflow state. Gate +execution admission, finish or product-cancel every old execution, and resolve +every invocation pinned to the exact old deployment before running: + +```bash +scripts/cutover-long-horizon-execution.sh \ + --database-admin-url postgresql://ADMIN_ROLE@DATABASE_HOST/DATABASE_NAME \ + --restate-admin-url https://RESTATE_ADMIN_HOST \ + --restate-ingress-url https://RESTATE_INGRESS_HOST \ + --old-deployment-id dp_EXACT_OLD_ID \ + --new-deployment-uri https://IMMUTABLE_NEW_HANDLER_URI \ + --archive-dir /explicit/precreated/empty/archive/directory +``` + +That first pass is read-only and must print zero nonterminal Postgres runs and +zero nonterminal invocations for the retired services or exact old deployment. +Review and retain the evidence, then rerun the same explicit target arguments +with `--confirm-destructive-cutover`. The confirmed pass archives the three +terminal execution tables, applies the repository-owned migration command, clears and +purges only `ExecutionRun`, `ExecutionTask`, and `ExecutionCompensation`, +removes the exact old deployment, registers the new immutable URI, and proves +that the full bounded execution handler inventory is present and every retired +service is absent. Inventory verification checks each Restate primitive type, +keeps every internal delivery surface private, and leaves only the +tenant-authorized `ExecutionSchedule` public. It never infers a +database, deployment, or archive target. + +Keep admission closed after the script returns. Require the singleton +fresh durable maintenance reconciliation, zero trigger/outbox dead letters, the parked-zero-hands +invariant, and one deterministic canary before reopening. Store the archive and +service inventory with the change record. + ## Hard product status cutover: `paused` to `idle` Product `idle` means a session is healthy between turns. Restate `paused` means @@ -311,7 +399,7 @@ Execute the cutover as a maintenance transaction: observation, migration-only Job and bootstrap, normal RestateDeployment readiness, then edge restoration. 2. Query `sys_invocation` and wait for active `Session`, `TurnExecution`, worker, - `ExecutionRun`, `ExecutionTask`, and `ExecutionCompensation` invocations to + and bounded execution controller/attempt/trigger invocations to finish. Cancel only through their product owners. The migration Job runs the complete current migration chain, so every hard-cut preflight in that chain must be satisfied before it starts. diff --git a/docs/schemas/moa-skill-v1.schema.json b/docs/schemas/moa-skill-v1.schema.json index 36bb27dd6..a3b4648b3 100644 --- a/docs/schemas/moa-skill-v1.schema.json +++ b/docs/schemas/moa-skill-v1.schema.json @@ -224,12 +224,14 @@ "additionalProperties": false, "required": [ "cancel_policy", + "input_wait_policy", "input_schema", "output_schema", "nodes" ], "properties": { "cancel_policy": { "$ref": "#/$defs/ExecutionCancelPolicy" }, + "input_wait_policy": { "$ref": "#/$defs/ExecutionWaitPolicy" }, "input_schema": {}, "output_schema": {}, "nodes": { @@ -464,19 +466,31 @@ { "type": "object", "additionalProperties": false, - "required": ["kind", "prompt"], + "required": ["kind", "prompt", "wait_policy"], "properties": { "kind": { "const": "review" }, - "prompt": { "type": "string" } + "prompt": { "type": "string" }, + "wait_policy": { "$ref": "#/$defs/ExecutionWaitPolicy" } } }, { "type": "object", "additionalProperties": false, - "required": ["kind", "signal_name"], + "required": ["kind", "signal_name", "wait_policy"], "properties": { "kind": { "const": "wait_signal" }, - "signal_name": { "type": "string" } + "signal_name": { "type": "string" }, + "wait_policy": { "$ref": "#/$defs/ExecutionWaitPolicy" } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["kind", "wake", "result"], + "properties": { + "kind": { "const": "wait_until" }, + "wake": { "$ref": "#/$defs/ExecutionTemporalTarget" }, + "result": {} } }, { @@ -490,6 +504,70 @@ } ] }, + "ExecutionWaitPolicy": { + "type": "object", + "additionalProperties": false, + "required": ["expiry", "on_expiry"], + "properties": { + "expiry": { "$ref": "#/$defs/ExecutionTemporalTarget" }, + "on_expiry": { "$ref": "#/$defs/ExecutionWaitExpiryAction" } + } + }, + "ExecutionTemporalTarget": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": ["kind", "at"], + "properties": { + "kind": { "const": "at" }, + "at": { "type": "string", "format": "date-time" } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["kind", "delay_seconds"], + "properties": { + "kind": { "const": "after" }, + "delay_seconds": { + "type": "integer", + "format": "uint64", + "minimum": 1 + } + } + } + ] + }, + "ExecutionWaitExpiryAction": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": ["kind"], + "properties": { + "kind": { "const": "fail_task" } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["kind"], + "properties": { + "kind": { "const": "fail_run" } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["kind", "output"], + "properties": { + "kind": { "const": "continue_with" }, + "output": {} + } + } + ] + }, "CapabilityReference": { "type": "object", "additionalProperties": false, diff --git a/k8s/base/20-orchestrator-deployment.yaml b/k8s/base/20-orchestrator-deployment.yaml index 6e260b256..14ac64ccc 100644 --- a/k8s/base/20-orchestrator-deployment.yaml +++ b/k8s/base/20-orchestrator-deployment.yaml @@ -5,6 +5,32 @@ metadata: namespace: moa-system spec: replicas: 6 + # The operator applies this HPA spec only to non-latest revisions that still + # own active invocations. Old versions retain one recovery replica but shed + # the six-replica serving footprint while long-lived journals drain. + autoscaling: + minReplicas: 1 + maxReplicas: 6 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 60 + behavior: + scaleUp: + stabilizationWindowSeconds: 0 + policies: + - type: Percent + value: 100 + periodSeconds: 60 + scaleDown: + stabilizationWindowSeconds: 120 + policies: + - type: Percent + value: 50 + periodSeconds: 60 restate: register: cluster: moa-restate @@ -53,7 +79,7 @@ spec: containerPort: 9080 - name: health containerPort: 9081 - - name: credentials + - name: internal containerPort: 10023 readinessProbe: httpGet: diff --git a/k8s/base/25-maintenance-deployment.yaml b/k8s/base/25-maintenance-deployment.yaml new file mode 100644 index 000000000..a4ffe6843 --- /dev/null +++ b/k8s/base/25-maintenance-deployment.yaml @@ -0,0 +1,118 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: moa-maintenance + namespace: moa-system + labels: + app.kubernetes.io/name: moa-maintenance + app.kubernetes.io/part-of: moa + app.kubernetes.io/component: maintenance +spec: + # Recreate avoids two complete fleet scanners during rollout. Correctness + # owners additionally use durable, expiring database leases so recovery from + # an unclean pod death remains safe. + replicas: 1 + strategy: + type: Recreate + revisionHistoryLimit: 3 + selector: + matchLabels: + app.kubernetes.io/name: moa-maintenance + app.kubernetes.io/part-of: moa + template: + metadata: + labels: + app.kubernetes.io/name: moa-maintenance + app.kubernetes.io/part-of: moa + app.kubernetes.io/component: maintenance + spec: + terminationGracePeriodSeconds: 600 + initContainers: + - name: wait-status-cutover + image: moa/orchestrator:latest + command: + - /bin/sh + - -ec + args: + - >- + exec /usr/local/bin/moa-orchestrator wait-status-cutover + --database-url "$MOA_CUTOVER_DATABASE_URL" + env: + - name: MOA_CUTOVER_DATABASE_URL + valueFrom: + secretKeyRef: + name: moa-postgres + key: url + containers: + - name: maintenance + image: moa/orchestrator:latest + args: ["maintenance"] + ports: + - name: health + containerPort: 9081 + readinessProbe: + httpGet: + path: /_health/ready + port: health + initialDelaySeconds: 5 + periodSeconds: 5 + livenessProbe: + httpGet: + path: /_health/live + port: health + initialDelaySeconds: 30 + periodSeconds: 15 + startupProbe: + httpGet: + path: /_health/live + port: health + failureThreshold: 30 + periodSeconds: 10 + envFrom: + - configMapRef: + name: moa-runtime-config + # Keep the first four entries aligned with the production security + # patch shared with the versioned orchestrator deployment. + env: + - name: RUST_LOG + value: warn,moa_orchestrator=info,moa_brain=info,moa_edge=info,async_openai::error=off + - name: MOA_SERVICE_INSTANCE_ID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: MOA_DATABASE_URL + valueFrom: + secretKeyRef: + name: moa-postgres + key: url + - name: MOA_DATABASE_MAINTENANCE_URL + valueFrom: + secretKeyRef: + name: moa-postgres-maintenance + key: url + optional: true + - name: MOA_DATABASE_MAX_CONNECTIONS + value: "4" + - name: MOA_DATABASE_BACKGROUND_MAX_CONNECTIONS + value: "4" + - name: MOA_DATABASE_CONNECT_TIMEOUT_SECONDS + value: "3" + - name: MOA_RESTATE_INGRESS_URL + value: http://restate.moa-restate.svc.cluster.local:8080 + - name: MOA_METRICS_EXPORTER + value: otlp + volumeMounts: + - name: kms-root-keys + mountPath: /var/run/secrets/moa-kms/root-keys + readOnly: true + resources: + requests: + cpu: "250m" + memory: 512Mi + limits: + cpu: "1" + memory: 2Gi + volumes: + - name: kms-root-keys + secret: + secretName: moa-kms-root-keys diff --git a/k8s/base/25-orchestrator-service.yaml b/k8s/base/25-orchestrator-service.yaml index 869201a88..28edb41f2 100644 --- a/k8s/base/25-orchestrator-service.yaml +++ b/k8s/base/25-orchestrator-service.yaml @@ -18,8 +18,8 @@ spec: - name: health port: 9081 targetPort: health - # Private connector credential ingress. ClusterIP plus the orchestrator + # Private connector-credential and provider-callback ingress. ClusterIP plus the orchestrator # NetworkPolicy limits this port to moa-edge pods in moa-system. - - name: credentials + - name: internal port: 10023 - targetPort: credentials + targetPort: internal diff --git a/k8s/base/50-edge-deployment.yaml b/k8s/base/50-edge-deployment.yaml index 53751e42b..39d0033d9 100644 --- a/k8s/base/50-edge-deployment.yaml +++ b/k8s/base/50-edge-deployment.yaml @@ -64,7 +64,7 @@ spec: value: http://restate.moa-restate.svc.cluster.local:8080 # Credential material takes a private, non-Restate path so it is # never persisted in the durable invocation journal. - - name: MOA_EDGE_CONNECTOR_CREDENTIAL_UPSTREAM + - name: MOA_EDGE_INTERNAL_INGRESS_UPSTREAM value: http://moa-orchestrator.moa-system.svc.cluster.local:10023 # Stays dark until the connector-management rollout checkpoint. - name: MOA_EDGE_CONNECTOR_MANAGEMENT_ENABLED diff --git a/k8s/base/kustomization.yaml b/k8s/base/kustomization.yaml index eccde9429..a37d2221f 100644 --- a/k8s/base/kustomization.yaml +++ b/k8s/base/kustomization.yaml @@ -7,6 +7,7 @@ resources: - 20-orchestrator-deployment.yaml - 21-session-status-migrator.yaml - 22-restate-bootstrap-job.yaml + - 25-maintenance-deployment.yaml - 25-orchestrator-service.yaml - 26-orchestrator-network-policy.yaml - 30-orchestrator-hpa.yaml diff --git a/k8s/overlays/production/kustomization.yaml b/k8s/overlays/production/kustomization.yaml index 89bdae93d..227c0ae45 100644 --- a/k8s/overlays/production/kustomization.yaml +++ b/k8s/overlays/production/kustomization.yaml @@ -20,6 +20,12 @@ patches: kind: RestateDeployment name: moa-orchestrator path: patches/orchestrator-security-profile.yaml + - target: + group: apps + version: v1 + kind: Deployment + name: moa-maintenance + path: patches/orchestrator-security-profile.yaml - target: group: restate.dev version: v1 @@ -32,6 +38,12 @@ patches: kind: RestateDeployment name: moa-orchestrator path: patches/orchestrator-observability.yaml + - target: + group: apps + version: v1 + kind: Deployment + name: moa-maintenance + path: patches/orchestrator-observability.yaml - target: group: apps version: v1 diff --git a/k8s/overlays/production/patches/runtime-security-profile.yaml b/k8s/overlays/production/patches/runtime-security-profile.yaml index 9cef521df..b51d13a58 100644 --- a/k8s/overlays/production/patches/runtime-security-profile.yaml +++ b/k8s/overlays/production/patches/runtime-security-profile.yaml @@ -10,10 +10,15 @@ data: MOA_SECURITY_PROFILE: cloud MOA_PERMISSIONS_DEFAULT_EFFECT: deny MOA_CLOUD_HANDS_DEFAULT_PROVIDER: e2b - # Production first starts maintenance owners while admission stays dark. - # Switching to `admit` is a later atomic rollout step after readiness proves - # the exact provider account, bucket, quotas, and reaper heartbeat. + # The singleton moa-maintenance Deployment owns reconciliation and reaping; + # versioned Restate handlers consume this policy without starting scanners. + # Production first keeps workspace admission dark. Switching to `admit` is a + # later atomic rollout step after maintenance readiness proves the exact + # provider account, bucket, quotas, and reaper heartbeat. MOA_SANDBOX_WORKSPACE_MODE: maintenance + # Safety-critical due delivery remains frequent. Full provider inventory and + # retention use their separate adaptive cadences inside the maintenance role. + MOA_EXECUTION_TRIGGER_RECONCILIATION_CADENCE_SECONDS: "30" MOA_SANDBOX_WORKSPACE_OPERATION_RETENTION_SECONDS: "604800" MOA_SANDBOX_WORKSPACE_MAXIMUM_OPERATION_SECONDS: "86400" MOA_SANDBOX_WORKSPACE_RECONCILIATION_CLAIM_TTL_SECONDS: "60" diff --git a/k8s/scripts/smoke.sh b/k8s/scripts/smoke.sh index 229f81bf9..641c08ac4 100755 --- a/k8s/scripts/smoke.sh +++ b/k8s/scripts/smoke.sh @@ -46,6 +46,64 @@ assert_occurrences() { || die "${description}: expected ${expected}, found ${observed}" } +container_image_pairs() { + awk ' + function indentation(line, copy) { + copy = line + sub(/[^ ].*$/, "", copy) + return length(copy) + } + function emit_item() { + if (item_name != "" && item_image != "") { + print item_name "=" item_image + } + item_name = "" + item_image = "" + } + { + indent = indentation($0) + if ($0 ~ /^[[:space:]]*containers:$/ || $0 ~ /^[[:space:]]*initContainers:$/) { + emit_item() + in_section = 1 + section_indent = indent + next + } + if (in_section && indent <= section_indent && $0 !~ /^[[:space:]]*-/) { + emit_item() + in_section = 0 + } + if (!in_section) { + next + } + if (indent == section_indent && $0 ~ /^[[:space:]]*-[[:space:]]/) { + emit_item() + } + if (indent == section_indent + 2 && $1 == "image:") { + item_image = $2 + } + if (indent == section_indent + 2 && $1 == "name:") { + item_name = $2 + } + } + END { emit_item() } + ' <<<"$1" +} + +assert_workload_image_contract() { + local content="$1" + local expected_image="$2" + local description="$3" + local pairs expected_name + shift 3 + pairs="$(container_image_pairs "${content}")" + assert_occurrences "${pairs}" "$#" "=${expected_image}" \ + "${description} has an unexpected number of containers using ${expected_image}" + for expected_name in "$@"; do + assert_contains "${pairs}" "${expected_name}=${expected_image}" \ + "${description} container ${expected_name} does not use ${expected_image}" + done +} + manifest_document() { local manifest="$1" local target_kind="$2" @@ -134,7 +192,7 @@ validate_manifests() { local work_dir local_manifest production_manifest jobs_manifest rendered_dir local local_orchestrator production_orchestrator local_edge production_edge local local_bootstrap production_bootstrap local_bootstrap_sa production_bootstrap_sa - local local_status_migrator production_status_migrator + local local_status_migrator production_status_migrator local_maintenance production_maintenance local local_orchestrator_service production_orchestrator_service local local_orchestrator_policy production_orchestrator_policy local local_edge_service production_edge_service @@ -143,7 +201,7 @@ validate_manifests() { local local_rustfs_pvc local_postgres_pvc local local_restate production_restate local local_orchestrator_readiness production_orchestrator_readiness - local rewrap_job application_content + local rewrap_job application_content local_orchestrator_image production_orchestrator_image work_dir="$(mktemp -d)" trap 'rm -rf -- "${work_dir}"' RETURN local_manifest="${work_dir}/local.yaml" @@ -184,6 +242,8 @@ validate_manifests() { production_bootstrap="$(manifest_document "${production_manifest}" Job moa-restate-bootstrap-image-revision)" local_status_migrator="$(manifest_document "${local_manifest}" Job moa-session-status-migrator-image-revision)" production_status_migrator="$(manifest_document "${production_manifest}" Job moa-session-status-migrator-image-revision)" + local_maintenance="$(manifest_document "${local_manifest}" Deployment moa-maintenance)" + production_maintenance="$(manifest_document "${production_manifest}" Deployment moa-maintenance)" local_bootstrap_sa="$(manifest_document "${local_manifest}" ServiceAccount moa-restate-bootstrap)" production_bootstrap_sa="$(manifest_document "${production_manifest}" ServiceAccount moa-restate-bootstrap)" local_orchestrator_readiness="$(readiness_probe_path "${local_orchestrator}")" @@ -229,11 +289,24 @@ validate_manifests() { assert_excludes "${orchestrator}" "MOA_DEREGISTER_""ON_SHUTDOWN" \ "normal orchestrator still owns shutdown deregistration" done - assert_occurrences "$(<"${local_manifest}")" 5 "image: moa/orchestrator:kind" \ - "local runtime, migration-only stage, and bootstrap must use the same orchestrator image" - assert_occurrences "$(<"${production_manifest}")" 6 \ - "image: moa/orchestrator@sha256:0000000000000000000000000000000000000000000000000000000000000000" \ - "unrendered production runtime, migration-only stage, and bootstrap must use the same immutable sentinel" + local_orchestrator_image="moa/orchestrator:kind" + production_orchestrator_image="moa/orchestrator@sha256:0000000000000000000000000000000000000000000000000000000000000000" + assert_workload_image_contract "${local_orchestrator}" "${local_orchestrator_image}" \ + "local orchestrator" orchestrator wait-status-cutover + assert_workload_image_contract "${local_maintenance}" "${local_orchestrator_image}" \ + "local maintenance runtime" maintenance wait-status-cutover + assert_workload_image_contract "${local_status_migrator}" "${local_orchestrator_image}" \ + "local status migrator" status-migrator database-migrations + assert_workload_image_contract "${local_bootstrap}" "${local_orchestrator_image}" \ + "local Restate bootstrap" bootstrap + assert_workload_image_contract "${production_orchestrator}" "${production_orchestrator_image}" \ + "production orchestrator" orchestrator wait-status-cutover prepare-hand-provider-credentials + assert_workload_image_contract "${production_maintenance}" "${production_orchestrator_image}" \ + "production maintenance runtime" maintenance wait-status-cutover prepare-hand-provider-credentials + assert_workload_image_contract "${production_status_migrator}" "${production_orchestrator_image}" \ + "production status migrator" status-migrator database-migrations + assert_workload_image_contract "${production_bootstrap}" "${production_orchestrator_image}" \ + "production Restate bootstrap" bootstrap assert_occurrences "${production_edge}" 1 \ "image: moa/edge@sha256:0000000000000000000000000000000000000000000000000000000000000000" \ "unrendered production edge must use the immutable sentinel" @@ -385,15 +458,15 @@ validate_manifests() { assert_excludes "${edge}" "MOA_KMS_" "edge unexpectedly receives KMS configuration" assert_excludes "${edge}" "moa-kms-root-keys" "edge unexpectedly mounts the KMS Secret" assert_excludes "${edge}" "/var/run/secrets/moa-kms" "edge unexpectedly exposes the KMS keyring" - assert_contains "${edge}" "name: MOA_EDGE_CONNECTOR_CREDENTIAL_UPSTREAM" \ - "edge is missing the private connector credential upstream" + assert_contains "${edge}" "name: MOA_EDGE_INTERNAL_INGRESS_UPSTREAM" \ + "edge is missing the private orchestrator ingress upstream" assert_contains "${edge}" "http://moa-orchestrator.moa-system.svc.cluster.local:10023" \ - "edge connector credential upstream does not target the private orchestrator listener" + "edge internal ingress upstream does not target the private orchestrator listener" done for orchestrator in "${local_orchestrator}" "${production_orchestrator}"; do assert_contains "${orchestrator}" "- --credential-port" \ - "orchestrator does not configure the private credential listener" - assert_contains "${orchestrator}" "name: credentials" \ + "orchestrator does not configure the private internal ingress listener" + assert_contains "${orchestrator}" "name: internal" \ "orchestrator pod does not declare its private credential port" assert_contains "${orchestrator}" "containerPort: 10023" \ "orchestrator private credential listener is not on the expected port" @@ -401,11 +474,11 @@ validate_manifests() { for service in "${local_orchestrator_service}" "${production_orchestrator_service}"; do assert_contains "${service}" "type: ClusterIP" \ "orchestrator Service is not explicitly internal-only" - assert_contains "${service}" "name: credentials" \ + assert_contains "${service}" "name: internal" \ "orchestrator Service does not route the private credential listener" assert_contains "${service}" "port: 10023" \ "orchestrator Service has the wrong credential port" - assert_contains "${service}" "targetPort: credentials" \ + assert_contains "${service}" "targetPort: internal" \ "orchestrator Service does not target the named credential port" done for service in "${local_edge_service}" "${production_edge_service}"; do diff --git a/k8s/scripts/validate-observability.sh b/k8s/scripts/validate-observability.sh index 5ce0196d6..0c5807a6b 100755 --- a/k8s/scripts/validate-observability.sh +++ b/k8s/scripts/validate-observability.sh @@ -47,6 +47,18 @@ EXPECTED_ALERTS=( MOAAuthzOutboxBacklogAge MOAAuthzOutboxDeadLetters MOABuiltinApprovalBacklogAge + MOAExecutionActiveAttemptStuck + MOAExecutionAdmissionSaturated + MOAExecutionExternalJobStuck + MOAExecutionMaintenanceReconcileStale + MOAExecutionOldestReadySLO + MOAExecutionOutboxDeadLetters + MOAExecutionOutboxLagHigh + MOAExecutionOverdueDeadlines + MOAExecutionQueueSampleSaturated + MOAExecutionRetentionStale + MOAExecutionTriggerDeadLetters + MOAExecutionTriggerLagHigh MOALLMFailoverElevated MOALineageDeadLettering MOALineageDrainTimeout @@ -63,6 +75,7 @@ EXPECTED_ALERTS=( MOARestateIngressRateLimited MOARestateInvocationTaskFailures MOARestateNodeScrapeDown + MOARestateOldDeploymentDrainAge MOARestatePartitionAppliedLSNLagHigh MOARestatePartitionLeaderMissing MOARestatePartitionStatusStale @@ -78,6 +91,7 @@ EXPECTED_ALERTS=( MOASandboxWorkspaceReaperBacklogAge MOASandboxWorkspaceReaperHeartbeatStale MOASandboxWorkspaceReaperUnready + MOASandboxParkedTaskRetainsActiveHand ) # Prometheus spellings of the server instruments verified against Restate @@ -463,6 +477,98 @@ ORCHESTRATOR_NETPOL_TEXT="$(<"${REPO_ROOT}/k8s/base/26-orchestrator-network-poli assert_excludes "${ORCHESTRATOR_NETPOL_TEXT}" "- Egress" \ "the orchestrator NetworkPolicy now restricts egress; OTLP push to the collector must be explicitly allowed or telemetry stops with no error anywhere" +echo "Checking long-horizon maintenance and draining-version contracts..." +python3 - "${REPO_ROOT}" <<'PY' || exit 1 +import pathlib +import sys + +import yaml + +root = pathlib.Path(sys.argv[1]) + +restate_deployment = yaml.safe_load( + (root / "k8s/base/20-orchestrator-deployment.yaml").read_text(encoding="utf-8") +) +autoscaling = restate_deployment.get("spec", {}).get("autoscaling") or {} +if autoscaling.get("minReplicas") != 1 or autoscaling.get("maxReplicas") != 6: + raise SystemExit( + "draining Restate revisions must autoscale between exactly one recovery " + "replica and the six-replica serving ceiling" + ) +if "scaleTargetRef" in autoscaling: + raise SystemExit( + "RestateDeployment autoscaling must omit scaleTargetRef; the operator injects it per revision" + ) +metrics = autoscaling.get("metrics") or [] +cpu_targets = [ + metric.get("resource", {}).get("target", {}).get("averageUtilization") + for metric in metrics + if metric.get("type") == "Resource" + and metric.get("resource", {}).get("name") == "cpu" +] +if cpu_targets != [60]: + raise SystemExit( + f"draining Restate revisions must use the single 60% CPU target, got {cpu_targets}" + ) +scale_down = autoscaling.get("behavior", {}).get("scaleDown") or {} +if scale_down.get("stabilizationWindowSeconds") != 120: + raise SystemExit("draining-version scale-down must use the reviewed 120s stabilization window") + +maintenance_path = root / "k8s/base/25-maintenance-deployment.yaml" +maintenance = yaml.safe_load(maintenance_path.read_text(encoding="utf-8")) +if maintenance.get("kind") != "Deployment" or maintenance.get("metadata", {}).get("name") != "moa-maintenance": + raise SystemExit(f"{maintenance_path} must define the moa-maintenance Deployment") +spec = maintenance.get("spec", {}) +if spec.get("replicas") != 1 or spec.get("strategy", {}).get("type") != "Recreate": + raise SystemExit("moa-maintenance must be a singleton using the Recreate rollout strategy") +pod_spec = spec.get("template", {}).get("spec", {}) +containers = pod_spec.get("containers") or [] +if len(containers) != 1: + raise SystemExit("moa-maintenance must have exactly one runtime container") +container = containers[0] +if container.get("args") != ["maintenance"]: + raise SystemExit("moa-maintenance must use the hard-break maintenance subcommand") +ports = {port.get("name") for port in container.get("ports") or []} +if ports != {"health"}: + raise SystemExit( + "moa-maintenance may expose only health; Restate, SCIM, and credential ingress belong to serving pods" + ) +env_names = {entry.get("name") for entry in container.get("env") or []} +for required in ( + "MOA_DATABASE_URL", + "MOA_DATABASE_MAINTENANCE_URL", + "MOA_RESTATE_INGRESS_URL", + "MOA_METRICS_EXPORTER", + "MOA_SERVICE_INSTANCE_ID", +): + if required not in env_names: + raise SystemExit(f"moa-maintenance is missing required environment binding {required}") + +base_kustomization = (root / "k8s/base/kustomization.yaml").read_text(encoding="utf-8") +if base_kustomization.count("25-maintenance-deployment.yaml") != 1: + raise SystemExit("base kustomization must include the maintenance Deployment exactly once") + +production_kustomization = ( + root / "k8s/overlays/production/kustomization.yaml" +).read_text(encoding="utf-8") +if production_kustomization.count("name: moa-maintenance") != 2: + raise SystemExit( + "production must apply both the security and observability patches to moa-maintenance" + ) +if production_kustomization.count("path: patches/orchestrator-security-profile.yaml") != 2: + raise SystemExit("maintenance and serving pods must share the production security patch") +if production_kustomization.count("path: patches/orchestrator-observability.yaml") != 2: + raise SystemExit("maintenance and serving pods must share the production OTLP patch") + +runtime_profile = ( + root / "k8s/overlays/production/patches/runtime-security-profile.yaml" +).read_text(encoding="utf-8") +if 'MOA_EXECUTION_TRIGGER_RECONCILIATION_CADENCE_SECONDS: "30"' not in runtime_profile: + raise SystemExit("production does not set the safety-critical trigger reconciliation cadence") + +print(" OK maintenance is singleton/private and old Restate revisions autoscale while draining") +PY + echo "Checking alert rules with promtool..." RULE_FILES=("${ALERTS_DIR}"/*.yaml) # kustomization.yaml lives in the same directory and is not a rule file. @@ -581,7 +687,9 @@ root = pathlib.Path(sys.argv[1]) dashboard_path = pathlib.Path(sys.argv[2]) allowed = set(sys.argv[3:]) allowed.add("up") -metric_pattern = re.compile(r"(?:restate_[a-z0-9_]+|\bup\b)") +# Match complete Restate server metric tokens, not the `restate_` substring in +# MOA-owned fleet gauges such as `moa_restate_draining_deployment_*`. +metric_pattern = re.compile(r"(?:(?', execution_source)) +if label_keys != {"phase", "queue", "resource", "sample", "scope"}: + raise SystemExit( + "long-horizon execution metric label vocabulary drifted: " + f"{sorted(label_keys)}" + ) +for forbidden in ( + "tenant_id", + "run_id", + "run_uid", + "task_id", + "task_uid", + "external_job_id", + "deployment_id", + "deployment_version", +): + if forbidden in execution_source: + raise SystemExit( + f"long-horizon execution metrics expose forbidden identity label {forbidden!r}" + ) + +alerts_path = root / "ops/prometheus/alerts/moa-long-horizon-execution.yaml" +alerts = yaml.safe_load(alerts_path.read_text(encoding="utf-8")) +rules = [ + rule + for group in alerts.get("spec", {}).get("groups") or [] + for rule in group.get("rules") or [] +] +if len(rules) != 12: + raise SystemExit(f"{alerts_path} must contain exactly twelve actionable alerts") +alert_expressions = "\n".join(rule.get("expr", "") for rule in rules) +required_alert_metrics = { + "moa_execution_active_attempt_oldest_age_seconds", + "moa_execution_admission_utilization_ratio", + "moa_execution_external_job_oldest_age_seconds", + "moa_execution_maintenance_last_success_age_seconds", + "moa_execution_maintenance_ready", + "moa_execution_retention_last_success_age_seconds", + "moa_execution_retention_ready", + "moa_execution_oldest_ready_age_seconds", + "moa_execution_outbox_dead_letters", + "moa_execution_outbox_lag_seconds", + "moa_execution_queue_sample_saturated", + "moa_execution_overdue_deadlines", + "moa_execution_trigger_dead_letters", + "moa_execution_trigger_lag_seconds", +} +observed_alert_metrics = set(re.findall(r"moa_execution_[a-z0-9_]+", alert_expressions)) +if observed_alert_metrics != required_alert_metrics: + raise SystemExit( + "long-horizon alert metric inventory drifted; " + f"missing={sorted(required_alert_metrics - observed_alert_metrics)}, " + f"extra={sorted(observed_alert_metrics - required_alert_metrics)}" + ) + +retention_rules = [rule for rule in rules if rule.get("alert") == "MOAExecutionRetentionStale"] +if len(retention_rules) != 1: + raise SystemExit("long-horizon alerts must define MOAExecutionRetentionStale exactly once") +retention_expr = retention_rules[0].get("expr", "") +for required_clause in ( + "absent(moa_execution_retention_ready)", + "max(moa_execution_retention_ready) == 0", + "max(moa_execution_retention_last_success_age_seconds) > 7200", +): + if required_clause not in retention_expr: + raise SystemExit( + "MOAExecutionRetentionStale must alert on missing, unready, and older-than-two-hour receipts" + ) + +kustomization = (root / "ops/prometheus/alerts/kustomization.yaml").read_text( + encoding="utf-8" +) +if kustomization.count("moa-long-horizon-execution.yaml") != 1: + raise SystemExit( + "moa-long-horizon-execution.yaml must appear exactly once in the alert kustomization" + ) + +print( + f" OK {len(expected)} low-cardinality metrics back {len(rules)} long-horizon alerts" +) +PY echo "Checking the sandbox workspace metrics/dashboard/alert contract..." python3 - "${REPO_ROOT}" <<'PY' || exit 1 import json @@ -804,8 +1041,8 @@ rules = [ for group in alerts.get("spec", {}).get("groups") or [] for rule in group.get("rules") or [] ] -if len(rules) != 7: - raise SystemExit(f"{alerts_path} must contain exactly seven actionable alerts") +if len(rules) != 8: + raise SystemExit(f"{alerts_path} must contain exactly eight actionable alerts") for rule in rules: expression = rule.get("expr", "") annotations = rule.get("annotations") or {} diff --git a/ops/prometheus/alerts/kustomization.yaml b/ops/prometheus/alerts/kustomization.yaml index e35ac4408..dbbe03261 100644 --- a/ops/prometheus/alerts/kustomization.yaml +++ b/ops/prometheus/alerts/kustomization.yaml @@ -9,5 +9,6 @@ kind: Kustomization resources: - moa-durability.yaml - moa-lineage.yaml + - moa-long-horizon-execution.yaml - moa-restate.yaml - sandbox-workspaces.yaml diff --git a/ops/prometheus/alerts/moa-long-horizon-execution.yaml b/ops/prometheus/alerts/moa-long-horizon-execution.yaml new file mode 100644 index 000000000..2b261c6b1 --- /dev/null +++ b/ops/prometheus/alerts/moa-long-horizon-execution.yaml @@ -0,0 +1,122 @@ +# Long-horizon execution and singleton-maintenance alerts. +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: moa-long-horizon-execution + namespace: observability + labels: + app.kubernetes.io/name: moa-long-horizon-execution-alerts + app.kubernetes.io/part-of: moa + moa.dev/rule-sync: mimir +spec: + groups: + - name: moa-long-horizon-execution + interval: 30s + rules: + - alert: MOAExecutionOverdueDeadlines + expr: max(moa_execution_overdue_deadlines) > 0 + for: 2m + labels: + severity: critical + annotations: + summary: Nonterminal executions remain past their deadline + description: "{{ $value }} execution runs are still nonterminal after their absolute deadline. Verify deadline triggers, controller activation, and terminal projection delivery." + + - alert: MOAExecutionOldestReadySLO + expr: max(moa_execution_oldest_ready_age_seconds) > 120 + for: 5m + labels: + severity: warning + annotations: + summary: Ready execution work is not being dispatched + description: "The oldest ready task has waited {{ $value }} seconds (threshold 120s). Check admission capacity, dispatcher ownership, and outbox delivery." + + - alert: MOAExecutionActiveAttemptStuck + expr: max(moa_execution_active_attempt_oldest_age_seconds) > 900 + for: 5m + labels: + severity: critical + annotations: + summary: An active execution attempt exceeded its lease SLO + description: "The oldest active task attempt is {{ $value }} seconds old (threshold 900s). Verify watchdog delivery and ambiguous-effect reconciliation before replaying work." + + - alert: MOAExecutionTriggerLagHigh + expr: max(moa_execution_trigger_lag_seconds) > 120 + for: 5m + labels: + severity: warning + annotations: + summary: Due execution triggers are delayed + description: "The oldest due undelivered trigger is {{ $value }} seconds late (threshold 120s). Check the maintenance owner, Restate ingress, and trigger claim expiry." + + - alert: MOAExecutionTriggerDeadLetters + expr: max(moa_execution_trigger_dead_letters) > 0 + for: 2m + labels: + severity: critical + annotations: + summary: Execution triggers entered dead-letter state + description: "{{ $value }} execution triggers require operator repair. Inspect immutable trigger IDs and generation fences before redelivery." + + - alert: MOAExecutionOutboxLagHigh + expr: max(moa_execution_outbox_lag_seconds) > 120 + for: 5m + labels: + severity: warning + annotations: + summary: Execution dispatch outbox is delayed + description: "The oldest undispatched execution outbox row is {{ $value }} seconds old (threshold 120s). Check claim ownership and Restate ingress availability." + + - alert: MOAExecutionOutboxDeadLetters + expr: max(moa_execution_outbox_dead_letters) > 0 + for: 2m + labels: + severity: critical + annotations: + summary: Execution dispatch rows entered dead-letter state + description: "{{ $value }} execution outbox rows require operator repair. Preserve the persisted generation and idempotency key during redelivery." + + - alert: MOAExecutionQueueSampleSaturated + expr: max by (queue, sample) (moa_execution_queue_sample_saturated) > 0 + for: 5m + labels: + severity: warning + annotations: + summary: Execution queue-health observation reached its cap + description: "The bounded {{ $labels.queue }} {{ $labels.sample }} sample is saturated, so its exported depth is a lower bound. Inspect the queue and repair capacity before treating the count as exact." + + - alert: MOAExecutionMaintenanceReconcileStale + expr: absent(moa_execution_maintenance_ready) or max(moa_execution_maintenance_ready) == 0 or max(moa_execution_maintenance_last_success_age_seconds) > 120 + for: 2m + labels: + severity: critical + annotations: + summary: Singleton execution maintenance reconciliation is stale + description: "The maintenance owner is absent, unready, or has not committed a successful bounded reconciliation receipt within 120 seconds. Due triggers and dispatch repair cannot be assumed healthy." + + - alert: MOAExecutionRetentionStale + expr: absent(moa_execution_retention_ready) or max(moa_execution_retention_ready) == 0 or max(moa_execution_retention_last_success_age_seconds) > 7200 + for: 10m + labels: + severity: warning + annotations: + summary: Execution retention is stale + description: "Execution retention is absent, unready, or its durable last-success receipt is older than two hours. The bounded retention owner normally runs at least once per hour; inspect its checkpoint and errors before terminal evidence exceeds policy." + + - alert: MOAExecutionAdmissionSaturated + expr: max by (resource, scope) (moa_execution_admission_utilization_ratio) > 0.9 + for: 15m + labels: + severity: warning + annotations: + summary: Execution admission is near a configured ceiling + description: "Execution {{ $labels.resource }} utilization at {{ $labels.scope }} scope is {{ $value | humanizePercentage }} (threshold 90%). Confirm fair queueing before increasing the durable limit." + + - alert: MOAExecutionExternalJobStuck + expr: max(moa_execution_external_job_oldest_age_seconds) > 86400 + for: 30m + labels: + severity: warning + annotations: + summary: An asynchronous external execution job is stale + description: "The oldest nonterminal external job is {{ $value }} seconds old (threshold 24h). Check callback delivery, sparse reconciliation, and provider status." diff --git a/ops/prometheus/alerts/moa-restate.yaml b/ops/prometheus/alerts/moa-restate.yaml index 5313032a6..6955abb3d 100644 --- a/ops/prometheus/alerts/moa-restate.yaml +++ b/ops/prometheus/alerts/moa-restate.yaml @@ -127,3 +127,12 @@ spec: annotations: summary: Restate has a partition without an effective leader description: "Only {{ $value }} effective partition leaders are visible. Compare with restate_num_partitions and inspect cluster membership before changing placement." + + - alert: MOARestateOldDeploymentDrainAge + expr: max(moa_restate_draining_deployment_oldest_age_seconds) > 3600 + for: 15m + labels: + severity: warning + annotations: + summary: An old Restate deployment revision is draining too long + description: "The oldest draining MOA service revision is {{ $value }} seconds old (threshold 1h). Check remaining invocations and the draining-revision HPA before retaining full old-version capacity." diff --git a/ops/prometheus/alerts/sandbox-workspaces.yaml b/ops/prometheus/alerts/sandbox-workspaces.yaml index e3f9d3df4..b264e36c9 100644 --- a/ops/prometheus/alerts/sandbox-workspaces.yaml +++ b/ops/prometheus/alerts/sandbox-workspaces.yaml @@ -30,7 +30,7 @@ spec: severity: critical annotations: summary: The supervised workspace reaper is unready - description: "At least one serving replica reports an unready workspace reaper. Readiness should drain it; verify the job result and do not disable maintenance while durable work remains." + description: "The singleton maintenance owner reports an unready workspace reaper or publishes no series. Verify its supervised job result and do not disable maintenance while durable work remains." runbook_url: https://github.com/hwuiwon/moa/blob/main/docs/19-data-operations.md#workspace-reaper-failure - alert: MOASandboxWorkspaceReaperHeartbeatStale @@ -40,7 +40,7 @@ spec: severity: critical annotations: summary: The workspace reaper heartbeat is stale - description: "The oldest workspace reaper heartbeat is {{ $value }} seconds old (threshold 120s). Treat the replica as unready and inspect the supervised job before restarting it." + description: "The workspace reaper heartbeat is {{ $value }} seconds old (threshold 120s). Treat maintenance as unready and inspect the supervised job before restarting it." runbook_url: https://github.com/hwuiwon/moa/blob/main/docs/19-data-operations.md#workspace-reaper-failure - alert: MOASandboxWorkspaceReaperBacklogAge @@ -82,3 +82,13 @@ spec: summary: Provider inventory differs from durable MOA ownership description: "{{ $value }} unresolved {{ $labels.classification }} findings exist for {{ $labels.provider_kind }}. Findings are quarantined and must never be auto-deleted; prove ownership and absence before resolving them." runbook_url: https://github.com/hwuiwon/moa/blob/main/docs/19-data-operations.md#provider-inventory-drift + + - alert: MOASandboxParkedTaskRetainsActiveHand + expr: max(moa_sandbox_workspace_parked_tasks_with_active_hands) > 0 + for: 1m + labels: + severity: critical + annotations: + summary: Parked execution work is retaining sandbox compute + description: "{{ $value }} parked execution tasks still own an active hand. Checkpoint-and-release is a correctness and cost invariant; fence the affected attempts before cleanup." + runbook_url: https://github.com/hwuiwon/moa/blob/main/docs/19-data-operations.md#workspace-reaper-failure diff --git a/scripts/cutover-long-horizon-execution.sh b/scripts/cutover-long-horizon-execution.sh new file mode 100755 index 000000000..9c1eb9b23 --- /dev/null +++ b/scripts/cutover-long-horizon-execution.sh @@ -0,0 +1,410 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +REPO_ROOT="$(cd -- "${SCRIPT_DIR}/.." && pwd -P)" + +DATABASE_ADMIN_URL="" +RESTATE_ADMIN_URL="" +RESTATE_INGRESS_URL="" +OLD_DEPLOYMENT_ID="" +NEW_DEPLOYMENT_URI="" +ARCHIVE_DIR="" +CONFIRMED=0 + +readonly OLD_SERVICES=(ExecutionRun ExecutionTask ExecutionCompensation) +readonly NEW_SERVICES=( + ExecutionRunController + ExecutionTaskAttempt + ExecutionCompensationAttempt + ExecutionTrigger + ExecutionDispatcher + ExecutionDispatchDrain + ExecutionDispatchReconciler + ExecutionRetention + ExecutionSchedule + DurableTimeout +) +readonly NEW_SERVICE_CONTRACT_JSON='[ + {"name":"ExecutionRunController","ty":"VirtualObject","public":false}, + {"name":"ExecutionTaskAttempt","ty":"Workflow","public":false}, + {"name":"ExecutionCompensationAttempt","ty":"Workflow","public":false}, + {"name":"ExecutionTrigger","ty":"Service","public":false}, + {"name":"ExecutionDispatcher","ty":"Service","public":false}, + {"name":"ExecutionDispatchDrain","ty":"VirtualObject","public":false}, + {"name":"ExecutionDispatchReconciler","ty":"Service","public":false}, + {"name":"ExecutionRetention","ty":"Service","public":false}, + {"name":"ExecutionSchedule","ty":"Service","public":true}, + {"name":"DurableTimeout","ty":"Service","public":false} +]' + +# This is the destructive first registration of the bounded execution family: the retired +# deployment contains only OLD_SERVICES. The stateless ExecutionDispatcher router and fleet-keyed +# ExecutionDispatchDrain are therefore registered directly; there is no compatibility deployment. + +usage() { + cat <<'USAGE' +Usage: scripts/cutover-long-horizon-execution.sh \ + --database-admin-url URL \ + --restate-admin-url URL \ + --restate-ingress-url URL \ + --old-deployment-id ID \ + --new-deployment-uri URI \ + --archive-dir ABSOLUTE_EMPTY_DIRECTORY \ + [--confirm-destructive-cutover] + +Hard-cuts the retired ExecutionRun, ExecutionTask, and ExecutionCompensation +Restate runtime to bounded execution activations. The script always performs +and prints its read-only Postgres and Restate preflight before considering any +mutation. Without --confirm-destructive-cutover it exits after preflight. + +The archive directory must already exist, be absolute, and be empty. The +database URL, Restate Admin/ingress URLs, old deployment ID, and new immutable +deployment URI are mandatory; no destructive target is inferred. +USAGE +} + +die() { + echo "cutover refused: $*" >&2 + exit 2 +} + +require_cmd() { + command -v "$1" >/dev/null 2>&1 || die "required command not found: $1" +} + +require_value() { + local option="$1" + local value="${2:-}" + [[ -n "${value}" ]] || die "${option} requires a non-empty value" +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --database-admin-url) + require_value "$1" "${2:-}" + DATABASE_ADMIN_URL="$2" + shift 2 + ;; + --restate-admin-url) + require_value "$1" "${2:-}" + RESTATE_ADMIN_URL="${2%/}" + shift 2 + ;; + --restate-ingress-url) + require_value "$1" "${2:-}" + RESTATE_INGRESS_URL="${2%/}" + shift 2 + ;; + --old-deployment-id) + require_value "$1" "${2:-}" + OLD_DEPLOYMENT_ID="$2" + shift 2 + ;; + --new-deployment-uri) + require_value "$1" "${2:-}" + NEW_DEPLOYMENT_URI="${2%/}" + shift 2 + ;; + --archive-dir) + require_value "$1" "${2:-}" + ARCHIVE_DIR="${2%/}" + shift 2 + ;; + --confirm-destructive-cutover) + CONFIRMED=1 + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + die "unknown argument: $1" + ;; + esac +done + +require_value --database-admin-url "${DATABASE_ADMIN_URL}" +require_value --restate-admin-url "${RESTATE_ADMIN_URL}" +require_value --restate-ingress-url "${RESTATE_INGRESS_URL}" +require_value --old-deployment-id "${OLD_DEPLOYMENT_ID}" +require_value --new-deployment-uri "${NEW_DEPLOYMENT_URI}" +require_value --archive-dir "${ARCHIVE_DIR}" + +for command in cargo curl find jq pg_dump psql restate seq tee tr wc; do + require_cmd "${command}" +done + +[[ "${DATABASE_ADMIN_URL}" == postgres://* || "${DATABASE_ADMIN_URL}" == postgresql://* ]] \ + || die "--database-admin-url must be an explicit postgres:// or postgresql:// URL" +[[ "${RESTATE_ADMIN_URL}" == http://* || "${RESTATE_ADMIN_URL}" == https://* ]] \ + || die "--restate-admin-url must be an explicit HTTP(S) URL" +[[ "${RESTATE_INGRESS_URL}" == http://* || "${RESTATE_INGRESS_URL}" == https://* ]] \ + || die "--restate-ingress-url must be an explicit HTTP(S) URL" +[[ "${NEW_DEPLOYMENT_URI}" == http://* || "${NEW_DEPLOYMENT_URI}" == https://* ]] \ + || die "--new-deployment-uri must be an explicit immutable HTTP(S) URI" +[[ "${OLD_DEPLOYMENT_ID}" =~ ^[A-Za-z0-9_-]+$ ]] \ + || die "--old-deployment-id contains unsafe characters" +[[ "${ARCHIVE_DIR}" == /* && "${ARCHIVE_DIR}" != / && "${ARCHIVE_DIR}" != *".."* ]] \ + || die "--archive-dir must be an explicit absolute non-root path without '..'" +[[ -d "${ARCHIVE_DIR}" ]] || die "--archive-dir must already exist" +[[ -z "$(find "${ARCHIVE_DIR}" -mindepth 1 -maxdepth 1 -print -quit)" ]] \ + || die "--archive-dir must be empty" + +restate_cli() { + RESTATE_ADMIN_URL="${RESTATE_ADMIN_URL}" \ + RESTATE_INGRESS_URL="${RESTATE_INGRESS_URL}" \ + RESTATE_CLI_CONFIG_HOME="${ARCHIVE_DIR}/restate-cli-config" \ + restate -e local "$@" +} + +restate_query() { + local query="$1" + curl -fsS \ + -X POST "${RESTATE_ADMIN_URL}/query" \ + -H "accept: application/json" \ + -H "content-type: application/json" \ + --data-binary "$(jq -cn --arg query "${query}" '{query: $query}')" +} + +echo "== Read-only preflight: Postgres nonterminal executions ==" +readonly NONTERMINAL_SQL=" +SELECT run_uid, status, updated_at +FROM moa.execution_run +WHERE status NOT IN ('completed', 'partial', 'blocked', 'unsupported', 'failed', 'cancelled') +ORDER BY updated_at, run_uid;" +psql -X "${DATABASE_ADMIN_URL}" --set=ON_ERROR_STOP=1 --pset=pager=off \ + --command "${NONTERMINAL_SQL}" +NONTERMINAL_RUNS="$( + psql -X "${DATABASE_ADMIN_URL}" --set=ON_ERROR_STOP=1 --tuples-only --no-align \ + --command "SELECT count(*) FROM moa.execution_run WHERE status NOT IN ('completed', 'partial', 'blocked', 'unsupported', 'failed', 'cancelled');" +)" + +echo "== Read-only preflight: old Restate deployment ==" +DEPLOYMENTS_JSON="$(curl -fsS "${RESTATE_ADMIN_URL}/deployments")" +jq --arg deployment_id "${OLD_DEPLOYMENT_ID}" \ + '.deployments[] | select(.id == $deployment_id)' <<<"${DEPLOYMENTS_JSON}" +OLD_DEPLOYMENT_MATCHES="$( + jq -r --arg deployment_id "${OLD_DEPLOYMENT_ID}" \ + '[.deployments[] | select(.id == $deployment_id)] | length' <<<"${DEPLOYMENTS_JSON}" +)" +[[ "${OLD_DEPLOYMENT_MATCHES}" == 1 ]] \ + || die "old deployment ID must resolve to exactly one registered deployment" +echo "== Read-only preflight: retired service deployments ==" +jq ' + [.deployments[] + | select(any(.services[]?; .name == "ExecutionRun" + or .name == "ExecutionTask" + or .name == "ExecutionCompensation")) + | {id, uri, services: [.services[]?.name]}] +' <<<"${DEPLOYMENTS_JSON}" +OLD_SERVICE_DEPLOYMENTS_VALID="$( + jq -r --arg deployment_id "${OLD_DEPLOYMENT_ID}" ' + [.deployments[] + | select(any(.services[]?; .name == "ExecutionRun" + or .name == "ExecutionTask" + or .name == "ExecutionCompensation"))] as $old + | ($old | length) == 1 + and ($old[0].id == $deployment_id) + and (["ExecutionRun", "ExecutionTask", "ExecutionCompensation"] + | all(. as $service | any($old[0].services[]?; .name == $service))) + ' <<<"${DEPLOYMENTS_JSON}" +)" +[[ "${OLD_SERVICE_DEPLOYMENTS_VALID}" == true ]] \ + || die "the exact old deployment must be the sole deployment containing all retired execution services" + +readonly OLD_INVOCATIONS_SQL=" +SELECT id, status, target_service_name, target_handler_name, + pinned_deployment_id, last_attempt_deployment_id +FROM sys_invocation +WHERE (target_service_name IN ('ExecutionRun', 'ExecutionTask', 'ExecutionCompensation') + OR pinned_deployment_id = '${OLD_DEPLOYMENT_ID}' + OR last_attempt_deployment_id = '${OLD_DEPLOYMENT_ID}') + AND status NOT IN ('completed', 'killed') +ORDER BY id;" +OLD_INVOCATIONS_JSON="$(restate_query "${OLD_INVOCATIONS_SQL}")" +jq '.rows' <<<"${OLD_INVOCATIONS_JSON}" +readonly OLD_INVOCATION_COUNT_SQL=" +SELECT count(*) AS invocation_count +FROM sys_invocation +WHERE (target_service_name IN ('ExecutionRun', 'ExecutionTask', 'ExecutionCompensation') + OR pinned_deployment_id = '${OLD_DEPLOYMENT_ID}' + OR last_attempt_deployment_id = '${OLD_DEPLOYMENT_ID}') + AND status NOT IN ('completed', 'killed');" +OLD_INVOCATION_COUNT_JSON="$(restate_query "${OLD_INVOCATION_COUNT_SQL}")" +OLD_INVOCATIONS="$(jq -er '.rows[0].invocation_count | tonumber' \ + <<<"${OLD_INVOCATION_COUNT_JSON}")" + +echo "== Read-only preflight: central migration position ==" +MIGRATION_POSITION="$( + psql -X "${DATABASE_ADMIN_URL}" --set=ON_ERROR_STOP=1 --tuples-only --no-align \ + --command "SELECT COALESCE(max(version), 0) FROM public.refinery_schema_history;" +)" +printf 'latest central migration: V%06d\n' "${MIGRATION_POSITION}" + +printf 'preflight totals: nonterminal_runs=%s old_service_or_deployment_invocations=%s\n' \ + "${NONTERMINAL_RUNS}" "${OLD_INVOCATIONS}" +[[ "${NONTERMINAL_RUNS}" == 0 ]] \ + || die "terminalize or cancel every legacy execution through the old product runtime" +[[ "${OLD_INVOCATIONS}" == 0 ]] \ + || die "retired execution services or the exact old deployment still own nonterminal invocations" +[[ "${MIGRATION_POSITION}" == 58 ]] \ + || die "the database must be at exactly V000058 before applying the V59/V60 hard cut" + +echo "Read-only preflight passed. No state has been changed." +[[ "${CONFIRMED}" == 1 ]] \ + || die "rerun with --confirm-destructive-cutover after reviewing the printed evidence" + +echo "== Archive terminal execution evidence ==" +ARCHIVE_FILE="${ARCHIVE_DIR}/terminal-execution-before-v59.sql" +printf '%s\n' "${DEPLOYMENTS_JSON}" >"${ARCHIVE_DIR}/restate-deployments-before-cutover.json" +readonly TERMINAL_INVOCATIONS_SQL=" +SELECT id, status, target_service_name, target_handler_name, + pinned_deployment_id, last_attempt_deployment_id +FROM sys_invocation +WHERE target_service_name IN ('ExecutionRun', 'ExecutionTask', 'ExecutionCompensation') + AND status IN ('completed', 'killed') +ORDER BY id;" +restate_query "${TERMINAL_INVOCATIONS_SQL}" \ + >"${ARCHIVE_DIR}/terminal-restate-invocations-before-cutover.json" +pg_dump \ + --dbname "${DATABASE_ADMIN_URL}" \ + --data-only \ + --no-owner \ + --no-privileges \ + --table moa.execution_run \ + --table moa.execution_task \ + --table moa.execution_compensation \ + --file "${ARCHIVE_FILE}" +[[ -s "${ARCHIVE_FILE}" ]] || die "terminal execution archive is empty" +{ + printf 'old_deployment_id=%s\n' "${OLD_DEPLOYMENT_ID}" + printf 'new_deployment_uri=%s\n' "${NEW_DEPLOYMENT_URI}" + printf 'nonterminal_runs=%s\n' "${NONTERMINAL_RUNS}" + printf 'old_deployment_invocations=%s\n' "${OLD_INVOCATIONS}" + printf 'archive_bytes=%s\n' "$(wc -c <"${ARCHIVE_FILE}" | tr -d ' ')" +} >"${ARCHIVE_DIR}/cutover-manifest.txt" + +echo "== Apply repository-owned V59/V60 migration chain ==" +( + cd -- "${REPO_ROOT}" + MOA_DATABASE_URL="${DATABASE_ADMIN_URL}" \ + MOA_DATABASE_ADMIN_URL="${DATABASE_ADMIN_URL}" \ + cargo run -p moa-orchestrator --bin moa-orchestrator-bin --locked -- migrate +) +APPLIED_MIGRATIONS="$( + psql -X "${DATABASE_ADMIN_URL}" --set=ON_ERROR_STOP=1 --tuples-only --no-align \ + --command "SELECT version, name FROM public.refinery_schema_history WHERE version IN (59, 60) ORDER BY version;" \ + | tee "${ARCHIVE_DIR}/applied-migrations.txt" +)" +[[ "${APPLIED_MIGRATIONS}" == $'59|long_horizon_execution\n60|sandbox_active_compute_capacity' ]] \ + || die "the repository runner did not record the exact V59/V60 identities" + +echo "== Reset only retired execution-service state and completed journals ==" +for service in "${OLD_SERVICES[@]}"; do + restate_cli --yes state clear "${service}" + for _attempt in $(seq 1 1000); do + TERMINAL_SERVICE_COUNT_JSON="$(restate_query " +SELECT count(*) AS invocation_count +FROM sys_invocation +WHERE target_service_name = '${service}' + AND status IN ('completed', 'killed');")" + TERMINAL_SERVICE_COUNT="$(jq -er '.rows[0].invocation_count | tonumber' \ + <<<"${TERMINAL_SERVICE_COUNT_JSON}")" + [[ "${TERMINAL_SERVICE_COUNT}" == 0 ]] && break + restate_cli --yes invocations purge --limit 500 "${service}" \ + >>"${ARCHIVE_DIR}/restate-invocation-purge.log" + done + [[ "${TERMINAL_SERVICE_COUNT}" == 0 ]] \ + || die "retired ${service} invocation history exceeded the bounded purge loop" +done + +echo "== Remove exact old deployment and register the new immutable endpoint ==" +curl -fsS \ + -X DELETE "${RESTATE_ADMIN_URL}/deployments/${OLD_DEPLOYMENT_ID}?force=true" \ + -o "${ARCHIVE_DIR}/old-deployment-removal.json" +for _attempt in $(seq 1 60); do + DEPLOYMENTS_JSON="$(curl -fsS "${RESTATE_ADMIN_URL}/deployments")" + if ! jq -e --arg deployment_id "${OLD_DEPLOYMENT_ID}" \ + '.deployments[] | select(.id == $deployment_id)' \ + <<<"${DEPLOYMENTS_JSON}" >/dev/null; then + break + fi + sleep 2 +done +if jq -e --arg deployment_id "${OLD_DEPLOYMENT_ID}" \ + '.deployments[] | select(.id == $deployment_id)' \ + <<<"${DEPLOYMENTS_JSON}" >/dev/null; then + die "old deployment remained registered after the bounded removal wait" +fi + +curl -fsS \ + -X POST "${RESTATE_ADMIN_URL}/deployments" \ + -H "content-type: application/json" \ + --data-binary "$(jq -cn --arg uri "${NEW_DEPLOYMENT_URI}" '{uri: $uri}')" \ + -o "${ARCHIVE_DIR}/new-deployment-registration.json" + +echo "== Verify the bounded-activation service inventory ==" +for _attempt in $(seq 1 60); do + DEPLOYMENTS_JSON="$(curl -fsS "${RESTATE_ADMIN_URL}/deployments")" + SERVICES_JSON="$(curl -fsS "${RESTATE_ADMIN_URL}/services")" + READY=1 + NEW_DEPLOYMENT_ID="$( + jq -r --arg uri "${NEW_DEPLOYMENT_URI}" ' + [.deployments[] | select(((.uri // "") | rtrimstr("/")) == $uri)] + | if length == 1 then .[0].id else "" end + ' <<<"${DEPLOYMENTS_JSON}" + )" + [[ -n "${NEW_DEPLOYMENT_ID}" ]] || READY=0 + for service in "${NEW_SERVICES[@]}"; do + jq -e --arg service "${service}" --arg uri "${NEW_DEPLOYMENT_URI}" \ + '.deployments[] + | select((.uri | rtrimstr("/")) == $uri) + | .services[]? + | select(.name == $service)' \ + <<<"${DEPLOYMENTS_JSON}" >/dev/null || READY=0 + done + for service in "${OLD_SERVICES[@]}"; do + if jq -e --arg service "${service}" \ + '.deployments[].services[]? | select(.name == $service)' \ + <<<"${DEPLOYMENTS_JSON}" >/dev/null; then + READY=0 + fi + done + if ! jq -e \ + --arg deployment_id "${NEW_DEPLOYMENT_ID}" \ + --argjson expected "${NEW_SERVICE_CONTRACT_JSON}" ' + .services as $services + | $expected + | all(. as $contract | + [$services[] + | select(.deployment_id == $deployment_id + and .name == $contract.name + and .ty == $contract.ty + and .public == $contract.public)] + | length == 1) + ' <<<"${SERVICES_JSON}" >/dev/null; then + READY=0 + fi + [[ "${READY}" == 1 ]] && break + sleep 2 +done +[[ "${READY}" == 1 ]] \ + || die "new handler type/privacy inventory was incomplete or a retired execution service remained registered" +jq -n \ + --argjson deployments "${DEPLOYMENTS_JSON}" \ + --argjson services "${SERVICES_JSON}" \ + --arg deployment_id "${NEW_DEPLOYMENT_ID}" ' + { + deployments: [$deployments.deployments[] | {id, uri, services: [.services[]?.name]}], + verified_new_handlers: [ + $services.services[] + | select(.deployment_id == $deployment_id) + | {name, ty, public} + ] + } + ' | tee "${ARCHIVE_DIR}/verified-service-inventory.json" + +echo "Long-horizon execution cutover complete. Keep admission gated until the" +echo "maintenance owner is ready and the archived evidence is stored durably." diff --git a/scripts/run-clean-e2e.sh b/scripts/run-clean-e2e.sh index 408c2f963..0e83a065c 100755 --- a/scripts/run-clean-e2e.sh +++ b/scripts/run-clean-e2e.sh @@ -13,11 +13,12 @@ LIVE=0 RUN_PROVIDERS=0 RUN_LONG_EVAL=0 RUN_BEHAVIOR_LAB_LIVE=0 +RUN_LONG_HORIZON=0 usage() { cat <<'USAGE' Usage: scripts/run-clean-e2e.sh [--live] [--providers] [--long-eval] - [--behavior-lab-live] + [--behavior-lab-live] [--long-horizon] Runs E2E tests against isolated state: - temporary Postgres database on the local compose Postgres service @@ -38,6 +39,9 @@ Options: the provider tests' own opt-in flags, such as MOA_RUN_LIVE_PROVIDER_TESTS=1. --long-eval Also run ignored long-conversation eval smoke. Requires --live. + --long-horizon + Run the deterministic accelerated-week execution suite. Requires + --live, but strips provider credentials and never spends budget. --behavior-lab-live Also run the billed Behavior Lab trial-to-score smoke. Requires --live, MOA_RUN_LIVE_PROVIDER_TESTS=1, a provider credential, and @@ -59,6 +63,9 @@ while [[ $# -gt 0 ]]; do --behavior-lab-live) RUN_BEHAVIOR_LAB_LIVE=1 ;; + --long-horizon) + RUN_LONG_HORIZON=1 + ;; -h|--help) usage exit 0 @@ -216,7 +223,7 @@ run() { run_without_provider_keys() { echo - echo ">> env -u MOA_ANTHROPIC_API_KEY -u MOA_OPENAI_API_KEY -u MOA_GOOGLE_API_KEY -u MOA_COHERE_API_KEY $*" + echo ">> env -u MOA_ANTHROPIC_API_KEY -u MOA_OPENAI_API_KEY -u MOA_GOOGLE_API_KEY -u MOA_COHERE_API_KEY -u MOA_ZEROENTROPY_API_KEY $*" local start=$SECONDS local status=0 begin_timing_phase "env -u provider keys $*" "env -u provider keys $*" "${start}" @@ -226,6 +233,7 @@ run_without_provider_keys() { -u MOA_OPENAI_API_KEY \ -u MOA_GOOGLE_API_KEY \ -u MOA_COHERE_API_KEY \ + -u MOA_ZEROENTROPY_API_KEY \ "$@" status=$? set -e @@ -558,6 +566,11 @@ if [[ "${RUN_LONG_EVAL}" -eq 1 && "${LIVE}" -ne 1 ]]; then exit 2 fi +if [[ "${RUN_LONG_HORIZON}" -eq 1 && "${LIVE}" -ne 1 ]]; then + echo "--long-horizon requires --live" >&2 + exit 2 +fi + # The billed Behavior Lab smoke spends real provider credit. Authorization and a # positive budget are both required, and they are checked here, before any # container or database is created, so an unauthorized run cannot get far enough @@ -790,6 +803,33 @@ if [[ "${LIVE}" -eq 1 ]]; then run_without_external_orchestrator cargo nextest run -p moa-orchestrator --locked --features "${ORCH_E2E_FEATURES}" --profile fixture-service-e2e --run-ignored ignored-only --no-tests fail + if [[ "${RUN_LONG_HORIZON}" -eq 1 ]]; then + # The suite owns its disposable Restate/Postgres/Valkey stack. Strip both + # external-stack discovery and provider keys so the lane remains hermetic + # and cannot accidentally consume billed provider credit. + run_without_external_orchestrator env \ + -u MOA_ANTHROPIC_API_KEY \ + -u MOA_OPENAI_API_KEY \ + -u MOA_GOOGLE_API_KEY \ + -u MOA_COHERE_API_KEY \ + -u MOA_ZEROENTROPY_API_KEY \ + -u MOA_FIDELITY_SIMULATOR_API_KEY \ + -u MOA_LLAMAPARSE_API_KEY \ + -u MOA_MERGE_API_KEY \ + -u MOA_NANGO_API_KEY \ + -u MOA_NEON_API_KEY \ + -u MOA_DATABASE_NEON_API_KEY \ + -u MOA_REDUCTO_API_KEY \ + -u MOA_TEST_MCP_DEPLOYMENT_API_KEY \ + -u MOA_TURBOPUFFER_API_KEY \ + -u MOA_UNSTRUCTURED_API_KEY \ + cargo nextest run -p moa-orchestrator --locked \ + --features "${ORCH_E2E_FEATURES}" \ + --profile long-horizon-execution \ + --run-ignored ignored-only \ + --no-tests fail + fi + run_without_external_orchestrator cargo nextest run -p moa-orchestrator --locked --features "${EXECUTION_EVAL_FEATURES}" --profile execution-eval-pr --run-ignored ignored-only --no-tests fail # Self-contained Behavior Lab lane. `OrchestratorTestFixture::with_execution_fixture` From ea7fe793270f255747ee92f89d8d3a0c0a81c959 Mon Sep 17 00:00:00 2001 From: Hwuiwon Kim Date: Wed, 12 Aug 2026 15:06:26 -0400 Subject: [PATCH 02/21] make long-horizon execution correct, observable, and cost-bounded --- .config/nextest.toml | 14 + .github/workflows/integration-tests.yml | 89 ++ Makefile | 38 +- crates/moa-artifacts/src/execution_plan.rs | 9 +- crates/moa-artifacts/src/validation.rs | 146 ++- .../src/validation/execution_plan.rs | 107 ++ .../artifacts_offline/definition_roundtrip.rs | 6 +- .../execution_plan_validation.rs | 115 ++- crates/moa-brain/src/compaction.rs | 2 +- .../src/execution_planning/request.rs | 2 +- .../src/execution_planning/routing.rs | 2 +- ...ution_planner.txt => execution_planner.md} | 5 +- ...ecution_router.txt => execution_router.md} | 0 .../prompts/{summarizer.txt => summarizer.md} | 0 crates/moa-core/src/events.rs | 4 - crates/moa-core/src/traits/mod.rs | 38 +- .../moa-core/src/types/sandbox_workspace.rs | 30 + crates/moa-edge/src/routes/session_stream.rs | 1 - .../examples/generate_execution_corpus.rs | 2 +- .../execution/contract-recorded.jsonl | 160 +-- .../tests/eval_offline/execution_snapshot.rs | 2 +- crates/moa-execution/src/bindings.rs | 3 + crates/moa-execution/src/compiler/estimate.rs | 8 + crates/moa-execution/src/compiler/mod.rs | 280 +++++- crates/moa-execution/src/completion.rs | 11 - .../src/interpreter/aggregate.rs | 362 ------- .../moa-execution/src/interpreter/catalog.rs | 26 + .../src/interpreter/materialize.rs | 315 ++---- crates/moa-execution/src/interpreter/mod.rs | 319 ++---- .../src/interpreter/projection.rs | 127 --- .../src/interpreter/temporal_wait.rs | 196 ---- .../moa-execution/src/interpreter/terminal.rs | 345 ------- crates/moa-execution/src/interpreter/tests.rs | 101 +- crates/moa-execution/src/lib.rs | 4 +- crates/moa-execution/src/replan.rs | 3 +- .../moa-execution/src/repository/capacity.rs | 155 ++- .../src/repository/compensation.rs | 377 +++---- .../src/repository/completion.rs | 70 +- .../src/repository/external_job.rs | 2 +- crates/moa-execution/src/repository/outbox.rs | 674 +++++++++++-- crates/moa-execution/src/repository/ready.rs | 282 +++++- crates/moa-execution/src/repository/run.rs | 218 ++++- .../moa-execution/src/repository/schedule.rs | 2 +- crates/moa-execution/src/repository/task.rs | 35 +- .../moa-execution/src/repository/terminal.rs | 6 +- .../src/repository/transition.rs | 61 +- .../moa-execution/src/repository/trigger.rs | 35 +- crates/moa-execution/src/state.rs | 147 ++- crates/moa-execution/tests/compiler.rs | 250 ++++- crates/moa-execution/tests/execution_db.rs | 6 + .../execution_db/active_run_capacity_db.rs | 4 +- .../execution_db/amendment_projection_db.rs | 1 + .../execution_db/compensation_attempts_db.rs | 256 ++++- .../execution_db/completion_projection_db.rs | 153 +++ .../execution_db/conditional_execution_db.rs | 560 +++++++++++ .../controller_wake_recovery_db.rs | 601 ++++++++++++ .../execution_db/execution_capacity_db.rs | 1 + .../execution_db/incremental_scheduler_db.rs | 111 +-- .../tests/execution_db/support.rs | 4 +- .../tests/execution_db/trigger_outbox_db.rs | 152 +++ .../execution_db/wait_entry_deadline_db.rs | 282 ++++++ crates/moa-execution/tests/interpreter.rs | 822 +++------------- crates/moa-hands/src/adapters/daytona/mod.rs | 13 +- .../src/adapters/daytona/workspace.rs | 2 + crates/moa-hands/src/adapters/e2b/mod.rs | 12 +- crates/moa-hands/src/adapters/e2b/tests.rs | 98 +- .../moa-hands/src/adapters/e2b/workspace.rs | 10 +- crates/moa-hands/src/adapters/local/mod.rs | 33 +- crates/moa-hands/src/adapters/local/tests.rs | 79 +- .../moa-hands/src/adapters/local/workspace.rs | 2 + crates/moa-hands/src/core/dispatch.rs | 30 +- crates/moa-hands/src/core/lifecycle.rs | 102 +- crates/moa-hands/src/core/lifecycle/tests.rs | 4 - crates/moa-hands/src/core/mod.rs | 2 +- crates/moa-hands/src/core/profile.rs | 4 - crates/moa-hands/src/core/reaper.rs | 4 - crates/moa-hands/src/core/recovery/tests.rs | 4 - .../src/core/sandbox_workspace/capacity.rs | 276 +++++- .../src/core/sandbox_workspace/lifecycle.rs | 346 ++++++- .../maintenance/inventory.rs | 44 +- .../core/sandbox_workspace/maintenance/mod.rs | 194 +++- .../repository/checkpoints.rs | 4 +- .../sandbox_workspace/repository/lifecycle.rs | 4 +- crates/moa-hands/src/core/telemetry.rs | 33 +- crates/moa-hands/src/lib.rs | 8 +- crates/moa-hands/tests/daytona_live.rs | 67 -- .../tests/docker_hardening_docker.rs | 3 - crates/moa-hands/tests/e2b_live.rs | 4 - .../hands_db/sandbox_workspace/capacity_db.rs | 251 ++++- .../hands_db/sandbox_workspace/dispatch_db.rs | 4 - .../sandbox_workspace/lifecycle_db.rs | 916 +++++++++++++++++- .../sandbox_workspace_recovery_offline.rs | 10 - .../ingest/prompts/{judge.txt => judge.md} | 0 crates/moa-memory/ingest/src/contradiction.rs | 2 +- .../V000059__long_horizon_execution.sql | 177 +++- ...00060__sandbox_active_compute_capacity.sql | 68 ++ .../execution_and_security_catalog.rs | 370 ++++++- .../execution_compensation.rs | 2 +- .../tests/run_idempotency_db/tenant_purge.rs | 16 +- crates/moa-observability/src/lib.rs | 19 +- .../moa-observability/src/runtime_metrics.rs | 184 ++-- .../src/external_job_ingress.rs | 1 + crates/moa-orchestrator/src/main.rs | 21 + .../src/objects/execution_run_controller.rs | 14 + .../execution_run_controller/advance.rs | 379 +++++--- .../execution_run_controller/settlement.rs | 17 +- .../objects/execution_run_controller/tests.rs | 204 ++-- .../moa-orchestrator/src/runtime/endpoint.rs | 19 + crates/moa-orchestrator/src/runtime/mod.rs | 1 + .../src/runtime/restate_drain.rs | 384 ++++++++ .../src/runtime/sandbox_workspace_rollout.rs | 69 +- .../src/services/execution/handlers.rs | 4 +- .../src/services/execution/support.rs | 10 +- .../src/services/execution/tests.rs | 6 +- .../services/execution_amendment_planner.rs | 556 +++++++++++ .../preparation.rs | 374 +++++++ .../execution_amendment_planner/tests.rs | 822 ++++++++++++++++ .../src/services/execution_dispatcher.rs | 404 +++++++- .../src/services/llm_gateway.rs | 14 + crates/moa-orchestrator/src/services/mod.rs | 1 + .../src/services/tool_executor.rs | 165 +++- .../src/workflows/attempt_slice.rs | 78 ++ .../execution_compensation_attempt.rs | 148 ++- .../external.rs | 7 +- .../yielding.rs | 11 +- .../src/workflows/execution_task_attempt.rs | 86 +- .../execution_task_attempt/active.rs | 5 +- .../execution_task_attempt/external.rs | 9 +- .../execution_task_attempt/watchdog.rs | 7 +- .../execution_task_attempt/yielding.rs | 206 +++- crates/moa-orchestrator/src/workflows/mod.rs | 1 + ...oordinator_worker_behavior_provider_e2e.rs | 2 +- .../tests/execution_run_service_e2e.rs | 4 +- .../admission_replay.rs | 2 +- .../bulk_and_recovery.rs | 2 +- .../compensation_recovery.rs | 2 +- .../observability.rs | 2 +- .../replan_and_completion.rs | 227 +++-- .../execution_run_service_e2e/routing.rs | 4 +- .../task_lifecycle.rs | 10 +- .../terminal_matrix.rs | 4 +- .../integration/action_policy_flow_e2e.rs | 2 +- .../long_horizon_execution_canary_live.rs | 15 +- .../long_horizon_execution_service_e2e.rs | 2 +- .../burst_admission.rs | 44 +- .../deadline_and_waits.rs | 2 +- .../pause_and_external.rs | 4 +- .../action_reviews_reaper_db.rs | 2 +- .../orchestrator_db/analytics_export_db.rs | 2 +- .../orchestrator_db/execution_service_db.rs | 4 +- .../check_architecture_boundaries/budgets.rs | 6 +- crates/xtask/src/execution_trace_manifest.rs | 7 + docs/01-architecture-overview.md | 2 +- docs/12-restate-architecture.md | 2 +- docs/17-observability.md | 20 +- .../artifacts/damaged-food-order.skill.yaml | 24 +- .../patterns/custom-logic.skill.yaml | 13 +- .../patterns/human-approval.skill.yaml | 4 +- .../patterns/parallel-review.skill.yaml | 4 +- .../artifacts/patterns/react-agent.skill.yaml | 2 +- .../artifacts/patterns/sequential.skill.yaml | 2 +- docs/schemas/moa-skill-v1.schema.json | 768 +++++++++++---- k8s/scripts/validate-observability.sh | 87 +- .../alerts/moa-long-horizon-execution.yaml | 37 +- ops/prometheus/alerts/moa-restate.yaml | 7 +- ops/prometheus/alerts/sandbox-workspaces.yaml | 7 +- scripts/cutover-long-horizon-execution.sh | 354 +++++-- scripts/run-clean-e2e.sh | 3 + 168 files changed, 12741 insertions(+), 4507 deletions(-) rename crates/moa-brain/src/prompts/{execution_planner.txt => execution_planner.md} (68%) rename crates/moa-brain/src/prompts/{execution_router.txt => execution_router.md} (100%) rename crates/moa-brain/src/prompts/{summarizer.txt => summarizer.md} (100%) delete mode 100644 crates/moa-execution/src/interpreter/aggregate.rs create mode 100644 crates/moa-execution/src/interpreter/catalog.rs delete mode 100644 crates/moa-execution/src/interpreter/projection.rs delete mode 100644 crates/moa-execution/src/interpreter/temporal_wait.rs delete mode 100644 crates/moa-execution/src/interpreter/terminal.rs create mode 100644 crates/moa-execution/tests/execution_db/conditional_execution_db.rs create mode 100644 crates/moa-execution/tests/execution_db/controller_wake_recovery_db.rs create mode 100644 crates/moa-execution/tests/execution_db/wait_entry_deadline_db.rs rename crates/moa-memory/ingest/prompts/{judge.txt => judge.md} (100%) create mode 100644 crates/moa-orchestrator/src/runtime/restate_drain.rs create mode 100644 crates/moa-orchestrator/src/services/execution_amendment_planner.rs create mode 100644 crates/moa-orchestrator/src/services/execution_amendment_planner/preparation.rs create mode 100644 crates/moa-orchestrator/src/services/execution_amendment_planner/tests.rs create mode 100644 crates/moa-orchestrator/src/workflows/attempt_slice.rs diff --git a/.config/nextest.toml b/.config/nextest.toml index 59f575f9f..b572c2549 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -107,6 +107,20 @@ junit.path = "target/nextest/long-horizon-execution/junit.xml" default-filter = 'package(moa-orchestrator) & binary(/^long_horizon_execution_service_e2e$/)' slow-timeout = { period = "60s", terminate-after = 10 } +# The thousand-common-wake case deliberately spends its first three minutes +# admitting and parking 1,000 runs before one shared absolute wake, then drains +# them in ~32 fleet-capped waves. That budget exceeds the lane's own ten-minute +# ceiling, so nextest was terminating the single gate that proves fleet cap and +# WFQ fairness instead of letting it fail on one of its own bounded phases. +# Every phase inside the case is bounded well below this ceiling, so a real +# regression still reports which phase stalled. +[[profile.long-horizon-execution.overrides]] +filter = ''' +binary(/^long_horizon_execution_service_e2e$/) + & test(/one_thousand_common_wakes_bound_capacity_invocations_and_oldest_ready_age/) +''' +slow-timeout = { period = "120s", terminate-after = 10 } + # Small deterministic crash/replay and worker-deadline cases run serially on # pull requests. Large fan-out, A/B rollout, node-loss, and chaos scenarios # remain in their existing local/nightly lanes. diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 49856e5f4..0f1cfeeda 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -2,6 +2,11 @@ name: integration tests on: pull_request: + schedule: + # Nightly slot reserved for the accelerated long-horizon execution suite. + # Every other job in this file is explicitly excluded from it, so adding + # this trigger does not turn the pull-request jobs into nightly jobs. + - cron: "41 3 * * *" workflow_dispatch: permissions: @@ -9,6 +14,7 @@ permissions: jobs: restate-recovery-matrix: + if: github.event_name != 'schedule' runs-on: ubuntu-latest timeout-minutes: 60 env: @@ -86,6 +92,7 @@ jobs: if-no-files-found: ignore restate-fixture-smoke: + if: github.event_name != 'schedule' runs-on: ubuntu-latest # The dedicated scripted fixture builds moa-orchestrator-bin inside the # test, which the old always-bailing invocation never paid for. @@ -187,6 +194,7 @@ jobs: run: docker compose down -v behavior-lab-scorecards: + if: github.event_name != 'schedule' runs-on: ubuntu-latest # Deterministic and unbilled by construction: every evaluator in the initial # product registry reads typed terminal evidence and calls no provider. The @@ -269,6 +277,7 @@ jobs: run: docker compose down -v clickhouse-analytics: + if: github.event_name != 'schedule' runs-on: ubuntu-latest # The named trigger for the ClickHouse analytics arm. That backend is wired # from `[clickhouse]` config presence and covered in the db lanes against a @@ -341,3 +350,83 @@ jobs: - name: Stop compose stack if: always() run: docker compose --profile clickhouse down -v + + long-horizon-execution: + # The named trigger for the accelerated long-horizon execution suite. Its 21 + # cases are the only coverage of pause/resume, external-job callback dedup, + # deadline and wait exactness, old-deployment drain, and disaster recovery, + # and they are `#[ignore]`d behind a profile that no Makefile target or CI + # job selected -- the same decay mode `.config/nextest.toml` documents for + # the Behavior Lab lanes. Nightly rather than per-pull-request because each + # case serially owns a disposable Restate/Postgres/Valkey stack. + # + # Deterministic and unbilled: the lane strips provider credentials and + # external-stack discovery, so it cannot reach a paid provider or silently + # become an ambient-stack test. + if: github.event.schedule == '41 3 * * *' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + timeout-minutes: 150 + env: + MOA_FIXTURE_RUST_LOG: info + RUST_BACKTRACE: "1" + SCCACHE_GHA_ENABLED: "true" + RUSTC_WRAPPER: sccache + steps: + - name: Check out repository + uses: actions/checkout@v6 + + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + + - name: Set up sccache + uses: mozilla-actions/sccache-action@v0.0.10 + + - name: Cache cargo artifacts + uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + + - name: Install nextest + uses: taiki-e/install-action@nextest + + # Identical selection to `make test-long-horizon` and to + # `scripts/run-clean-e2e.sh --long-horizon`: same profile, same features, + # same run-ignored/no-tests contract. + - name: Run accelerated long-horizon execution suite + run: make test-long-horizon + + - name: Publish long-horizon JUnit report + if: always() + uses: actions/upload-artifact@v6 + with: + name: long-horizon-execution-junit + path: target/nextest/long-horizon-execution/junit.xml + if-no-files-found: warn + + - name: Capture Docker fixture diagnostics + if: failure() + run: | + diagnostics="${RUNNER_TEMP}/long-horizon-docker.log" + { + echo "== docker ps -a ==" + docker ps -a --no-trunc + while IFS= read -r container_id; do + [ -n "${container_id}" ] || continue + echo + echo "== docker inspect ${container_id} ==" + docker inspect --format \ + 'name={{.Name}} image={{.Config.Image}} status={{.State.Status}} exit={{.State.ExitCode}} error={{json .State.Error}} health={{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' \ + "${container_id}" + echo + echo "== docker logs ${container_id} ==" + docker logs "${container_id}" + done < <(docker ps -aq) + } 2>&1 | tee "${diagnostics}" || true + + - name: Upload Docker fixture diagnostics + if: failure() + uses: actions/upload-artifact@v6 + with: + name: long-horizon-docker-diagnostics + path: ${{ runner.temp }}/long-horizon-docker.log + if-no-files-found: ignore diff --git a/Makefile b/Makefile index c8df6b1ef..ab1afa778 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: dev fga-bootstrap dev-down dev-wipe dev-logs dev-restate-ui dev-status test-fast test-affected test-ci test-db-session test-db-memory test-authz-pentest test-clickhouse test-service-e2e test-behavior-lab test-behavior-lab-live test-provider-e2e build-timings e2e-clean e2e-clean-live loadtest-mock loadtest-live loadtest-capacity loadtest-capacity-edge loadtest-capacity-direct-append loadtest-capacity-brackets chaos-smoke chaos-matrix codegraph +.PHONY: dev fga-bootstrap dev-down dev-wipe dev-logs dev-restate-ui dev-status test-fast test-affected test-ci test-db-session test-db-memory test-authz-pentest test-clickhouse test-long-horizon test-service-e2e test-behavior-lab test-behavior-lab-live test-provider-e2e build-timings e2e-clean e2e-clean-live loadtest-mock loadtest-live loadtest-capacity loadtest-capacity-edge loadtest-capacity-direct-append loadtest-capacity-brackets chaos-smoke chaos-matrix codegraph codegraph: @./scripts/codegraph init @@ -92,6 +92,42 @@ test-clickhouse: MOA_CLICKHOUSE_PASSWORD=$${MOA_CLICKHOUSE_PASSWORD:-dev} \ cargo nextest run --locked --profile clickhouse-docker --run-ignored all +# The accelerated long-horizon execution suite: pause/resume, external-job +# callback dedup, deadline/wait exactness, deployment drain, and disaster +# recovery. Every case owns a disposable Restate/Postgres/Valkey stack, so it +# needs Docker but neither the compose stack nor the clean-E2E harness. +# +# Deterministic and unbilled. Provider credentials and external-stack discovery +# are stripped from the environment so the lane stays hermetic and cannot spend +# provider credit even when a developer shell exports keys. `scripts/run-clean-e2e.sh +# --long-horizon` runs the identical selection inside the full gate; this target +# exists so the lane is reachable — and schedulable — on its own. +test-long-horizon: + @command -v cargo-nextest >/dev/null 2>&1 || { echo "cargo-nextest is required; install with: cargo install cargo-nextest --locked"; exit 127; } + env \ + -u MOA_RESTATE_INGRESS_URL \ + -u MOA_RESTATE_ADMIN_URL \ + -u RESTATE_ADMIN_URL \ + -u MOA_RESTATE_DEPLOYMENT_URI \ + -u MOA_ANTHROPIC_API_KEY \ + -u MOA_OPENAI_API_KEY \ + -u MOA_GOOGLE_API_KEY \ + -u MOA_COHERE_API_KEY \ + -u MOA_ZEROENTROPY_API_KEY \ + -u MOA_FIDELITY_SIMULATOR_API_KEY \ + -u MOA_LLAMAPARSE_API_KEY \ + -u MOA_MERGE_API_KEY \ + -u MOA_NANGO_API_KEY \ + -u MOA_NEON_API_KEY \ + -u MOA_DATABASE_NEON_API_KEY \ + -u MOA_REDUCTO_API_KEY \ + -u MOA_TEST_MCP_DEPLOYMENT_API_KEY \ + -u MOA_TURBOPUFFER_API_KEY \ + -u MOA_UNSTRUCTURED_API_KEY \ + cargo nextest run -p moa-orchestrator --locked \ + --features provider-overrides,integration \ + --profile long-horizon-execution --run-ignored ignored-only --no-tests fail + test-service-e2e: e2e-clean-live # The deterministic Behavior Lab lanes (behavior-lab-service-e2e and diff --git a/crates/moa-artifacts/src/execution_plan.rs b/crates/moa-artifacts/src/execution_plan.rs index 566192fee..a3dca250e 100644 --- a/crates/moa-artifacts/src/execution_plan.rs +++ b/crates/moa-artifacts/src/execution_plan.rs @@ -418,10 +418,11 @@ pub enum ExecutionTemporalTarget { #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] pub enum ExecutionWaitExpiryAction { - /// Fail only the waiting logical task. + /// Fail the waiting logical task. + /// + /// Task failure propagates to the run through the normal terminal projection, so a + /// separate run-level variant would be indistinguishable at the settlement site. FailTask, - /// Fail the complete execution run. - FailRun, /// Settle the wait successfully with a declared structured output. ContinueWith { /// Structured output supplied to downstream nodes. @@ -535,8 +536,6 @@ pub enum InputAudience { pub enum ExecutionFailureClass { /// Transient failure eligible for retry policy. Retryable, - /// A required predecessor ended in terminal failure. - DependencyFailed, /// Task input was invalid. InvalidInput, /// Task output did not satisfy its schema. diff --git a/crates/moa-artifacts/src/validation.rs b/crates/moa-artifacts/src/validation.rs index b20274549..7c836f352 100644 --- a/crates/moa-artifacts/src/validation.rs +++ b/crates/moa-artifacts/src/validation.rs @@ -4,7 +4,7 @@ mod connectors; mod execution_plan; mod json; -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeSet, HashMap, HashSet}; use moa_core::canonical_json::canonical_json_bytes; use moa_core::types::guardrails::GuardrailMode; @@ -934,9 +934,9 @@ fn validate_execution_plan_at( allow_absolute_temporal_targets: bool, report: &mut ValidationReport, ) { - execution_plan::validate_temporal_target( - &format!("{root}.input_wait_policy.expiry"), - &definition.input_wait_policy.expiry, + execution_plan::validate_input_wait_policy( + &format!("{root}.input_wait_policy"), + &definition.input_wait_policy, allow_absolute_temporal_targets, report, ); @@ -972,6 +972,134 @@ fn validate_execution_plan_at( validate_execution_dag(root, &definition.nodes, &node_ids, report); validate_terminal_output(root, &definition.nodes, report); + validate_condition_scope(root, &definition.nodes, report); +} + +/// Confines conditional nodes to effectful leaves whose output nothing reads. +/// +/// A node whose `when` evaluates false is committed as `skipped` with a JSON `null` +/// aggregate output. Nothing may read that value, and no requirement may be served +/// exclusively by conditional nodes, because every branch evaluating false would leave +/// the requirement with no node eligible to satisfy it. +fn validate_condition_scope(root: &str, nodes: &[ExecutionNode], report: &mut ValidationReport) { + let conditional_ids = nodes + .iter() + .filter(|node| node.when.is_some()) + .map(|node| node.id.as_str()) + .collect::>(); + if conditional_ids.is_empty() { + return; + } + + for (index, node) in nodes.iter().enumerate() { + let node_root = format!("{root}.nodes[{index}]"); + validate_no_conditional_reference( + &format!("{node_root}.input"), + &node.input, + &conditional_ids, + report, + ); + match &node.operation { + ExecutionOperation::Map { items, .. } | ExecutionOperation::Reduce { items, .. } => { + validate_no_conditional_reference( + &format!("{node_root}.operation.items"), + items, + &conditional_ids, + report, + ); + } + ExecutionOperation::Output { value } => validate_no_conditional_reference( + &format!("{node_root}.operation.value"), + value, + &conditional_ids, + report, + ), + ExecutionOperation::WaitUntil { result, .. } => validate_no_conditional_reference( + &format!("{node_root}.operation.result"), + result, + &conditional_ids, + report, + ), + ExecutionOperation::Capability { .. } + | ExecutionOperation::Agent { .. } + | ExecutionOperation::Review { .. } + | ExecutionOperation::WaitSignal { .. } => {} + } + } + + let mut unconditional_requirements = HashSet::new(); + let mut conditional_requirements = BTreeSet::new(); + for node in nodes { + for requirement_id in &node.requirement_ids { + if node.when.is_some() { + conditional_requirements.insert(requirement_id.as_str()); + } else { + unconditional_requirements.insert(requirement_id.as_str()); + } + } + } + for requirement_id in conditional_requirements { + if !unconditional_requirements.contains(requirement_id) { + report.push_error( + format!("{root}.nodes"), + format!( + "every node serving requirement `{requirement_id}` is conditional, so all \ + branches evaluating false would leave it with no eligible node" + ), + ); + } + } +} + +/// Rejects any `$ref` whose source node declares a condition. +fn validate_no_conditional_reference( + path: &str, + value: &Value, + conditional_ids: &HashSet<&str>, + report: &mut ValidationReport, +) { + match value { + Value::Array(values) => { + for (index, value) in values.iter().enumerate() { + validate_no_conditional_reference( + &format!("{path}[{index}]"), + value, + conditional_ids, + report, + ); + } + } + Value::Object(object) => { + if object.len() == 1 + && let Some(reference) = object.get("$ref").and_then(Value::as_str) + { + if execution_reference_node(reference) + .ok() + .flatten() + .is_some_and(|node_id| conditional_ids.contains(node_id)) + { + report.push_error( + path, + "a conditional node's output cannot be read; it is null when the \ + condition is false", + ); + } + return; + } + if object.keys().any(|key| key.starts_with('$')) { + return; + } + for (key, value) in object { + validate_no_conditional_reference( + &format!("{path}.{key}"), + value, + conditional_ids, + report, + ); + } + } + Value::Null | Value::Bool(_) | Value::Number(_) | Value::String(_) => {} + } } fn validate_execution_node( @@ -1006,6 +1134,10 @@ fn validate_execution_node( } } + // A false condition commits a skipped node with a JSON `null` aggregate output, so a + // conditional node is an effectful leaf: it may be depended on for ordering, but its + // output is unreadable and it cannot be the plan's terminal output. See + // `validate_condition_scope` for the plan-wide half of the same contract. if let Some(condition) = &node.when { let reference = match condition { ExecutionCondition::Exists { reference } @@ -1017,6 +1149,12 @@ fn validate_execution_node( node, report, ); + if matches!(node.operation, ExecutionOperation::Output { .. }) { + report.push_error( + format!("{root}.when"), + "the terminal output node must not be conditional", + ); + } } let map_input_scope = matches!(node.operation, ExecutionOperation::Map { .. }); diff --git a/crates/moa-artifacts/src/validation/execution_plan.rs b/crates/moa-artifacts/src/validation/execution_plan.rs index b5dd374bb..b5819f791 100644 --- a/crates/moa-artifacts/src/validation/execution_plan.rs +++ b/crates/moa-artifacts/src/validation/execution_plan.rs @@ -234,6 +234,36 @@ fn validate_wait_policy( } } +/// Validates the plan-level expiry policy for runtime `NeedsInput` outcomes. +/// +/// This one policy settles whichever logical task returned `NeedsInput`, so a +/// declared `continue_with` output has no single node `output_schema` to be checked +/// against. It is rejected here rather than deferred to run materialization, where +/// the schema check is a non-retryable failure. +pub(super) fn validate_input_wait_policy( + root: &str, + policy: &ExecutionWaitPolicy, + allow_absolute_temporal_targets: bool, + report: &mut ValidationReport, +) { + validate_temporal_target( + &format!("{root}.expiry"), + &policy.expiry, + allow_absolute_temporal_targets, + report, + ); + if matches!( + policy.on_expiry, + ExecutionWaitExpiryAction::ContinueWith { .. } + ) { + report.push_error( + format!("{root}.on_expiry"), + "input wait expiry must fail the waiting task; continue_with cannot be validated \ + against the output schema of the node that requested input", + ); + } +} + pub(super) fn validate_temporal_target( root: &str, target: &ExecutionTemporalTarget, @@ -251,3 +281,80 @@ pub(super) fn validate_temporal_target( ExecutionTemporalTarget::At { .. } | ExecutionTemporalTarget::After { .. } => {} } } + +#[cfg(test)] +mod tests { + use serde_json::json; + + use crate::execution_plan::{ + ExecutionCancelPolicy, ExecutionNode, ExecutionOperation, ExecutionPlanDefinition, + ExecutionTemporalTarget, ExecutionWaitExpiryAction, ExecutionWaitPolicy, RetryPolicy, + }; + use crate::validation::validate_execution_plan_definition; + + // Pins: `input_wait_policy.on_expiry` is the one wait policy with no owning node, + // so a `continue_with` output has nothing to validate against. Before this check + // it compiled cleanly and the schema violation surfaced at run materialization as + // a non-retryable infrastructure error against whichever task happened to ask for + // input. Both directions are asserted so the rejection is provably about + // `continue_with` and not about the surrounding fixture. + #[test] + fn input_wait_policy_rejects_continue_with_but_accepts_a_failing_action() { + let continued = plan(ExecutionWaitExpiryAction::ContinueWith { + output: json!({ "approved": true }), + }); + + let report = validate_execution_plan_definition(&continued); + + assert!( + report.errors.iter().any(|error| { + error.path == "execution_plan.input_wait_policy.on_expiry" + && error.message.contains("must fail the waiting task") + }), + "continue_with must be refused for the plan-level input wait policy: {report:?}" + ); + + for action in [ + ExecutionWaitExpiryAction::FailTask, + ExecutionWaitExpiryAction::FailTask, + ] { + let report = validate_execution_plan_definition(&plan(action)); + assert!( + report.errors.is_empty(), + "a failing input wait expiry must still validate: {report:?}" + ); + } + } + + fn plan(on_expiry: ExecutionWaitExpiryAction) -> ExecutionPlanDefinition { + ExecutionPlanDefinition { + cancel_policy: ExecutionCancelPolicy::RetainEffects, + input_wait_policy: ExecutionWaitPolicy { + expiry: ExecutionTemporalTarget::After { + delay_seconds: 3_600, + }, + on_expiry, + }, + input_schema: json!({ "type": "object" }), + output_schema: json!({ "type": "object" }), + nodes: vec![ExecutionNode { + id: "output".to_string(), + requirement_ids: vec!["req_output".to_string()], + depends_on: Vec::new(), + when: None, + input: json!({}), + output_schema: json!({ "type": "object" }), + operation: ExecutionOperation::Output { + value: json!({ "$ref": "$.input" }), + }, + compensation: None, + retry: RetryPolicy { + max_attempts: 1, + initial_backoff_ms: 0, + max_backoff_ms: 0, + }, + budget: None, + }], + } + } +} diff --git a/crates/moa-artifacts/tests/artifacts_offline/definition_roundtrip.rs b/crates/moa-artifacts/tests/artifacts_offline/definition_roundtrip.rs index 1abf63963..15ccfa170 100644 --- a/crates/moa-artifacts/tests/artifacts_offline/definition_roundtrip.rs +++ b/crates/moa-artifacts/tests/artifacts_offline/definition_roundtrip.rs @@ -529,7 +529,7 @@ definition: cancel_policy: retain_effects input_wait_policy: expiry: { kind: after, delay_seconds: 3600 } - on_expiry: { kind: fail_run } + on_expiry: { kind: fail_task } input_schema: { type: object } output_schema: { type: object } nodes: @@ -621,7 +621,7 @@ definition: cancel_policy: retain_effects input_wait_policy: expiry: { kind: after, delay_seconds: 3600 } - on_expiry: { kind: fail_run } + on_expiry: { kind: fail_task } input_schema: { type: object } output_schema: { type: object } nodes: @@ -698,7 +698,7 @@ definition: cancel_policy: retain_effects input_wait_policy: expiry: { kind: after, delay_seconds: 3600 } - on_expiry: { kind: fail_run } + on_expiry: { kind: fail_task } input_schema: { type: object } output_schema: { type: object } nodes: diff --git a/crates/moa-artifacts/tests/artifacts_offline/execution_plan_validation.rs b/crates/moa-artifacts/tests/artifacts_offline/execution_plan_validation.rs index a8d55f91c..3e5852dd3 100644 --- a/crates/moa-artifacts/tests/artifacts_offline/execution_plan_validation.rs +++ b/crates/moa-artifacts/tests/artifacts_offline/execution_plan_validation.rs @@ -96,14 +96,14 @@ fn all_eight_execution_operations_round_trip_exact_json_and_yaml() { ( ExecutionOperation::Review { prompt: "Approve the report?".to_string(), - wait_policy: wait_policy(ExecutionWaitExpiryAction::FailRun), + wait_policy: wait_policy(ExecutionWaitExpiryAction::FailTask), }, json!({ "kind": "review", "prompt": "Approve the report?", "wait_policy": { "expiry": { "kind": "after", "delay_seconds": 3600 }, - "on_expiry": { "kind": "fail_run" } + "on_expiry": { "kind": "fail_task" } } }), ), @@ -173,8 +173,8 @@ fn wait_expiry_actions_round_trip_with_canonical_tagged_shapes() { json!({ "kind": "fail_task" }), ), ( - ExecutionWaitExpiryAction::FailRun, - json!({ "kind": "fail_run" }), + ExecutionWaitExpiryAction::FailTask, + json!({ "kind": "fail_task" }), ), ( ExecutionWaitExpiryAction::ContinueWith { @@ -280,7 +280,7 @@ fn reusable_plan_templates_reject_absolute_temporal_targets_at_every_wait_surfac prompt: "Approve?".to_string(), wait_policy: ExecutionWaitPolicy { expiry: absolute_target(), - on_expiry: ExecutionWaitExpiryAction::FailRun, + on_expiry: ExecutionWaitExpiryAction::FailTask, }, }; cases.push(( @@ -341,7 +341,7 @@ fn skill_schema_and_rust_types_require_the_same_wait_contract() { let wait_operations = [ ExecutionOperation::Review { prompt: "Approve?".to_string(), - wait_policy: wait_policy(ExecutionWaitExpiryAction::FailRun), + wait_policy: wait_policy(ExecutionWaitExpiryAction::FailTask), }, ExecutionOperation::WaitSignal { signal_name: "ready".to_string(), @@ -350,8 +350,8 @@ fn skill_schema_and_rust_types_require_the_same_wait_contract() { }), }, ExecutionOperation::WaitUntil { - wake: ExecutionTemporalTarget::At { - at: at("2030-01-02T02:00:00Z"), + wake: ExecutionTemporalTarget::After { + delay_seconds: 3_600, }, result: json!({ "ready": true }), }, @@ -364,6 +364,24 @@ fn skill_schema_and_rust_types_require_the_same_wait_contract() { ); } + // A skill is a reusable template, so `validate_skill` passes + // `allow_absolute_temporal_targets = false` and rejects every absolute target. + // The schema used to advertise the `at` branch anyway, which meant it accepted + // documents the canonical validator always refused. Both layers now agree. + let absolute_timer = json!({ + "kind": "wait_until", + "wake": { "kind": "at", "at": "2030-01-02T02:00:00Z" }, + "result": { "ready": true } + }); + assert!( + serde_json::from_value::(absolute_timer.clone()).is_ok(), + "the canonical Rust type still carries the absolute branch for compiled plans" + ); + assert!( + !operation_validator.is_valid(&absolute_timer), + "skill schema must reject an absolute temporal target that skill validation always refuses" + ); + let stale_review = json!({ "kind": "review", "prompt": "Approve?" @@ -403,7 +421,7 @@ fn skill_schema_and_rust_types_require_the_same_wait_contract() { plan_validator.is_valid(&plan_json), "skill schema must accept the canonical Rust plan" ); - let mut stale_plan = plan_json; + let mut stale_plan = plan_json.clone(); stale_plan .as_object_mut() .expect("plan is an object") @@ -416,6 +434,34 @@ fn skill_schema_and_rust_types_require_the_same_wait_contract() { !plan_validator.is_valid(&stale_plan), "skill schema must reject a plan without input_wait_policy" ); + + // The plan-level policy settles whichever task returned NeedsInput, so a declared + // continue_with output has no node output_schema to be checked against. The + // canonical validator refuses it, and the schema must refuse it at the same + // place — while still accepting continue_with on a node-owned wait, which does + // have an owning schema. + let mut continued_input_wait = plan_json; + continued_input_wait["input_wait_policy"]["on_expiry"] = + json!({ "kind": "continue_with", "output": { "approved": true } }); + assert!( + !plan_validator.is_valid(&continued_input_wait), + "skill schema must reject continue_with on the plan-level input wait policy" + ); + let mut definition = + serde_json::from_value::(continued_input_wait.clone()) + .expect("plan-level continue_with is still a well-formed value"); + assert!( + validate_execution_plan_definition(&definition) + .errors + .iter() + .any(|error| error.path == "execution_plan.input_wait_policy.on_expiry"), + "canonical validation must reject continue_with on the input wait policy" + ); + definition.input_wait_policy.on_expiry = ExecutionWaitExpiryAction::FailTask; + assert!( + plan_validator.is_valid(&serde_json::to_value(&definition).expect("serialize plan")), + "schema must still accept a failing input wait expiry" + ); } #[test] @@ -502,7 +548,7 @@ fn skill_reference_paths_cover_agent_map_and_reducer_agents_only() { "cancel_policy": "retain_effects", "input_wait_policy": { "expiry": { "kind": "after", "delay_seconds": 3600 }, - "on_expiry": { "kind": "fail_run" } + "on_expiry": { "kind": "fail_task" } }, "input_schema": { "type": "object" }, "output_schema": { "type": "object" }, @@ -628,7 +674,7 @@ fn execution_plan_round_trips_without_a_nested_version() { encoded["input_wait_policy"], json!({ "expiry": { "kind": "after", "delay_seconds": 3600 }, - "on_expiry": { "kind": "fail_run" } + "on_expiry": { "kind": "fail_task" } }) ); assert_eq!( @@ -1300,6 +1346,51 @@ fn execution_task_outcome_enforces_512_character_citation_id_limit() { ); } +#[test] +fn conditional_nodes_are_effectful_leaves_nothing_reads_or_depends_on_for_completion() { + // Pins: the artifact layer refuses a plan whose meaning needs a conditional node to have + // run, before the plan ever reaches a compiler. A false condition commits a skipped node + // with a null output, so reading that output, making it the terminal output, or serving a + // requirement only through conditional nodes each describes a run nobody can satisfy. + let mut read_output = valid_plan(); + read_output.nodes[0].when = Some(ExecutionCondition::Exists { + reference: ExecutionReference { + path: "$.input.order_id".to_string(), + }, + }); + assert_error( + &validate_execution_plan_definition(&read_output), + "execution_plan.nodes[1].operation.value", + "a conditional node's output cannot be read; it is null when the condition is false", + ); + + let mut conditional_output = valid_plan(); + conditional_output.nodes[1].when = Some(ExecutionCondition::Exists { + reference: ExecutionReference { + path: "$.input.order_id".to_string(), + }, + }); + assert_error( + &validate_execution_plan_definition(&conditional_output), + "execution_plan.nodes[1].when", + "the terminal output node must not be conditional", + ); + + let mut only_conditional = valid_plan(); + only_conditional.nodes[0].when = Some(ExecutionCondition::Exists { + reference: ExecutionReference { + path: "$.input.order_id".to_string(), + }, + }); + only_conditional.nodes[0].requirement_ids = vec!["req_branch".to_string()]; + assert_error( + &validate_execution_plan_definition(&only_conditional), + "execution_plan.nodes", + "every node serving requirement `req_branch` is conditional, so all branches \ + evaluating false would leave it with no eligible node", + ); +} + #[test] fn conditions_use_the_same_reference_visibility_rules() { // Pins: conditional execution cannot inspect undeclared node output. @@ -1372,7 +1463,7 @@ fn task_outcome_variants_round_trip_without_extra_envelope_fields() { fn valid_plan() -> ExecutionPlanDefinition { ExecutionPlanDefinition { cancel_policy: ExecutionCancelPolicy::RetainEffects, - input_wait_policy: wait_policy(ExecutionWaitExpiryAction::FailRun), + input_wait_policy: wait_policy(ExecutionWaitExpiryAction::FailTask), input_schema: json!({ "type": "object" }), output_schema: json!({ "type": "object" }), nodes: vec![ diff --git a/crates/moa-brain/src/compaction.rs b/crates/moa-brain/src/compaction.rs index c8b536802..53a5f010e 100644 --- a/crates/moa-brain/src/compaction.rs +++ b/crates/moa-brain/src/compaction.rs @@ -249,7 +249,7 @@ fn compaction_request( CompletionRequest { model: None, messages: vec![ - ContextMessage::system(include_str!("prompts/summarizer.txt")), + ContextMessage::system(include_str!("prompts/summarizer.md")), ContextMessage::user(prompt), ], tools: Vec::new(), diff --git a/crates/moa-brain/src/execution_planning/request.rs b/crates/moa-brain/src/execution_planning/request.rs index f6a8c165a..e5b9541d6 100644 --- a/crates/moa-brain/src/execution_planning/request.rs +++ b/crates/moa-brain/src/execution_planning/request.rs @@ -22,7 +22,7 @@ use serde_json::Value; pub const EXECUTION_PLANNER_PROMPT_VERSION: &str = "execution-planner-v7"; /// Fixed maximum collected planner output tokens. pub const EXECUTION_PLANNER_MAX_OUTPUT_TOKENS: usize = 32_768; -const EXECUTION_PLANNER_PROMPT: &str = include_str!("../prompts/execution_planner.txt"); +const EXECUTION_PLANNER_PROMPT: &str = include_str!("../prompts/execution_planner.md"); /// Immutable inputs for initial plan generation or exact template instantiation. #[derive(Clone, Debug)] diff --git a/crates/moa-brain/src/execution_planning/routing.rs b/crates/moa-brain/src/execution_planning/routing.rs index 6da0b63df..8f8a258db 100644 --- a/crates/moa-brain/src/execution_planning/routing.rs +++ b/crates/moa-brain/src/execution_planning/routing.rs @@ -37,7 +37,7 @@ const EXECUTION_ROUTER_MAX_MISSING_INPUT_BYTES: usize = 256; /// Maximum installed-skill names serialized into the classifier prompt as a /// coverage hint. Bounds the per-turn user message regardless of tenant size. pub const EXECUTION_ROUTER_MAX_SKILL_NAMES: usize = 24; -const EXECUTION_ROUTER_PROMPT: &str = include_str!("../prompts/execution_router.txt"); +const EXECUTION_ROUTER_PROMPT: &str = include_str!("../prompts/execution_router.md"); const ROUTER_STAGE_METADATA_KEY: &str = "moa.pipeline.stage"; const OPENAI_REASONING_EFFORT_METADATA_KEY: &str = "_moa.openai.reasoning_effort"; diff --git a/crates/moa-brain/src/prompts/execution_planner.txt b/crates/moa-brain/src/prompts/execution_planner.md similarity index 68% rename from crates/moa-brain/src/prompts/execution_planner.txt rename to crates/moa-brain/src/prompts/execution_planner.md index eb1615220..b16ac15aa 100644 --- a/crates/moa-brain/src/prompts/execution_planner.txt +++ b/crates/moa-brain/src/prompts/execution_planner.md @@ -6,8 +6,9 @@ Compiler invariants: - Set `goal.objective` to the frozen `objective` byte-for-byte. - Choose exactly one explicit `plan.cancel_policy`: `retain_effects` or `compensate_committed`. - Treat the frozen `budget.deadline_at` as the absolute Durable-run deadline. Never emit a wait, retry window, or active task whose bound reaches or exceeds it. -- Use `WaitUntil` for an absolute calendar-time delay. Its `wake_at` must be before the run deadline, and its declared `result` is the structured value made available to downstream nodes after the timer fires. -- Give every `Review` and `WaitSignal` an explicit wait policy. Represent human and external waits only with these storage-backed wait operations; never keep an `Agent` or `Capability` active while waiting for a person, callback, schedule, or retry time. +- Use `WaitUntil` for a calendar-time delay. Its `wake` is a tagged temporal target, either `{"kind":"at","at":""}` for an exact instant or `{"kind":"after","delay_seconds":}` for a delay measured from the moment the node starts waiting. Emit exactly those fields for the chosen shape and nothing else. The resolved wake time must land before the run deadline. Its declared `result` is the structured value made available to downstream nodes after the timer fires. +- Give every `Review` and `WaitSignal` an explicit `wait_policy` of `{"expiry": , "on_expiry": }`, using the same two temporal-target shapes. An expiry action is either `{"kind":"fail_task"}` or `{"kind":"continue_with","output":}`. Represent human and external waits only with these storage-backed wait operations; never keep an `Agent` or `Capability` active while waiting for a person, callback, schedule, or retry time. +- Always set `plan.input_wait_policy`. It is required, and it governs every task that pauses for runtime input rather than one named node, so its `on_expiry` accepts only `{"kind":"fail_task"}` — never `continue_with`. - Decompose long work into bounded active tasks separated by durable nodes. Never plan a continuously running multi-hour or multi-day model call, tool call, shell process, network connection, or sandbox; use a registered asynchronous capability when the catalog explicitly provides one. - Set every node's `compensation` explicitly. Use `null` unless the node is a direct side-effecting `Capability` whose exact catalog entry advertises the same compensator and bounded input mapping, and that compensator has `requires_sandbox=false`. Never add compensation to reads, agents, maps, reduces, reviews, signals, or outputs, never use a sandbox-backed compensator, and never invent rollback authority. - An amendment must preserve compensation for work that is running or committed and must not weaken the run's cancellation policy. diff --git a/crates/moa-brain/src/prompts/execution_router.txt b/crates/moa-brain/src/prompts/execution_router.md similarity index 100% rename from crates/moa-brain/src/prompts/execution_router.txt rename to crates/moa-brain/src/prompts/execution_router.md diff --git a/crates/moa-brain/src/prompts/summarizer.txt b/crates/moa-brain/src/prompts/summarizer.md similarity index 100% rename from crates/moa-brain/src/prompts/summarizer.txt rename to crates/moa-brain/src/prompts/summarizer.md diff --git a/crates/moa-core/src/events.rs b/crates/moa-core/src/events.rs index 50e420039..fec800eb1 100644 --- a/crates/moa-core/src/events.rs +++ b/crates/moa-core/src/events.rs @@ -52,8 +52,6 @@ pub enum ExecutionProgressPhase { Pausing, /// The run is fully paused and consumes no active execution capacity. Paused, - /// A late callback or activation was fenced without advancing canonical state. - StaleWork, } /// Audience expected to resolve the run's current public blocker. @@ -62,8 +60,6 @@ pub enum ExecutionProgressPhase { pub enum ExecutionBlockerAudience { /// The owning user must provide input. User, - /// Another authorized agent must provide input. - Agent, /// A tenant reviewer must decide. TenantReviewer, /// An external actor or callback must signal. diff --git a/crates/moa-core/src/traits/mod.rs b/crates/moa-core/src/traits/mod.rs index 0638cc634..0d17d9d2e 100644 --- a/crates/moa-core/src/traits/mod.rs +++ b/crates/moa-core/src/traits/mod.rs @@ -734,10 +734,35 @@ pub trait HandProvider: Send + Sync { /// Returns the current hand status. async fn status(&self, handle: &HandHandle) -> Result; - /// Pauses a provisioned hand. - async fn pause(&self, handle: &HandHandle) -> Result<()>; + /// Reports whether [`HandProvider::suspend`] actually releases compute here. + /// + /// Callers must consult this before choosing the suspend path, because a + /// provider that cannot release compute needs a different — and differently + /// priced — decision, not a failed call. The default is `false`: a provider + /// only opts in when its suspend genuinely stops billing for CPU and memory. + fn supports_suspend(&self) -> bool { + false + } + + /// Releases a hand's compute while keeping its filesystem for a later resume. + /// + /// This is an optional warm tier, never a durability primitive. Callers must + /// have published a portable checkpoint *before* suspending, which is what + /// makes an evicted or failed suspension a pure cache miss rather than data + /// loss. Failure is therefore non-fatal: the caller degrades to + /// checkpoint-and-destroy and restores from the published head. + /// + /// A provider that cannot actually release compute must return + /// [`MoaError::Unsupported`] and leave [`HandProvider::supports_suspend`] + /// false rather than approximating it with a freeze that keeps the resource + /// billed. Resuming a suspended hand goes through [`HandProvider::resume`]. + async fn suspend(&self, _handle: &HandHandle) -> Result<()> { + Err(MoaError::Unsupported( + "compute suspension is not supported by this hand provider".to_string(), + )) + } - /// Resumes a paused hand. + /// Resumes a paused or suspended hand. async fn resume(&self, handle: &HandHandle) -> Result<()>; /// Destroys a provisioned hand. @@ -811,6 +836,13 @@ pub trait SandboxStorageProvider: Send + Sync { } /// Reconciles an operation whose provider outcome could not be confirmed. + /// + /// Reconciliation proves only what the provider already did to storage. It + /// must never release compute as a side effect: the caller may still be + /// executing on the reconciled hand, and a compute-releasing commit destroys + /// its hand as a separate exact step after the durable publication CAS. A + /// confirmed commit or checkpoint therefore always reports + /// `WorkspacePostCommitState::AttachmentRetained`. async fn reconcile_workspace_operation( &self, request: WorkspaceReconcileRequest, diff --git a/crates/moa-core/src/types/sandbox_workspace.rs b/crates/moa-core/src/types/sandbox_workspace.rs index 10898bc7b..78c21837d 100644 --- a/crates/moa-core/src/types/sandbox_workspace.rs +++ b/crates/moa-core/src/types/sandbox_workspace.rs @@ -244,6 +244,36 @@ pub enum ExecutionHandReleaseOwner { }, } +/// What a continuation boundary actually did with the attempt's sandbox compute. +/// +/// A continuation is not a wait: the next slice is enqueued immediately, so the +/// boundary tries to keep the sandbox rather than destroy and re-provision it +/// milliseconds later. The three keep-or-not outcomes have materially different +/// cost and different follow-up work for the caller, so they are reported +/// explicitly instead of being flattened into success. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "disposition", rename_all = "snake_case", deny_unknown_fields)] +pub enum ExecutionHandContinuationDisposition { + /// The attempt owned no durable workspace or no live lease; nothing was kept. + NoComputeOwned, + /// Compute was released in-path and the filesystem kept for the next slice. + /// + /// The lease stays live so the next slice reattaches, and the active-compute + /// capacity charge went back to the fleet. + Suspended, + /// The provider cannot release compute, so the hand stays hot on a short bound. + /// + /// The reaper owns the deadline; a slice that does not arrive inside it loses + /// the sandbox and restores from the checkpoint published here. + RetainedHot, + /// Suspension was attempted and failed; the caller must fall back to release. + /// + /// The checkpoint is published either way, so the caller finishes the + /// ordinary checkpoint-and-destroy path rather than leaving a hand hot on a + /// bet that already lost. + SuspendFailed, +} + /// Durable proof that one exact execution attempt released its sandbox compute. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] diff --git a/crates/moa-edge/src/routes/session_stream.rs b/crates/moa-edge/src/routes/session_stream.rs index 48a92f6dd..6a2a73c64 100644 --- a/crates/moa-edge/src/routes/session_stream.rs +++ b/crates/moa-edge/src/routes/session_stream.rs @@ -499,7 +499,6 @@ fn sse_event_name(event: &Event) -> &'static str { moa_core::events::ExecutionProgressPhase::WaitingReview => "execution_review_wait", moa_core::events::ExecutionProgressPhase::WaitingSignal => "execution_signal_wait", moa_core::events::ExecutionProgressPhase::WaitingTimer => "execution_timer_wait", - moa_core::events::ExecutionProgressPhase::StaleWork => "execution_stale", moa_core::events::ExecutionProgressPhase::Running => "execution_progress", }, Event::ExecutionInputRequired(_) => "execution_input_request", diff --git a/crates/moa-eval/examples/generate_execution_corpus.rs b/crates/moa-eval/examples/generate_execution_corpus.rs index 11b802ccf..5120ce18d 100644 --- a/crates/moa-eval/examples/generate_execution_corpus.rs +++ b/crates/moa-eval/examples/generate_execution_corpus.rs @@ -477,7 +477,7 @@ fn contract_case(index: usize) -> ExecutionContractCase { expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { delay_seconds: 86_400, }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, }, input_schema: json!({ "type": "object", diff --git a/crates/moa-eval/scenarios/execution/contract-recorded.jsonl b/crates/moa-eval/scenarios/execution/contract-recorded.jsonl index 87a7023bb..697377061 100644 --- a/crates/moa-eval/scenarios/execution/contract-recorded.jsonl +++ b/crates/moa-eval/scenarios/execution/contract-recorded.jsonl @@ -1,80 +1,80 @@ -{"schema_version":1,"case_id":"contract-000","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (000).","requirements":[{"id":"req-screen-000","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-000","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-000","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-000","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-000-a","issuer-000-b","issuer-000-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-000","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-000","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-000","description":"Require complete map coverage","requirement_ids":["req-screen-000"],"constraint_ids":["constraint-exclusions-000"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-000","description":"Require citations for every issuer","requirement_ids":["req-report-000"],"constraint_ids":["constraint-definition-000"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-000"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-000-a"},{"ticker":"issuer-000-b"},{"ticker":"issuer-000-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-000"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-000"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-000-a","issuer-000-b","issuer-000-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-001","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (001).","requirements":[{"id":"req-screen-001","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-001","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-001","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-001","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-001-a","issuer-001-b","issuer-001-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-001","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-001","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-001","description":"Require complete map coverage","requirement_ids":["req-screen-001"],"constraint_ids":["constraint-exclusions-001"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-001","description":"Require citations for every issuer","requirement_ids":["req-report-001"],"constraint_ids":["constraint-definition-001"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-001"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-001-a"},{"ticker":"issuer-001-b"},{"ticker":"issuer-001-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-001"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-001"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-001-a","issuer-001-b","issuer-001-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-002","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (002).","requirements":[{"id":"req-screen-002","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-002","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-002","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-002","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-002-a","issuer-002-b","issuer-002-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-002","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-002","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-002","description":"Require complete map coverage","requirement_ids":["req-screen-002"],"constraint_ids":["constraint-exclusions-002"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-002","description":"Require citations for every issuer","requirement_ids":["req-report-002"],"constraint_ids":["constraint-definition-002"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-002"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-002-a"},{"ticker":"issuer-002-b"},{"ticker":"issuer-002-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-002"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-002"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-002-a","issuer-002-b","issuer-002-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-003","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (003).","requirements":[{"id":"req-screen-003","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-003","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-003","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-003","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-003-a","issuer-003-b","issuer-003-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-003","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-003","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-003","description":"Require complete map coverage","requirement_ids":["req-screen-003"],"constraint_ids":["constraint-exclusions-003"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-003","description":"Require citations for every issuer","requirement_ids":["req-report-003"],"constraint_ids":["constraint-definition-003"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-003"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-003-a"},{"ticker":"issuer-003-b"},{"ticker":"issuer-003-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-003"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-003"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-003-a","issuer-003-b","issuer-003-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-004","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (004).","requirements":[{"id":"req-screen-004","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-004","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-004","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-004","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-004-a","issuer-004-b","issuer-004-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-004","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-004","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-004","description":"Require complete map coverage","requirement_ids":["req-screen-004"],"constraint_ids":["constraint-exclusions-004"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-004","description":"Require citations for every issuer","requirement_ids":["req-report-004"],"constraint_ids":["constraint-definition-004"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-004"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-004-a"},{"ticker":"issuer-004-b"},{"ticker":"issuer-004-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-004"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-004"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-004-a","issuer-004-b","issuer-004-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-005","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (005).","requirements":[{"id":"req-screen-005","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-005","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-005","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-005","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-005-a","issuer-005-b","issuer-005-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-005","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-005","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-005","description":"Require complete map coverage","requirement_ids":["req-screen-005"],"constraint_ids":["constraint-exclusions-005"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-005","description":"Require citations for every issuer","requirement_ids":["req-report-005"],"constraint_ids":["constraint-definition-005"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-005"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-005-a"},{"ticker":"issuer-005-b"},{"ticker":"issuer-005-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-005"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-005"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-005-a","issuer-005-b","issuer-005-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-006","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (006).","requirements":[{"id":"req-screen-006","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-006","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-006","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-006","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-006-a","issuer-006-b","issuer-006-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-006","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-006","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-006","description":"Require complete map coverage","requirement_ids":["req-screen-006"],"constraint_ids":["constraint-exclusions-006"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-006","description":"Require citations for every issuer","requirement_ids":["req-report-006"],"constraint_ids":["constraint-definition-006"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-006"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-006-a"},{"ticker":"issuer-006-b"},{"ticker":"issuer-006-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-006"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-006"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-006-a","issuer-006-b","issuer-006-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-007","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (007).","requirements":[{"id":"req-screen-007","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-007","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-007","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-007","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-007-a","issuer-007-b","issuer-007-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-007","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-007","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-007","description":"Require complete map coverage","requirement_ids":["req-screen-007"],"constraint_ids":["constraint-exclusions-007"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-007","description":"Require citations for every issuer","requirement_ids":["req-report-007"],"constraint_ids":["constraint-definition-007"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-007"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-007-a"},{"ticker":"issuer-007-b"},{"ticker":"issuer-007-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-007"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-007"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-007-a","issuer-007-b","issuer-007-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-008","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (008).","requirements":[{"id":"req-screen-008","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-008","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-008","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-008","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-008-a","issuer-008-b","issuer-008-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-008","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-008","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-008","description":"Require complete map coverage","requirement_ids":["req-screen-008"],"constraint_ids":["constraint-exclusions-008"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-008","description":"Require citations for every issuer","requirement_ids":["req-report-008"],"constraint_ids":["constraint-definition-008"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-008"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-008-a"},{"ticker":"issuer-008-b"},{"ticker":"issuer-008-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-008"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-008"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-008-a","issuer-008-b","issuer-008-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-009","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (009).","requirements":[{"id":"req-screen-009","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-009","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-009","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-009","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-009-a","issuer-009-b","issuer-009-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-009","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-009","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-009","description":"Require complete map coverage","requirement_ids":["req-screen-009"],"constraint_ids":["constraint-exclusions-009"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-009","description":"Require citations for every issuer","requirement_ids":["req-report-009"],"constraint_ids":["constraint-definition-009"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-009"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-009-a"},{"ticker":"issuer-009-b"},{"ticker":"issuer-009-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-009"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-009"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-009-a","issuer-009-b","issuer-009-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-010","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (010).","requirements":[{"id":"req-screen-010","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-010","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-010","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-010","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-010-a","issuer-010-b","issuer-010-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-010","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-010","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-010","description":"Require complete map coverage","requirement_ids":["req-screen-010"],"constraint_ids":["constraint-exclusions-010"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-010","description":"Require citations for every issuer","requirement_ids":["req-report-010"],"constraint_ids":["constraint-definition-010"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-010"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-010-a"},{"ticker":"issuer-010-b"},{"ticker":"issuer-010-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-010"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-010"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-010-a","issuer-010-b","issuer-010-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-011","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (011).","requirements":[{"id":"req-screen-011","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-011","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-011","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-011","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-011-a","issuer-011-b","issuer-011-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-011","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-011","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-011","description":"Require complete map coverage","requirement_ids":["req-screen-011"],"constraint_ids":["constraint-exclusions-011"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-011","description":"Require citations for every issuer","requirement_ids":["req-report-011"],"constraint_ids":["constraint-definition-011"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-011"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-011-a"},{"ticker":"issuer-011-b"},{"ticker":"issuer-011-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-011"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-011"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-011-a","issuer-011-b","issuer-011-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-012","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (012).","requirements":[{"id":"req-screen-012","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-012","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-012","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-012","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-012-a","issuer-012-b","issuer-012-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-012","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-012","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-012","description":"Require complete map coverage","requirement_ids":["req-screen-012"],"constraint_ids":["constraint-exclusions-012"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-012","description":"Require citations for every issuer","requirement_ids":["req-report-012"],"constraint_ids":["constraint-definition-012"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-012"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-012-a"},{"ticker":"issuer-012-b"},{"ticker":"issuer-012-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-012"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-012"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-012-a","issuer-012-b","issuer-012-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-013","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (013).","requirements":[{"id":"req-screen-013","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-013","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-013","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-013","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-013-a","issuer-013-b","issuer-013-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-013","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-013","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-013","description":"Require complete map coverage","requirement_ids":["req-screen-013"],"constraint_ids":["constraint-exclusions-013"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-013","description":"Require citations for every issuer","requirement_ids":["req-report-013"],"constraint_ids":["constraint-definition-013"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-013"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-013-a"},{"ticker":"issuer-013-b"},{"ticker":"issuer-013-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-013"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-013"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-013-a","issuer-013-b","issuer-013-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-014","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (014).","requirements":[{"id":"req-screen-014","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-014","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-014","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-014","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-014-a","issuer-014-b","issuer-014-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-014","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-014","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-014","description":"Require complete map coverage","requirement_ids":["req-screen-014"],"constraint_ids":["constraint-exclusions-014"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-014","description":"Require citations for every issuer","requirement_ids":["req-report-014"],"constraint_ids":["constraint-definition-014"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-014"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-014-a"},{"ticker":"issuer-014-b"},{"ticker":"issuer-014-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-014"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-014"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-014-a","issuer-014-b","issuer-014-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-015","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (015).","requirements":[{"id":"req-screen-015","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-015","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-015","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-015","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-015-a","issuer-015-b","issuer-015-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-015","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-015","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-015","description":"Require complete map coverage","requirement_ids":["req-screen-015"],"constraint_ids":["constraint-exclusions-015"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-015","description":"Require citations for every issuer","requirement_ids":["req-report-015"],"constraint_ids":["constraint-definition-015"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-015"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-015-a"},{"ticker":"issuer-015-b"},{"ticker":"issuer-015-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-015"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-015"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-015-a","issuer-015-b","issuer-015-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-016","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (016).","requirements":[{"id":"req-screen-016","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-016","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-016","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-016","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-016-a","issuer-016-b","issuer-016-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-016","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-016","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-016","description":"Require complete map coverage","requirement_ids":["req-screen-016"],"constraint_ids":["constraint-exclusions-016"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-016","description":"Require citations for every issuer","requirement_ids":["req-report-016"],"constraint_ids":["constraint-definition-016"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-016"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-016-a"},{"ticker":"issuer-016-b"},{"ticker":"issuer-016-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-016"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-016"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-016-a","issuer-016-b","issuer-016-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-017","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (017).","requirements":[{"id":"req-screen-017","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-017","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-017","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-017","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-017-a","issuer-017-b","issuer-017-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-017","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-017","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-017","description":"Require complete map coverage","requirement_ids":["req-screen-017"],"constraint_ids":["constraint-exclusions-017"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-017","description":"Require citations for every issuer","requirement_ids":["req-report-017"],"constraint_ids":["constraint-definition-017"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-017"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-017-a"},{"ticker":"issuer-017-b"},{"ticker":"issuer-017-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-017"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-017"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-017-a","issuer-017-b","issuer-017-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-018","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (018).","requirements":[{"id":"req-screen-018","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-018","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-018","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-018","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-018-a","issuer-018-b","issuer-018-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-018","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-018","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-018","description":"Require complete map coverage","requirement_ids":["req-screen-018"],"constraint_ids":["constraint-exclusions-018"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-018","description":"Require citations for every issuer","requirement_ids":["req-report-018"],"constraint_ids":["constraint-definition-018"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-018"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-018-a"},{"ticker":"issuer-018-b"},{"ticker":"issuer-018-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-018"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-018"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-018-a","issuer-018-b","issuer-018-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-019","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (019).","requirements":[{"id":"req-screen-019","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-019","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-019","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-019","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-019-a","issuer-019-b","issuer-019-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-019","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-019","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-019","description":"Require complete map coverage","requirement_ids":["req-screen-019"],"constraint_ids":["constraint-exclusions-019"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-019","description":"Require citations for every issuer","requirement_ids":["req-report-019"],"constraint_ids":["constraint-definition-019"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-019"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-019-a"},{"ticker":"issuer-019-b"},{"ticker":"issuer-019-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-019"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-019"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-019-a","issuer-019-b","issuer-019-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-020","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (020).","requirements":[{"id":"req-screen-020","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-020","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-020","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-020","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-020-a","issuer-020-b","issuer-020-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-020","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-020","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-020","description":"Require complete map coverage","requirement_ids":["req-screen-020"],"constraint_ids":["constraint-exclusions-020"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-020","description":"Require citations for every issuer","requirement_ids":["req-report-020"],"constraint_ids":["constraint-definition-020"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-020"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-020-a"},{"ticker":"issuer-020-b"},{"ticker":"issuer-020-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-020"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-020"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-020-a","issuer-020-b","issuer-020-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-021","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (021).","requirements":[{"id":"req-screen-021","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-021","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-021","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-021","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-021-a","issuer-021-b","issuer-021-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-021","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-021","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-021","description":"Require complete map coverage","requirement_ids":["req-screen-021"],"constraint_ids":["constraint-exclusions-021"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-021","description":"Require citations for every issuer","requirement_ids":["req-report-021"],"constraint_ids":["constraint-definition-021"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-021"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-021-a"},{"ticker":"issuer-021-b"},{"ticker":"issuer-021-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-021"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-021"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-021-a","issuer-021-b","issuer-021-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-022","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (022).","requirements":[{"id":"req-screen-022","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-022","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-022","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-022","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-022-a","issuer-022-b","issuer-022-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-022","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-022","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-022","description":"Require complete map coverage","requirement_ids":["req-screen-022"],"constraint_ids":["constraint-exclusions-022"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-022","description":"Require citations for every issuer","requirement_ids":["req-report-022"],"constraint_ids":["constraint-definition-022"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-022"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-022-a"},{"ticker":"issuer-022-b"},{"ticker":"issuer-022-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-022"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-022"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-022-a","issuer-022-b","issuer-022-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-023","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (023).","requirements":[{"id":"req-screen-023","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-023","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-023","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-023","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-023-a","issuer-023-b","issuer-023-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-023","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-023","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-023","description":"Require complete map coverage","requirement_ids":["req-screen-023"],"constraint_ids":["constraint-exclusions-023"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-023","description":"Require citations for every issuer","requirement_ids":["req-report-023"],"constraint_ids":["constraint-definition-023"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-023"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-023-a"},{"ticker":"issuer-023-b"},{"ticker":"issuer-023-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-023"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-023"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-023-a","issuer-023-b","issuer-023-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-024","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (024).","requirements":[{"id":"req-screen-024","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-024","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-024","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-024","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-024-a","issuer-024-b","issuer-024-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-024","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-024","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-024","description":"Require complete map coverage","requirement_ids":["req-screen-024"],"constraint_ids":["constraint-exclusions-024"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-024","description":"Require citations for every issuer","requirement_ids":["req-report-024"],"constraint_ids":["constraint-definition-024"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-024"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-024-a"},{"ticker":"issuer-024-b"},{"ticker":"issuer-024-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-024"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-024"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-024-a","issuer-024-b","issuer-024-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-025","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (025).","requirements":[{"id":"req-screen-025","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-025","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-025","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-025","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-025-a","issuer-025-b","issuer-025-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-025","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-025","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-025","description":"Require complete map coverage","requirement_ids":["req-screen-025"],"constraint_ids":["constraint-exclusions-025"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-025","description":"Require citations for every issuer","requirement_ids":["req-report-025"],"constraint_ids":["constraint-definition-025"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-025"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-025-a"},{"ticker":"issuer-025-b"},{"ticker":"issuer-025-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-025"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-025"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-025-a","issuer-025-b","issuer-025-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-026","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (026).","requirements":[{"id":"req-screen-026","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-026","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-026","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-026","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-026-a","issuer-026-b","issuer-026-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-026","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-026","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-026","description":"Require complete map coverage","requirement_ids":["req-screen-026"],"constraint_ids":["constraint-exclusions-026"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-026","description":"Require citations for every issuer","requirement_ids":["req-report-026"],"constraint_ids":["constraint-definition-026"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-026"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-026-a"},{"ticker":"issuer-026-b"},{"ticker":"issuer-026-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-026"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-026"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-026-a","issuer-026-b","issuer-026-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-027","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (027).","requirements":[{"id":"req-screen-027","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-027","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-027","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-027","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-027-a","issuer-027-b","issuer-027-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-027","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-027","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-027","description":"Require complete map coverage","requirement_ids":["req-screen-027"],"constraint_ids":["constraint-exclusions-027"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-027","description":"Require citations for every issuer","requirement_ids":["req-report-027"],"constraint_ids":["constraint-definition-027"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-027"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-027-a"},{"ticker":"issuer-027-b"},{"ticker":"issuer-027-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-027"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-027"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-027-a","issuer-027-b","issuer-027-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-028","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (028).","requirements":[{"id":"req-screen-028","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-028","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-028","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-028","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-028-a","issuer-028-b","issuer-028-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-028","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-028","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-028","description":"Require complete map coverage","requirement_ids":["req-screen-028"],"constraint_ids":["constraint-exclusions-028"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-028","description":"Require citations for every issuer","requirement_ids":["req-report-028"],"constraint_ids":["constraint-definition-028"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-028"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-028-a"},{"ticker":"issuer-028-b"},{"ticker":"issuer-028-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-028"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-028"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-028-a","issuer-028-b","issuer-028-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-029","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (029).","requirements":[{"id":"req-screen-029","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-029","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-029","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-029","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-029-a","issuer-029-b","issuer-029-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-029","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-029","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-029","description":"Require complete map coverage","requirement_ids":["req-screen-029"],"constraint_ids":["constraint-exclusions-029"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-029","description":"Require citations for every issuer","requirement_ids":["req-report-029"],"constraint_ids":["constraint-definition-029"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-029"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-029-a"},{"ticker":"issuer-029-b"},{"ticker":"issuer-029-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-029"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-029"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-029-a","issuer-029-b","issuer-029-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-030","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (030).","requirements":[{"id":"req-screen-030","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-030","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-030","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-030","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-030-a","issuer-030-b","issuer-030-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-030","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-030","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-030","description":"Require complete map coverage","requirement_ids":["req-screen-030"],"constraint_ids":["constraint-exclusions-030"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-030","description":"Require citations for every issuer","requirement_ids":["req-report-030"],"constraint_ids":["constraint-definition-030"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-030"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-030-a"},{"ticker":"issuer-030-b"},{"ticker":"issuer-030-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-030"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-030"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-030-a","issuer-030-b","issuer-030-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-031","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (031).","requirements":[{"id":"req-screen-031","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-031","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-031","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-031","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-031-a","issuer-031-b","issuer-031-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-031","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-031","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-031","description":"Require complete map coverage","requirement_ids":["req-screen-031"],"constraint_ids":["constraint-exclusions-031"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-031","description":"Require citations for every issuer","requirement_ids":["req-report-031"],"constraint_ids":["constraint-definition-031"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-031"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-031-a"},{"ticker":"issuer-031-b"},{"ticker":"issuer-031-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-031"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-031"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-031-a","issuer-031-b","issuer-031-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-032","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (032).","requirements":[{"id":"req-screen-032","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-032","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-032","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-032","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-032-a","issuer-032-b","issuer-032-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-032","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-032","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-032","description":"Require complete map coverage","requirement_ids":["req-screen-032"],"constraint_ids":["constraint-exclusions-032"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-032","description":"Require citations for every issuer","requirement_ids":["req-report-032"],"constraint_ids":["constraint-definition-032"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-032"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-032-a"},{"ticker":"issuer-032-b"},{"ticker":"issuer-032-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-032"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-032"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-032-a","issuer-032-b","issuer-032-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-033","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (033).","requirements":[{"id":"req-screen-033","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-033","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-033","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-033","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-033-a","issuer-033-b","issuer-033-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-033","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-033","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-033","description":"Require complete map coverage","requirement_ids":["req-screen-033"],"constraint_ids":["constraint-exclusions-033"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-033","description":"Require citations for every issuer","requirement_ids":["req-report-033"],"constraint_ids":["constraint-definition-033"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-033"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-033-a"},{"ticker":"issuer-033-b"},{"ticker":"issuer-033-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-033"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-033"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-033-a","issuer-033-b","issuer-033-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-034","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (034).","requirements":[{"id":"req-screen-034","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-034","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-034","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-034","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-034-a","issuer-034-b","issuer-034-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-034","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-034","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-034","description":"Require complete map coverage","requirement_ids":["req-screen-034"],"constraint_ids":["constraint-exclusions-034"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-034","description":"Require citations for every issuer","requirement_ids":["req-report-034"],"constraint_ids":["constraint-definition-034"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-034"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-034-a"},{"ticker":"issuer-034-b"},{"ticker":"issuer-034-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-034"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-034"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-034-a","issuer-034-b","issuer-034-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-035","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (035).","requirements":[{"id":"req-screen-035","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-035","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-035","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-035","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-035-a","issuer-035-b","issuer-035-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-035","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-035","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-035","description":"Require complete map coverage","requirement_ids":["req-screen-035"],"constraint_ids":["constraint-exclusions-035"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-035","description":"Require citations for every issuer","requirement_ids":["req-report-035"],"constraint_ids":["constraint-definition-035"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-035"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-035-a"},{"ticker":"issuer-035-b"},{"ticker":"issuer-035-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-035"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-035"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-035-a","issuer-035-b","issuer-035-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-036","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (036).","requirements":[{"id":"req-screen-036","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-036","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-036","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-036","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-036-a","issuer-036-b","issuer-036-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-036","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-036","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-036","description":"Require complete map coverage","requirement_ids":["req-screen-036"],"constraint_ids":["constraint-exclusions-036"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-036","description":"Require citations for every issuer","requirement_ids":["req-report-036"],"constraint_ids":["constraint-definition-036"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-036"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-036-a"},{"ticker":"issuer-036-b"},{"ticker":"issuer-036-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-036"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-036"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-036-a","issuer-036-b","issuer-036-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-037","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (037).","requirements":[{"id":"req-screen-037","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-037","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-037","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-037","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-037-a","issuer-037-b","issuer-037-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-037","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-037","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-037","description":"Require complete map coverage","requirement_ids":["req-screen-037"],"constraint_ids":["constraint-exclusions-037"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-037","description":"Require citations for every issuer","requirement_ids":["req-report-037"],"constraint_ids":["constraint-definition-037"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-037"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-037-a"},{"ticker":"issuer-037-b"},{"ticker":"issuer-037-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-037"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-037"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-037-a","issuer-037-b","issuer-037-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-038","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (038).","requirements":[{"id":"req-screen-038","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-038","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-038","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-038","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-038-a","issuer-038-b","issuer-038-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-038","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-038","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-038","description":"Require complete map coverage","requirement_ids":["req-screen-038"],"constraint_ids":["constraint-exclusions-038"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-038","description":"Require citations for every issuer","requirement_ids":["req-report-038"],"constraint_ids":["constraint-definition-038"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-038"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-038-a"},{"ticker":"issuer-038-b"},{"ticker":"issuer-038-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-038"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-038"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-038-a","issuer-038-b","issuer-038-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-039","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (039).","requirements":[{"id":"req-screen-039","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-039","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-039","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-039","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-039-a","issuer-039-b","issuer-039-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-039","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-039","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-039","description":"Require complete map coverage","requirement_ids":["req-screen-039"],"constraint_ids":["constraint-exclusions-039"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-039","description":"Require citations for every issuer","requirement_ids":["req-report-039"],"constraint_ids":["constraint-definition-039"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-039"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-039-a"},{"ticker":"issuer-039-b"},{"ticker":"issuer-039-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-039"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-039"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-039-a","issuer-039-b","issuer-039-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-040","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (040).","requirements":[{"id":"req-screen-040","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-040","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-040","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-040","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-040-a","issuer-040-b","issuer-040-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-040","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-040","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-040","description":"Require complete map coverage","requirement_ids":["req-screen-040"],"constraint_ids":["constraint-exclusions-040"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-040","description":"Require citations for every issuer","requirement_ids":["req-report-040"],"constraint_ids":["constraint-definition-040"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-040"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-040-a"},{"ticker":"issuer-040-b"},{"ticker":"issuer-040-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-040"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-040"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-040-a","issuer-040-b","issuer-040-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-041","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (041).","requirements":[{"id":"req-screen-041","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-041","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-041","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-041","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-041-a","issuer-041-b","issuer-041-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-041","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-041","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-041","description":"Require complete map coverage","requirement_ids":["req-screen-041"],"constraint_ids":["constraint-exclusions-041"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-041","description":"Require citations for every issuer","requirement_ids":["req-report-041"],"constraint_ids":["constraint-definition-041"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-041"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-041-a"},{"ticker":"issuer-041-b"},{"ticker":"issuer-041-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-041"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-041"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-041-a","issuer-041-b","issuer-041-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-042","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (042).","requirements":[{"id":"req-screen-042","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-042","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-042","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-042","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-042-a","issuer-042-b","issuer-042-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-042","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-042","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-042","description":"Require complete map coverage","requirement_ids":["req-screen-042"],"constraint_ids":["constraint-exclusions-042"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-042","description":"Require citations for every issuer","requirement_ids":["req-report-042"],"constraint_ids":["constraint-definition-042"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-042"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-042-a"},{"ticker":"issuer-042-b"},{"ticker":"issuer-042-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-042"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-042"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-042-a","issuer-042-b","issuer-042-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-043","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (043).","requirements":[{"id":"req-screen-043","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-043","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-043","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-043","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-043-a","issuer-043-b","issuer-043-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-043","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-043","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-043","description":"Require complete map coverage","requirement_ids":["req-screen-043"],"constraint_ids":["constraint-exclusions-043"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-043","description":"Require citations for every issuer","requirement_ids":["req-report-043"],"constraint_ids":["constraint-definition-043"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-043"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-043-a"},{"ticker":"issuer-043-b"},{"ticker":"issuer-043-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-043"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-043"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-043-a","issuer-043-b","issuer-043-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-044","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (044).","requirements":[{"id":"req-screen-044","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-044","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-044","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-044","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-044-a","issuer-044-b","issuer-044-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-044","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-044","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-044","description":"Require complete map coverage","requirement_ids":["req-screen-044"],"constraint_ids":["constraint-exclusions-044"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-044","description":"Require citations for every issuer","requirement_ids":["req-report-044"],"constraint_ids":["constraint-definition-044"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-044"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-044-a"},{"ticker":"issuer-044-b"},{"ticker":"issuer-044-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-044"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-044"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-044-a","issuer-044-b","issuer-044-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-045","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (045).","requirements":[{"id":"req-screen-045","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-045","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-045","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-045","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-045-a","issuer-045-b","issuer-045-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-045","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-045","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-045","description":"Require complete map coverage","requirement_ids":["req-screen-045"],"constraint_ids":["constraint-exclusions-045"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-045","description":"Require citations for every issuer","requirement_ids":["req-report-045"],"constraint_ids":["constraint-definition-045"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-045"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-045-a"},{"ticker":"issuer-045-b"},{"ticker":"issuer-045-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-045"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-045"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-045-a","issuer-045-b","issuer-045-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-046","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (046).","requirements":[{"id":"req-screen-046","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-046","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-046","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-046","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-046-a","issuer-046-b","issuer-046-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-046","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-046","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-046","description":"Require complete map coverage","requirement_ids":["req-screen-046"],"constraint_ids":["constraint-exclusions-046"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-046","description":"Require citations for every issuer","requirement_ids":["req-report-046"],"constraint_ids":["constraint-definition-046"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-046"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-046-a"},{"ticker":"issuer-046-b"},{"ticker":"issuer-046-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-046"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-046"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-046-a","issuer-046-b","issuer-046-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-047","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (047).","requirements":[{"id":"req-screen-047","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-047","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-047","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-047","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-047-a","issuer-047-b","issuer-047-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-047","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-047","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-047","description":"Require complete map coverage","requirement_ids":["req-screen-047"],"constraint_ids":["constraint-exclusions-047"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-047","description":"Require citations for every issuer","requirement_ids":["req-report-047"],"constraint_ids":["constraint-definition-047"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-047"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-047-a"},{"ticker":"issuer-047-b"},{"ticker":"issuer-047-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-047"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-047"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-047-a","issuer-047-b","issuer-047-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-048","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (048).","requirements":[{"id":"req-screen-048","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-048","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-048","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-048","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-048-a","issuer-048-b","issuer-048-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-048","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-048","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-048","description":"Require complete map coverage","requirement_ids":["req-screen-048"],"constraint_ids":["constraint-exclusions-048"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-048","description":"Require citations for every issuer","requirement_ids":["req-report-048"],"constraint_ids":["constraint-definition-048"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-048"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-048-a"},{"ticker":"issuer-048-b"},{"ticker":"issuer-048-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-048"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-048"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-048-a","issuer-048-b","issuer-048-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-049","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (049).","requirements":[{"id":"req-screen-049","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-049","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-049","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-049","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-049-a","issuer-049-b","issuer-049-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-049","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-049","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-049","description":"Require complete map coverage","requirement_ids":["req-screen-049"],"constraint_ids":["constraint-exclusions-049"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-049","description":"Require citations for every issuer","requirement_ids":["req-report-049"],"constraint_ids":["constraint-definition-049"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-049"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-049-a"},{"ticker":"issuer-049-b"},{"ticker":"issuer-049-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-049"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-049"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-049-a","issuer-049-b","issuer-049-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-050","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (050).","requirements":[{"id":"req-screen-050","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-050","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-050","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-050","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-050-a","issuer-050-b","issuer-050-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-050","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-050","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-050","description":"Require complete map coverage","requirement_ids":["req-screen-050"],"constraint_ids":["constraint-exclusions-050"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-050","description":"Require citations for every issuer","requirement_ids":["req-report-050"],"constraint_ids":["constraint-definition-050"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-050"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-050-a"},{"ticker":"issuer-050-b"},{"ticker":"issuer-050-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-050"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-050"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-050-a","issuer-050-b","issuer-050-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-051","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (051).","requirements":[{"id":"req-screen-051","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-051","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-051","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-051","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-051-a","issuer-051-b","issuer-051-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-051","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-051","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-051","description":"Require complete map coverage","requirement_ids":["req-screen-051"],"constraint_ids":["constraint-exclusions-051"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-051","description":"Require citations for every issuer","requirement_ids":["req-report-051"],"constraint_ids":["constraint-definition-051"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-051"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-051-a"},{"ticker":"issuer-051-b"},{"ticker":"issuer-051-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-051"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-051"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-051-a","issuer-051-b","issuer-051-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-052","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (052).","requirements":[{"id":"req-screen-052","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-052","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-052","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-052","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-052-a","issuer-052-b","issuer-052-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-052","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-052","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-052","description":"Require complete map coverage","requirement_ids":["req-screen-052"],"constraint_ids":["constraint-exclusions-052"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-052","description":"Require citations for every issuer","requirement_ids":["req-report-052"],"constraint_ids":["constraint-definition-052"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-052"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-052-a"},{"ticker":"issuer-052-b"},{"ticker":"issuer-052-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-052"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-052"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-052-a","issuer-052-b","issuer-052-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-053","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (053).","requirements":[{"id":"req-screen-053","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-053","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-053","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-053","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-053-a","issuer-053-b","issuer-053-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-053","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-053","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-053","description":"Require complete map coverage","requirement_ids":["req-screen-053"],"constraint_ids":["constraint-exclusions-053"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-053","description":"Require citations for every issuer","requirement_ids":["req-report-053"],"constraint_ids":["constraint-definition-053"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-053"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-053-a"},{"ticker":"issuer-053-b"},{"ticker":"issuer-053-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-053"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-053"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-053-a","issuer-053-b","issuer-053-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-054","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (054).","requirements":[{"id":"req-screen-054","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-054","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-054","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-054","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-054-a","issuer-054-b","issuer-054-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-054","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-054","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-054","description":"Require complete map coverage","requirement_ids":["req-screen-054"],"constraint_ids":["constraint-exclusions-054"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-054","description":"Require citations for every issuer","requirement_ids":["req-report-054"],"constraint_ids":["constraint-definition-054"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-054"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-054-a"},{"ticker":"issuer-054-b"},{"ticker":"issuer-054-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-054"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-054"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-054-a","issuer-054-b","issuer-054-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-055","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (055).","requirements":[{"id":"req-screen-055","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-055","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-055","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-055","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-055-a","issuer-055-b","issuer-055-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-055","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-055","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-055","description":"Require complete map coverage","requirement_ids":["req-screen-055"],"constraint_ids":["constraint-exclusions-055"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-055","description":"Require citations for every issuer","requirement_ids":["req-report-055"],"constraint_ids":["constraint-definition-055"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-055"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-055-a"},{"ticker":"issuer-055-b"},{"ticker":"issuer-055-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-055"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-055"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-055-a","issuer-055-b","issuer-055-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-056","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (056).","requirements":[{"id":"req-screen-056","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-056","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-056","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-056","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-056-a","issuer-056-b","issuer-056-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-056","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-056","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-056","description":"Require complete map coverage","requirement_ids":["req-screen-056"],"constraint_ids":["constraint-exclusions-056"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-056","description":"Require citations for every issuer","requirement_ids":["req-report-056"],"constraint_ids":["constraint-definition-056"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-056"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-056-a"},{"ticker":"issuer-056-b"},{"ticker":"issuer-056-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-056"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-056"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-056-a","issuer-056-b","issuer-056-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-057","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (057).","requirements":[{"id":"req-screen-057","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-057","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-057","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-057","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-057-a","issuer-057-b","issuer-057-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-057","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-057","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-057","description":"Require complete map coverage","requirement_ids":["req-screen-057"],"constraint_ids":["constraint-exclusions-057"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-057","description":"Require citations for every issuer","requirement_ids":["req-report-057"],"constraint_ids":["constraint-definition-057"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-057"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-057-a"},{"ticker":"issuer-057-b"},{"ticker":"issuer-057-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-057"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-057"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-057-a","issuer-057-b","issuer-057-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-058","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (058).","requirements":[{"id":"req-screen-058","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-058","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-058","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-058","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-058-a","issuer-058-b","issuer-058-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-058","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-058","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-058","description":"Require complete map coverage","requirement_ids":["req-screen-058"],"constraint_ids":["constraint-exclusions-058"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-058","description":"Require citations for every issuer","requirement_ids":["req-report-058"],"constraint_ids":["constraint-definition-058"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-058"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-058-a"},{"ticker":"issuer-058-b"},{"ticker":"issuer-058-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-058"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-058"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-058-a","issuer-058-b","issuer-058-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-059","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (059).","requirements":[{"id":"req-screen-059","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-059","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-059","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-059","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-059-a","issuer-059-b","issuer-059-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-059","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-059","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-059","description":"Require complete map coverage","requirement_ids":["req-screen-059"],"constraint_ids":["constraint-exclusions-059"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-059","description":"Require citations for every issuer","requirement_ids":["req-report-059"],"constraint_ids":["constraint-definition-059"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-059"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-059-a"},{"ticker":"issuer-059-b"},{"ticker":"issuer-059-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-059"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-059"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-059-a","issuer-059-b","issuer-059-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-060","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (060).","requirements":[{"id":"req-screen-060","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-060","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-060","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-060","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-060-a","issuer-060-b","issuer-060-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-060","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-060","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-060","description":"Require complete map coverage","requirement_ids":["req-screen-060"],"constraint_ids":["constraint-exclusions-060"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-060","description":"Require citations for every issuer","requirement_ids":["req-report-060"],"constraint_ids":["constraint-definition-060"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-060"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-060-a"},{"ticker":"issuer-060-b"},{"ticker":"issuer-060-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-060"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-060"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-060-a","issuer-060-b","issuer-060-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-061","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (061).","requirements":[{"id":"req-screen-061","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-061","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-061","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-061","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-061-a","issuer-061-b","issuer-061-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-061","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-061","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-061","description":"Require complete map coverage","requirement_ids":["req-screen-061"],"constraint_ids":["constraint-exclusions-061"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-061","description":"Require citations for every issuer","requirement_ids":["req-report-061"],"constraint_ids":["constraint-definition-061"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-061"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-061-a"},{"ticker":"issuer-061-b"},{"ticker":"issuer-061-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-061"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-061"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-061-a","issuer-061-b","issuer-061-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-062","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (062).","requirements":[{"id":"req-screen-062","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-062","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-062","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-062","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-062-a","issuer-062-b","issuer-062-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-062","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-062","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-062","description":"Require complete map coverage","requirement_ids":["req-screen-062"],"constraint_ids":["constraint-exclusions-062"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-062","description":"Require citations for every issuer","requirement_ids":["req-report-062"],"constraint_ids":["constraint-definition-062"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-062"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-062-a"},{"ticker":"issuer-062-b"},{"ticker":"issuer-062-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-062"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-062"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-062-a","issuer-062-b","issuer-062-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-063","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (063).","requirements":[{"id":"req-screen-063","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-063","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-063","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-063","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-063-a","issuer-063-b","issuer-063-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-063","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-063","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-063","description":"Require complete map coverage","requirement_ids":["req-screen-063"],"constraint_ids":["constraint-exclusions-063"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-063","description":"Require citations for every issuer","requirement_ids":["req-report-063"],"constraint_ids":["constraint-definition-063"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-063"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-063-a"},{"ticker":"issuer-063-b"},{"ticker":"issuer-063-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-063"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-063"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-063-a","issuer-063-b","issuer-063-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-064","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (064).","requirements":[{"id":"req-screen-064","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-064","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-064","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-064","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-064-a","issuer-064-b","issuer-064-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-064","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-064","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-064","description":"Require complete map coverage","requirement_ids":["req-screen-064"],"constraint_ids":["constraint-exclusions-064"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-064","description":"Require citations for every issuer","requirement_ids":["req-report-064"],"constraint_ids":["constraint-definition-064"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-064"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-064-a"},{"ticker":"issuer-064-b"},{"ticker":"issuer-064-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-064"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-064"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-064-a","issuer-064-b","issuer-064-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-065","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (065).","requirements":[{"id":"req-screen-065","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-065","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-065","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-065","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-065-a","issuer-065-b","issuer-065-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-065","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-065","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-065","description":"Require complete map coverage","requirement_ids":["req-screen-065"],"constraint_ids":["constraint-exclusions-065"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-065","description":"Require citations for every issuer","requirement_ids":["req-report-065"],"constraint_ids":["constraint-definition-065"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-065"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-065-a"},{"ticker":"issuer-065-b"},{"ticker":"issuer-065-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-065"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-065"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-065-a","issuer-065-b","issuer-065-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-066","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (066).","requirements":[{"id":"req-screen-066","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-066","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-066","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-066","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-066-a","issuer-066-b","issuer-066-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-066","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-066","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-066","description":"Require complete map coverage","requirement_ids":["req-screen-066"],"constraint_ids":["constraint-exclusions-066"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-066","description":"Require citations for every issuer","requirement_ids":["req-report-066"],"constraint_ids":["constraint-definition-066"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-066"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-066-a"},{"ticker":"issuer-066-b"},{"ticker":"issuer-066-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-066"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-066"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-066-a","issuer-066-b","issuer-066-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-067","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (067).","requirements":[{"id":"req-screen-067","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-067","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-067","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-067","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-067-a","issuer-067-b","issuer-067-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-067","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-067","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-067","description":"Require complete map coverage","requirement_ids":["req-screen-067"],"constraint_ids":["constraint-exclusions-067"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-067","description":"Require citations for every issuer","requirement_ids":["req-report-067"],"constraint_ids":["constraint-definition-067"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-067"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-067-a"},{"ticker":"issuer-067-b"},{"ticker":"issuer-067-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-067"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-067"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-067-a","issuer-067-b","issuer-067-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-068","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (068).","requirements":[{"id":"req-screen-068","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-068","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-068","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-068","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-068-a","issuer-068-b","issuer-068-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-068","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-068","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-068","description":"Require complete map coverage","requirement_ids":["req-screen-068"],"constraint_ids":["constraint-exclusions-068"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-068","description":"Require citations for every issuer","requirement_ids":["req-report-068"],"constraint_ids":["constraint-definition-068"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-068"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-068-a"},{"ticker":"issuer-068-b"},{"ticker":"issuer-068-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-068"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-068"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-068-a","issuer-068-b","issuer-068-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-069","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (069).","requirements":[{"id":"req-screen-069","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-069","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-069","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-069","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-069-a","issuer-069-b","issuer-069-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-069","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-069","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-069","description":"Require complete map coverage","requirement_ids":["req-screen-069"],"constraint_ids":["constraint-exclusions-069"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-069","description":"Require citations for every issuer","requirement_ids":["req-report-069"],"constraint_ids":["constraint-definition-069"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-069"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-069-a"},{"ticker":"issuer-069-b"},{"ticker":"issuer-069-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-069"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-069"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-069-a","issuer-069-b","issuer-069-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-070","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (070).","requirements":[{"id":"req-screen-070","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-070","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-070","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-070","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-070-a","issuer-070-b","issuer-070-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-070","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-070","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-070","description":"Require complete map coverage","requirement_ids":["req-screen-070"],"constraint_ids":["constraint-exclusions-070"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-070","description":"Require citations for every issuer","requirement_ids":["req-report-070"],"constraint_ids":["constraint-definition-070"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-070"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-070-a"},{"ticker":"issuer-070-b"},{"ticker":"issuer-070-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-070"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-070"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-070-a","issuer-070-b","issuer-070-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-071","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (071).","requirements":[{"id":"req-screen-071","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-071","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-071","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-071","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-071-a","issuer-071-b","issuer-071-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-071","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-071","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-071","description":"Require complete map coverage","requirement_ids":["req-screen-071"],"constraint_ids":["constraint-exclusions-071"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-071","description":"Require citations for every issuer","requirement_ids":["req-report-071"],"constraint_ids":["constraint-definition-071"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-071"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-071-a"},{"ticker":"issuer-071-b"},{"ticker":"issuer-071-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-071"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-071"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-071-a","issuer-071-b","issuer-071-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-072","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (072).","requirements":[{"id":"req-screen-072","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-072","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-072","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-072","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-072-a","issuer-072-b","issuer-072-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-072","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-072","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-072","description":"Require complete map coverage","requirement_ids":["req-screen-072"],"constraint_ids":["constraint-exclusions-072"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-072","description":"Require citations for every issuer","requirement_ids":["req-report-072"],"constraint_ids":["constraint-definition-072"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-072"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-072-a"},{"ticker":"issuer-072-b"},{"ticker":"issuer-072-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-072"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-072"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-072-a","issuer-072-b","issuer-072-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-073","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (073).","requirements":[{"id":"req-screen-073","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-073","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-073","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-073","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-073-a","issuer-073-b","issuer-073-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-073","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-073","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-073","description":"Require complete map coverage","requirement_ids":["req-screen-073"],"constraint_ids":["constraint-exclusions-073"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-073","description":"Require citations for every issuer","requirement_ids":["req-report-073"],"constraint_ids":["constraint-definition-073"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-073"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-073-a"},{"ticker":"issuer-073-b"},{"ticker":"issuer-073-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-073"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-073"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-073-a","issuer-073-b","issuer-073-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-074","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (074).","requirements":[{"id":"req-screen-074","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-074","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-074","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-074","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-074-a","issuer-074-b","issuer-074-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-074","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-074","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-074","description":"Require complete map coverage","requirement_ids":["req-screen-074"],"constraint_ids":["constraint-exclusions-074"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-074","description":"Require citations for every issuer","requirement_ids":["req-report-074"],"constraint_ids":["constraint-definition-074"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-074"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-074-a"},{"ticker":"issuer-074-b"},{"ticker":"issuer-074-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-074"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-074"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-074-a","issuer-074-b","issuer-074-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-075","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (075).","requirements":[{"id":"req-screen-075","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-075","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-075","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-075","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-075-a","issuer-075-b","issuer-075-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-075","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-075","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-075","description":"Require complete map coverage","requirement_ids":["req-screen-075"],"constraint_ids":["constraint-exclusions-075"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-075","description":"Require citations for every issuer","requirement_ids":["req-report-075"],"constraint_ids":["constraint-definition-075"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-075"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-075-a"},{"ticker":"issuer-075-b"},{"ticker":"issuer-075-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-075"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-075"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-075-a","issuer-075-b","issuer-075-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-076","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (076).","requirements":[{"id":"req-screen-076","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-076","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-076","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-076","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-076-a","issuer-076-b","issuer-076-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-076","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-076","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-076","description":"Require complete map coverage","requirement_ids":["req-screen-076"],"constraint_ids":["constraint-exclusions-076"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-076","description":"Require citations for every issuer","requirement_ids":["req-report-076"],"constraint_ids":["constraint-definition-076"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-076"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-076-a"},{"ticker":"issuer-076-b"},{"ticker":"issuer-076-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-076"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-076"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-076-a","issuer-076-b","issuer-076-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-077","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (077).","requirements":[{"id":"req-screen-077","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-077","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-077","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-077","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-077-a","issuer-077-b","issuer-077-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-077","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-077","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-077","description":"Require complete map coverage","requirement_ids":["req-screen-077"],"constraint_ids":["constraint-exclusions-077"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-077","description":"Require citations for every issuer","requirement_ids":["req-report-077"],"constraint_ids":["constraint-definition-077"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-077"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-077-a"},{"ticker":"issuer-077-b"},{"ticker":"issuer-077-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-077"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-077"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-077-a","issuer-077-b","issuer-077-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-078","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (078).","requirements":[{"id":"req-screen-078","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-078","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-078","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-078","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-078-a","issuer-078-b","issuer-078-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-078","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-078","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-078","description":"Require complete map coverage","requirement_ids":["req-screen-078"],"constraint_ids":["constraint-exclusions-078"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-078","description":"Require citations for every issuer","requirement_ids":["req-report-078"],"constraint_ids":["constraint-definition-078"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-078"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-078-a"},{"ticker":"issuer-078-b"},{"ticker":"issuer-078-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-078"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-078"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-078-a","issuer-078-b","issuer-078-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-079","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (079).","requirements":[{"id":"req-screen-079","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-079","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-079","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-079","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-079-a","issuer-079-b","issuer-079-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-079","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-079","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-079","description":"Require complete map coverage","requirement_ids":["req-screen-079"],"constraint_ids":["constraint-exclusions-079"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-079","description":"Require citations for every issuer","requirement_ids":["req-report-079"],"constraint_ids":["constraint-definition-079"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_run"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-079"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-079-a"},{"ticker":"issuer-079-b"},{"ticker":"issuer-079-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-079"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-079"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-079-a","issuer-079-b","issuer-079-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-000","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (000).","requirements":[{"id":"req-screen-000","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-000","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-000","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-000","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-000-a","issuer-000-b","issuer-000-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-000","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-000","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-000","description":"Require complete map coverage","requirement_ids":["req-screen-000"],"constraint_ids":["constraint-exclusions-000"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-000","description":"Require citations for every issuer","requirement_ids":["req-report-000"],"constraint_ids":["constraint-definition-000"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-000"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-000-a"},{"ticker":"issuer-000-b"},{"ticker":"issuer-000-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-000"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-000"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-000-a","issuer-000-b","issuer-000-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-001","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (001).","requirements":[{"id":"req-screen-001","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-001","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-001","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-001","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-001-a","issuer-001-b","issuer-001-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-001","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-001","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-001","description":"Require complete map coverage","requirement_ids":["req-screen-001"],"constraint_ids":["constraint-exclusions-001"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-001","description":"Require citations for every issuer","requirement_ids":["req-report-001"],"constraint_ids":["constraint-definition-001"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-001"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-001-a"},{"ticker":"issuer-001-b"},{"ticker":"issuer-001-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-001"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-001"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-001-a","issuer-001-b","issuer-001-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-002","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (002).","requirements":[{"id":"req-screen-002","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-002","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-002","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-002","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-002-a","issuer-002-b","issuer-002-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-002","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-002","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-002","description":"Require complete map coverage","requirement_ids":["req-screen-002"],"constraint_ids":["constraint-exclusions-002"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-002","description":"Require citations for every issuer","requirement_ids":["req-report-002"],"constraint_ids":["constraint-definition-002"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-002"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-002-a"},{"ticker":"issuer-002-b"},{"ticker":"issuer-002-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-002"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-002"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-002-a","issuer-002-b","issuer-002-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-003","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (003).","requirements":[{"id":"req-screen-003","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-003","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-003","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-003","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-003-a","issuer-003-b","issuer-003-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-003","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-003","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-003","description":"Require complete map coverage","requirement_ids":["req-screen-003"],"constraint_ids":["constraint-exclusions-003"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-003","description":"Require citations for every issuer","requirement_ids":["req-report-003"],"constraint_ids":["constraint-definition-003"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-003"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-003-a"},{"ticker":"issuer-003-b"},{"ticker":"issuer-003-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-003"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-003"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-003-a","issuer-003-b","issuer-003-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-004","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (004).","requirements":[{"id":"req-screen-004","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-004","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-004","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-004","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-004-a","issuer-004-b","issuer-004-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-004","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-004","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-004","description":"Require complete map coverage","requirement_ids":["req-screen-004"],"constraint_ids":["constraint-exclusions-004"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-004","description":"Require citations for every issuer","requirement_ids":["req-report-004"],"constraint_ids":["constraint-definition-004"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-004"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-004-a"},{"ticker":"issuer-004-b"},{"ticker":"issuer-004-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-004"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-004"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-004-a","issuer-004-b","issuer-004-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-005","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (005).","requirements":[{"id":"req-screen-005","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-005","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-005","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-005","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-005-a","issuer-005-b","issuer-005-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-005","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-005","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-005","description":"Require complete map coverage","requirement_ids":["req-screen-005"],"constraint_ids":["constraint-exclusions-005"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-005","description":"Require citations for every issuer","requirement_ids":["req-report-005"],"constraint_ids":["constraint-definition-005"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-005"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-005-a"},{"ticker":"issuer-005-b"},{"ticker":"issuer-005-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-005"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-005"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-005-a","issuer-005-b","issuer-005-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-006","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (006).","requirements":[{"id":"req-screen-006","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-006","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-006","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-006","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-006-a","issuer-006-b","issuer-006-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-006","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-006","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-006","description":"Require complete map coverage","requirement_ids":["req-screen-006"],"constraint_ids":["constraint-exclusions-006"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-006","description":"Require citations for every issuer","requirement_ids":["req-report-006"],"constraint_ids":["constraint-definition-006"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-006"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-006-a"},{"ticker":"issuer-006-b"},{"ticker":"issuer-006-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-006"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-006"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-006-a","issuer-006-b","issuer-006-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-007","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (007).","requirements":[{"id":"req-screen-007","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-007","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-007","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-007","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-007-a","issuer-007-b","issuer-007-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-007","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-007","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-007","description":"Require complete map coverage","requirement_ids":["req-screen-007"],"constraint_ids":["constraint-exclusions-007"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-007","description":"Require citations for every issuer","requirement_ids":["req-report-007"],"constraint_ids":["constraint-definition-007"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-007"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-007-a"},{"ticker":"issuer-007-b"},{"ticker":"issuer-007-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-007"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-007"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-007-a","issuer-007-b","issuer-007-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-008","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (008).","requirements":[{"id":"req-screen-008","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-008","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-008","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-008","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-008-a","issuer-008-b","issuer-008-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-008","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-008","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-008","description":"Require complete map coverage","requirement_ids":["req-screen-008"],"constraint_ids":["constraint-exclusions-008"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-008","description":"Require citations for every issuer","requirement_ids":["req-report-008"],"constraint_ids":["constraint-definition-008"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-008"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-008-a"},{"ticker":"issuer-008-b"},{"ticker":"issuer-008-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-008"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-008"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-008-a","issuer-008-b","issuer-008-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-009","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (009).","requirements":[{"id":"req-screen-009","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-009","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-009","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-009","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-009-a","issuer-009-b","issuer-009-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-009","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-009","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-009","description":"Require complete map coverage","requirement_ids":["req-screen-009"],"constraint_ids":["constraint-exclusions-009"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-009","description":"Require citations for every issuer","requirement_ids":["req-report-009"],"constraint_ids":["constraint-definition-009"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-009"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-009-a"},{"ticker":"issuer-009-b"},{"ticker":"issuer-009-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-009"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-009"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-009-a","issuer-009-b","issuer-009-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-010","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (010).","requirements":[{"id":"req-screen-010","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-010","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-010","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-010","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-010-a","issuer-010-b","issuer-010-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-010","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-010","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-010","description":"Require complete map coverage","requirement_ids":["req-screen-010"],"constraint_ids":["constraint-exclusions-010"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-010","description":"Require citations for every issuer","requirement_ids":["req-report-010"],"constraint_ids":["constraint-definition-010"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-010"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-010-a"},{"ticker":"issuer-010-b"},{"ticker":"issuer-010-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-010"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-010"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-010-a","issuer-010-b","issuer-010-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-011","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (011).","requirements":[{"id":"req-screen-011","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-011","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-011","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-011","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-011-a","issuer-011-b","issuer-011-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-011","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-011","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-011","description":"Require complete map coverage","requirement_ids":["req-screen-011"],"constraint_ids":["constraint-exclusions-011"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-011","description":"Require citations for every issuer","requirement_ids":["req-report-011"],"constraint_ids":["constraint-definition-011"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-011"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-011-a"},{"ticker":"issuer-011-b"},{"ticker":"issuer-011-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-011"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-011"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-011-a","issuer-011-b","issuer-011-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-012","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (012).","requirements":[{"id":"req-screen-012","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-012","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-012","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-012","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-012-a","issuer-012-b","issuer-012-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-012","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-012","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-012","description":"Require complete map coverage","requirement_ids":["req-screen-012"],"constraint_ids":["constraint-exclusions-012"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-012","description":"Require citations for every issuer","requirement_ids":["req-report-012"],"constraint_ids":["constraint-definition-012"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-012"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-012-a"},{"ticker":"issuer-012-b"},{"ticker":"issuer-012-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-012"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-012"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-012-a","issuer-012-b","issuer-012-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-013","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (013).","requirements":[{"id":"req-screen-013","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-013","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-013","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-013","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-013-a","issuer-013-b","issuer-013-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-013","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-013","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-013","description":"Require complete map coverage","requirement_ids":["req-screen-013"],"constraint_ids":["constraint-exclusions-013"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-013","description":"Require citations for every issuer","requirement_ids":["req-report-013"],"constraint_ids":["constraint-definition-013"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-013"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-013-a"},{"ticker":"issuer-013-b"},{"ticker":"issuer-013-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-013"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-013"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-013-a","issuer-013-b","issuer-013-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-014","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (014).","requirements":[{"id":"req-screen-014","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-014","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-014","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-014","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-014-a","issuer-014-b","issuer-014-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-014","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-014","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-014","description":"Require complete map coverage","requirement_ids":["req-screen-014"],"constraint_ids":["constraint-exclusions-014"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-014","description":"Require citations for every issuer","requirement_ids":["req-report-014"],"constraint_ids":["constraint-definition-014"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-014"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-014-a"},{"ticker":"issuer-014-b"},{"ticker":"issuer-014-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-014"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-014"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-014-a","issuer-014-b","issuer-014-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-015","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (015).","requirements":[{"id":"req-screen-015","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-015","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-015","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-015","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-015-a","issuer-015-b","issuer-015-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-015","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-015","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-015","description":"Require complete map coverage","requirement_ids":["req-screen-015"],"constraint_ids":["constraint-exclusions-015"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-015","description":"Require citations for every issuer","requirement_ids":["req-report-015"],"constraint_ids":["constraint-definition-015"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-015"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-015-a"},{"ticker":"issuer-015-b"},{"ticker":"issuer-015-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-015"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-015"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-015-a","issuer-015-b","issuer-015-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-016","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (016).","requirements":[{"id":"req-screen-016","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-016","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-016","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-016","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-016-a","issuer-016-b","issuer-016-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-016","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-016","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-016","description":"Require complete map coverage","requirement_ids":["req-screen-016"],"constraint_ids":["constraint-exclusions-016"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-016","description":"Require citations for every issuer","requirement_ids":["req-report-016"],"constraint_ids":["constraint-definition-016"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-016"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-016-a"},{"ticker":"issuer-016-b"},{"ticker":"issuer-016-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-016"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-016"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-016-a","issuer-016-b","issuer-016-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-017","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (017).","requirements":[{"id":"req-screen-017","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-017","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-017","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-017","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-017-a","issuer-017-b","issuer-017-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-017","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-017","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-017","description":"Require complete map coverage","requirement_ids":["req-screen-017"],"constraint_ids":["constraint-exclusions-017"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-017","description":"Require citations for every issuer","requirement_ids":["req-report-017"],"constraint_ids":["constraint-definition-017"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-017"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-017-a"},{"ticker":"issuer-017-b"},{"ticker":"issuer-017-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-017"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-017"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-017-a","issuer-017-b","issuer-017-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-018","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (018).","requirements":[{"id":"req-screen-018","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-018","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-018","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-018","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-018-a","issuer-018-b","issuer-018-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-018","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-018","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-018","description":"Require complete map coverage","requirement_ids":["req-screen-018"],"constraint_ids":["constraint-exclusions-018"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-018","description":"Require citations for every issuer","requirement_ids":["req-report-018"],"constraint_ids":["constraint-definition-018"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-018"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-018-a"},{"ticker":"issuer-018-b"},{"ticker":"issuer-018-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-018"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-018"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-018-a","issuer-018-b","issuer-018-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-019","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (019).","requirements":[{"id":"req-screen-019","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-019","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-019","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-019","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-019-a","issuer-019-b","issuer-019-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-019","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-019","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-019","description":"Require complete map coverage","requirement_ids":["req-screen-019"],"constraint_ids":["constraint-exclusions-019"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-019","description":"Require citations for every issuer","requirement_ids":["req-report-019"],"constraint_ids":["constraint-definition-019"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-019"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-019-a"},{"ticker":"issuer-019-b"},{"ticker":"issuer-019-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-019"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-019"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-019-a","issuer-019-b","issuer-019-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-020","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (020).","requirements":[{"id":"req-screen-020","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-020","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-020","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-020","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-020-a","issuer-020-b","issuer-020-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-020","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-020","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-020","description":"Require complete map coverage","requirement_ids":["req-screen-020"],"constraint_ids":["constraint-exclusions-020"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-020","description":"Require citations for every issuer","requirement_ids":["req-report-020"],"constraint_ids":["constraint-definition-020"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-020"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-020-a"},{"ticker":"issuer-020-b"},{"ticker":"issuer-020-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-020"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-020"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-020-a","issuer-020-b","issuer-020-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-021","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (021).","requirements":[{"id":"req-screen-021","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-021","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-021","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-021","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-021-a","issuer-021-b","issuer-021-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-021","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-021","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-021","description":"Require complete map coverage","requirement_ids":["req-screen-021"],"constraint_ids":["constraint-exclusions-021"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-021","description":"Require citations for every issuer","requirement_ids":["req-report-021"],"constraint_ids":["constraint-definition-021"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-021"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-021-a"},{"ticker":"issuer-021-b"},{"ticker":"issuer-021-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-021"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-021"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-021-a","issuer-021-b","issuer-021-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-022","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (022).","requirements":[{"id":"req-screen-022","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-022","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-022","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-022","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-022-a","issuer-022-b","issuer-022-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-022","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-022","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-022","description":"Require complete map coverage","requirement_ids":["req-screen-022"],"constraint_ids":["constraint-exclusions-022"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-022","description":"Require citations for every issuer","requirement_ids":["req-report-022"],"constraint_ids":["constraint-definition-022"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-022"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-022-a"},{"ticker":"issuer-022-b"},{"ticker":"issuer-022-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-022"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-022"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-022-a","issuer-022-b","issuer-022-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-023","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (023).","requirements":[{"id":"req-screen-023","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-023","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-023","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-023","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-023-a","issuer-023-b","issuer-023-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-023","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-023","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-023","description":"Require complete map coverage","requirement_ids":["req-screen-023"],"constraint_ids":["constraint-exclusions-023"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-023","description":"Require citations for every issuer","requirement_ids":["req-report-023"],"constraint_ids":["constraint-definition-023"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-023"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-023-a"},{"ticker":"issuer-023-b"},{"ticker":"issuer-023-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-023"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-023"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-023-a","issuer-023-b","issuer-023-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-024","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (024).","requirements":[{"id":"req-screen-024","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-024","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-024","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-024","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-024-a","issuer-024-b","issuer-024-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-024","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-024","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-024","description":"Require complete map coverage","requirement_ids":["req-screen-024"],"constraint_ids":["constraint-exclusions-024"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-024","description":"Require citations for every issuer","requirement_ids":["req-report-024"],"constraint_ids":["constraint-definition-024"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-024"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-024-a"},{"ticker":"issuer-024-b"},{"ticker":"issuer-024-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-024"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-024"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-024-a","issuer-024-b","issuer-024-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-025","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (025).","requirements":[{"id":"req-screen-025","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-025","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-025","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-025","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-025-a","issuer-025-b","issuer-025-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-025","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-025","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-025","description":"Require complete map coverage","requirement_ids":["req-screen-025"],"constraint_ids":["constraint-exclusions-025"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-025","description":"Require citations for every issuer","requirement_ids":["req-report-025"],"constraint_ids":["constraint-definition-025"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-025"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-025-a"},{"ticker":"issuer-025-b"},{"ticker":"issuer-025-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-025"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-025"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-025-a","issuer-025-b","issuer-025-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-026","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (026).","requirements":[{"id":"req-screen-026","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-026","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-026","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-026","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-026-a","issuer-026-b","issuer-026-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-026","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-026","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-026","description":"Require complete map coverage","requirement_ids":["req-screen-026"],"constraint_ids":["constraint-exclusions-026"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-026","description":"Require citations for every issuer","requirement_ids":["req-report-026"],"constraint_ids":["constraint-definition-026"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-026"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-026-a"},{"ticker":"issuer-026-b"},{"ticker":"issuer-026-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-026"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-026"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-026-a","issuer-026-b","issuer-026-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-027","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (027).","requirements":[{"id":"req-screen-027","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-027","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-027","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-027","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-027-a","issuer-027-b","issuer-027-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-027","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-027","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-027","description":"Require complete map coverage","requirement_ids":["req-screen-027"],"constraint_ids":["constraint-exclusions-027"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-027","description":"Require citations for every issuer","requirement_ids":["req-report-027"],"constraint_ids":["constraint-definition-027"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-027"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-027-a"},{"ticker":"issuer-027-b"},{"ticker":"issuer-027-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-027"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-027"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-027-a","issuer-027-b","issuer-027-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-028","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (028).","requirements":[{"id":"req-screen-028","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-028","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-028","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-028","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-028-a","issuer-028-b","issuer-028-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-028","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-028","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-028","description":"Require complete map coverage","requirement_ids":["req-screen-028"],"constraint_ids":["constraint-exclusions-028"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-028","description":"Require citations for every issuer","requirement_ids":["req-report-028"],"constraint_ids":["constraint-definition-028"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-028"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-028-a"},{"ticker":"issuer-028-b"},{"ticker":"issuer-028-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-028"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-028"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-028-a","issuer-028-b","issuer-028-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-029","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (029).","requirements":[{"id":"req-screen-029","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-029","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-029","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-029","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-029-a","issuer-029-b","issuer-029-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-029","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-029","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-029","description":"Require complete map coverage","requirement_ids":["req-screen-029"],"constraint_ids":["constraint-exclusions-029"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-029","description":"Require citations for every issuer","requirement_ids":["req-report-029"],"constraint_ids":["constraint-definition-029"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-029"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-029-a"},{"ticker":"issuer-029-b"},{"ticker":"issuer-029-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-029"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-029"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-029-a","issuer-029-b","issuer-029-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-030","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (030).","requirements":[{"id":"req-screen-030","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-030","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-030","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-030","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-030-a","issuer-030-b","issuer-030-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-030","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-030","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-030","description":"Require complete map coverage","requirement_ids":["req-screen-030"],"constraint_ids":["constraint-exclusions-030"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-030","description":"Require citations for every issuer","requirement_ids":["req-report-030"],"constraint_ids":["constraint-definition-030"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-030"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-030-a"},{"ticker":"issuer-030-b"},{"ticker":"issuer-030-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-030"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-030"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-030-a","issuer-030-b","issuer-030-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-031","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (031).","requirements":[{"id":"req-screen-031","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-031","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-031","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-031","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-031-a","issuer-031-b","issuer-031-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-031","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-031","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-031","description":"Require complete map coverage","requirement_ids":["req-screen-031"],"constraint_ids":["constraint-exclusions-031"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-031","description":"Require citations for every issuer","requirement_ids":["req-report-031"],"constraint_ids":["constraint-definition-031"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-031"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-031-a"},{"ticker":"issuer-031-b"},{"ticker":"issuer-031-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-031"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-031"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-031-a","issuer-031-b","issuer-031-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-032","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (032).","requirements":[{"id":"req-screen-032","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-032","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-032","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-032","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-032-a","issuer-032-b","issuer-032-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-032","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-032","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-032","description":"Require complete map coverage","requirement_ids":["req-screen-032"],"constraint_ids":["constraint-exclusions-032"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-032","description":"Require citations for every issuer","requirement_ids":["req-report-032"],"constraint_ids":["constraint-definition-032"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-032"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-032-a"},{"ticker":"issuer-032-b"},{"ticker":"issuer-032-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-032"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-032"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-032-a","issuer-032-b","issuer-032-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-033","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (033).","requirements":[{"id":"req-screen-033","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-033","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-033","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-033","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-033-a","issuer-033-b","issuer-033-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-033","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-033","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-033","description":"Require complete map coverage","requirement_ids":["req-screen-033"],"constraint_ids":["constraint-exclusions-033"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-033","description":"Require citations for every issuer","requirement_ids":["req-report-033"],"constraint_ids":["constraint-definition-033"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-033"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-033-a"},{"ticker":"issuer-033-b"},{"ticker":"issuer-033-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-033"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-033"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-033-a","issuer-033-b","issuer-033-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-034","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (034).","requirements":[{"id":"req-screen-034","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-034","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-034","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-034","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-034-a","issuer-034-b","issuer-034-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-034","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-034","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-034","description":"Require complete map coverage","requirement_ids":["req-screen-034"],"constraint_ids":["constraint-exclusions-034"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-034","description":"Require citations for every issuer","requirement_ids":["req-report-034"],"constraint_ids":["constraint-definition-034"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-034"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-034-a"},{"ticker":"issuer-034-b"},{"ticker":"issuer-034-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-034"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-034"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-034-a","issuer-034-b","issuer-034-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-035","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (035).","requirements":[{"id":"req-screen-035","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-035","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-035","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-035","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-035-a","issuer-035-b","issuer-035-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-035","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-035","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-035","description":"Require complete map coverage","requirement_ids":["req-screen-035"],"constraint_ids":["constraint-exclusions-035"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-035","description":"Require citations for every issuer","requirement_ids":["req-report-035"],"constraint_ids":["constraint-definition-035"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-035"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-035-a"},{"ticker":"issuer-035-b"},{"ticker":"issuer-035-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-035"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-035"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-035-a","issuer-035-b","issuer-035-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-036","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (036).","requirements":[{"id":"req-screen-036","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-036","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-036","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-036","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-036-a","issuer-036-b","issuer-036-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-036","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-036","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-036","description":"Require complete map coverage","requirement_ids":["req-screen-036"],"constraint_ids":["constraint-exclusions-036"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-036","description":"Require citations for every issuer","requirement_ids":["req-report-036"],"constraint_ids":["constraint-definition-036"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-036"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-036-a"},{"ticker":"issuer-036-b"},{"ticker":"issuer-036-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-036"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-036"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-036-a","issuer-036-b","issuer-036-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-037","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (037).","requirements":[{"id":"req-screen-037","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-037","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-037","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-037","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-037-a","issuer-037-b","issuer-037-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-037","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-037","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-037","description":"Require complete map coverage","requirement_ids":["req-screen-037"],"constraint_ids":["constraint-exclusions-037"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-037","description":"Require citations for every issuer","requirement_ids":["req-report-037"],"constraint_ids":["constraint-definition-037"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-037"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-037-a"},{"ticker":"issuer-037-b"},{"ticker":"issuer-037-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-037"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-037"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-037-a","issuer-037-b","issuer-037-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-038","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (038).","requirements":[{"id":"req-screen-038","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-038","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-038","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-038","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-038-a","issuer-038-b","issuer-038-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-038","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-038","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-038","description":"Require complete map coverage","requirement_ids":["req-screen-038"],"constraint_ids":["constraint-exclusions-038"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-038","description":"Require citations for every issuer","requirement_ids":["req-report-038"],"constraint_ids":["constraint-definition-038"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-038"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-038-a"},{"ticker":"issuer-038-b"},{"ticker":"issuer-038-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-038"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-038"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-038-a","issuer-038-b","issuer-038-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-039","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (039).","requirements":[{"id":"req-screen-039","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-039","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-039","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-039","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-039-a","issuer-039-b","issuer-039-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-039","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-039","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-039","description":"Require complete map coverage","requirement_ids":["req-screen-039"],"constraint_ids":["constraint-exclusions-039"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-039","description":"Require citations for every issuer","requirement_ids":["req-report-039"],"constraint_ids":["constraint-definition-039"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-039"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-039-a"},{"ticker":"issuer-039-b"},{"ticker":"issuer-039-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-039"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-039"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-039-a","issuer-039-b","issuer-039-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-040","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (040).","requirements":[{"id":"req-screen-040","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-040","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-040","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-040","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-040-a","issuer-040-b","issuer-040-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-040","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-040","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-040","description":"Require complete map coverage","requirement_ids":["req-screen-040"],"constraint_ids":["constraint-exclusions-040"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-040","description":"Require citations for every issuer","requirement_ids":["req-report-040"],"constraint_ids":["constraint-definition-040"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-040"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-040-a"},{"ticker":"issuer-040-b"},{"ticker":"issuer-040-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-040"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-040"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-040-a","issuer-040-b","issuer-040-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-041","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (041).","requirements":[{"id":"req-screen-041","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-041","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-041","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-041","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-041-a","issuer-041-b","issuer-041-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-041","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-041","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-041","description":"Require complete map coverage","requirement_ids":["req-screen-041"],"constraint_ids":["constraint-exclusions-041"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-041","description":"Require citations for every issuer","requirement_ids":["req-report-041"],"constraint_ids":["constraint-definition-041"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-041"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-041-a"},{"ticker":"issuer-041-b"},{"ticker":"issuer-041-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-041"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-041"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-041-a","issuer-041-b","issuer-041-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-042","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (042).","requirements":[{"id":"req-screen-042","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-042","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-042","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-042","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-042-a","issuer-042-b","issuer-042-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-042","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-042","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-042","description":"Require complete map coverage","requirement_ids":["req-screen-042"],"constraint_ids":["constraint-exclusions-042"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-042","description":"Require citations for every issuer","requirement_ids":["req-report-042"],"constraint_ids":["constraint-definition-042"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-042"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-042-a"},{"ticker":"issuer-042-b"},{"ticker":"issuer-042-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-042"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-042"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-042-a","issuer-042-b","issuer-042-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-043","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (043).","requirements":[{"id":"req-screen-043","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-043","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-043","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-043","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-043-a","issuer-043-b","issuer-043-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-043","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-043","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-043","description":"Require complete map coverage","requirement_ids":["req-screen-043"],"constraint_ids":["constraint-exclusions-043"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-043","description":"Require citations for every issuer","requirement_ids":["req-report-043"],"constraint_ids":["constraint-definition-043"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-043"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-043-a"},{"ticker":"issuer-043-b"},{"ticker":"issuer-043-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-043"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-043"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-043-a","issuer-043-b","issuer-043-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-044","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (044).","requirements":[{"id":"req-screen-044","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-044","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-044","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-044","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-044-a","issuer-044-b","issuer-044-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-044","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-044","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-044","description":"Require complete map coverage","requirement_ids":["req-screen-044"],"constraint_ids":["constraint-exclusions-044"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-044","description":"Require citations for every issuer","requirement_ids":["req-report-044"],"constraint_ids":["constraint-definition-044"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-044"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-044-a"},{"ticker":"issuer-044-b"},{"ticker":"issuer-044-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-044"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-044"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-044-a","issuer-044-b","issuer-044-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-045","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (045).","requirements":[{"id":"req-screen-045","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-045","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-045","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-045","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-045-a","issuer-045-b","issuer-045-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-045","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-045","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-045","description":"Require complete map coverage","requirement_ids":["req-screen-045"],"constraint_ids":["constraint-exclusions-045"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-045","description":"Require citations for every issuer","requirement_ids":["req-report-045"],"constraint_ids":["constraint-definition-045"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-045"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-045-a"},{"ticker":"issuer-045-b"},{"ticker":"issuer-045-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-045"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-045"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-045-a","issuer-045-b","issuer-045-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-046","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (046).","requirements":[{"id":"req-screen-046","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-046","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-046","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-046","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-046-a","issuer-046-b","issuer-046-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-046","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-046","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-046","description":"Require complete map coverage","requirement_ids":["req-screen-046"],"constraint_ids":["constraint-exclusions-046"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-046","description":"Require citations for every issuer","requirement_ids":["req-report-046"],"constraint_ids":["constraint-definition-046"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-046"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-046-a"},{"ticker":"issuer-046-b"},{"ticker":"issuer-046-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-046"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-046"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-046-a","issuer-046-b","issuer-046-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-047","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (047).","requirements":[{"id":"req-screen-047","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-047","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-047","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-047","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-047-a","issuer-047-b","issuer-047-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-047","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-047","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-047","description":"Require complete map coverage","requirement_ids":["req-screen-047"],"constraint_ids":["constraint-exclusions-047"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-047","description":"Require citations for every issuer","requirement_ids":["req-report-047"],"constraint_ids":["constraint-definition-047"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-047"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-047-a"},{"ticker":"issuer-047-b"},{"ticker":"issuer-047-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-047"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-047"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-047-a","issuer-047-b","issuer-047-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-048","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (048).","requirements":[{"id":"req-screen-048","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-048","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-048","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-048","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-048-a","issuer-048-b","issuer-048-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-048","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-048","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-048","description":"Require complete map coverage","requirement_ids":["req-screen-048"],"constraint_ids":["constraint-exclusions-048"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-048","description":"Require citations for every issuer","requirement_ids":["req-report-048"],"constraint_ids":["constraint-definition-048"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-048"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-048-a"},{"ticker":"issuer-048-b"},{"ticker":"issuer-048-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-048"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-048"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-048-a","issuer-048-b","issuer-048-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-049","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (049).","requirements":[{"id":"req-screen-049","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-049","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-049","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-049","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-049-a","issuer-049-b","issuer-049-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-049","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-049","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-049","description":"Require complete map coverage","requirement_ids":["req-screen-049"],"constraint_ids":["constraint-exclusions-049"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-049","description":"Require citations for every issuer","requirement_ids":["req-report-049"],"constraint_ids":["constraint-definition-049"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-049"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-049-a"},{"ticker":"issuer-049-b"},{"ticker":"issuer-049-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-049"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-049"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-049-a","issuer-049-b","issuer-049-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-050","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (050).","requirements":[{"id":"req-screen-050","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-050","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-050","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-050","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-050-a","issuer-050-b","issuer-050-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-050","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-050","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-050","description":"Require complete map coverage","requirement_ids":["req-screen-050"],"constraint_ids":["constraint-exclusions-050"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-050","description":"Require citations for every issuer","requirement_ids":["req-report-050"],"constraint_ids":["constraint-definition-050"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-050"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-050-a"},{"ticker":"issuer-050-b"},{"ticker":"issuer-050-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-050"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-050"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-050-a","issuer-050-b","issuer-050-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-051","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (051).","requirements":[{"id":"req-screen-051","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-051","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-051","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-051","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-051-a","issuer-051-b","issuer-051-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-051","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-051","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-051","description":"Require complete map coverage","requirement_ids":["req-screen-051"],"constraint_ids":["constraint-exclusions-051"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-051","description":"Require citations for every issuer","requirement_ids":["req-report-051"],"constraint_ids":["constraint-definition-051"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-051"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-051-a"},{"ticker":"issuer-051-b"},{"ticker":"issuer-051-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-051"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-051"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-051-a","issuer-051-b","issuer-051-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-052","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (052).","requirements":[{"id":"req-screen-052","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-052","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-052","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-052","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-052-a","issuer-052-b","issuer-052-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-052","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-052","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-052","description":"Require complete map coverage","requirement_ids":["req-screen-052"],"constraint_ids":["constraint-exclusions-052"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-052","description":"Require citations for every issuer","requirement_ids":["req-report-052"],"constraint_ids":["constraint-definition-052"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-052"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-052-a"},{"ticker":"issuer-052-b"},{"ticker":"issuer-052-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-052"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-052"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-052-a","issuer-052-b","issuer-052-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-053","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (053).","requirements":[{"id":"req-screen-053","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-053","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-053","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-053","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-053-a","issuer-053-b","issuer-053-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-053","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-053","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-053","description":"Require complete map coverage","requirement_ids":["req-screen-053"],"constraint_ids":["constraint-exclusions-053"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-053","description":"Require citations for every issuer","requirement_ids":["req-report-053"],"constraint_ids":["constraint-definition-053"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-053"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-053-a"},{"ticker":"issuer-053-b"},{"ticker":"issuer-053-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-053"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-053"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-053-a","issuer-053-b","issuer-053-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-054","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (054).","requirements":[{"id":"req-screen-054","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-054","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-054","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-054","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-054-a","issuer-054-b","issuer-054-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-054","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-054","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-054","description":"Require complete map coverage","requirement_ids":["req-screen-054"],"constraint_ids":["constraint-exclusions-054"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-054","description":"Require citations for every issuer","requirement_ids":["req-report-054"],"constraint_ids":["constraint-definition-054"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-054"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-054-a"},{"ticker":"issuer-054-b"},{"ticker":"issuer-054-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-054"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-054"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-054-a","issuer-054-b","issuer-054-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-055","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (055).","requirements":[{"id":"req-screen-055","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-055","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-055","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-055","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-055-a","issuer-055-b","issuer-055-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-055","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-055","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-055","description":"Require complete map coverage","requirement_ids":["req-screen-055"],"constraint_ids":["constraint-exclusions-055"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-055","description":"Require citations for every issuer","requirement_ids":["req-report-055"],"constraint_ids":["constraint-definition-055"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-055"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-055-a"},{"ticker":"issuer-055-b"},{"ticker":"issuer-055-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-055"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-055"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-055-a","issuer-055-b","issuer-055-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-056","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (056).","requirements":[{"id":"req-screen-056","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-056","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-056","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-056","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-056-a","issuer-056-b","issuer-056-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-056","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-056","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-056","description":"Require complete map coverage","requirement_ids":["req-screen-056"],"constraint_ids":["constraint-exclusions-056"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-056","description":"Require citations for every issuer","requirement_ids":["req-report-056"],"constraint_ids":["constraint-definition-056"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-056"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-056-a"},{"ticker":"issuer-056-b"},{"ticker":"issuer-056-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-056"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-056"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-056-a","issuer-056-b","issuer-056-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-057","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (057).","requirements":[{"id":"req-screen-057","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-057","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-057","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-057","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-057-a","issuer-057-b","issuer-057-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-057","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-057","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-057","description":"Require complete map coverage","requirement_ids":["req-screen-057"],"constraint_ids":["constraint-exclusions-057"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-057","description":"Require citations for every issuer","requirement_ids":["req-report-057"],"constraint_ids":["constraint-definition-057"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-057"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-057-a"},{"ticker":"issuer-057-b"},{"ticker":"issuer-057-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-057"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-057"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-057-a","issuer-057-b","issuer-057-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-058","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (058).","requirements":[{"id":"req-screen-058","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-058","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-058","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-058","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-058-a","issuer-058-b","issuer-058-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-058","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-058","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-058","description":"Require complete map coverage","requirement_ids":["req-screen-058"],"constraint_ids":["constraint-exclusions-058"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-058","description":"Require citations for every issuer","requirement_ids":["req-report-058"],"constraint_ids":["constraint-definition-058"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-058"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-058-a"},{"ticker":"issuer-058-b"},{"ticker":"issuer-058-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-058"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-058"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-058-a","issuer-058-b","issuer-058-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-059","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (059).","requirements":[{"id":"req-screen-059","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-059","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-059","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-059","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-059-a","issuer-059-b","issuer-059-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-059","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-059","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-059","description":"Require complete map coverage","requirement_ids":["req-screen-059"],"constraint_ids":["constraint-exclusions-059"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-059","description":"Require citations for every issuer","requirement_ids":["req-report-059"],"constraint_ids":["constraint-definition-059"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-059"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-059-a"},{"ticker":"issuer-059-b"},{"ticker":"issuer-059-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-059"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-059"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-059-a","issuer-059-b","issuer-059-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-060","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (060).","requirements":[{"id":"req-screen-060","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-060","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-060","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-060","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-060-a","issuer-060-b","issuer-060-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-060","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-060","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-060","description":"Require complete map coverage","requirement_ids":["req-screen-060"],"constraint_ids":["constraint-exclusions-060"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-060","description":"Require citations for every issuer","requirement_ids":["req-report-060"],"constraint_ids":["constraint-definition-060"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-060"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-060-a"},{"ticker":"issuer-060-b"},{"ticker":"issuer-060-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-060"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-060"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-060-a","issuer-060-b","issuer-060-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-061","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (061).","requirements":[{"id":"req-screen-061","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-061","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-061","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-061","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-061-a","issuer-061-b","issuer-061-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-061","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-061","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-061","description":"Require complete map coverage","requirement_ids":["req-screen-061"],"constraint_ids":["constraint-exclusions-061"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-061","description":"Require citations for every issuer","requirement_ids":["req-report-061"],"constraint_ids":["constraint-definition-061"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-061"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-061-a"},{"ticker":"issuer-061-b"},{"ticker":"issuer-061-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-061"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-061"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-061-a","issuer-061-b","issuer-061-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-062","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (062).","requirements":[{"id":"req-screen-062","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-062","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-062","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-062","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-062-a","issuer-062-b","issuer-062-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-062","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-062","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-062","description":"Require complete map coverage","requirement_ids":["req-screen-062"],"constraint_ids":["constraint-exclusions-062"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-062","description":"Require citations for every issuer","requirement_ids":["req-report-062"],"constraint_ids":["constraint-definition-062"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-062"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-062-a"},{"ticker":"issuer-062-b"},{"ticker":"issuer-062-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-062"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-062"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-062-a","issuer-062-b","issuer-062-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-063","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (063).","requirements":[{"id":"req-screen-063","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-063","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-063","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-063","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-063-a","issuer-063-b","issuer-063-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-063","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-063","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-063","description":"Require complete map coverage","requirement_ids":["req-screen-063"],"constraint_ids":["constraint-exclusions-063"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-063","description":"Require citations for every issuer","requirement_ids":["req-report-063"],"constraint_ids":["constraint-definition-063"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-063"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-063-a"},{"ticker":"issuer-063-b"},{"ticker":"issuer-063-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-063"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-063"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-063-a","issuer-063-b","issuer-063-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-064","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (064).","requirements":[{"id":"req-screen-064","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-064","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-064","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-064","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-064-a","issuer-064-b","issuer-064-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-064","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-064","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-064","description":"Require complete map coverage","requirement_ids":["req-screen-064"],"constraint_ids":["constraint-exclusions-064"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-064","description":"Require citations for every issuer","requirement_ids":["req-report-064"],"constraint_ids":["constraint-definition-064"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-064"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-064-a"},{"ticker":"issuer-064-b"},{"ticker":"issuer-064-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-064"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-064"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-064-a","issuer-064-b","issuer-064-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-065","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (065).","requirements":[{"id":"req-screen-065","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-065","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-065","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-065","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-065-a","issuer-065-b","issuer-065-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-065","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-065","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-065","description":"Require complete map coverage","requirement_ids":["req-screen-065"],"constraint_ids":["constraint-exclusions-065"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-065","description":"Require citations for every issuer","requirement_ids":["req-report-065"],"constraint_ids":["constraint-definition-065"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-065"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-065-a"},{"ticker":"issuer-065-b"},{"ticker":"issuer-065-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-065"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-065"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-065-a","issuer-065-b","issuer-065-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-066","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (066).","requirements":[{"id":"req-screen-066","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-066","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-066","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-066","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-066-a","issuer-066-b","issuer-066-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-066","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-066","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-066","description":"Require complete map coverage","requirement_ids":["req-screen-066"],"constraint_ids":["constraint-exclusions-066"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-066","description":"Require citations for every issuer","requirement_ids":["req-report-066"],"constraint_ids":["constraint-definition-066"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-066"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-066-a"},{"ticker":"issuer-066-b"},{"ticker":"issuer-066-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-066"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-066"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-066-a","issuer-066-b","issuer-066-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-067","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (067).","requirements":[{"id":"req-screen-067","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-067","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-067","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-067","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-067-a","issuer-067-b","issuer-067-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-067","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-067","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-067","description":"Require complete map coverage","requirement_ids":["req-screen-067"],"constraint_ids":["constraint-exclusions-067"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-067","description":"Require citations for every issuer","requirement_ids":["req-report-067"],"constraint_ids":["constraint-definition-067"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-067"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-067-a"},{"ticker":"issuer-067-b"},{"ticker":"issuer-067-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-067"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-067"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-067-a","issuer-067-b","issuer-067-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-068","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (068).","requirements":[{"id":"req-screen-068","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-068","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-068","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-068","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-068-a","issuer-068-b","issuer-068-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-068","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-068","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-068","description":"Require complete map coverage","requirement_ids":["req-screen-068"],"constraint_ids":["constraint-exclusions-068"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-068","description":"Require citations for every issuer","requirement_ids":["req-report-068"],"constraint_ids":["constraint-definition-068"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-068"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-068-a"},{"ticker":"issuer-068-b"},{"ticker":"issuer-068-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-068"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-068"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-068-a","issuer-068-b","issuer-068-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-069","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (069).","requirements":[{"id":"req-screen-069","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-069","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-069","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-069","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-069-a","issuer-069-b","issuer-069-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-069","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-069","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-069","description":"Require complete map coverage","requirement_ids":["req-screen-069"],"constraint_ids":["constraint-exclusions-069"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-069","description":"Require citations for every issuer","requirement_ids":["req-report-069"],"constraint_ids":["constraint-definition-069"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-069"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-069-a"},{"ticker":"issuer-069-b"},{"ticker":"issuer-069-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-069"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-069"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-069-a","issuer-069-b","issuer-069-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-070","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (070).","requirements":[{"id":"req-screen-070","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-070","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-070","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-070","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-070-a","issuer-070-b","issuer-070-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-070","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-070","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-070","description":"Require complete map coverage","requirement_ids":["req-screen-070"],"constraint_ids":["constraint-exclusions-070"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-070","description":"Require citations for every issuer","requirement_ids":["req-report-070"],"constraint_ids":["constraint-definition-070"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-070"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-070-a"},{"ticker":"issuer-070-b"},{"ticker":"issuer-070-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-070"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-070"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-070-a","issuer-070-b","issuer-070-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-071","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (071).","requirements":[{"id":"req-screen-071","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-071","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-071","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-071","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-071-a","issuer-071-b","issuer-071-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-071","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-071","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-071","description":"Require complete map coverage","requirement_ids":["req-screen-071"],"constraint_ids":["constraint-exclusions-071"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-071","description":"Require citations for every issuer","requirement_ids":["req-report-071"],"constraint_ids":["constraint-definition-071"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-071"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-071-a"},{"ticker":"issuer-071-b"},{"ticker":"issuer-071-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-071"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-071"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-071-a","issuer-071-b","issuer-071-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-072","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (072).","requirements":[{"id":"req-screen-072","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-072","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-072","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-072","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-072-a","issuer-072-b","issuer-072-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-072","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-072","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-072","description":"Require complete map coverage","requirement_ids":["req-screen-072"],"constraint_ids":["constraint-exclusions-072"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-072","description":"Require citations for every issuer","requirement_ids":["req-report-072"],"constraint_ids":["constraint-definition-072"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-072"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-072-a"},{"ticker":"issuer-072-b"},{"ticker":"issuer-072-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-072"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-072"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-072-a","issuer-072-b","issuer-072-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-073","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (073).","requirements":[{"id":"req-screen-073","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-073","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-073","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-073","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-073-a","issuer-073-b","issuer-073-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-073","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-073","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-073","description":"Require complete map coverage","requirement_ids":["req-screen-073"],"constraint_ids":["constraint-exclusions-073"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-073","description":"Require citations for every issuer","requirement_ids":["req-report-073"],"constraint_ids":["constraint-definition-073"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-073"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-073-a"},{"ticker":"issuer-073-b"},{"ticker":"issuer-073-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-073"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-073"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-073-a","issuer-073-b","issuer-073-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-074","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (074).","requirements":[{"id":"req-screen-074","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-074","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-074","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-074","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-074-a","issuer-074-b","issuer-074-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-074","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-074","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-074","description":"Require complete map coverage","requirement_ids":["req-screen-074"],"constraint_ids":["constraint-exclusions-074"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-074","description":"Require citations for every issuer","requirement_ids":["req-report-074"],"constraint_ids":["constraint-definition-074"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-074"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-074-a"},{"ticker":"issuer-074-b"},{"ticker":"issuer-074-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-074"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-074"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-074-a","issuer-074-b","issuer-074-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-075","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (075).","requirements":[{"id":"req-screen-075","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-075","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-075","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-075","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-075-a","issuer-075-b","issuer-075-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-075","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-075","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-075","description":"Require complete map coverage","requirement_ids":["req-screen-075"],"constraint_ids":["constraint-exclusions-075"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-075","description":"Require citations for every issuer","requirement_ids":["req-report-075"],"constraint_ids":["constraint-definition-075"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-075"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-075-a"},{"ticker":"issuer-075-b"},{"ticker":"issuer-075-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-075"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-075"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-075-a","issuer-075-b","issuer-075-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-076","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (076).","requirements":[{"id":"req-screen-076","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-076","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-076","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-076","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-076-a","issuer-076-b","issuer-076-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-076","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-076","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-076","description":"Require complete map coverage","requirement_ids":["req-screen-076"],"constraint_ids":["constraint-exclusions-076"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-076","description":"Require citations for every issuer","requirement_ids":["req-report-076"],"constraint_ids":["constraint-definition-076"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-076"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-076-a"},{"ticker":"issuer-076-b"},{"ticker":"issuer-076-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-076"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-076"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-076-a","issuer-076-b","issuer-076-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-077","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (077).","requirements":[{"id":"req-screen-077","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-077","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-077","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-077","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-077-a","issuer-077-b","issuer-077-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-077","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-077","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-077","description":"Require complete map coverage","requirement_ids":["req-screen-077"],"constraint_ids":["constraint-exclusions-077"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-077","description":"Require citations for every issuer","requirement_ids":["req-report-077"],"constraint_ids":["constraint-definition-077"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-077"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-077-a"},{"ticker":"issuer-077-b"},{"ticker":"issuer-077-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-077"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-077"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-077-a","issuer-077-b","issuer-077-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-078","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (078).","requirements":[{"id":"req-screen-078","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-078","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-078","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-078","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-078-a","issuer-078-b","issuer-078-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-078","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-078","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-078","description":"Require complete map coverage","requirement_ids":["req-screen-078"],"constraint_ids":["constraint-exclusions-078"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-078","description":"Require citations for every issuer","requirement_ids":["req-report-078"],"constraint_ids":["constraint-definition-078"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-078"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-078-a"},{"ticker":"issuer-078-b"},{"ticker":"issuer-078-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-078"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-078"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-078-a","issuer-078-b","issuer-078-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-079","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (079).","requirements":[{"id":"req-screen-079","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-079","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-079","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-079","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-079-a","issuer-079-b","issuer-079-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-079","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-079","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-079","description":"Require complete map coverage","requirement_ids":["req-screen-079"],"constraint_ids":["constraint-exclusions-079"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-079","description":"Require citations for every issuer","requirement_ids":["req-report-079"],"constraint_ids":["constraint-definition-079"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-079"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-079-a"},{"ticker":"issuer-079-b"},{"ticker":"issuer-079-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-079"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-079"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-079-a","issuer-079-b","issuer-079-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} diff --git a/crates/moa-eval/tests/eval_offline/execution_snapshot.rs b/crates/moa-eval/tests/eval_offline/execution_snapshot.rs index 4a6349aa5..3d96f8d4a 100644 --- a/crates/moa-eval/tests/eval_offline/execution_snapshot.rs +++ b/crates/moa-eval/tests/eval_offline/execution_snapshot.rs @@ -482,7 +482,7 @@ fn canonical_plan(catalog_hash: ExecutionHash) -> CanonicalExecutionPlan { expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::At { at: fixed_time() + chrono::TimeDelta::hours(1), }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, }, input_schema: json!({ "type": "object" }), output_schema: json!({ "type": "object" }), diff --git a/crates/moa-execution/src/bindings.rs b/crates/moa-execution/src/bindings.rs index 37ee4c231..96ac2fc52 100644 --- a/crates/moa-execution/src/bindings.rs +++ b/crates/moa-execution/src/bindings.rs @@ -29,6 +29,9 @@ pub fn resolve_bindings(value: &Value, context: &BindingContext<'_>) -> Result, diff --git a/crates/moa-execution/src/compiler/estimate.rs b/crates/moa-execution/src/compiler/estimate.rs index bb1d9a49c..6b9d2fb36 100644 --- a/crates/moa-execution/src/compiler/estimate.rs +++ b/crates/moa-execution/src/compiler/estimate.rs @@ -2,6 +2,14 @@ use super::*; +/// Sums the worst case over *every* node, including nodes that declare a condition. +/// +/// A branchy plan therefore reserves the union of its branches even though at most one +/// of them can run. That over-reservation is deliberate: the estimate is what the owning +/// user approves and what the budget ledger fences, and it must be an upper bound that +/// holds before any condition has been evaluated. Narrowing it to the taken branch would +/// require knowing the run input at approval time and would let a run exceed the amount +/// its user actually saw. pub(super) fn estimate_plan( goal: &ExecutionGoalContract, plan: &ExecutionPlanDefinition, diff --git a/crates/moa-execution/src/compiler/mod.rs b/crates/moa-execution/src/compiler/mod.rs index f8deb1416..d84b81996 100644 --- a/crates/moa-execution/src/compiler/mod.rs +++ b/crates/moa-execution/src/compiler/mod.rs @@ -202,6 +202,7 @@ pub fn compile(request: CompileExecutionRequest) -> CompileExecutionOutcome { validate_goal_plan_links(&request.goal, &request.plan, &mut report); validate_plan_activation_bound(&request.plan, &request.config, &mut report); validate_completion_activation_bounds(&request.goal, &request.config, &mut report); + validate_condition_scope(&request.goal, &request.plan, &mut report); validate_catalog(&request.catalog, &mut report); validate_authorization(&request.authorization, &mut report); validate_schemas(&request.goal, &request.plan, &mut report); @@ -346,6 +347,7 @@ pub fn validate_amendment(request: ValidateAmendmentRequest) -> AmendmentValidat append_artifact_reports(&request.goal, &definition, &mut report); validate_goal_plan_links(&request.goal, &definition, &mut report); + validate_condition_scope(&request.goal, &definition, &mut report); validate_plan_activation_bound(&definition, &request.config, &mut report); validate_completion_activation_bounds(&request.goal, &request.config, &mut report); validate_schemas(&request.goal, &definition, &mut report); @@ -444,6 +446,247 @@ fn append_execution_config_validation( } } +/// Confines conditional nodes to the exact semantics the runtime implements. +/// +/// A false condition commits `node_status = 'skipped'` with a JSON `null` aggregate +/// output. Version one therefore treats a conditional node as an *effectful leaf*: +/// other nodes may depend on it for ordering, but nothing may read its output, and +/// nothing may make the run's success contingent on it having run. Every rule below +/// rejects a plan whose meaning would otherwise depend on a value or an outcome that +/// a skipped branch cannot produce: +/// +/// - a condition may read only run input or a *declared dependency's* output, so the +/// value it tests is guaranteed to exist by the time the node is activated; +/// - no `$ref` may read a conditional node's output, because the skipped value is +/// `null` and neither `resolve_reference` nor a capability input schema accepts it; +/// - a conditional node may not be the plan's `Output` operation, whose value is the +/// run's deliverable; +/// - a conditional node may not appear in a `RequiredNodes` completion check, which +/// counts a skipped node as failed and would turn a legitimately false branch into +/// a partial run; and +/// - a requirement served *only* by conditional nodes is rejected, because all +/// branches evaluating false would leave it with no eligible node at all. +fn validate_condition_scope( + goal: &ExecutionGoalContract, + plan: &ExecutionPlanDefinition, + report: &mut ExecutionValidationReport, +) { + let conditional_ids = plan + .nodes + .iter() + .filter(|node| node.when.is_some()) + .map(|node| node.id.as_str()) + .collect::>(); + if conditional_ids.is_empty() { + return; + } + + for (index, node) in plan.nodes.iter().enumerate() { + let root = format!("plan.nodes[{index}]"); + if let Some(condition) = &node.when { + let reference = match condition { + moa_artifacts::execution_plan::ExecutionCondition::Exists { reference } + | moa_artifacts::execution_plan::ExecutionCondition::Equals { reference, .. } => { + reference + } + }; + validate_condition_reference_visibility( + &format!("{root}.when.reference.$ref"), + &reference.path, + node, + report, + ); + if matches!(node.operation, ExecutionOperation::Output { .. }) { + report.error( + "conditional_output_node", + format!("{root}.when"), + "the terminal output node must not be conditional", + ); + } + for check in &goal.completion_checks { + let CompletionCheckKind::RequiredNodes { node_ids } = &check.kind else { + continue; + }; + if node_ids.contains(&node.id) { + report.error( + "conditional_required_node", + format!("{root}.when"), + format!( + "completion check `{}` requires node `{}`, which a false condition \ + would skip and the check would count as failed", + check.id, node.id + ), + ); + } + } + for coverage in goal + .coverage + .iter() + .filter(|coverage| coverage.map_node_id == node.id) + { + report.error( + "conditional_coverage_node", + format!("{root}.when"), + format!( + "coverage requirement `{}` measures node `{}`, which a false condition \ + would skip and the coverage would count as unmet", + coverage.id, node.id + ), + ); + } + } + validate_no_conditional_reference( + &format!("{root}.input"), + &node.input, + &conditional_ids, + report, + ); + match &node.operation { + ExecutionOperation::Map { items, .. } | ExecutionOperation::Reduce { items, .. } => { + validate_no_conditional_reference( + &format!("{root}.operation.items"), + items, + &conditional_ids, + report, + ); + } + ExecutionOperation::Output { value } => validate_no_conditional_reference( + &format!("{root}.operation.value"), + value, + &conditional_ids, + report, + ), + ExecutionOperation::WaitUntil { result, .. } => validate_no_conditional_reference( + &format!("{root}.operation.result"), + result, + &conditional_ids, + report, + ), + ExecutionOperation::Capability { .. } + | ExecutionOperation::Agent { .. } + | ExecutionOperation::Review { .. } + | ExecutionOperation::WaitSignal { .. } => {} + } + } + + for (index, coverage) in goal.coverage.iter().enumerate() { + validate_no_conditional_reference( + &format!("goal.coverage[{index}].expected_items"), + &coverage.expected_items, + &conditional_ids, + report, + ); + } + + let mut unconditional_requirements = BTreeSet::new(); + let mut conditional_requirements = BTreeMap::new(); + for node in &plan.nodes { + for requirement_id in &node.requirement_ids { + if node.when.is_some() { + conditional_requirements + .entry(requirement_id.as_str()) + .or_insert(node.id.as_str()); + } else { + unconditional_requirements.insert(requirement_id.as_str()); + } + } + } + for (requirement_id, node_id) in conditional_requirements { + if !unconditional_requirements.contains(requirement_id) { + report.error( + "requirement_only_conditional", + format!("plan.nodes.{node_id}.requirement_ids"), + format!( + "every node serving requirement `{requirement_id}` is conditional, so all \ + branches evaluating false would leave it with no eligible node" + ), + ); + } + } +} + +/// Rejects a condition that reads anything other than run input or a declared dependency. +fn validate_condition_reference_visibility( + path: &str, + reference: &str, + node: &ExecutionNode, + report: &mut ExecutionValidationReport, +) { + let Some(node_id) = condition_reference_node(reference) else { + return; + }; + if node_id == node.id { + report.error( + "condition_reference_not_visible", + path, + "a node condition cannot reference its own output", + ); + } else if !node.depends_on.iter().any(|id| id == node_id) { + report.error( + "condition_reference_not_visible", + path, + "a node condition may only read run input or a declared dependency output", + ); + } +} + +fn condition_reference_node(reference: &str) -> Option<&str> { + reference + .strip_prefix("$.nodes.") + .and_then(|rest| rest.split_once(".output")) + .map(|(node_id, _)| node_id) +} + +/// Rejects any `$ref` whose source node declares a condition. +fn validate_no_conditional_reference( + path: &str, + value: &Value, + conditional_ids: &BTreeSet<&str>, + report: &mut ExecutionValidationReport, +) { + match value { + Value::Array(values) => { + for (index, value) in values.iter().enumerate() { + validate_no_conditional_reference( + &format!("{path}[{index}]"), + value, + conditional_ids, + report, + ); + } + } + Value::Object(object) => { + if object.len() == 1 + && let Some(reference) = object.get("$ref").and_then(Value::as_str) + { + if condition_reference_node(reference) + .is_some_and(|node_id| conditional_ids.contains(node_id)) + { + report.error( + "conditional_output_read", + path, + "a conditional node's output cannot be read; it is null when the \ + condition is false", + ); + } + return; + } + if object.keys().any(|key| key.starts_with('$')) { + return; + } + for (key, value) in object { + validate_no_conditional_reference( + &format!("{path}.{key}"), + value, + conditional_ids, + report, + ); + } + } + Value::Null | Value::Bool(_) | Value::Number(_) | Value::String(_) => {} + } +} + fn validate_temporal_contract( plan: &ExecutionPlanDefinition, deadline_at: Option>, @@ -486,7 +729,7 @@ fn validate_temporal_contract( ); } - validate_wait_policy( + validate_input_wait_policy( &plan.input_wait_policy, "plan.input_wait_policy", now, @@ -559,6 +802,34 @@ fn validate_wait_policy( ); } +/// Validates the plan-level expiry policy for runtime `NeedsInput` outcomes. +/// +/// Unlike a `Review` or `WaitSignal` policy, this one policy settles whichever +/// logical task returned `NeedsInput`, so no single declared output can be checked +/// against a specific node's `output_schema` at compile time. `ContinueWith` is +/// therefore rejected outright rather than deferred to the materialization +/// transaction, where the schema check is a non-retryable failure. +fn validate_input_wait_policy( + policy: &moa_artifacts::execution_plan::ExecutionWaitPolicy, + path: &str, + now: DateTime, + deadline_at: DateTime, + report: &mut ExecutionValidationReport, +) { + validate_wait_policy(policy, path, now, deadline_at, report); + if matches!( + policy.on_expiry, + ExecutionWaitExpiryAction::ContinueWith { .. } + ) { + report.error( + "unsupported_input_wait_expiry", + format!("{path}.on_expiry"), + "input wait expiry must fail the waiting task; continue_with cannot be validated \ + against the output schema of the node that requested input", + ); + } +} + fn validate_temporal_target( target: &ExecutionTemporalTarget, path: &str, @@ -583,6 +854,13 @@ fn validate_temporal_target( ); } } + // A relative delay is resolved against the clock at wait entry, which is + // never earlier than compile time, so the remaining horizon at wait entry is + // never larger than it is here. `delay < remaining` is therefore a necessary + // condition, and rejecting a delay that fails it rejects only delays that can + // never be legal. It is not a sufficient one: a delay accepted here can still + // be illegal once the wait is actually entered, and only wait entry can fence + // that. See `interpreter::resolve_temporal_target`. ExecutionTemporalTarget::After { delay_seconds } => { let remaining_seconds = deadline_at .signed_duration_since(now) diff --git a/crates/moa-execution/src/completion.rs b/crates/moa-execution/src/completion.rs index b503b1b7d..b38fbdbd3 100644 --- a/crates/moa-execution/src/completion.rs +++ b/crates/moa-execution/src/completion.rs @@ -338,15 +338,6 @@ pub fn execution_terminal_reason( } terminal_reason_from_limit(*reason) } - ExecutionTerminalCause::SchedulerNoProgress => match projection { - TerminalProjection::Unsupported { .. } => ExecutionTerminalReason::UnsupportedPlan, - TerminalProjection::Partial { .. } - | TerminalProjection::Blocked { .. } - | TerminalProjection::Failed { .. } => ExecutionTerminalReason::NoProgress, - TerminalProjection::Completed { .. } | TerminalProjection::Cancelled { .. } => { - return invalid_terminal_combination(cause, projection); - } - }, ExecutionTerminalCause::TaskFailure { class } => match projection { TerminalProjection::Unsupported { .. } => ExecutionTerminalReason::UnsupportedPlan, TerminalProjection::Blocked { .. } => ExecutionTerminalReason::Blocked, @@ -356,7 +347,6 @@ pub fn execution_terminal_reason( } ExecutionFailureClass::BudgetExceeded => ExecutionTerminalReason::BudgetExceeded, ExecutionFailureClass::Retryable - | ExecutionFailureClass::DependencyFailed | ExecutionFailureClass::InvalidInput | ExecutionFailureClass::InvalidOutput | ExecutionFailureClass::AuthorizationDenied @@ -370,7 +360,6 @@ pub fn execution_terminal_reason( } ExecutionFailureClass::BudgetExceeded => ExecutionTerminalReason::BudgetExceeded, ExecutionFailureClass::Retryable - | ExecutionFailureClass::DependencyFailed | ExecutionFailureClass::InvalidInput | ExecutionFailureClass::InvalidOutput | ExecutionFailureClass::AuthorizationDenied diff --git a/crates/moa-execution/src/interpreter/aggregate.rs b/crates/moa-execution/src/interpreter/aggregate.rs deleted file mode 100644 index 9895b0c67..000000000 --- a/crates/moa-execution/src/interpreter/aggregate.rs +++ /dev/null @@ -1,362 +0,0 @@ -//! Condition evaluation and deterministic aggregate-node derivation. - -use super::*; - -pub(super) fn apply_false_conditions( - request: &ScheduleRequest, - statuses: &mut BTreeMap, - outputs: &mut BTreeMap, -) -> Result<()> { - let mut changed = true; - while changed { - changed = false; - for node in &request.plan.definition.nodes { - if effective_status(statuses, &node.id) != Some(ExecutionNodeStatus::Pending) - || node.when.is_none() - || !node.depends_on.iter().all(|id| { - matches!( - effective_status(statuses, id), - Some(ExecutionNodeStatus::Completed | ExecutionNodeStatus::Skipped) - ) - }) - { - continue; - } - let dependencies = node.depends_on.iter().cloned().collect::>(); - let context = BindingContext { - run_input: &request.run_input, - node_outputs: outputs, - dependencies: &dependencies, - item: None, - item_key: None, - }; - if let Some(condition) = &node.when - && !evaluate_condition(condition, &context)? - { - statuses.insert(node.id.clone(), ExecutionNodeStatus::Skipped); - outputs.insert(node.id.clone(), Value::Null); - changed = true; - } - } - } - Ok(()) -} - -pub(super) fn derive_aggregate_nodes( - request: &ScheduleRequest, - statuses: &mut BTreeMap, - outputs: &mut BTreeMap, -) -> Result<()> { - let mut changed = true; - while changed { - changed = false; - for node in &request.plan.definition.nodes { - if matches!( - effective_status(statuses, &node.id), - Some( - ExecutionNodeStatus::Skipped - | ExecutionNodeStatus::Failed - | ExecutionNodeStatus::Cancelled - ) - ) || !node.depends_on.iter().all(|dependency| { - matches!( - effective_status(statuses, dependency), - Some(ExecutionNodeStatus::Completed | ExecutionNodeStatus::Skipped) - ) - }) { - continue; - } - - let aggregate = match &node.operation { - ExecutionOperation::Map { - items, - item_key, - max_items, - .. - } => derive_map_output(MapDerivationRequest { - schedule: request, - node, - outputs, - items, - item_key_pointer: item_key, - max_items: *max_items, - })?, - ExecutionOperation::Reduce { - items, - max_items, - batch_size, - .. - } => derive_reduce_output(ReduceDerivationRequest { - schedule: request, - node, - outputs, - items, - max_items: *max_items, - batch_size: *batch_size, - })?, - ExecutionOperation::Capability { .. } - | ExecutionOperation::Agent { .. } - | ExecutionOperation::Review { .. } - | ExecutionOperation::WaitSignal { .. } - | ExecutionOperation::WaitUntil { .. } - | ExecutionOperation::Output { .. } => AggregateState::Pending, - }; - - match aggregate { - AggregateState::Pending => { - if matches!( - node.operation, - ExecutionOperation::Map { .. } | ExecutionOperation::Reduce { .. } - ) && effective_status(statuses, &node.id) - == Some(ExecutionNodeStatus::Completed) - { - return Err(Error::InvalidProjection { - message: format!( - "aggregate node {} is completed before all deterministic work is terminal", - node.id - ), - }); - } - } - AggregateState::Completed(output) => { - if outputs.get(&node.id) != Some(&output) - || effective_status(statuses, &node.id) - != Some(ExecutionNodeStatus::Completed) - { - outputs.insert(node.id.clone(), output); - statuses.insert(node.id.clone(), ExecutionNodeStatus::Completed); - changed = true; - } - } - AggregateState::Failed => { - statuses.insert(node.id.clone(), ExecutionNodeStatus::Failed); - changed = true; - } - AggregateState::Cancelled => { - statuses.insert(node.id.clone(), ExecutionNodeStatus::Cancelled); - changed = true; - } - } - } - } - Ok(()) -} - -pub(super) enum AggregateState { - Pending, - Completed(Value), - Failed, - Cancelled, -} - -pub(super) struct MapDerivationRequest<'a> { - schedule: &'a ScheduleRequest, - node: &'a ExecutionNode, - outputs: &'a BTreeMap, - items: &'a Value, - item_key_pointer: &'a str, - max_items: u64, -} - -pub(super) fn derive_map_output(request: MapDerivationRequest<'_>) -> Result { - let MapDerivationRequest { - schedule, - node, - outputs, - items, - item_key_pointer, - max_items, - } = request; - let dependencies = node.depends_on.iter().cloned().collect::>(); - let resolved = resolve_bindings( - items, - &BindingContext { - run_input: &schedule.run_input, - node_outputs: outputs, - dependencies: &dependencies, - item: None, - item_key: None, - }, - )?; - let values = resolved.as_array().ok_or_else(|| Error::Binding { - path: format!("node.{}.operation.items", node.id), - message: "map items must resolve to an array".to_string(), - })?; - let count = u64::try_from(values.len()).map_err(|_| Error::ArithmeticOverflow { - context: format!("map {} item count", node.id), - })?; - if count > max_items { - return Err(Error::InvalidProjection { - message: format!("map {} exceeds max_items", node.id), - }); - } - - let mut expected = BTreeSet::new(); - for item in values { - let key = extract_map_key(item, item_key_pointer)?; - if !expected.insert(key) { - return Err(Error::InvalidProjection { - message: format!("map {} produced duplicate item keys", node.id), - }); - } - } - let tasks = schedule - .projection - .tasks - .iter() - .filter(|task| task.node_id == node.id) - .collect::>(); - if tasks.iter().any(|task| !expected.contains(&task.item_key)) { - return Err(Error::InvalidProjection { - message: format!("map {} projection contains an unexpected item key", node.id), - }); - } - if expected.iter().any(|key| { - !tasks - .iter() - .any(|task| task.item_key == *key && is_terminal_task_status(task.status)) - }) { - return Ok(AggregateState::Pending); - } - - let aggregate = serde_json::to_value(map_output(node, &schedule.projection)?)?; - validate_instance( - &node.output_schema, - &aggregate, - &format!("node.{}.output", node.id), - )?; - Ok(AggregateState::Completed(aggregate)) -} - -pub(super) struct ReduceDerivationRequest<'a> { - schedule: &'a ScheduleRequest, - node: &'a ExecutionNode, - outputs: &'a BTreeMap, - items: &'a Value, - max_items: u64, - batch_size: u32, -} - -pub(super) fn derive_reduce_output(request: ReduceDerivationRequest<'_>) -> Result { - let ReduceDerivationRequest { - schedule, - node, - outputs, - items, - max_items, - batch_size, - } = request; - let dependencies = node.depends_on.iter().cloned().collect::>(); - let resolved = resolve_bindings( - items, - &BindingContext { - run_input: &schedule.run_input, - node_outputs: outputs, - dependencies: &dependencies, - item: None, - item_key: None, - }, - )?; - let round_items = resolved.as_array().cloned().ok_or_else(|| Error::Binding { - path: format!("node.{}.operation.items", node.id), - message: "reduce items must resolve to an array".to_string(), - })?; - let count = u64::try_from(round_items.len()).map_err(|_| Error::ArithmeticOverflow { - context: format!("reduce {} item count", node.id), - })?; - if count > max_items { - return Err(Error::InvalidProjection { - message: format!("reduce {} exceeds max_items", node.id), - }); - } - let Some(single) = round_items.first().filter(|_| round_items.len() == 1) else { - if round_items.is_empty() { - return Err(Error::InvalidProjection { - message: format!("reduce {} requires at least one item", node.id), - }); - } - return derive_reduce_rounds(schedule, node, round_items, batch_size); - }; - validate_instance( - &node.output_schema, - single, - &format!("node.{}.output", node.id), - )?; - Ok(AggregateState::Completed(single.clone())) -} - -pub(super) fn derive_reduce_rounds( - request: &ScheduleRequest, - node: &ExecutionNode, - mut round_items: Vec, - batch_size: u32, -) -> Result { - let batch_size = usize::try_from(batch_size).map_err(|_| Error::ArithmeticOverflow { - context: format!("reduce {} batch_size", node.id), - })?; - let mut round = 1_u32; - loop { - let mut completed = Vec::new(); - for (batch_index, _) in round_items.chunks(batch_size).enumerate() { - let batch_index = - u64::try_from(batch_index).map_err(|_| Error::ArithmeticOverflow { - context: format!("reduce {} batch index", node.id), - })?; - let item_key = format!("r{round}:b{batch_index}"); - let Some(task) = request - .projection - .tasks - .iter() - .find(|task| task.node_id == node.id && task.item_key == item_key) - else { - return Ok(AggregateState::Pending); - }; - match task.status { - ExecutionTaskStatus::Completed => { - let output = - completed_output(task).ok_or_else(|| Error::InvalidProjection { - message: format!( - "completed reducer task {} has no output", - task.task_id - ), - })?; - validate_instance( - &node.output_schema, - &output, - &format!("node.{}.reduce.{item_key}", node.id), - )?; - completed.push(output); - } - ExecutionTaskStatus::Failed | ExecutionTaskStatus::UnknownOutcome => { - return Ok(AggregateState::Failed); - } - ExecutionTaskStatus::Cancelled => return Ok(AggregateState::Cancelled), - ExecutionTaskStatus::Skipped => { - return Err(Error::InvalidProjection { - message: format!("reducer task {} cannot be skipped", task.task_id), - }); - } - ExecutionTaskStatus::Pending - | ExecutionTaskStatus::Ready - | ExecutionTaskStatus::Reserved - | ExecutionTaskStatus::Dispatching - | ExecutionTaskStatus::Running - | ExecutionTaskStatus::WaitingInput - | ExecutionTaskStatus::WaitingReview - | ExecutionTaskStatus::WaitingSignal - | ExecutionTaskStatus::WaitingTimer - | ExecutionTaskStatus::WaitingExternal - | ExecutionTaskStatus::WaitingReplan => return Ok(AggregateState::Pending), - } - } - if completed.len() == 1 { - return Ok(AggregateState::Completed(completed.remove(0))); - } - round_items = completed; - round = round - .checked_add(1) - .ok_or_else(|| Error::ArithmeticOverflow { - context: format!("reduce {} round", node.id), - })?; - } -} diff --git a/crates/moa-execution/src/interpreter/catalog.rs b/crates/moa-execution/src/interpreter/catalog.rs new file mode 100644 index 000000000..cb0d4262c --- /dev/null +++ b/crates/moa-execution/src/interpreter/catalog.rs @@ -0,0 +1,26 @@ +//! Capability-catalog validation for materialization inputs. + +use super::*; + +pub(super) fn validate_scheduler_catalog(catalog: &ExecutionCapabilityCatalog) -> Result<()> { + let mut previous = None; + for capability in &catalog.capabilities { + if capability.estimate.tasks != 1 { + return Err(Error::InvalidProjection { + message: format!( + "capability {}@{} must reserve exactly one logical task", + capability.reference.name, capability.reference.version + ), + }); + } + let key = canonical_sort_key(&capability.reference)?; + if previous.as_ref().is_some_and(|previous| key <= *previous) { + return Err(Error::InvalidProjection { + message: "scheduler capability catalog must be sorted and duplicate-free" + .to_string(), + }); + } + previous = Some(key); + } + Ok(()) +} diff --git a/crates/moa-execution/src/interpreter/materialize.rs b/crates/moa-execution/src/interpreter/materialize.rs index fb12c8386..e0409289f 100644 --- a/crates/moa-execution/src/interpreter/materialize.rs +++ b/crates/moa-execution/src/interpreter/materialize.rs @@ -56,9 +56,10 @@ pub(super) fn materialize_node_page( source_exhausted: true, reduce_cursor: None, terminal_output: None, + condition_skipped: false, }); } - let mut tasks = materialize_node(request, node, outputs)?; + let mut tasks = materialize_single_task(request, node, outputs)?; let limit = usize::try_from(limit).map_err(|_| Error::ArithmeticOverflow { context: format!("node {} materialization page limit", node.id), })?; @@ -73,6 +74,7 @@ pub(super) fn materialize_node_page( source_exhausted, reduce_cursor: None, terminal_output: None, + condition_skipped: false, }) } @@ -159,6 +161,7 @@ fn materialize_reduce_page( round_input_count: actual_count, }), terminal_output: Some(output), + condition_skipped: false, }); } let start = usize::try_from(cursor.batch_cursor) @@ -283,6 +286,7 @@ fn materialize_reduce_page( round_input_count, }), terminal_output: None, + condition_skipped: false, }) } @@ -337,11 +341,30 @@ fn materialize_map_page( source_exhausted: true, reduce_cursor: None, terminal_output: Some(output), + condition_skipped: false, }); } let start = usize::try_from(cursor).map_err(|_| Error::ArithmeticOverflow { context: format!("map {} materialization cursor", node.id), })?; + // Duplicate item keys are checked across the WHOLE resolved array, not just this + // page. A per-page check misses a duplicate that spans a page boundary, and that + // case does not fail loudly: both items derive the same `ExecutionTaskId`, the + // insert hits `ON CONFLICT ... DO NOTHING`, materialization returns `Conflict`, and + // the controller reschedules the same page forever until the run deadline. Paying + // one O(n) scan on the first page turns that livelock back into a deterministic + // plan error, which is what the pre-paging implementation did. + if start == 0 { + let mut all_keys = BTreeSet::new(); + for item in values.iter() { + let item_key = extract_map_key(item, item_key_pointer)?; + if !all_keys.insert(item_key.clone()) { + return Err(Error::InvalidProjection { + message: format!("map {} produced duplicate item key {item_key}", node.id), + }); + } + } + } let end_u64 = cursor.saturating_add(u64::from(limit)).min(count); let end = usize::try_from(end_u64).map_err(|_| Error::ArithmeticOverflow { context: format!("map {} materialization page end", node.id), @@ -378,279 +401,55 @@ fn materialize_map_page( source_exhausted: end_u64 == count, reduce_cursor: None, terminal_output: None, + condition_skipped: false, }) } -pub(super) fn materialize_node( +/// Materializes the single logical task of one non-aggregate node. +/// +/// Map and reduce nodes are paged by `materialize_map_page` and +/// `materialize_reduce_page` before this is reached. +fn materialize_single_task( request: &ScheduleRequest, node: &ExecutionNode, outputs: &BTreeMap, ) -> Result> { - match &node.operation { - ExecutionOperation::Map { - items, - item_key, - max_items, - task, - .. - } => materialize_map(MapMaterializationRequest { - schedule: request, - node, - outputs, - items, - item_key_pointer: item_key, - max_items: *max_items, - task, - }), - ExecutionOperation::Reduce { - items, - max_items, - reducer, - batch_size, - } => materialize_reduce(ReduceMaterializationRequest { - schedule: request, - node, - outputs, - items, - max_items: *max_items, - reducer, - batch_size: *batch_size, - }), - ExecutionOperation::Capability { .. } - | ExecutionOperation::Agent { .. } - | ExecutionOperation::Review { .. } - | ExecutionOperation::WaitSignal { .. } - | ExecutionOperation::WaitUntil { .. } - | ExecutionOperation::Output { .. } => { - if request - .projection - .tasks - .iter() - .any(|task| task.node_id == node.id && task.item_key.is_empty()) - { - return Ok(Vec::new()); - } - let dependencies = node.depends_on.iter().cloned().collect::>(); - let context = BindingContext { - run_input: &request.run_input, - node_outputs: outputs, - dependencies: &dependencies, - item: None, - item_key: None, - }; - let input = resolve_bindings(&node.input, &context)?; - let kind = logical_kind(&node.operation, &context)?; - validate_capability_input(request, &node.operation, &input)?; - if let LogicalTaskKind::Output { value } = &kind { - validate_instance( - &node.output_schema, - value, - &format!("node.{}.output", node.id), - )?; - validate_instance(&request.plan.definition.output_schema, value, "plan.output")?; - } - let reservation = - operation_reservation(request, &node.operation, node.retry.max_attempts)?; - Ok(vec![logical_task( - request, - node, - String::new(), - input, - kind, - reservation, - )?]) - } - } -} - -pub(super) struct MapMaterializationRequest<'a> { - schedule: &'a ScheduleRequest, - node: &'a ExecutionNode, - outputs: &'a BTreeMap, - items: &'a Value, - item_key_pointer: &'a str, - max_items: u64, - task: &'a MapTask, -} - -pub(super) fn materialize_map(request: MapMaterializationRequest<'_>) -> Result> { - let MapMaterializationRequest { - schedule, - node, - outputs, - items, - item_key_pointer, - max_items, - task, - } = request; - let dependencies = node.depends_on.iter().cloned().collect::>(); - let base = BindingContext { - run_input: &schedule.run_input, - node_outputs: outputs, - dependencies: &dependencies, - item: None, - item_key: None, - }; - let resolved = resolve_bindings(items, &base)?; - let values = resolved.as_array().ok_or_else(|| Error::Binding { - path: format!("node.{}.operation.items", node.id), - message: "map items must resolve to an array".to_string(), - })?; - let count = u64::try_from(values.len()).map_err(|_| Error::ArithmeticOverflow { - context: format!("map {} item count", node.id), - })?; - if count > max_items { - return Err(Error::InvalidProjection { - message: format!("map {} exceeds max_items", node.id), - }); - } - let mut seen = BTreeSet::new(); - let mut ready = Vec::new(); - for item in values { - let item_key = extract_map_key(item, item_key_pointer)?; - if !seen.insert(item_key.clone()) { - return Err(Error::InvalidProjection { - message: format!("map {} produced duplicate item key {item_key}", node.id), - }); - } - if schedule - .projection - .tasks - .iter() - .any(|existing| existing.node_id == node.id && existing.item_key == item_key) - { - continue; - } - let context = BindingContext { - item: Some(item), - item_key: Some(&item_key), - ..base - }; - let input = resolve_bindings(&node.input, &context)?; - let kind = map_kind(task); - validate_map_capability_input(schedule, task, &input)?; - let reservation = map_task_reservation(schedule, task, node.retry.max_attempts)?; - ready.push(logical_task( - schedule, - node, - item_key, - input, - kind, - reservation, - )?); + if request + .projection + .tasks + .iter() + .any(|task| task.node_id == node.id && task.item_key.is_empty()) + { + return Ok(Vec::new()); } - Ok(ready) -} - -pub(super) struct ReduceMaterializationRequest<'a> { - schedule: &'a ScheduleRequest, - node: &'a ExecutionNode, - outputs: &'a BTreeMap, - items: &'a Value, - max_items: u64, - reducer: &'a ExecutionReducer, - batch_size: u32, -} - -pub(super) fn materialize_reduce( - request: ReduceMaterializationRequest<'_>, -) -> Result> { - let ReduceMaterializationRequest { - schedule, - node, - outputs, - items, - max_items, - reducer, - batch_size, - } = request; let dependencies = node.depends_on.iter().cloned().collect::>(); let context = BindingContext { - run_input: &schedule.run_input, + run_input: &request.run_input, node_outputs: outputs, dependencies: &dependencies, item: None, item_key: None, }; - let resolved = resolve_bindings(items, &context)?; - let mut round_items = resolved.as_array().cloned().ok_or_else(|| Error::Binding { - path: format!("node.{}.operation.items", node.id), - message: "reduce items must resolve to an array".to_string(), - })?; - let count = u64::try_from(round_items.len()).map_err(|_| Error::ArithmeticOverflow { - context: format!("reduce {} item count", node.id), - })?; - if count > max_items { - return Err(Error::InvalidProjection { - message: format!("reduce {} exceeds max_items", node.id), - }); - } - if round_items.len() <= 1 { - return Ok(Vec::new()); - } - - let batch_size = usize::try_from(batch_size).map_err(|_| Error::ArithmeticOverflow { - context: format!("reduce {} batch_size", node.id), - })?; - let mut round = 1_u32; - loop { - let mut ready = Vec::new(); - let mut completed = Vec::new(); - for (batch_index, batch) in round_items.chunks(batch_size).enumerate() { - let batch_index = - u64::try_from(batch_index).map_err(|_| Error::ArithmeticOverflow { - context: format!("reduce {} batch index", node.id), - })?; - let item_key = format!("r{round}:b{batch_index}"); - if let Some(existing) = schedule - .projection - .tasks - .iter() - .find(|task| task.node_id == node.id && task.item_key == item_key) - { - if let Some(output) = completed_output(existing) { - validate_instance( - &node.output_schema, - &output, - &format!("node.{}.reduce.{item_key}", node.id), - )?; - completed.push(output); - } - continue; - } - let input = json!({ - "round": round, - "batch_index": batch_index, - "items": batch, - }); - validate_reducer_capability_input(schedule, reducer, &input)?; - let reservation = reducer_reservation(schedule, reducer, node.retry.max_attempts)?; - ready.push(logical_task( - schedule, - node, - item_key, - input, - reducer_kind(reducer), - reservation, - )?); - } - if !ready.is_empty() { - return Ok(ready); - } - let expected = round_items.len().div_ceil(batch_size); - if completed.len() != expected { - return Ok(Vec::new()); - } - if completed.len() == 1 { - return Ok(Vec::new()); - } - round_items = completed; - round = round - .checked_add(1) - .ok_or_else(|| Error::ArithmeticOverflow { - context: format!("reduce {} round", node.id), - })?; + let input = resolve_bindings(&node.input, &context)?; + let kind = logical_kind(&node.operation, &context)?; + validate_capability_input(request, &node.operation, &input)?; + if let LogicalTaskKind::Output { value } = &kind { + validate_instance( + &node.output_schema, + value, + &format!("node.{}.output", node.id), + )?; + validate_instance(&request.plan.definition.output_schema, value, "plan.output")?; } + let reservation = operation_reservation(request, &node.operation, node.retry.max_attempts)?; + Ok(vec![logical_task( + request, + node, + String::new(), + input, + kind, + reservation, + )?]) } pub(super) fn logical_task( diff --git a/crates/moa-execution/src/interpreter/mod.rs b/crates/moa-execution/src/interpreter/mod.rs index 16a750824..5892ddfff 100644 --- a/crates/moa-execution/src/interpreter/mod.rs +++ b/crates/moa-execution/src/interpreter/mod.rs @@ -1,20 +1,13 @@ -//! Pure execution-plan scheduling and logical-task materialization. +//! Pure bounded logical-task materialization for execution-plan nodes. -mod aggregate; +mod catalog; mod compensation; mod materialize; -mod projection; mod reservation; -mod temporal_wait; -mod terminal; -use aggregate::*; +use catalog::*; pub use compensation::resolve_compensation_input; -use materialize::*; -use projection::*; use reservation::*; -use temporal_wait::*; -use terminal::*; /// Derives the bounded reservation for one persisted completion verifier. pub(crate) fn verifier_turn_reservation( @@ -28,9 +21,9 @@ use std::collections::{BTreeMap, BTreeSet}; use chrono::{DateTime, Utc}; use moa_artifacts::execution_plan::{ - CapabilityReference, CompletionCheckKind, ExecutionFailureClass, ExecutionGoalContract, - ExecutionNode, ExecutionOperation, ExecutionReducer, ExecutionTaskOutcome, ExecutionTaskResult, - ExecutionTemporalTarget, MapTask, RetryPolicy, + CapabilityReference, ExecutionFailureClass, ExecutionGoalContract, ExecutionNode, + ExecutionOperation, ExecutionReducer, ExecutionTaskOutcome, ExecutionTaskResult, + ExecutionTemporalTarget, MapTask, }; use moa_config::ExecutionConfig; @@ -44,19 +37,10 @@ use crate::{ budget::BudgetLedger, capability::{ ExecutionCapability, ExecutionCapabilityCatalog, ExecutionEstimate, canonical_sort_key, - catalog_hash, task_output_hash, }, compiler::CanonicalExecutionPlan, - completion::{ - CompletionEvaluationRequest, CompletionStatus, completed_output, evaluate_completion, - map_output, node_outputs, terminal_projection_from_evaluation, - }, schema::validate_instance, - state::{ - ExecutionNodeStatus, ExecutionProjection, ExecutionTaskFailure, ExecutionTaskId, - ExecutionTaskStatus, LogicalTask, LogicalTaskKind, ScheduleDecision, TerminalProjection, - VerifierTaskSummary, WaitingReason, task_status_from_outcome, - }, + state::{ExecutionProjection, ExecutionTaskId, LogicalTask, LogicalTaskKind}, }; /// Validates a completed task outcome against its concrete plan-node output contract. @@ -121,7 +105,7 @@ fn task_output_schema(node: &ExecutionNode) -> &Value { } } -/// Complete pure input to one scheduler evaluation. +/// Complete pure input to one bounded materialization evaluation. #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] #[serde(deny_unknown_fields)] pub struct ScheduleRequest { @@ -141,20 +125,10 @@ pub struct ScheduleRequest { pub config: ExecutionConfig, /// Current pure run-level budget ledger. pub budget_ledger: BudgetLedger, - /// Deterministic scheduler time. + /// Deterministic materialization time. pub now: DateTime, } -/// One scheduler decision paired with the exact effective projection it evaluated. -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -#[serde(deny_unknown_fields)] -pub struct ScheduleOutcome { - /// Ready, waiting, terminal, or no-progress scheduler decision. - pub decision: ScheduleDecision, - /// Projection after deterministic conditions and aggregate nodes were derived. - pub effective_projection: ExecutionProjection, -} - /// One bounded deterministic logical-task page for a single eligible node. #[derive(Clone, Debug, PartialEq)] pub struct NodeMaterializationPage { @@ -168,6 +142,8 @@ pub struct NodeMaterializationPage { pub reduce_cursor: Option, /// Aggregate output for a source that completes without creating a logical task. pub terminal_output: Option, + /// Whether the node's declared condition evaluated false and the node must be skipped. + pub condition_skipped: bool, } /// Exact reduce-round source position used to derive one materialization page. @@ -227,15 +203,65 @@ pub fn materialize_node_page( message: format!("node `{node_id}` is missing a direct dependency output"), }); } + // The condition is evaluated here, in the wrapper, so a false branch never enters + // map or reduce paging at all: no items are resolved, no reduce round is opened, + // and the node's whole source is declared exhausted in one empty page. It is + // evaluated only at cursor zero because a condition that has already admitted its + // first page must not be re-litigated mid-source; every input it can read is + // immutable for the life of the node. + if cursor == 0 + && let Some(condition) = &node.when + { + let dependencies = node.depends_on.iter().cloned().collect::>(); + let context = BindingContext { + run_input: &request.run_input, + node_outputs: referenced_outputs, + dependencies: &dependencies, + item: None, + item_key: None, + }; + if !evaluate_condition(condition, &context)? { + return Ok(NodeMaterializationPage { + tasks: Vec::new(), + next_cursor: 0, + source_exhausted: true, + reduce_cursor: None, + terminal_output: None, + condition_skipped: true, + }); + } + } materialize::materialize_node_page(request, node, referenced_outputs, cursor, limit, reduce) } +/// Outcome of fencing one resolved temporal target against the run deadline. +/// +/// A relative target is resolved against wait entry, not compile time, so a delay that was +/// legal when the plan compiled can land past the deadline by the time the wait is entered. +/// That is a normal product outcome for a long-horizon run, not an invalid plan, so it is +/// reported as a value rather than an error. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum TemporalTargetResolution { + /// The target resolves strictly before the run deadline. + Due(DateTime), + /// The target resolves at or after the run deadline and cannot be waited on. + DeadlineExceeded { + /// Instant the wait would have become due. + due_at: DateTime, + /// Absolute deadline of the owning run. + run_deadline_at: DateTime, + }, +} + /// Resolves an exact or wait-entry-relative temporal target and fences it by the run deadline. -pub fn resolve_temporal_target( +/// +/// Fails only on impossible input; a target past the run deadline is returned as +/// [`TemporalTargetResolution::DeadlineExceeded`] for the caller to project as a typed failure. +pub fn resolve_temporal_target_within_deadline( target: &ExecutionTemporalTarget, wait_entered_at: DateTime, run_deadline_at: DateTime, -) -> Result> { +) -> Result { let due_at = match target { ExecutionTemporalTarget::At { at } => *at, ExecutionTemporalTarget::After { delay_seconds } => { @@ -261,208 +287,31 @@ pub fn resolve_temporal_target( })? } }; - if due_at >= run_deadline_at { - return Err(Error::InvalidProjection { - message: "temporal target must be earlier than the run deadline".to_string(), - }); - } - Ok(due_at) -} - -/// Returns map nodes whose first deterministic materialization contains zero items. -/// -/// The repository uses these node IDs to persist a zero-fan-out marker even though -/// the scheduler has no logical task row to return for an empty map. -pub fn ready_empty_map_nodes(request: &ScheduleRequest) -> Result> { - validate_projection(request)?; - let mut outputs = node_outputs(&request.plan, &request.projection)?; - let mut statuses = request.projection.node_statuses.clone(); - apply_false_conditions(request, &mut statuses, &mut outputs)?; - derive_aggregate_nodes(request, &mut statuses, &mut outputs)?; - apply_false_conditions(request, &mut statuses, &mut outputs)?; - - let mut node_ids = Vec::new(); - for node in &request.plan.definition.nodes { - let ExecutionOperation::Map { - items, max_items, .. - } = &node.operation - else { - continue; - }; - if effective_status(&statuses, &node.id) != Some(ExecutionNodeStatus::Completed) - || request - .projection - .tasks - .iter() - .any(|task| task.node_id == node.id) - { - continue; - } - let dependencies = node.depends_on.iter().cloned().collect::>(); - let resolved = resolve_bindings( - items, - &BindingContext { - run_input: &request.run_input, - node_outputs: &outputs, - dependencies: &dependencies, - item: None, - item_key: None, - }, - )?; - let values = resolved.as_array().ok_or_else(|| Error::Binding { - path: format!("node.{}.operation.items", node.id), - message: "map items must resolve to an array".to_string(), - })?; - let count = u64::try_from(values.len()).map_err(|_| Error::ArithmeticOverflow { - context: format!("map {} item count", node.id), - })?; - if count > *max_items { - return Err(Error::InvalidProjection { - message: format!("map {} exceeds max_items", node.id), - }); + Ok(if due_at >= run_deadline_at { + TemporalTargetResolution::DeadlineExceeded { + due_at, + run_deadline_at, } - if values.is_empty() { - node_ids.push(node.id.clone()); - } - } - node_ids.sort(); - Ok(node_ids) + } else { + TemporalTargetResolution::Due(due_at) + }) } -/// Returns one decision together with the exact effective projection it evaluated. -pub fn schedule(mut request: ScheduleRequest) -> Result { - validate_projection(&request)?; - let mut outputs = node_outputs(&request.plan, &request.projection)?; - let mut statuses = request.projection.node_statuses.clone(); - apply_false_conditions(&request, &mut statuses, &mut outputs)?; - derive_aggregate_nodes(&request, &mut statuses, &mut outputs)?; - apply_false_conditions(&request, &mut statuses, &mut outputs)?; - request.projection.node_statuses = statuses.clone(); - - let ordinary_terminal = request - .plan - .definition - .nodes - .iter() - .all(|node| effective_status(&statuses, &node.id).is_some_and(is_terminal_node_status)); - if ordinary_terminal { - let effective_projection = request.projection.clone(); - return schedule_verifiers_or_complete(request).map(|decision| ScheduleOutcome { - decision, - effective_projection, - }); - } - - if request - .budget_ledger - .limit - .deadline_at - .is_some_and(|deadline| request.now >= deadline) - { - let decision = completion_terminal( - &request, - terminal_output(&request.plan, &request.projection), - )?; - return Ok(ScheduleOutcome { - decision, - effective_projection: request.projection, - }); - } - - if let Some(settlement) = ready_wait_settlement(&request, &outputs)? { - return Ok(ScheduleOutcome { - decision: ScheduleDecision::SettleWait(settlement), - effective_projection: request.projection, - }); - } - - let mut ready = Vec::new(); - let mut dependency_waits = BTreeSet::new(); - for node in &request.plan.definition.nodes { - if effective_status(&statuses, &node.id) != Some(ExecutionNodeStatus::Pending) { - continue; - } - let dependency_statuses = node - .depends_on - .iter() - .map(|id| (id, effective_status(&statuses, id))) - .collect::>(); - if dependency_statuses.iter().any(|(_, status)| { - matches!( - status, - Some(ExecutionNodeStatus::Failed | ExecutionNodeStatus::Cancelled) - ) - }) { - return Ok(ScheduleOutcome { - decision: ScheduleDecision::Terminal(TerminalProjection::Failed { - failure: ExecutionTaskFailure { - class: ExecutionFailureClass::DependencyFailed, - message: format!("node {} has a terminal failed dependency", node.id), - capability_ref: operation_capability(&node.operation), - }, - }), - effective_projection: request.projection, - }); - } - if !dependency_statuses.iter().all(|(_, status)| { - matches!( - status, - Some(ExecutionNodeStatus::Completed | ExecutionNodeStatus::Skipped) - ) - }) { - dependency_waits.insert(node.id.clone()); - continue; - } - - let mut materialized = materialize_node(&request, node, &outputs)?; - ready.append(&mut materialized); - } - - if !ready.is_empty() { - ready.sort_by(|left, right| { - (&left.node_id, &left.item_key, left.task_id).cmp(&( - &right.node_id, - &right.item_key, - right.task_id, - )) - }); - let mut ledger = request.budget_ledger.clone(); - for task in &ready { - if ledger.try_reserve(task.reservation).is_err() { - return Ok(ScheduleOutcome { - decision: budget_terminal(terminal_output(&request.plan, &request.projection)), - effective_projection: request.projection, - }); - } - } - return Ok(ScheduleOutcome { - decision: ScheduleDecision::Ready(ready), - effective_projection: request.projection, - }); - } - - let waiting = waiting_reasons(&request, dependency_waits); - if !waiting.is_empty() { - return Ok(ScheduleOutcome { - decision: ScheduleDecision::Waiting(waiting), - effective_projection: request.projection, - }); +/// Resolves a temporal target that must already be strictly before the run deadline. +/// +/// Callers settling an entered wait use this: the target was fenced at wait entry, so a +/// deadline violation here is a corrupted projection rather than a product outcome. +pub fn resolve_temporal_target( + target: &ExecutionTemporalTarget, + wait_entered_at: DateTime, + run_deadline_at: DateTime, +) -> Result> { + match resolve_temporal_target_within_deadline(target, wait_entered_at, run_deadline_at)? { + TemporalTargetResolution::Due(due_at) => Ok(due_at), + TemporalTargetResolution::DeadlineExceeded { .. } => Err(Error::InvalidProjection { + message: "temporal target must be earlier than the run deadline".to_string(), + }), } - - let mut pending_node_ids = request - .plan - .definition - .nodes - .iter() - .filter(|node| !effective_status(&statuses, &node.id).is_some_and(is_terminal_node_status)) - .map(|node| node.id.clone()) - .collect::>(); - pending_node_ids.sort(); - pending_node_ids.dedup(); - Ok(ScheduleOutcome { - decision: ScheduleDecision::NoProgress { pending_node_ids }, - effective_projection: request.projection, - }) } #[cfg(test)] diff --git a/crates/moa-execution/src/interpreter/projection.rs b/crates/moa-execution/src/interpreter/projection.rs deleted file mode 100644 index 02a875c3b..000000000 --- a/crates/moa-execution/src/interpreter/projection.rs +++ /dev/null @@ -1,127 +0,0 @@ -//! Projection and capability-catalog validation for scheduler inputs. - -use super::*; - -pub(super) fn validate_projection(request: &ScheduleRequest) -> Result<()> { - validate_scheduler_catalog(&request.catalog)?; - let canonical_catalog_hash = catalog_hash(&request.catalog.capabilities)?; - if canonical_catalog_hash != request.plan.catalog_hash - || request.catalog.catalog_hash != canonical_catalog_hash - { - return Err(Error::InvalidProjection { - message: "scheduler capability catalog hash does not match the canonical plan" - .to_string(), - }); - } - - for task in &request.projection.tasks { - if task.attempt == 0 || task.generation == 0 { - return Err(Error::InvalidProjection { - message: format!("task {} has a zero attempt or generation", task.task_id), - }); - } - let expected = ExecutionTaskId::derive(request.run_uid, &task.node_id, &task.item_key)?; - if task.task_id != expected { - return Err(Error::InvalidProjection { - message: format!( - "task {} does not match its framed logical identity", - task.task_id - ), - }); - } - if let Some(outcome) = &task.outcome { - if outcome.schema_version != 1 { - return Err(Error::InvalidProjection { - message: format!("task {} outcome schema_version must equal 1", task.task_id), - }); - } - let expected_status = - task_status_from_outcome(outcome, task.status == ExecutionTaskStatus::Running); - if task.status != expected_status { - return Err(Error::InvalidProjection { - message: format!( - "task {} status does not match its persisted outcome", - task.task_id - ), - }); - } - } else if matches!( - task.status, - ExecutionTaskStatus::Completed - | ExecutionTaskStatus::Failed - | ExecutionTaskStatus::UnknownOutcome - | ExecutionTaskStatus::Cancelled - | ExecutionTaskStatus::WaitingInput - | ExecutionTaskStatus::WaitingReplan - ) { - return Err(Error::InvalidProjection { - message: format!( - "task {} terminal/waiting status has no outcome", - task.task_id - ), - }); - } - - let Some(node) = request - .plan - .definition - .nodes - .iter() - .find(|node| node.id == task.node_id) - else { - if task.node_id.starts_with("@check/") { - continue; - } - if task.status == ExecutionTaskStatus::Cancelled { - continue; - } - return Err(Error::InvalidProjection { - message: format!("task {} references an unknown plan node", task.task_id), - }); - }; - if task.attempt > node.retry.max_attempts { - return Err(Error::InvalidProjection { - message: format!("task {} exceeds its retry policy", task.task_id), - }); - } - if let Some(reference) = operation_capability(&node.operation) { - let capability = find_capability(&request.catalog, &reference)?; - validate_instance( - &capability.input_schema, - &task.input, - &format!("task.{}.input", task.task_id), - )?; - if let Some(output) = completed_output(task) { - validate_instance( - &capability.output_schema, - &output, - &format!("task.{}.output", task.task_id), - )?; - } - } - } - Ok(()) -} - -pub(super) fn validate_scheduler_catalog(catalog: &ExecutionCapabilityCatalog) -> Result<()> { - let mut previous = None; - for capability in &catalog.capabilities { - if capability.estimate.tasks != 1 { - return Err(Error::InvalidProjection { - message: format!( - "capability {}@{} must reserve exactly one logical task", - capability.reference.name, capability.reference.version - ), - }); - } - let key = canonical_sort_key(&capability.reference)?; - if previous.as_ref().is_some_and(|previous| key <= *previous) { - return Err(Error::InvalidProjection { - message: "scheduler capability catalog must be sorted and duplicate-free" - .to_string(), - }); - } - previous = Some(key); - } - Ok(()) -} diff --git a/crates/moa-execution/src/interpreter/temporal_wait.rs b/crates/moa-execution/src/interpreter/temporal_wait.rs deleted file mode 100644 index 04cc94300..000000000 --- a/crates/moa-execution/src/interpreter/temporal_wait.rs +++ /dev/null @@ -1,196 +0,0 @@ -//! Pure projection and due-settlement decisions for storage-only execution waits. - -use std::collections::{BTreeMap, BTreeSet}; - -use chrono::{DateTime, Utc}; -use moa_artifacts::execution_plan::{ - ExecutionOperation, ExecutionTaskResult, ExecutionTemporalTarget, -}; -use serde_json::Value; - -use super::ScheduleRequest; -use crate::{ - Result, - bindings::{BindingContext, resolve_bindings}, - state::{ExecutionNodeStatus, ExecutionTaskStatus, WaitSettlement, WaitingReason}, -}; - -pub(super) fn waiting_reasons( - request: &ScheduleRequest, - dependency_waits: BTreeSet, -) -> Vec { - let by_id = request - .plan - .definition - .nodes - .iter() - .map(|node| (node.id.as_str(), node)) - .collect::>(); - let mut waiting = Vec::new(); - if request.projection.tasks.iter().any(|task| { - matches!( - task.status, - ExecutionTaskStatus::Pending - | ExecutionTaskStatus::Ready - | ExecutionTaskStatus::Reserved - | ExecutionTaskStatus::Dispatching - | ExecutionTaskStatus::Running - | ExecutionTaskStatus::WaitingReplan - ) - }) { - waiting.push(WaitingReason::RunningTasks); - } - for task in &request.projection.tasks { - if task.status == ExecutionTaskStatus::WaitingExternal { - waiting.push(WaitingReason::External { - task_id: task.task_id, - }); - } - if task.status == ExecutionTaskStatus::WaitingInput - && let Some(outcome) = &task.outcome - && let ExecutionTaskResult::NeedsInput { question, audience } = &outcome.result - { - waiting.push(WaitingReason::Input { - task_id: task.task_id, - audience: audience.clone(), - question: question.clone(), - wait_policy: request.plan.definition.input_wait_policy.clone(), - }); - } - if request.projection.node_statuses.get(&task.node_id) - == Some(&ExecutionNodeStatus::Waiting) - && let Some(node) = by_id.get(task.node_id.as_str()) - { - match &node.operation { - ExecutionOperation::Review { - prompt, - wait_policy, - } => waiting.push(WaitingReason::Review { - task_id: task.task_id, - prompt: prompt.clone(), - wait_policy: wait_policy.clone(), - }), - ExecutionOperation::WaitSignal { - signal_name, - wait_policy, - } => { - waiting.push(WaitingReason::Signal { - task_id: task.task_id, - signal_name: signal_name.clone(), - wait_policy: wait_policy.clone(), - }); - } - ExecutionOperation::WaitUntil { wake, .. } => { - waiting.push(WaitingReason::Timer { - task_id: task.task_id, - wake: wake.clone(), - }); - } - ExecutionOperation::Capability { .. } - | ExecutionOperation::Agent { .. } - | ExecutionOperation::Map { .. } - | ExecutionOperation::Reduce { .. } - | ExecutionOperation::Output { .. } => {} - } - } - } - if !dependency_waits.is_empty() { - waiting.push(WaitingReason::Dependencies { - node_ids: dependency_waits.into_iter().collect(), - }); - } - waiting -} - -pub(super) fn ready_wait_settlement( - request: &ScheduleRequest, - outputs: &BTreeMap, -) -> Result> { - let by_id = request - .plan - .definition - .nodes - .iter() - .map(|node| (node.id.as_str(), node)) - .collect::>(); - let mut settlements = Vec::new(); - for task in &request.projection.tasks { - if task.status == ExecutionTaskStatus::WaitingInput - && temporal_target_is_due( - &request.plan.definition.input_wait_policy.expiry, - request.now, - ) - { - settlements.push(( - task.task_id, - WaitSettlement::WaitExpired { - task_id: task.task_id, - action: request.plan.definition.input_wait_policy.on_expiry.clone(), - }, - )); - continue; - } - let Some(node) = by_id.get(task.node_id.as_str()) else { - continue; - }; - let settlement = match &node.operation { - ExecutionOperation::Review { wait_policy, .. } - if matches!( - task.status, - ExecutionTaskStatus::Running | ExecutionTaskStatus::WaitingReview - ) && temporal_target_is_due(&wait_policy.expiry, request.now) => - { - Some(WaitSettlement::WaitExpired { - task_id: task.task_id, - action: wait_policy.on_expiry.clone(), - }) - } - ExecutionOperation::WaitSignal { wait_policy, .. } - if matches!( - task.status, - ExecutionTaskStatus::Running | ExecutionTaskStatus::WaitingSignal - ) && temporal_target_is_due(&wait_policy.expiry, request.now) => - { - Some(WaitSettlement::WaitExpired { - task_id: task.task_id, - action: wait_policy.on_expiry.clone(), - }) - } - ExecutionOperation::WaitUntil { wake, result } - if matches!( - task.status, - ExecutionTaskStatus::Running | ExecutionTaskStatus::WaitingTimer - ) && temporal_target_is_due(wake, request.now) => - { - let dependencies = node.depends_on.iter().cloned().collect::>(); - let output = resolve_bindings( - result, - &BindingContext { - run_input: &request.run_input, - node_outputs: outputs, - dependencies: &dependencies, - item: None, - item_key: None, - }, - )?; - Some(WaitSettlement::TimerElapsed { - task_id: task.task_id, - output, - }) - } - _ => None, - }; - if let Some(settlement) = settlement { - settlements.push((task.task_id, settlement)); - } - } - settlements.sort_by_key(|(task_id, _)| *task_id); - Ok(settlements - .into_iter() - .next() - .map(|(_, settlement)| settlement)) -} - -fn temporal_target_is_due(target: &ExecutionTemporalTarget, now: DateTime) -> bool { - matches!(target, ExecutionTemporalTarget::At { at } if now >= *at) -} diff --git a/crates/moa-execution/src/interpreter/terminal.rs b/crates/moa-execution/src/interpreter/terminal.rs deleted file mode 100644 index 9c192ca98..000000000 --- a/crates/moa-execution/src/interpreter/terminal.rs +++ /dev/null @@ -1,345 +0,0 @@ -//! Terminal completion and verifier scheduler decisions. - -use super::temporal_wait::waiting_reasons; -use super::*; - -pub(super) fn schedule_verifiers_or_complete(request: ScheduleRequest) -> Result { - let terminal = terminal_output(&request.plan, &request.projection); - let preliminary = evaluate_completion(CompletionEvaluationRequest { - goal: request.goal.clone(), - plan: request.plan.clone(), - run_input: request.run_input.clone(), - projection: request.projection.clone(), - terminal_output: terminal.clone(), - budget_ledger: request.budget_ledger.clone(), - now: request.now, - })?; - let unresolved = preliminary.unsatisfied_requirement_ids; - let summaries = verifier_summaries(&request.plan, &request.projection)?; - let mut ready = Vec::new(); - for check in &request.goal.completion_checks { - let CompletionCheckKind::AgentVerifier { - instructions, - max_turns, - } = &check.kind - else { - continue; - }; - let node_id = format!("@check/{}", check.id); - let item_key = format!("check:{}", check.id); - if request - .projection - .tasks - .iter() - .any(|task| task.node_id == node_id) - { - continue; - } - let reservation = turn_reservation(&request.config, *max_turns, 1, true)?; - ready.push(LogicalTask { - task_id: ExecutionTaskId::derive(request.run_uid, &node_id, &item_key)?, - node_id, - item_key, - requirement_ids: unresolved.clone(), - plan_revision: request.projection.plan_revision, - generation: 1, - input: json!({ - "goal": &request.goal, - "check_id": &check.id, - "description": &check.description, - "terminal_output": &terminal, - "task_summaries": &summaries, - }), - kind: LogicalTaskKind::CompletionVerifier { - check_id: check.id.clone(), - instructions: instructions.clone(), - max_turns: *max_turns, - }, - compensation: None, - retry: RetryPolicy { - max_attempts: 1, - initial_backoff_ms: 0, - max_backoff_ms: 0, - }, - reservation, - }); - } - if !ready.is_empty() { - let mut ledger = request.budget_ledger.clone(); - for task in &ready { - if ledger.try_reserve(task.reservation).is_err() { - return Ok(budget_terminal(terminal)); - } - } - return Ok(ScheduleDecision::Ready(ready)); - } - if request.projection.tasks.iter().any(|task| { - task.node_id.starts_with("@check/") - && !matches!( - task.status, - ExecutionTaskStatus::Completed - | ExecutionTaskStatus::Failed - | ExecutionTaskStatus::UnknownOutcome - | ExecutionTaskStatus::Cancelled - ) - }) { - let waiting = waiting_reasons(&request, BTreeSet::new()); - return Ok(ScheduleDecision::Waiting(if waiting.is_empty() { - vec![WaitingReason::RunningTasks] - } else { - waiting - })); - } - completion_terminal(&request, terminal) -} - -pub(super) fn verifier_summaries( - plan: &CanonicalExecutionPlan, - projection: &ExecutionProjection, -) -> Result> { - let mut summaries = projection - .tasks - .iter() - .filter(|task| !task.node_id.starts_with("@check/")) - .filter(|task| is_terminal_task_status(task.status)) - .map(|task| { - let output_hash = completed_output(task) - .as_ref() - .map(task_output_hash) - .transpose()?; - let failure = task - .outcome - .as_ref() - .and_then(|outcome| match &outcome.result { - ExecutionTaskResult::Failed { class, message } => Some(ExecutionTaskFailure { - class: class.clone(), - message: message.clone(), - capability_ref: plan - .definition - .nodes - .iter() - .find(|node| node.id == task.node_id) - .and_then(|node| operation_capability(&node.operation)), - }), - ExecutionTaskResult::Cancelled { reason } => Some(ExecutionTaskFailure { - class: ExecutionFailureClass::Cancelled, - message: reason.clone(), - capability_ref: None, - }), - ExecutionTaskResult::UnknownOutcome { message } => Some(ExecutionTaskFailure { - class: ExecutionFailureClass::Terminal, - message: message.clone(), - capability_ref: plan - .definition - .nodes - .iter() - .find(|node| node.id == task.node_id) - .and_then(|node| operation_capability(&node.operation)), - }), - ExecutionTaskResult::Completed { .. } - | ExecutionTaskResult::NeedsInput { .. } - | ExecutionTaskResult::NeedsReplan { .. } => None, - }); - let mut citation_source_ids = task - .outcome - .as_ref() - .and_then(|outcome| match &outcome.result { - ExecutionTaskResult::Completed { citations, .. } => Some( - citations - .iter() - .filter(|citation| !citation.source_id.trim().is_empty()) - .map(|citation| citation.source_id.clone()) - .collect::>(), - ), - _ => None, - }) - .unwrap_or_default(); - citation_source_ids.sort(); - citation_source_ids.dedup(); - Ok(VerifierTaskSummary { - task_id: task.task_id, - node_id: task.node_id.clone(), - item_key: task.item_key.clone(), - status: task.status, - output_hash, - failure, - citation_source_ids, - }) - }) - .collect::>>()?; - summaries.sort_by(|left, right| { - (&left.node_id, &left.item_key, left.task_id).cmp(&( - &right.node_id, - &right.item_key, - right.task_id, - )) - }); - Ok(summaries) -} - -pub(super) fn budget_terminal(output: Option) -> ScheduleDecision { - match output { - Some(output) => ScheduleDecision::Terminal(TerminalProjection::Partial { - output: Some(output), - gaps: vec!["execution budget cannot reserve required work".to_string()], - }), - None => ScheduleDecision::Terminal(TerminalProjection::Failed { - failure: ExecutionTaskFailure { - class: ExecutionFailureClass::BudgetExceeded, - message: "required task reservation exceeds the remaining run budget".to_string(), - capability_ref: None, - }, - }), - } -} - -pub(super) fn completion_terminal( - request: &ScheduleRequest, - terminal_output: Option, -) -> Result { - if request.plan.definition.nodes.iter().all(|node| { - request.projection.node_statuses.get(&node.id) == Some(&ExecutionNodeStatus::Cancelled) - }) { - return Ok(ScheduleDecision::Terminal(TerminalProjection::Cancelled { - reason: "all execution nodes were cancelled".to_string(), - })); - } - let evaluation = evaluate_completion(CompletionEvaluationRequest { - goal: request.goal.clone(), - plan: request.plan.clone(), - run_input: request.run_input.clone(), - projection: request.projection.clone(), - terminal_output: terminal_output.clone(), - budget_ledger: request.budget_ledger.clone(), - now: request.now, - })?; - let failure = (evaluation.status == CompletionStatus::Failed) - .then(|| terminal_failure(request, &evaluation.gaps)); - let terminal = terminal_projection_from_evaluation( - &evaluation, - terminal_output, - None, - failure, - (evaluation.status == CompletionStatus::Unsupported) - .then(|| "required execution paths are unsupported".to_string()), - )?; - Ok(ScheduleDecision::Terminal(terminal)) -} - -pub(super) fn terminal_failure(request: &ScheduleRequest, gaps: &[String]) -> ExecutionTaskFailure { - request - .projection - .tasks - .iter() - .find_map(|task| { - task.outcome - .as_ref() - .and_then(|outcome| match &outcome.result { - ExecutionTaskResult::Failed { class, message } => Some(ExecutionTaskFailure { - class: class.clone(), - message: message.clone(), - capability_ref: request - .plan - .definition - .nodes - .iter() - .find(|node| node.id == task.node_id) - .and_then(|node| operation_capability(&node.operation)), - }), - ExecutionTaskResult::UnknownOutcome { message } => Some(ExecutionTaskFailure { - class: ExecutionFailureClass::Terminal, - message: message.clone(), - capability_ref: request - .plan - .definition - .nodes - .iter() - .find(|node| node.id == task.node_id) - .and_then(|node| operation_capability(&node.operation)), - }), - _ => None, - }) - }) - .unwrap_or_else(|| ExecutionTaskFailure { - class: ExecutionFailureClass::Terminal, - message: if gaps.is_empty() { - "execution did not produce a complete result".to_string() - } else { - gaps.join("; ") - }, - capability_ref: None, - }) -} - -pub(super) fn terminal_output( - plan: &CanonicalExecutionPlan, - projection: &ExecutionProjection, -) -> Option { - let output_node_id = plan - .definition - .nodes - .iter() - .find(|node| matches!(node.operation, ExecutionOperation::Output { .. })) - .map(|node| node.id.as_str())?; - projection - .tasks - .iter() - .filter(|task| task.node_id == output_node_id) - .filter(|task| task.item_key.is_empty()) - .filter(|task| task.status == ExecutionTaskStatus::Completed) - .find_map(completed_output) -} - -pub(super) fn operation_capability(operation: &ExecutionOperation) -> Option { - match operation { - ExecutionOperation::Capability { reference } => Some(reference.clone()), - ExecutionOperation::Map { - task: MapTask::Capability { reference }, - .. - } - | ExecutionOperation::Reduce { - reducer: ExecutionReducer::Capability { reference }, - .. - } => Some(reference.clone()), - ExecutionOperation::Agent { .. } - | ExecutionOperation::Map { .. } - | ExecutionOperation::Reduce { .. } - | ExecutionOperation::Review { .. } - | ExecutionOperation::WaitSignal { .. } - | ExecutionOperation::WaitUntil { .. } - | ExecutionOperation::Output { .. } => None, - } -} - -pub(super) fn effective_status( - statuses: &BTreeMap, - node_id: &str, -) -> Option { - Some( - statuses - .get(node_id) - .copied() - .unwrap_or(ExecutionNodeStatus::Pending), - ) -} - -pub(super) const fn is_terminal_node_status(status: ExecutionNodeStatus) -> bool { - matches!( - status, - ExecutionNodeStatus::Completed - | ExecutionNodeStatus::Skipped - | ExecutionNodeStatus::Failed - | ExecutionNodeStatus::Cancelled - ) -} - -pub(super) const fn is_terminal_task_status(status: ExecutionTaskStatus) -> bool { - matches!( - status, - ExecutionTaskStatus::Completed - | ExecutionTaskStatus::Skipped - | ExecutionTaskStatus::Failed - | ExecutionTaskStatus::UnknownOutcome - | ExecutionTaskStatus::Cancelled - ) -} diff --git a/crates/moa-execution/src/interpreter/tests.rs b/crates/moa-execution/src/interpreter/tests.rs index e75bf6c97..6a0b38f9e 100644 --- a/crates/moa-execution/src/interpreter/tests.rs +++ b/crates/moa-execution/src/interpreter/tests.rs @@ -1,6 +1,6 @@ -//! Unit tests for pure execution scheduling. +//! Unit tests for pure logical-task materialization. -use chrono::{Duration, Utc}; +use chrono::Utc; use moa_artifacts::execution_plan::{ CapabilityReference, CompensationInputMapping, ExecutionBudgetLimit, ExecutionCancelPolicy, ExecutionCompensation, ExecutionGoalContract, ExecutionNode, ExecutionOperation, @@ -8,6 +8,7 @@ use moa_artifacts::execution_plan::{ ExecutionWaitPolicy, MapTask, RetryPolicy, }; +use super::materialize::logical_task; use super::*; use crate::{ capability::{ExecutionCapabilityCatalog, ExecutionEstimate, ExecutionHash}, @@ -16,8 +17,8 @@ use crate::{ #[test] fn map_execution_task_validates_the_item_output_schema() { - // Pins: each materialized map task validates its own result before the - // scheduler builds and validates the aggregate map-node output. + // Pins: each materialized map task validates its own result against the node's + // item output schema rather than the aggregate map-node output schema. let item_schema = serde_json::json!({"type": "object", "required": ["symbol"]}); let catalog = ExecutionCapabilityCatalog::build(Vec::new()).expect("build empty catalog"); let plan = CanonicalExecutionPlan { @@ -93,98 +94,6 @@ fn map_execution_task_validates_the_item_output_schema() { )); } -#[test] -fn empty_map_is_reported_as_first_materialization_without_a_logical_task() { - // Pins: a valid zero-item map produces a durable marker candidate even though - // `schedule` cannot return a task row for it. - let catalog = ExecutionCapabilityCatalog::build(Vec::new()).expect("build empty catalog"); - let map_node = ExecutionNode { - id: "empty-map".to_string(), - requirement_ids: Vec::new(), - depends_on: Vec::new(), - when: None, - input: serde_json::json!({}), - output_schema: serde_json::json!({}), - operation: ExecutionOperation::Map { - items: serde_json::json!([]), - item_key: String::new(), - max_items: 4, - item_output_schema: serde_json::json!({}), - task: MapTask::Agent { - instructions: "inspect".to_string(), - skill_refs: Vec::new(), - capability_refs: Vec::new(), - max_turns: 1, - }, - }, - compensation: None, - retry: RetryPolicy { - max_attempts: 1, - initial_backoff_ms: 1, - max_backoff_ms: 1, - }, - budget: None, - }; - let request = ScheduleRequest { - run_uid: Uuid::now_v7(), - goal: ExecutionGoalContract { - objective: "accept empty input".to_string(), - requirements: Vec::new(), - deliverables: Vec::new(), - coverage: Vec::new(), - constraints: Vec::new(), - completion_checks: Vec::new(), - }, - plan: CanonicalExecutionPlan { - definition: ExecutionPlanDefinition { - cancel_policy: ExecutionCancelPolicy::RetainEffects, - input_wait_policy: test_input_wait_policy(), - input_schema: serde_json::json!({}), - output_schema: serde_json::json!({}), - nodes: vec![map_node], - }, - plan_hash: ExecutionHash::from_bytes([1; 32]), - catalog_hash: catalog.catalog_hash, - estimate: ExecutionEstimate::default(), - report: ExecutionValidationReport::default(), - }, - catalog, - run_input: serde_json::json!({}), - projection: ExecutionProjection { - plan_revision: 1, - node_statuses: BTreeMap::new(), - tasks: Vec::new(), - }, - config: ExecutionConfig::default(), - budget_ledger: BudgetLedger::new(ExecutionBudgetLimit { - max_cost_microusd: None, - max_tokens: None, - max_tasks: Some(10), - max_tool_calls: None, - max_retrieved_bytes: None, - deadline_at: Some(Utc::now() + Duration::hours(1)), - }), - now: Utc::now(), - }; - - assert_eq!( - ready_empty_map_nodes(&request).expect("derive empty map marker"), - vec!["empty-map".to_string()] - ); - - let mut nonempty = request; - let ExecutionOperation::Map { items, .. } = &mut nonempty.plan.definition.nodes[0].operation - else { - unreachable!("test plan node must remain a map"); - }; - *items = serde_json::json!([{"id": 1}]); - assert!( - ready_empty_map_nodes(&nonempty) - .expect("derive nonempty map") - .is_empty() - ); -} - #[test] fn only_direct_capability_task_materializes_compensation_contract() { // Pins: compensation belongs only to a direct capability node; aggregate task diff --git a/crates/moa-execution/src/lib.rs b/crates/moa-execution/src/lib.rs index 8026653d4..14169192e 100644 --- a/crates/moa-execution/src/lib.rs +++ b/crates/moa-execution/src/lib.rs @@ -12,7 +12,7 @@ pub mod compiler; pub mod completion; /// Crate error contract. pub mod error; -/// Pure execution scheduling. +/// Pure bounded logical-task materialization. pub mod interpreter; /// Deterministic replan-stop evaluation. pub mod replan; @@ -44,7 +44,7 @@ pub use completion::{ pub use error::Error; pub use interpreter::{ NodeMaterializationPage, ReduceMaterializationCursor, ReduceMaterializationPageInput, - ScheduleRequest, materialize_node_page, ready_empty_map_nodes, schedule, + ScheduleRequest, materialize_node_page, }; pub use replan::{ReplanDecision, ReplanEvaluationRequest, ReplanStopReason, evaluate_replan_stop}; pub use state::{ExecutionSourceKind, ExecutionTerminalReason}; diff --git a/crates/moa-execution/src/replan.rs b/crates/moa-execution/src/replan.rs index e9cd52749..466f5e820 100644 --- a/crates/moa-execution/src/replan.rs +++ b/crates/moa-execution/src/replan.rs @@ -99,7 +99,8 @@ pub enum ReplanStopReason { } /// Immediately knowable resource exhaustion that prevents another replan attempt. -#[derive(Clone, Debug, Eq, PartialEq)] +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] pub struct ReplanExhaustion { /// Typed terminal stop reason. pub reason: ReplanStopReason, diff --git a/crates/moa-execution/src/repository/capacity.rs b/crates/moa-execution/src/repository/capacity.rs index 2cbf4cc42..36c3a9757 100644 --- a/crates/moa-execution/src/repository/capacity.rs +++ b/crates/moa-execution/src/repository/capacity.rs @@ -16,7 +16,12 @@ use super::{ trigger::{ExecutionTriggerKind, NewExecutionTrigger, create_trigger_with_dispatch_in_conn}, }; -const MAX_ADMISSION_BATCH: u32 = 1_000; +/// Upper bound on one admission transaction's item count. +/// +/// Admission holds `FOR UPDATE` on the single fleet `active_tasks` bucket row for the whole batch, +/// and every multi-dimension execution transaction prelocks that same row. The cap keeps that +/// fleet-wide lock hold proportional to one dispatch window; the dispatcher loops for more. +const MAX_ADMISSION_BATCH: u32 = 64; const FAIRNESS_QUANTUM: i64 = 1_000_000; const CAPACITY_RESERVATION_NAMESPACE: Uuid = Uuid::from_u128(0x5b72_581c_d6f1_5a0b_9097_a267_eb1c_18d4); @@ -87,6 +92,15 @@ pub enum ExecutionCapacityOwner { Trigger { trigger_uid: Uuid }, /// Provider-owned external job capacity. ExternalJob { external_job_uid: Uuid }, + /// Active-task capacity owned by one exact compensation attempt generation. + Compensation { + /// Registered compensation the attempt rolls back. + compensation_id: Uuid, + /// Logical compensation generation that owns the attempt. + compensation_generation: u64, + /// Bounded attempt generation inside that compensation generation. + compensation_attempt_generation: u64, + }, } /// Generic capacity request shared by run, trigger, and external-job transactions. @@ -165,6 +179,40 @@ pub(super) fn parked_run_capacity_request( } } +/// Builds the exact active-task receipt owned by one compensation attempt generation. +/// +/// The reservation identity is derived from the immutable owner fence, so a replayed admission +/// resolves to the same receipt instead of racing +/// `execution_capacity_reservation_compensation_owner_uidx`. +#[must_use] +pub(super) fn compensation_attempt_capacity_request( + tenant_id: TenantId, + run_uid: Uuid, + controller_generation: u64, + compensation_id: Uuid, + compensation_generation: u64, + compensation_attempt_generation: u64, + expires_at: Option>, +) -> ExecutionCapacityRequest { + let owner_name = format!( + "active_tasks:{run_uid}:{controller_generation}:{compensation_id}:\ + {compensation_generation}:{compensation_attempt_generation}" + ); + ExecutionCapacityRequest { + reservation_uid: Uuid::new_v5(&CAPACITY_RESERVATION_NAMESPACE, owner_name.as_bytes()), + tenant_id, + run_uid: Some(run_uid), + controller_generation: Some(controller_generation), + dimension: ExecutionCapacityDimension::ActiveTasks, + owner: ExecutionCapacityOwner::Compensation { + compensation_id, + compensation_generation, + compensation_attempt_generation, + }, + expires_at, + } +} + /// One task attempt admitted atomically with capacity, outbox, and watchdog state. #[derive(Clone, Debug, Eq, PartialEq)] pub struct ExecutionAdmissionItem { @@ -554,7 +602,8 @@ pub(super) async fn reserve_capacity_in_tx( .await?; let existing = sqlx::query( "SELECT tenant_id, run_uid, controller_generation, resource_dimension, state, \ - trigger_uid, external_job_uid \ + trigger_uid, external_job_uid, compensation_id, compensation_generation, \ + compensation_attempt_generation \ FROM moa.execution_capacity_reservation WHERE reservation_uid = $1 FOR UPDATE", ) .bind(request.reservation_uid) @@ -583,7 +632,15 @@ pub(super) async fn reserve_capacity_in_tx( && existing .try_get::, _>("external_job_uid") .map_err(row_error)? - == owner_external_job_uid(request.owner); + == owner_external_job_uid(request.owner) + && existing + .try_get::, _>("compensation_id") + .map_err(row_error)? + == owner_compensation_uid(request.owner) + && optional_u64(&existing, "compensation_generation")? + == owner_compensation_generation(request.owner) + && optional_u64(&existing, "compensation_attempt_generation")? + == owner_compensation_attempt_generation(request.owner); if !matches { return Err(Error::InvalidRepositoryData { message: "capacity reservation UID is bound to different immutable coordinates" @@ -606,14 +663,26 @@ pub(super) async fn reserve_capacity_in_tx( sqlx::query( "INSERT INTO moa.execution_capacity_reservation (\ reservation_uid, tenant_id, run_uid, trigger_uid, external_job_uid, \ + compensation_id, compensation_generation, compensation_attempt_generation, \ controller_generation, resource_dimension, quantity, expires_at\ - ) VALUES ($1, $2, $3, $4, $5, $6, $7, 1, $8)", + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, 1, $11)", ) .bind(request.reservation_uid) .bind(request.tenant_id.0) .bind(request.run_uid) .bind(owner_trigger_uid(request.owner)) .bind(owner_external_job_uid(request.owner)) + .bind(owner_compensation_uid(request.owner)) + .bind( + owner_compensation_generation(request.owner) + .map(|generation| to_i64(generation, "capacity compensation generation")) + .transpose()?, + ) + .bind( + owner_compensation_attempt_generation(request.owner) + .map(|generation| to_i64(generation, "capacity compensation attempt generation")) + .transpose()?, + ) .bind( request .controller_generation @@ -811,7 +880,8 @@ pub(super) async fn release_capacity_in_tx( lock_existing_capacity_bucket(conn, "tenant", Some(request.tenant_id.0), dimension).await?; let row = sqlx::query( "SELECT tenant_id, run_uid, controller_generation, resource_dimension, state, \ - trigger_uid, external_job_uid \ + trigger_uid, external_job_uid, compensation_id, compensation_generation, \ + compensation_attempt_generation \ FROM moa.execution_capacity_reservation WHERE reservation_uid = $1 FOR UPDATE", ) .bind(request.reservation_uid) @@ -838,7 +908,15 @@ pub(super) async fn release_capacity_in_tx( && row .try_get::, _>("external_job_uid") .map_err(row_error)? - == owner_external_job_uid(request.owner); + == owner_external_job_uid(request.owner) + && row + .try_get::, _>("compensation_id") + .map_err(row_error)? + == owner_compensation_uid(request.owner) + && optional_u64(&row, "compensation_generation")? + == owner_compensation_generation(request.owner) + && optional_u64(&row, "compensation_attempt_generation")? + == owner_compensation_attempt_generation(request.owner); if !matches { return Ok(CapacityReleaseOutcome::Stale); } @@ -1058,7 +1136,8 @@ async fn reactivate_active_run_capacity_in_tx( Ok(CapacityReserveOutcome::Reserved) } -async fn capacity_bucket_has_room( +/// Reports whether one locked capacity bucket can admit a further unit. +pub(super) async fn capacity_bucket_has_room( conn: &mut PgConnection, scope_kind: &str, tenant_id: Option, @@ -1165,6 +1244,9 @@ fn validate_generic_capacity_request(request: &ExecutionCapacityRequest) -> Resu ) | ( ExecutionCapacityDimension::ExternalJobs, ExecutionCapacityOwner::ExternalJob { .. } + ) | ( + ExecutionCapacityDimension::ActiveTasks, + ExecutionCapacityOwner::Compensation { .. } ) ); let run_fence_valid = match ( @@ -1180,6 +1262,15 @@ fn validate_generic_capacity_request(request: &ExecutionCapacityRequest) -> Resu ExecutionCapacityOwner::Run => true, ExecutionCapacityOwner::Trigger { trigger_uid } => !trigger_uid.is_nil(), ExecutionCapacityOwner::ExternalJob { external_job_uid } => !external_job_uid.is_nil(), + ExecutionCapacityOwner::Compensation { + compensation_id, + compensation_generation, + compensation_attempt_generation, + } => { + !compensation_id.is_nil() + && compensation_generation > 0 + && compensation_attempt_generation > 0 + } }; if !valid || !run_fence_valid || !owner_identity_valid || request.reservation_uid.is_nil() { return Err(Error::InvalidRepositoryInput { @@ -1192,14 +1283,53 @@ fn validate_generic_capacity_request(request: &ExecutionCapacityRequest) -> Resu const fn owner_trigger_uid(owner: ExecutionCapacityOwner) -> Option { match owner { ExecutionCapacityOwner::Trigger { trigger_uid } => Some(trigger_uid), - ExecutionCapacityOwner::Run | ExecutionCapacityOwner::ExternalJob { .. } => None, + ExecutionCapacityOwner::Run + | ExecutionCapacityOwner::ExternalJob { .. } + | ExecutionCapacityOwner::Compensation { .. } => None, } } const fn owner_external_job_uid(owner: ExecutionCapacityOwner) -> Option { match owner { ExecutionCapacityOwner::ExternalJob { external_job_uid } => Some(external_job_uid), - ExecutionCapacityOwner::Run | ExecutionCapacityOwner::Trigger { .. } => None, + ExecutionCapacityOwner::Run + | ExecutionCapacityOwner::Trigger { .. } + | ExecutionCapacityOwner::Compensation { .. } => None, + } +} + +const fn owner_compensation_uid(owner: ExecutionCapacityOwner) -> Option { + match owner { + ExecutionCapacityOwner::Compensation { + compensation_id, .. + } => Some(compensation_id), + ExecutionCapacityOwner::Run + | ExecutionCapacityOwner::Trigger { .. } + | ExecutionCapacityOwner::ExternalJob { .. } => None, + } +} + +const fn owner_compensation_generation(owner: ExecutionCapacityOwner) -> Option { + match owner { + ExecutionCapacityOwner::Compensation { + compensation_generation, + .. + } => Some(compensation_generation), + ExecutionCapacityOwner::Run + | ExecutionCapacityOwner::Trigger { .. } + | ExecutionCapacityOwner::ExternalJob { .. } => None, + } +} + +const fn owner_compensation_attempt_generation(owner: ExecutionCapacityOwner) -> Option { + match owner { + ExecutionCapacityOwner::Compensation { + compensation_attempt_generation, + .. + } => Some(compensation_attempt_generation), + ExecutionCapacityOwner::Run + | ExecutionCapacityOwner::Trigger { .. } + | ExecutionCapacityOwner::ExternalJob { .. } => None, } } @@ -1418,7 +1548,11 @@ async fn increment_capacity( Ok(()) } -async fn advance_tenant_fairness( +/// Charges one admitted attempt against the tenant's weighted-fair virtual finish. +/// +/// Forward and compensation admission both call this, so a tenant's rollback work consumes the +/// same weighted-fair credit as its forward work instead of jumping the queue for free. +pub(super) async fn advance_tenant_fairness( conn: &mut ScopedConn<'_>, tenant_id: Uuid, now: DateTime, @@ -1435,7 +1569,6 @@ async fn advance_tenant_fairness( UPDATE moa.execution_tenant_dispatch_state AS state \ SET virtual_finish = GREATEST(state.virtual_finish, active_floor.value) \ + ($2::NUMERIC / state.weight), \ - deficit = state.deficit + state.weight - 1, \ last_dispatched_at = $3, version = state.version + 1, updated_at = NOW() \ FROM active_floor WHERE state.tenant_id = $1", ) diff --git a/crates/moa-execution/src/repository/compensation.rs b/crates/moa-execution/src/repository/compensation.rs index 7e67c0090..3de24a563 100644 --- a/crates/moa-execution/src/repository/compensation.rs +++ b/crates/moa-execution/src/repository/compensation.rs @@ -3,8 +3,10 @@ use super::*; use super::{ capacity::{ - ExecutionCapacityDimension, prelock_capacity_dimensions_in_tx, - release_owned_run_capacity_in_tx, + CapacityReleaseOutcome, CapacityReserveOutcome, ExecutionCapacityDimension, + advance_tenant_fairness, capacity_bucket_has_room, compensation_attempt_capacity_request, + prelock_capacity_dimensions_in_tx, prelock_existing_capacity_dimensions_in_tx, + release_capacity_in_tx, release_owned_run_capacity_in_tx, reserve_capacity_in_tx, }, external_job::{ ExecutionExternalJobCancellationRequestOutcome, ExecutionExternalJobIntentReleaseOutcome, @@ -29,7 +31,7 @@ use super::{ }, trigger::{ ExecutionTriggerKind, ExecutionTriggerWrite, NewExecutionTrigger, - create_trigger_with_dispatch_in_conn, + create_trigger_with_dispatch_in_conn, release_trigger_capacity_in_conn, trigger_from_row, }, }; use crate::{ @@ -593,10 +595,15 @@ impl ExecutionRepository { conn.commit().await.map_err(storage_error)?; return Ok(PendingTerminalAdvanceOutcome::Conflict); }; - let expected_stop_reason = match &pending.terminal_evidence.cause { - ExecutionTerminalCause::ReplanStop { reason } => reason.as_str(), - _ => unreachable!("validated replan-stop terminal cause"), + let ExecutionTerminalCause::ReplanStop { + reason: expected_stop_reason, + } = &pending.terminal_evidence.cause + else { + return Err(Error::InvalidRepositoryData { + message: "replan-stop fence lost its validated terminal cause".to_string(), + }); }; + let expected_stop_reason = expected_stop_reason.as_str(); let intent_exact = intent.try_get::("tenant_id").map_err(row_error)? == run.tenant_id.0 && required_u64(&intent, "controller_generation")? == controller_generation @@ -754,11 +761,11 @@ impl ExecutionRepository { return Ok(CompensationAttemptAdmissionOutcome::NotFound); }; let visible_run = run_from_row(&run_row)?; - lock_compensation_capacity( - &mut conn, + prelock_capacity_dimensions_in_tx( + conn.as_mut(), + config, visible_run.tenant_id, - config.max_fleet_active_tasks, - config.max_tenant_active_tasks, + &[ExecutionCapacityDimension::ActiveTasks], ) .await?; let Some(run_row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) @@ -852,18 +859,17 @@ impl ExecutionRepository { } let attempt_generation = required_u64(&row, "attempt_generation")?; let dispatch_uid = Uuid::now_v7(); - let reservation_uid = Uuid::now_v7(); let watchdog_uid = Uuid::now_v7(); - insert_compensation_capacity_reservation( + let reservation_uid = reserve_compensation_attempt_capacity( &mut conn, - reservation_uid, + config, &run, ®istration, attempt_generation, deadline, + now, ) .await?; - increment_compensation_capacity(&mut conn, run.tenant_id).await?; let watchdog = create_trigger_with_dispatch_in_conn( conn.as_mut(), config, @@ -1132,7 +1138,12 @@ impl ExecutionRepository { conn.commit().await.map_err(storage_error)?; return Ok(CompensationAttemptWriteOutcome::NotFound); }; - lock_capacity_for_release(&mut conn, tenant_id).await?; + prelock_existing_capacity_dimensions_in_tx( + conn.as_mut(), + tenant_id, + &[ExecutionCapacityDimension::ActiveTasks], + ) + .await?; let Some(run_row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) .bind(request.run_uid) .fetch_optional(conn.as_mut()) @@ -1180,7 +1191,7 @@ impl ExecutionRepository { } release_unbound_compensation_external_intent_in_conn(&mut conn, &run, ¤t).await?; let resource_fence = compensation_cancel_resource_fence(request); - release_compensation_capacity(&mut conn, &run, resource_fence).await?; + release_compensation_attempt_capacity(&mut conn, &run, resource_fence).await?; supersede_compensation_triggers(&mut conn, resource_fence, None).await?; let updated = sqlx::query( "UPDATE moa.execution_compensation SET attempt_state='idle', \ @@ -1253,7 +1264,12 @@ impl ExecutionRepository { conn.commit().await.map_err(storage_error)?; return Ok(CompensationAttemptWriteOutcome::NotFound); }; - lock_capacity_for_release(&mut conn, tenant_id).await?; + prelock_existing_capacity_dimensions_in_tx( + conn.as_mut(), + tenant_id, + &[ExecutionCapacityDimension::ActiveTasks], + ) + .await?; let Some(run_row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) .bind(request.run_uid) .fetch_optional(conn.as_mut()) @@ -1305,7 +1321,7 @@ impl ExecutionRepository { return Ok(CompensationAttemptWriteOutcome::Conflict); } let resource_fence = compensation_cancel_resource_fence(request); - release_compensation_capacity(&mut conn, &run, resource_fence).await?; + release_compensation_attempt_capacity(&mut conn, &run, resource_fence).await?; supersede_compensation_triggers(&mut conn, resource_fence, None).await?; let updated = sqlx::query( "UPDATE moa.execution_compensation SET attempt_state='idle', \ @@ -1377,7 +1393,12 @@ impl ExecutionRepository { conn.commit().await.map_err(storage_error)?; return Ok(CompensationAttemptWriteOutcome::NotFound); }; - lock_capacity_for_release(&mut conn, tenant_id).await?; + prelock_existing_capacity_dimensions_in_tx( + conn.as_mut(), + tenant_id, + &[ExecutionCapacityDimension::ActiveTasks], + ) + .await?; let Some((run, row)) = load_fenced_compensation_for_cancel(&mut conn, request).await? else { conn.commit().await.map_err(storage_error)?; @@ -1426,7 +1447,7 @@ impl ExecutionRepository { } release_unbound_compensation_external_intent_in_conn(&mut conn, &run, ¤t).await?; let resource_fence = compensation_cancel_resource_fence(request); - release_compensation_capacity(&mut conn, &run, resource_fence).await?; + release_compensation_attempt_capacity(&mut conn, &run, resource_fence).await?; supersede_compensation_triggers(&mut conn, resource_fence, None).await?; persisted.review_audit.push(CompensationReviewAuditEntry { review_uid, @@ -1513,7 +1534,12 @@ impl ExecutionRepository { conn.commit().await.map_err(storage_error)?; return Ok(CompensationAttemptExternalOutcome::NotFound); }; - lock_capacity_for_release(&mut conn, tenant_id).await?; + prelock_existing_capacity_dimensions_in_tx( + conn.as_mut(), + tenant_id, + &[ExecutionCapacityDimension::ActiveTasks], + ) + .await?; let Some((run, row)) = load_fenced_compensation_for_cancel(&mut conn, request).await? else { conn.commit().await.map_err(storage_error)?; @@ -1571,7 +1597,7 @@ impl ExecutionRepository { return Ok(CompensationAttemptExternalOutcome::Stale); } let resource_fence = compensation_cancel_resource_fence(request); - release_compensation_capacity(&mut conn, &run, resource_fence).await?; + release_compensation_attempt_capacity(&mut conn, &run, resource_fence).await?; supersede_compensation_triggers(&mut conn, resource_fence, None).await?; let row = sqlx::query( "UPDATE moa.execution_compensation SET attempt_state='waiting_external', \ @@ -1834,7 +1860,12 @@ impl ExecutionRepository { conn.commit().await.map_err(storage_error)?; return Ok(CompensationAttemptWriteOutcome::NotFound); }; - lock_capacity_for_release(&mut conn, tenant_id).await?; + prelock_existing_capacity_dimensions_in_tx( + conn.as_mut(), + tenant_id, + &[ExecutionCapacityDimension::ActiveTasks], + ) + .await?; let loaded = if let Some(request) = cancellation_request { load_fenced_compensation_for_cancel(&mut conn, request).await? } else { @@ -1907,7 +1938,7 @@ impl ExecutionRepository { let resource_fence = cancellation_request .map(compensation_cancel_resource_fence) .unwrap_or(fence); - release_compensation_capacity(&mut conn, &run, resource_fence).await?; + release_compensation_attempt_capacity(&mut conn, &run, resource_fence).await?; supersede_compensation_triggers(&mut conn, resource_fence, None).await?; let updated = sqlx::query( "UPDATE moa.execution_compensation SET status=$6, attempt_state=$7, attempt=$8, \ @@ -2239,195 +2270,78 @@ fn checked_retry_at(config: &ExecutionConfig, now: DateTime) -> Result, - tenant_id: TenantId, - fleet_limit: u32, - tenant_limit: u32, -) -> Result<()> { - for (scope_kind, owner, limit) in [ - ("fleet", None, fleet_limit), - ("tenant", Some(tenant_id.0), tenant_limit), - ] { - sqlx::query( - "INSERT INTO moa.execution_capacity_bucket (capacity_bucket_uid, scope_kind, \ - tenant_id, resource_dimension, limit_value) VALUES ($1, $2, $3, \ - 'active_tasks', $4) ON CONFLICT DO NOTHING", - ) - .bind(Uuid::now_v7()) - .bind(scope_kind) - .bind(owner) - .bind(i64::from(limit)) - .execute(conn.as_mut()) - .await - .map_err(sqlx_error)?; - sqlx::query( - "UPDATE moa.execution_capacity_bucket SET limit_value=$4, version=version+1, \ - updated_at=NOW() WHERE scope_kind=$1 AND tenant_id IS NOT DISTINCT FROM $2 \ - AND resource_dimension=$3 RETURNING capacity_bucket_uid", - ) - .bind(scope_kind) - .bind(owner) - .bind("active_tasks") - .bind(i64::from(limit)) - .fetch_one(conn.as_mut()) - .await - .map_err(sqlx_error)?; - } - Ok(()) -} - -async fn lock_capacity_for_release(conn: &mut ScopedConn<'_>, tenant_id: TenantId) -> Result<()> { - for (scope_kind, owner) in [("fleet", None), ("tenant", Some(tenant_id.0))] { - sqlx::query( - "SELECT capacity_bucket_uid FROM moa.execution_capacity_bucket \ - WHERE scope_kind=$1 AND tenant_id IS NOT DISTINCT FROM $2 \ - AND resource_dimension='active_tasks' FOR UPDATE", - ) - .bind(scope_kind) - .bind(owner) - .fetch_one(conn.as_mut()) - .await - .map_err(sqlx_error)?; - } - Ok(()) -} - +/// Reports whether the locked `active_tasks` buckets can admit one compensation attempt. async fn compensation_capacity_available( conn: &mut ScopedConn<'_>, tenant_id: TenantId, ) -> Result { - let rows: Vec<(String, i64, i64)> = sqlx::query_as( - "SELECT scope_kind, limit_value, reserved_quantity \ - FROM moa.execution_capacity_bucket WHERE resource_dimension='active_tasks' \ - AND ((scope_kind='fleet' AND tenant_id IS NULL) \ - OR (scope_kind='tenant' AND tenant_id=$1))", - ) - .bind(tenant_id.0) - .fetch_all(conn.as_mut()) - .await - .map_err(sqlx_error)?; - Ok(rows.len() == 2 && rows.iter().all(|(_, limit, reserved)| reserved < limit)) + let dimension = ExecutionCapacityDimension::ActiveTasks.as_str(); + let fleet = capacity_bucket_has_room(conn.as_mut(), "fleet", None, dimension).await?; + let tenant = + capacity_bucket_has_room(conn.as_mut(), "tenant", Some(tenant_id.0), dimension).await?; + Ok(fleet && tenant) } -async fn increment_compensation_capacity( - conn: &mut ScopedConn<'_>, - tenant_id: TenantId, -) -> Result<()> { - for (scope_kind, owner) in [("fleet", None), ("tenant", Some(tenant_id.0))] { - let updated = sqlx::query( - "UPDATE moa.execution_capacity_bucket SET reserved_quantity=reserved_quantity+1, \ - version=version+1, updated_at=NOW() WHERE scope_kind=$1 \ - AND tenant_id IS NOT DISTINCT FROM $2 AND resource_dimension='active_tasks' \ - AND reserved_quantity < limit_value", - ) - .bind(scope_kind) - .bind(owner) - .execute(conn.as_mut()) - .await - .map_err(sqlx_error)?; - if updated.rows_affected() != 1 { - return Err(Error::InvalidRepositoryData { - message: format!("locked {scope_kind} compensation capacity was over-admitted"), - }); - } - } - Ok(()) -} - -async fn insert_compensation_capacity_reservation( +/// Reserves the exact active-task receipt for one compensation attempt and charges fairness. +/// +/// Compensation shares the `active_tasks` dimension with forward attempts, so it also shares the +/// weighted-fair accounting: an admitted rollback advances the tenant's virtual finish exactly +/// like a forward dispatch instead of consuming fleet capacity outside the scheduler. +async fn reserve_compensation_attempt_capacity( conn: &mut ScopedConn<'_>, - reservation_uid: Uuid, + config: &ExecutionConfig, run: &ExecutionRunRecord, registration: &CompensationRegistrationProjection, attempt_generation: u64, deadline: DateTime, -) -> Result<()> { - sqlx::query( - "INSERT INTO moa.execution_capacity_reservation (reservation_uid, tenant_id, run_uid, \ - compensation_id, controller_generation, compensation_generation, \ - compensation_attempt_generation, resource_dimension, quantity, expires_at) \ - VALUES ($1,$2,$3,$4,$5,$6,$7,'active_tasks',1,$8)", - ) - .bind(reservation_uid) - .bind(run.tenant_id.0) - .bind(run.run_uid) - .bind(registration.compensation_id.as_uuid()) - .bind(to_i64(run.controller_generation, "controller generation")?) - .bind(to_i64(registration.generation, "compensation generation")?) - .bind(to_i64( + now: DateTime, +) -> Result { + let request = compensation_attempt_capacity_request( + run.tenant_id, + run.run_uid, + run.controller_generation, + registration.compensation_id.as_uuid(), + registration.generation, attempt_generation, - "compensation attempt generation", - )?) - .bind(deadline) - .execute(conn.as_mut()) - .await - .map_err(sqlx_error)?; - Ok(()) + Some(deadline), + ); + match reserve_capacity_in_tx(conn.as_mut(), config, request).await? { + CapacityReserveOutcome::Reserved | CapacityReserveOutcome::Replayed => {} + CapacityReserveOutcome::Saturated => { + return Err(Error::InvalidRepositoryData { + message: "locked active-task capacity rejected an admitted compensation attempt" + .to_string(), + }); + } + } + advance_tenant_fairness(conn, run.tenant_id.0, now).await?; + Ok(request.reservation_uid) } -async fn release_compensation_capacity( +/// Releases the exact active-task receipt owned by one settled compensation attempt. +async fn release_compensation_attempt_capacity( conn: &mut ScopedConn<'_>, run: &ExecutionRunRecord, fence: CompensationAttemptFence, ) -> Result<()> { - let reservation_uid: Option = sqlx::query_scalar( - "SELECT reservation_uid FROM moa.execution_capacity_reservation \ - WHERE tenant_id=$1 AND run_uid=$2 AND compensation_id=$3 \ - AND controller_generation=$4 AND compensation_generation=$5 \ - AND compensation_attempt_generation=$6 AND resource_dimension='active_tasks' \ - AND state IN ('reserved','reconciling') FOR UPDATE", - ) - .bind(run.tenant_id.0) - .bind(fence.run_uid) - .bind(fence.compensation_id.as_uuid()) - .bind(to_i64( + let request = compensation_attempt_capacity_request( + run.tenant_id, + fence.run_uid, fence.controller_generation, - "controller generation", - )?) - .bind(to_i64( + fence.compensation_id.as_uuid(), fence.compensation_generation, - "compensation generation", - )?) - .bind(to_i64( fence.attempt_generation, - "compensation attempt generation", - )?) - .fetch_optional(conn.as_mut()) - .await - .map_err(sqlx_error)?; - let Some(reservation_uid) = reservation_uid else { - return Err(Error::InvalidRepositoryData { - message: "active compensation slice lost its capacity reservation".to_string(), - }); - }; - for (scope_kind, owner) in [("fleet", None), ("tenant", Some(run.tenant_id.0))] { - let updated = sqlx::query( - "UPDATE moa.execution_capacity_bucket SET reserved_quantity=reserved_quantity-1, \ - version=version+1, updated_at=NOW() WHERE scope_kind=$1 \ - AND tenant_id IS NOT DISTINCT FROM $2 AND resource_dimension='active_tasks' \ - AND reserved_quantity >= 1", - ) - .bind(scope_kind) - .bind(owner) - .execute(conn.as_mut()) - .await - .map_err(sqlx_error)?; - if updated.rows_affected() != 1 { - return Err(Error::InvalidRepositoryData { - message: format!("{scope_kind} active compensation capacity underflow"), - }); + None, + ); + match release_capacity_in_tx(conn.as_mut(), request).await? { + // A replayed settlement legitimately observes its own already-released receipt. + CapacityReleaseOutcome::Released | CapacityReleaseOutcome::AlreadyReleased => Ok(()), + CapacityReleaseOutcome::NotFound | CapacityReleaseOutcome::Stale => { + Err(Error::InvalidRepositoryData { + message: "active compensation slice lost its capacity reservation".to_string(), + }) } } - sqlx::query( - "UPDATE moa.execution_capacity_reservation SET state='released', released_at=NOW(), \ - updated_at=NOW() WHERE reservation_uid=$1", - ) - .bind(reservation_uid) - .execute(conn.as_mut()) - .await - .map_err(sqlx_error)?; - Ok(()) } fn compensation_dispatch( @@ -2518,7 +2432,7 @@ async fn load_existing_compensation_admission( "SELECT trigger_uid, due_at, payload FROM moa.execution_trigger \ WHERE run_uid=$1 AND compensation_id=$2 AND trigger_kind='compensation_watchdog' \ AND controller_generation=$3 AND compensation_generation=$4 \ - AND compensation_attempt_generation=$5 AND state IN ('pending','dispatching')", + AND compensation_attempt_generation=$5 AND state = 'pending'", ) .bind(run.run_uid) .bind(registration.compensation_id.as_uuid()) @@ -2596,8 +2510,14 @@ async fn drive_pending_terminal_compensation_in_conn( .fetch_one(conn.as_mut()) .await .map_err(sqlx_error)?; + // `manual_repair_required` is deliberately NOT rejected here. Settling a compensation + // attempt with a non-retryable failure sets that flag on the run, so rejecting it made + // the very next controller activation return a terminal repository error and the run + // sat in `compensating` forever instead of terminalizing `Failed`/`CompensationFailed`. + // The flag means "stop driving automatically and hand this to an operator", which is a + // `ManualRepair` outcome, not an invalid state — see the check below the registration + // load, which needs the row to report which compensation is stuck. if run.status != ExecutionRunStatus::Compensating - || run.manual_repair_required || run.pending_terminal.is_none() || nonterminal_forward_exists { @@ -2619,10 +2539,12 @@ async fn drive_pending_terminal_compensation_in_conn( }; let registration = compensation_from_row(&row)?; let attempt_state = compensation_attempt_state_from_row(&row)?; - if matches!( - registration.status, - CompensationStatus::Failed | CompensationStatus::UnknownOutcome - ) { + if run.manual_repair_required + || matches!( + registration.status, + CompensationStatus::Failed | CompensationStatus::UnknownOutcome + ) + { return Ok(PendingCompensationDrive::ManualRepair(registration)); } if attempt_state == CompensationAttemptState::Dispatching { @@ -2717,18 +2639,17 @@ async fn drive_pending_terminal_compensation_in_conn( let deadline = checked_attempt_deadline(config, now)?; let attempt_generation = required_u64(&row, "attempt_generation")?; let dispatch_uid = Uuid::now_v7(); - let reservation_uid = Uuid::now_v7(); let watchdog_uid = Uuid::now_v7(); - insert_compensation_capacity_reservation( + let reservation_uid = reserve_compensation_attempt_capacity( conn, - reservation_uid, + config, run, ®istration, attempt_generation, deadline, + now, ) .await?; - increment_compensation_capacity(conn, run.tenant_id).await?; let watchdog = create_trigger_with_dispatch_in_conn( conn.as_mut(), config, @@ -2942,7 +2863,7 @@ async fn compensation_attempt_resources_match( AND trigger.compensation_generation=$6 \ AND trigger.compensation_attempt_generation=$7 \ AND trigger.trigger_kind='compensation_watchdog' \ - AND trigger.state IN ('pending','dispatching'))", + AND trigger.state = 'pending')", ) .bind(request.capacity_reservation_uid) .bind(request.tenant_id.0) @@ -2985,7 +2906,7 @@ async fn canonical_active_compensation_release_request( AND trigger.compensation_generation=reservation.compensation_generation \ AND trigger.compensation_attempt_generation=reservation.compensation_attempt_generation \ AND trigger.trigger_kind='compensation_watchdog' \ - AND trigger.state IN ('pending','dispatching') \ + AND trigger.state = 'pending' \ WHERE reservation.tenant_id=$1 AND reservation.run_uid=$2 \ AND reservation.compensation_id=$3 AND reservation.controller_generation=$4 \ AND reservation.compensation_generation=$5 \ @@ -3445,7 +3366,7 @@ pub(super) async fn begin_compensation_external_not_started_release_in_conn( AND trigger.compensation_generation=reservation.compensation_generation \ AND trigger.compensation_attempt_generation=reservation.compensation_attempt_generation \ AND trigger.trigger_kind='compensation_watchdog' \ - AND trigger.state IN ('pending','dispatching') \ + AND trigger.state = 'pending' \ WHERE reservation.tenant_id=$1 AND reservation.run_uid=$2 \ AND reservation.compensation_id=$3 AND reservation.controller_generation=$4 \ AND reservation.compensation_generation=$5 \ @@ -3608,7 +3529,12 @@ pub(super) async fn settle_unstarted_compensation_attempt_in_conn( if tenant_id != request.tenant_id { return Ok(CompensationAttemptWriteOutcome::Conflict); } - lock_capacity_for_release(conn, tenant_id).await?; + prelock_existing_capacity_dimensions_in_tx( + conn.as_mut(), + tenant_id, + &[ExecutionCapacityDimension::ActiveTasks], + ) + .await?; let run_row = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) .bind(request.run_uid) .fetch_one(conn.as_mut()) @@ -3641,7 +3567,7 @@ pub(super) async fn settle_unstarted_compensation_attempt_in_conn( AND EXISTS (SELECT 1 FROM moa.execution_trigger WHERE trigger_uid=$8 AND run_uid=$3 \ AND compensation_id=$4 AND controller_generation=$5 AND compensation_generation=$6 \ AND compensation_attempt_generation=$7 AND trigger_kind='compensation_watchdog' \ - AND state IN ('pending','dispatching'))", + AND state = 'pending')", ) .bind(request.capacity_reservation_uid) .bind(request.tenant_id.0) @@ -3681,7 +3607,7 @@ pub(super) async fn settle_unstarted_compensation_attempt_in_conn( dispatch_uid: request.dispatch_uid, }; release_unbound_compensation_external_intent_in_conn(conn, &run, ¤t).await?; - release_compensation_capacity(conn, &run, fence).await?; + release_compensation_attempt_capacity(conn, &run, fence).await?; supersede_compensation_triggers(conn, fence, None).await?; let updated = sqlx::query( "UPDATE moa.execution_compensation SET attempt_state='idle', \ @@ -3753,12 +3679,20 @@ async fn supersede_compensation_triggers( .execute(conn.as_mut()) .await .map_err(sqlx_error)?; - sqlx::query( - "UPDATE moa.execution_trigger SET state='superseded', claim_owner=NULL, claimed_at=NULL, \ - claim_expires_at=NULL, updated_at=NOW() WHERE run_uid=$1 AND compensation_id=$2 \ + // `RETURNING *` rather than a bare UPDATE: every superseded trigger still owns a + // `scheduled_triggers` capacity receipt, and superseding the row does not release it. + // Leaking it wedges the run permanently — both terminal-drain branches probe for any + // held `active_tasks`/`scheduled_triggers`/`external_jobs` reservation and hard-error + // ("failed compensation retained non-lifetime capacity" / "completed compensation + // retained non-lifetime capacity"), so a compensating run could never finalize whether + // its undo failed OR succeeded. `trigger.rs::supersede_trigger_in_conn` releases on + // every arm; this bulk path must do the same. + let superseded = sqlx::query( + "UPDATE moa.execution_trigger SET state='superseded', updated_at=NOW() \ + WHERE run_uid=$1 AND compensation_id=$2 \ AND controller_generation=$3 AND compensation_generation=$4 \ AND compensation_attempt_generation=$5 AND ($6::TEXT IS NULL OR trigger_kind <> $6) \ - AND state IN ('pending','dispatching')", + AND state = 'pending' RETURNING *", ) .bind(fence.run_uid) .bind(fence.compensation_id.as_uuid()) @@ -3775,9 +3709,13 @@ async fn supersede_compensation_triggers( "compensation attempt generation", )?) .bind(except) - .execute(conn.as_mut()) + .fetch_all(conn.as_mut()) .await .map_err(sqlx_error)?; + for row in &superseded { + let trigger = trigger_from_row(row)?; + release_trigger_capacity_in_conn(conn.as_mut(), &trigger).await?; + } Ok(()) } @@ -4524,7 +4462,7 @@ async fn advance_pending_terminal_page_in_conn( let should_compensate = has_registrations && !retain_cancelled_effects; let active_trigger_exists: bool = sqlx::query_scalar( "SELECT EXISTS (SELECT 1 FROM moa.execution_trigger WHERE run_uid=$1 \ - AND state IN ('pending','dispatching'))", + AND state = 'pending')", ) .bind(run.run_uid) .fetch_one(conn.as_mut()) @@ -4835,7 +4773,7 @@ async fn enqueue_pending_terminal_task_cancellation( AND trigger.controller_generation=reservation.controller_generation \ AND trigger.attempt_generation=reservation.attempt_generation \ AND trigger.trigger_kind='task_watchdog' \ - AND trigger.state IN ('pending','dispatching') \ + AND trigger.state = 'pending' \ WHERE reservation.run_uid=$1 AND reservation.task_id=$2 \ AND reservation.controller_generation=$3 AND reservation.attempt_generation=$4 \ AND reservation.resource_dimension='active_tasks' \ @@ -4963,7 +4901,7 @@ async fn enqueue_pending_terminal_compensation_cancellation( AND trigger.compensation_generation=reservation.compensation_generation \ AND trigger.compensation_attempt_generation=reservation.compensation_attempt_generation \ AND trigger.trigger_kind='compensation_watchdog' \ - AND trigger.state IN ('pending','dispatching') \ + AND trigger.state = 'pending' \ WHERE reservation.run_uid=$1 AND reservation.compensation_id=$2 \ AND reservation.controller_generation=$3 AND reservation.compensation_generation=$4 \ AND reservation.compensation_attempt_generation=$5 \ @@ -5205,9 +5143,9 @@ async fn supersede_storage_task_waits( task: &ExecutionTaskRecord, ) -> Result<()> { let trigger_uids = sqlx::query_scalar::<_, Uuid>( - "UPDATE moa.execution_trigger SET state='superseded', claimed_at=NULL, \ - claimed_by=NULL, updated_at=NOW() WHERE run_uid=$1 AND task_id=$2 \ - AND trigger_kind <> 'task_watchdog' AND state IN ('pending','dispatching') \ + "UPDATE moa.execution_trigger SET state='superseded', updated_at=NOW() \ + WHERE run_uid=$1 AND task_id=$2 \ + AND trigger_kind <> 'task_watchdog' AND state = 'pending' \ RETURNING trigger_uid", ) .bind(task.run_uid) @@ -5217,8 +5155,9 @@ async fn supersede_storage_task_waits( .map_err(sqlx_error)?; if !trigger_uids.is_empty() { sqlx::query( - "UPDATE moa.execution_dispatch_outbox SET state='superseded', claimed_at=NULL, \ - claimed_by=NULL, updated_at=NOW() WHERE trigger_uid=ANY($1::UUID[]) \ + "UPDATE moa.execution_dispatch_outbox SET state='superseded', claim_owner=NULL, \ + claimed_at=NULL, claim_expires_at=NULL, updated_at=NOW() \ + WHERE trigger_uid=ANY($1::UUID[]) \ AND state IN ('pending','dispatching')", ) .bind(&trigger_uids) diff --git a/crates/moa-execution/src/repository/completion.rs b/crates/moa-execution/src/repository/completion.rs index 1cc9c1429..b1a7223fb 100644 --- a/crates/moa-execution/src/repository/completion.rs +++ b/crates/moa-execution/src/repository/completion.rs @@ -569,8 +569,10 @@ impl ExecutionRepository { .extend(replan_stop_gaps(intent.stop_reason, Some(&intent.detail))); evaluation.gaps.sort(); evaluation.gaps.dedup(); + let typed_failure = + load_earliest_typed_task_failure(conn.as_mut(), run.run_uid).await?; let terminal_projection = - terminal_projection_for_evaluation(&evaluation, terminal_output)?; + terminal_projection_for_evaluation(&evaluation, terminal_output, typed_failure)?; let cause = ExecutionTerminalCause::ReplanStop { reason: intent.stop_reason, }; @@ -600,7 +602,9 @@ impl ExecutionRepository { receipt: intent.receipt(), }); } - let terminal_projection = terminal_projection_for_evaluation(&evaluation, terminal_output)?; + let typed_failure = load_earliest_typed_task_failure(conn.as_mut(), run.run_uid).await?; + let terminal_projection = + terminal_projection_for_evaluation(&evaluation, terminal_output, typed_failure)?; if evaluation.status != CompletionStatus::Completed { let cause = ExecutionTerminalCause::Completion { limit_stop: evaluation.limit_stop, @@ -1529,18 +1533,72 @@ fn validate_completion_runtime_bounds( Ok(()) } +/// Loads the earliest typed task failure so a run terminal keeps its real class. +/// +/// Without this the run terminal reports a generic [`ExecutionFailureClass::Terminal`], +/// so a run that died on a rate limit, an unsupported capability, or an authorization +/// denial is indistinguishable from any other failure at the product surface. One +/// bounded lookup on the failing task restores the attribution. +async fn load_earliest_typed_task_failure( + conn: &mut sqlx::PgConnection, + run_uid: Uuid, +) -> Result> { + let row = sqlx::query( + "SELECT current_outcome \ + FROM moa.execution_task \ + WHERE run_uid = $1 \ + AND status IN ('failed', 'unknown_outcome') \ + AND current_outcome IS NOT NULL \ + ORDER BY completed_at NULLS LAST, task_id \ + LIMIT 1", + ) + .bind(run_uid) + .fetch_optional(conn) + .await + .map_err(sqlx_error)?; + let Some(row) = row else { + return Ok(None); + }; + let outcome: Value = row.try_get("current_outcome").map_err(row_error)?; + let outcome: ExecutionTaskOutcome = + serde_json::from_value(outcome).map_err(|error| Error::InvalidRepositoryData { + message: format!("terminal failure outcome is undecodable: {error}"), + })?; + Ok(match outcome.result { + ExecutionTaskResult::Failed { class, message } => Some(ExecutionTaskFailure { + class, + message, + capability_ref: None, + }), + _ => None, + }) +} + fn terminal_projection_for_evaluation( evaluation: &CompletionEvaluation, output: Option, + typed_failure: Option, ) -> Result { crate::completion::terminal_projection_from_evaluation( evaluation, output, None, - (evaluation.status == CompletionStatus::Failed).then(|| ExecutionTaskFailure { - class: ExecutionFailureClass::Terminal, - message: evaluation.gaps.join("; "), - capability_ref: None, + (evaluation.status == CompletionStatus::Failed).then(|| { + // Prefer the failing task's own class over a flattened `Terminal`, but keep + // the evaluation gaps as the message: they explain which requirements were + // unmet, which the task-level message does not. + typed_failure.map_or_else( + || ExecutionTaskFailure { + class: ExecutionFailureClass::Terminal, + message: evaluation.gaps.join("; "), + capability_ref: None, + }, + |failure| ExecutionTaskFailure { + class: failure.class, + message: evaluation.gaps.join("; "), + capability_ref: failure.capability_ref, + }, + ) }), (evaluation.status == CompletionStatus::Unsupported) .then(|| "required execution paths are unsupported".to_string()), diff --git a/crates/moa-execution/src/repository/external_job.rs b/crates/moa-execution/src/repository/external_job.rs index 054ff83fc..7f1447860 100644 --- a/crates/moa-execution/src/repository/external_job.rs +++ b/crates/moa-execution/src/repository/external_job.rs @@ -992,7 +992,7 @@ async fn replace_external_reconcile_trigger_in_conn( FROM moa.execution_trigger WHERE tenant_id=$1 AND run_uid=$2 \ AND task_id IS NOT DISTINCT FROM $3 \ AND compensation_id IS NOT DISTINCT FROM $4 \ - AND trigger_kind='external_reconcile' AND state IN ('pending','dispatching') \ + AND trigger_kind='external_reconcile' AND state = 'pending' \ FOR UPDATE", ) .bind(job.tenant_id.0) diff --git a/crates/moa-execution/src/repository/outbox.rs b/crates/moa-execution/src/repository/outbox.rs index aecccd06c..c0a9a182a 100644 --- a/crates/moa-execution/src/repository/outbox.rs +++ b/crates/moa-execution/src/repository/outbox.rs @@ -22,6 +22,24 @@ use super::{ const MAX_CLAIM_BATCH_SIZE: u32 = 1_000; const MAX_HEALTH_SAMPLE_SIZE: u32 = 100_000; +/// Per-phase census of the live nonterminal fleet. +/// +/// The status list is spelled out rather than bound as a parameter so it matches the +/// `execution_run_nonterminal_idx` predicate literally: the planner cannot prove a bound +/// array implies that predicate, and a parameterized form loses the index-only scan. The +/// list is pinned against [`ExecutionRunPhaseDimension::ALL`] by an offline test, and the +/// index leads on `status` so the grouped aggregate needs no sort. The scan is proportional +/// to the live nonterminal set, not to run history, because the index is partial. +const RUN_PHASE_CENSUS_SQL: &str = r#" +SELECT status, count(*)::BIGINT AS run_count +FROM moa.execution_run +WHERE status IN ( + 'awaiting_confirmation', 'queued', 'running', 'waiting_input', + 'waiting_review', 'waiting_signal', 'waiting_timer', 'waiting_external', + 'waiting_replan', 'pause_requested', 'pausing', 'paused', 'compensating' +) +GROUP BY status +"#; const MAX_ERROR_CHARS: usize = 4_096; const MAX_MAINTENANCE_ERROR_BYTES: usize = 4_096; @@ -208,6 +226,12 @@ pub struct ExecutionDispatchRecord { pub claim_expires_at: Option>, /// Number of bounded delivery attempts. pub delivery_attempts: u32, + /// Monotonic repair generation, incremented once per recovery requeue. + /// + /// Delivery identity is `dispatch_uid` while this is zero and + /// `{dispatch_uid}:{repair_epoch}` afterwards, so a repaired row cannot attach to the + /// completed invocation its original identity already memoized. + pub repair_epoch: u32, /// Successful delivery time. pub delivered_at: Option>, /// Latest bounded delivery error. @@ -258,8 +282,13 @@ impl ExecutionDispatchRetryPolicy { pub enum ExecutionDispatchFailureOutcome { /// The exact claim was released behind this durable retry time. RetryScheduled { not_before_at: DateTime }, - /// The exact claim exhausted its delivery budget. + /// The exact claim exhausted its delivery budget and its owner was repaired. DeadLettered, + /// The exact claim exhausted its delivery budget while its owner was already settled. + /// + /// The dead letter still commits: refusing it would re-poison every drain that reclaims + /// the row. The caller is expected to surface this as an operator-visible anomaly. + DeadLetteredWithoutOwnerRepair, /// The row was absent, already terminal, or owned by another claim. StaleClaim, } @@ -275,8 +304,196 @@ pub struct ExecutionQueueBacklogSample { pub saturated: bool, } +/// Bounded long-horizon resource governed by execution admission. +/// +/// The variants are exactly the durable +/// `moa.execution_capacity_bucket.resource_dimension` discriminators. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ExecutionAdmissionResourceDimension { + /// Nonterminal runs that are not fully parked. + ActiveRuns, + /// Forward and compensation attempts holding active-compute reservations. + ActiveTasks, + /// Runs retained in storage-only waiting or paused states. + ParkedRuns, + /// Pending durable trigger rows. + ScheduledTriggers, + /// Nonterminal asynchronous provider jobs. + ExternalJobs, +} + +impl ExecutionAdmissionResourceDimension { + /// Every bounded dimension, in durable label order. + pub const ALL: [Self; 5] = [ + Self::ActiveRuns, + Self::ActiveTasks, + Self::ParkedRuns, + Self::ScheduledTriggers, + Self::ExternalJobs, + ]; + + /// Returns the canonical database label. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::ActiveRuns => "active_runs", + Self::ActiveTasks => "active_tasks", + Self::ParkedRuns => "parked_runs", + Self::ScheduledTriggers => "scheduled_triggers", + Self::ExternalJobs => "external_jobs", + } + } +} + +impl FromStr for ExecutionAdmissionResourceDimension { + type Err = Error; + + fn from_str(value: &str) -> Result { + match value { + "active_runs" => Ok(Self::ActiveRuns), + "active_tasks" => Ok(Self::ActiveTasks), + "parked_runs" => Ok(Self::ParkedRuns), + "scheduled_triggers" => Ok(Self::ScheduledTriggers), + "external_jobs" => Ok(Self::ExternalJobs), + _ => Err(Error::InvalidRepositoryData { + message: format!("unknown execution admission resource `{value}`"), + }), + } + } +} + +/// Ceiling utilization and tenant concentration for one bounded admission resource. +/// +/// Ratios are aggregated inside the database, so no tenant identity leaves the query. +#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Serialize)] +pub struct ExecutionAdmissionUtilizationSample { + /// Bounded resource whose ceilings were observed. + pub resource: ExecutionAdmissionResourceDimension, + /// Utilization of the shared fleet ceiling. + pub fleet_ratio: f64, + /// Highest utilization observed across tenant-scoped ceilings. + pub tenant_peak_ratio: f64, + /// Largest single tenant's share of everything tenants currently hold. + /// + /// This is a different question from `tenant_peak_ratio`, which measures a tenant + /// against its own ceiling. A small tenant can sit at `tenant_peak_ratio` 1.0 while + /// holding almost none of the fleet, and a large tenant can hold the entire fleet + /// while far from its own ceiling. Only this ratio answers whether one tenant is + /// crowding out the others. + pub tenant_max_share_ratio: f64, +} + +/// Bounded nonterminal run phase reported by the fleet run census. +/// +/// The variants are exactly the nonterminal `moa.execution_run.status` discriminators +/// carried by the `execution_run_nonterminal_idx` predicate. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ExecutionRunPhaseDimension { + /// The displayed plan and estimate require owning-user confirmation. + AwaitingConfirmation, + /// The run is accepted and may materialize or reserve work. + Queued, + /// At least one task may be executing. + Running, + /// The run is waiting for task input. + WaitingInput, + /// The run is waiting for a tenant review decision. + WaitingReview, + /// The run is waiting for a named external signal. + WaitingSignal, + /// The run is waiting for an exact durable timer. + WaitingTimer, + /// The run is waiting for an asynchronous external job. + WaitingExternal, + /// The run is waiting for a compiler-validated amendment. + WaitingReplan, + /// An authorized caller requested a safe pause. + PauseRequested, + /// Active work is reaching safe checkpoint boundaries before pausing. + Pausing, + /// The run is durably parked without active compute. + Paused, + /// Forward work is fenced while committed effects are undone in reverse order. + Compensating, +} + +impl ExecutionRunPhaseDimension { + /// Every bounded nonterminal phase, in durable status order. + pub const ALL: [Self; 13] = [ + Self::AwaitingConfirmation, + Self::Queued, + Self::Running, + Self::WaitingInput, + Self::WaitingReview, + Self::WaitingSignal, + Self::WaitingTimer, + Self::WaitingExternal, + Self::WaitingReplan, + Self::PauseRequested, + Self::Pausing, + Self::Paused, + Self::Compensating, + ]; + + /// Returns the canonical database label. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::AwaitingConfirmation => "awaiting_confirmation", + Self::Queued => "queued", + Self::Running => "running", + Self::WaitingInput => "waiting_input", + Self::WaitingReview => "waiting_review", + Self::WaitingSignal => "waiting_signal", + Self::WaitingTimer => "waiting_timer", + Self::WaitingExternal => "waiting_external", + Self::WaitingReplan => "waiting_replan", + Self::PauseRequested => "pause_requested", + Self::Pausing => "pausing", + Self::Paused => "paused", + Self::Compensating => "compensating", + } + } +} + +impl FromStr for ExecutionRunPhaseDimension { + type Err = Error; + + fn from_str(value: &str) -> Result { + match value { + "awaiting_confirmation" => Ok(Self::AwaitingConfirmation), + "queued" => Ok(Self::Queued), + "running" => Ok(Self::Running), + "waiting_input" => Ok(Self::WaitingInput), + "waiting_review" => Ok(Self::WaitingReview), + "waiting_signal" => Ok(Self::WaitingSignal), + "waiting_timer" => Ok(Self::WaitingTimer), + "waiting_external" => Ok(Self::WaitingExternal), + "waiting_replan" => Ok(Self::WaitingReplan), + "pause_requested" => Ok(Self::PauseRequested), + "pausing" => Ok(Self::Pausing), + "paused" => Ok(Self::Paused), + "compensating" => Ok(Self::Compensating), + _ => Err(Error::InvalidRepositoryData { + message: format!("unknown nonterminal execution run status `{value}`"), + }), + } + } +} + +/// Live run count for one bounded nonterminal phase. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct ExecutionRunPhaseSample { + /// Bounded nonterminal phase. + pub phase: ExecutionRunPhaseDimension, + /// Runs currently in that phase across the fleet. + pub run_count: u64, +} + /// Bounded trigger/outbox health observed in one scoped transaction. -#[derive(Clone, Debug, Eq, PartialEq)] +#[derive(Clone, Debug, PartialEq)] pub struct ExecutionQueueHealthSnapshot { /// Canonical database observation time. pub observed_at: DateTime, @@ -284,10 +501,18 @@ pub struct ExecutionQueueHealthSnapshot { pub due_triggers: ExecutionQueueBacklogSample, /// Pending or claim-expired dispatches eligible for delivery. pub claimable_dispatches: ExecutionQueueBacklogSample, - /// Trigger deliveries that exhausted their retry policy. - pub dead_letter_triggers: ExecutionQueueBacklogSample, /// Outbox deliveries that exhausted their retry policy. pub dead_letter_dispatches: ExecutionQueueBacklogSample, + /// Nonterminal runs whose absolute deadline has elapsed, capped at the sample limit. + pub overdue_deadlines: u32, + /// Start time of the oldest active forward or compensation attempt. + pub oldest_active_attempt_at: Option>, + /// Live run count for every bounded nonterminal phase, including idle zeroes. + pub run_phases: Vec, + /// Creation time of the oldest nonterminal asynchronous external job. + pub oldest_external_job_at: Option>, + /// Ceiling utilization for every bounded admission resource, including idle zeroes. + pub admission_utilization: Vec, } /// Database clock and earliest indexed deadline for pending dispatch work. @@ -470,15 +695,19 @@ impl ExecutionRepository { r#" SELECT claimable_at FROM ( - SELECT not_before_at AS claimable_at, dispatch_uid - FROM moa.execution_dispatch_outbox - WHERE state = 'pending' AND not_before_at <= now() + (SELECT not_before_at AS claimable_at, created_at, dispatch_uid + FROM moa.execution_dispatch_outbox + WHERE state = 'pending' AND not_before_at <= now() + ORDER BY not_before_at, created_at, dispatch_uid + LIMIT $1) UNION ALL - SELECT claim_expires_at AS claimable_at, dispatch_uid - FROM moa.execution_dispatch_outbox - WHERE state = 'dispatching' AND claim_expires_at <= now() + (SELECT claim_expires_at AS claimable_at, created_at, dispatch_uid + FROM moa.execution_dispatch_outbox + WHERE state = 'dispatching' AND claim_expires_at <= now() + ORDER BY claim_expires_at, created_at, dispatch_uid + LIMIT $1) ) AS claimable - ORDER BY claimable_at, dispatch_uid + ORDER BY claimable_at, created_at, dispatch_uid LIMIT $1 "#, ) @@ -486,12 +715,12 @@ impl ExecutionRepository { .fetch_all(conn.as_mut()) .await .map_err(sqlx_error)?; - let dead_letter_triggers = sqlx::query_scalar::<_, DateTime>( + let dead_letter_dispatches = sqlx::query_scalar::<_, DateTime>( r#" SELECT created_at - FROM moa.execution_trigger + FROM moa.execution_dispatch_outbox WHERE state = 'dead_letter' - ORDER BY created_at, tenant_id, trigger_uid + ORDER BY created_at, tenant_id, dispatch_uid LIMIT $1 "#, ) @@ -499,16 +728,114 @@ impl ExecutionRepository { .fetch_all(conn.as_mut()) .await .map_err(sqlx_error)?; - let dead_letter_dispatches = sqlx::query_scalar::<_, DateTime>( + // The exact-deadline invariant guard. Counting is capped rather than exhaustive + // because the alert only distinguishes zero from nonzero, so the cap bounds the + // work without changing the answer. `execution_run_overdue_deadline_idx` leads on + // `budget_deadline_at` and carries this exact status predicate, so the capped scan + // is index-only. + let overdue_deadlines = sqlx::query_scalar::<_, i32>( + r#" + SELECT count(*)::INTEGER + FROM ( + SELECT 1 + FROM moa.execution_run + WHERE status IN ( + 'awaiting_confirmation', 'queued', 'running', 'waiting_input', + 'waiting_review', 'waiting_signal', 'waiting_timer', + 'waiting_external', 'waiting_replan', 'pause_requested', + 'pausing', 'paused', 'compensating' + ) + AND budget_deadline_at IS NOT NULL + AND budget_deadline_at <= now() + LIMIT $1 + ) AS overdue + "#, + ) + .bind(i64::from(sample_limit)) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + // Compensation attempts hold the same active-compute reservation as forward attempts, + // so a stuck rollback must not be invisible to the stuck-attempt alert. Each branch + // takes its own ordered minimum so the aggregate reduces two rows, not two scans. + let oldest_active_attempt_at = sqlx::query_scalar::<_, Option>>( + r#" + SELECT min(attempt_started_at) + FROM ( + (SELECT attempt_started_at + FROM moa.execution_task + WHERE status = 'running' AND attempt_state = 'running' + AND attempt_started_at IS NOT NULL + ORDER BY attempt_started_at + LIMIT 1) + UNION ALL + (SELECT attempt_started_at + FROM moa.execution_compensation + WHERE status = 'running' AND attempt_state = 'running' + AND attempt_started_at IS NOT NULL + ORDER BY attempt_started_at + LIMIT 1) + ) AS active_attempt + "#, + ) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + // The per-phase census of the live fleet. This is the one aggregate here that is not + // capped: a census that stops counting reports a number no dashboard can sum, and the + // partial nonterminal index bounds the scan to live work regardless of run history. + let run_phase_rows = sqlx::query_as::<_, (String, i64)>(RUN_PHASE_CENSUS_SQL) + .fetch_all(conn.as_mut()) + .await + .map_err(sqlx_error)?; + // The oldest live asynchronous job, including `unbound` jobs that were never handed to + // a provider. `execution_external_job_reconcile_idx` cannot serve this: it excludes + // `unbound` and rows with no `next_reconcile_at`, which is exactly where a job that + // never started is stranded. `execution_external_job_live_age_idx` leads on `created_at` + // and carries this state predicate, so the ordered lookup reads one index tuple. + let oldest_external_job_at = sqlx::query_scalar::<_, DateTime>( r#" SELECT created_at - FROM moa.execution_dispatch_outbox - WHERE state = 'dead_letter' - ORDER BY created_at, tenant_id, dispatch_uid - LIMIT $1 + FROM moa.execution_external_job + WHERE state IN ( + 'unbound', 'starting', 'running', 'waiting_reconcile', 'cancel_requested' + ) + ORDER BY created_at, external_job_uid + LIMIT 1 + "#, + ) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + // One aggregate over the bounded capacity ledger. Every ratio is reduced inside the + // database, so no tenant identifier reaches the metric layer. The share denominator is + // the tenant total rather than the fleet bucket's counter so the ratio stays within + // [0, 1] by construction instead of depending on cross-scope bookkeeping agreeing. + let utilization_rows = sqlx::query_as::<_, (String, f64, f64, f64)>( + r#" + SELECT resource_dimension, + COALESCE( + max(reserved_quantity::DOUBLE PRECISION + / limit_value::DOUBLE PRECISION) + FILTER (WHERE scope_kind = 'fleet'), 0.0 + ) AS fleet_ratio, + COALESCE( + max(reserved_quantity::DOUBLE PRECISION + / limit_value::DOUBLE PRECISION) + FILTER (WHERE scope_kind = 'tenant'), 0.0 + ) AS tenant_peak_ratio, + COALESCE( + max(reserved_quantity) + FILTER (WHERE scope_kind = 'tenant')::DOUBLE PRECISION + / NULLIF( + sum(reserved_quantity) FILTER (WHERE scope_kind = 'tenant'), 0 + )::DOUBLE PRECISION, + 0.0 + ) AS tenant_max_share_ratio + FROM moa.execution_capacity_bucket + GROUP BY resource_dimension "#, ) - .bind(fetch_limit) .fetch_all(conn.as_mut()) .await .map_err(sqlx_error)?; @@ -517,8 +844,12 @@ impl ExecutionRepository { observed_at, due_triggers: backlog_sample(due_triggers, sample_limit), claimable_dispatches: backlog_sample(claimable_dispatches, sample_limit), - dead_letter_triggers: backlog_sample(dead_letter_triggers, sample_limit), dead_letter_dispatches: backlog_sample(dead_letter_dispatches, sample_limit), + overdue_deadlines: to_u32(overdue_deadlines, "overdue execution deadlines")?, + oldest_active_attempt_at, + run_phases: run_phase_samples(&run_phase_rows)?, + oldest_external_job_at, + admission_utilization: admission_utilization_samples(&utilization_rows)?, }) } @@ -535,6 +866,10 @@ impl ExecutionRepository { } /// Claims a bounded due batch, including claims abandoned past their expiry. + /// + /// Each eligibility branch is ordered and capped on its own partial index — pending rows + /// by `not_before_at`, abandoned claims by `claim_expires_at` — so neither branch reads + /// beyond one bounded index window. Only the merged head is then locked. pub async fn claim_due_dispatches( &self, scope: ExecutionScope, @@ -547,19 +882,36 @@ impl ExecutionRepository { let mut conn = scope.begin(&self.pool).await?; let rows = sqlx::query( r#" - WITH claimable AS ( + WITH head AS ( SELECT dispatch_uid - FROM moa.execution_dispatch_outbox + FROM ( + (SELECT dispatch_uid, not_before_at AS claimable_at, created_at + FROM moa.execution_dispatch_outbox + WHERE state = 'pending' AND not_before_at <= now() + ORDER BY not_before_at, created_at, dispatch_uid + LIMIT $1) + UNION ALL + (SELECT dispatch_uid, claim_expires_at AS claimable_at, created_at + FROM moa.execution_dispatch_outbox + WHERE state = 'dispatching' AND claim_expires_at <= now() + ORDER BY claim_expires_at, created_at, dispatch_uid + LIMIT $1) + ) AS candidate + ORDER BY claimable_at, created_at, dispatch_uid + LIMIT $1 + ), + claimable AS ( + SELECT claimed.dispatch_uid + FROM moa.execution_dispatch_outbox AS claimed + JOIN head ON head.dispatch_uid = claimed.dispatch_uid WHERE ( - state = 'pending' - AND not_before_at <= now() + claimed.state = 'pending' + AND claimed.not_before_at <= now() ) OR ( - state = 'dispatching' - AND claim_expires_at <= now() + claimed.state = 'dispatching' + AND claimed.claim_expires_at <= now() ) - ORDER BY not_before_at, created_at, dispatch_uid - LIMIT $1 - FOR UPDATE SKIP LOCKED + FOR UPDATE OF claimed SKIP LOCKED ) UPDATE moa.execution_dispatch_outbox AS dispatch SET state = 'dispatching', @@ -707,11 +1059,13 @@ impl ExecutionRepository { let last_error = error.chars().take(MAX_ERROR_CHARS).collect::(); let requires_durable_retry = dispatch_requires_durable_retry(dispatch_kind); let outcome = if attempts >= retry.max_attempts && !requires_durable_retry { - if dispatch_kind == ExecutionDispatchKind::TaskAttempt { - repair_dead_lettered_task_dispatch_in_conn(&mut conn, &dispatch).await?; + let owner_repaired = if dispatch_kind == ExecutionDispatchKind::TaskAttempt { + repair_dead_lettered_task_dispatch_in_conn(&mut conn, &dispatch).await? } else if dispatch_kind == ExecutionDispatchKind::CompensationAttempt { - repair_dead_lettered_compensation_dispatch_in_conn(&mut conn, &dispatch).await?; - } + repair_dead_lettered_compensation_dispatch_in_conn(&mut conn, &dispatch).await? + } else { + true + }; sqlx::query( r#" UPDATE moa.execution_dispatch_outbox @@ -726,7 +1080,11 @@ impl ExecutionRepository { .execute(conn.as_mut()) .await .map_err(sqlx_error)?; - ExecutionDispatchFailureOutcome::DeadLettered + if owner_repaired { + ExecutionDispatchFailureOutcome::DeadLettered + } else { + ExecutionDispatchFailureOutcome::DeadLetteredWithoutOwnerRepair + } } else { let delay = if attempts >= retry.max_attempts { retry.maximum_delay @@ -769,10 +1127,14 @@ fn dispatch_requires_durable_retry(kind: ExecutionDispatchKind) -> bool { } } +/// Settles the never-started task attempt owning one dead-lettered dispatch. +/// +/// Returns whether the exact `Dispatching` owner was repaired. A `false` result means the +/// owner was already settled by another writer, which must not block the dead letter. async fn repair_dead_lettered_task_dispatch_in_conn( conn: &mut super::ScopedConn<'_>, dispatch: &ExecutionDispatchRecord, -) -> Result<()> { +) -> Result { let request = serde_json::from_value::(dispatch.payload.clone()) .map_err(|error| Error::InvalidRepositoryData { message: format!("invalid dead-letter task-attempt payload: {error}"), @@ -808,22 +1170,25 @@ async fn repair_dead_lettered_task_dispatch_in_conn( settled_at, ) .await?; - match outcome { + Ok(match outcome { TaskAttemptSettlementOutcome::Applied { .. } - | TaskAttemptSettlementOutcome::Replayed { .. } => Ok(()), + | TaskAttemptSettlementOutcome::Replayed { .. } => true, + // The owner is already gone, superseded, or settled by another writer, so there is + // nothing left to repair. Dead-lettering must still commit: refusing here would + // re-poison every drain that reclaims this row and stop the fleet's only pump. TaskAttemptSettlementOutcome::NotFound | TaskAttemptSettlementOutcome::Stale - | TaskAttemptSettlementOutcome::InvalidState => Err(Error::InvalidRepositoryData { - message: "task-attempt dead letter could not repair its exact Dispatching owner" - .to_string(), - }), - } + | TaskAttemptSettlementOutcome::InvalidState => false, + }) } +/// Settles the never-started compensation attempt owning one dead-lettered dispatch. +/// +/// Returns whether the exact `Dispatching` owner was repaired; see the task twin. async fn repair_dead_lettered_compensation_dispatch_in_conn( conn: &mut super::ScopedConn<'_>, dispatch: &ExecutionDispatchRecord, -) -> Result<()> { +) -> Result { let request = serde_json::from_value::(dispatch.payload.clone()) .map_err(|error| Error::InvalidRepositoryData { @@ -850,18 +1215,13 @@ async fn repair_dead_lettered_compensation_dispatch_in_conn( conn, &request, settled_at, ) .await?; - match outcome { + Ok(match outcome { super::compensation::CompensationAttemptWriteOutcome::Applied(_) - | super::compensation::CompensationAttemptWriteOutcome::Replayed(_) => Ok(()), + | super::compensation::CompensationAttemptWriteOutcome::Replayed(_) => true, + // See the task twin: an already-settled owner must not block the dead letter itself. super::compensation::CompensationAttemptWriteOutcome::Conflict - | super::compensation::CompensationAttemptWriteOutcome::NotFound => { - Err(Error::InvalidRepositoryData { - message: - "compensation-attempt dead letter could not repair its exact Dispatching owner" - .to_string(), - }) - } - } + | super::compensation::CompensationAttemptWriteOutcome::NotFound => false, + }) } async fn settle_execution_maintenance( @@ -967,6 +1327,55 @@ fn maintenance_checkpoint_from_row( }) } +/// Expands observed ledger rows to every bounded dimension, defaulting absent ones to idle. +/// +/// A dimension with no bucket row has never been reserved. It must still be reported, or the +/// gauge would be absent on a quiet fleet and its `absent()` alert would fire. +fn admission_utilization_samples( + rows: &[(String, f64, f64, f64)], +) -> Result> { + for (resource, _, _, _) in rows { + resource.parse::()?; + } + Ok(ExecutionAdmissionResourceDimension::ALL + .into_iter() + .map(|resource| { + let observed = rows + .iter() + .find(|(label, _, _, _)| label == resource.as_str()); + ExecutionAdmissionUtilizationSample { + resource, + fleet_ratio: observed.map_or(0.0, |(_, fleet, _, _)| *fleet), + tenant_peak_ratio: observed.map_or(0.0, |(_, _, tenant_peak, _)| *tenant_peak), + tenant_max_share_ratio: observed.map_or(0.0, |(_, _, _, share)| *share), + } + }) + .collect()) +} + +/// Expands observed census rows to every bounded phase, defaulting absent ones to zero. +/// +/// `GROUP BY status` returns no row for a phase holding no runs, so an idle phase would +/// leave its gauge unwritten and its `absent()` alert would fire on a quiet fleet. An +/// unmodelled durable status fails the whole snapshot rather than silently vanishing from +/// a census whose sum is supposed to equal the live nonterminal fleet. +fn run_phase_samples(rows: &[(String, i64)]) -> Result> { + for (status, _) in rows { + status.parse::()?; + } + ExecutionRunPhaseDimension::ALL + .into_iter() + .map(|phase| { + let observed = rows.iter().find(|(label, _)| label == phase.as_str()); + let run_count = match observed { + Some((_, count)) => super::to_u64(*count, "execution run phase count")?, + None => 0, + }; + Ok(ExecutionRunPhaseSample { phase, run_count }) + }) + .collect() +} + fn backlog_sample( mut timestamps: Vec>, sample_limit: u32, @@ -1052,7 +1461,9 @@ pub async fn enqueue_dispatch_in_conn( /// Requeues one previously accepted dispatch without changing its immutable identity. /// /// The caller must first establish the authoritative generation fence while holding the -/// corresponding trigger row lock. A non-delivered replay is left untouched. +/// corresponding trigger row lock. A non-delivered replay is left untouched. The repair +/// epoch advances so redelivery cannot attach to the memoized completed invocation the +/// previous delivery identity already produced. pub(super) async fn requeue_delivered_dispatch_in_conn( conn: &mut PgConnection, request: &NewExecutionDispatch, @@ -1062,6 +1473,7 @@ pub(super) async fn requeue_delivered_dispatch_in_conn( r#" UPDATE moa.execution_dispatch_outbox SET state = 'pending', delivered_at = NULL, delivery_attempts = 0, + repair_epoch = repair_epoch + 1, claim_owner = NULL, claimed_at = NULL, claim_expires_at = NULL, last_error = NULL, updated_at = now() WHERE dispatch_uid = $1 AND state = 'delivered' @@ -1085,6 +1497,9 @@ pub(super) async fn requeue_delivered_dispatch_in_conn( } /// Requeues a bounded page of accepted run activations whose authoritative wake remains queued. +/// +/// The repair epoch advances so the replacement activation carries a delivery identity the +/// original completed controller invocation cannot memoize. pub(super) async fn requeue_current_run_activations_in_conn( conn: &mut PgConnection, batch_size: u32, @@ -1117,6 +1532,7 @@ pub(super) async fn requeue_current_run_activations_in_conn( ) UPDATE moa.execution_dispatch_outbox AS dispatch SET state = 'pending', delivered_at = NULL, delivery_attempts = 0, + repair_epoch = dispatch.repair_epoch + 1, claim_owner = NULL, claimed_at = NULL, claim_expires_at = NULL, last_error = NULL, updated_at = now() FROM candidates @@ -1136,7 +1552,19 @@ pub(super) async fn requeue_current_run_activations_in_conn( /// Requeues accepted deliveries only while their exact bounded work has not started. /// /// A running attempt is deliberately excluded: once effects may have begun, its -/// watchdog owns ambiguity resolution and the dispatcher must not replay it. +/// watchdog owns ambiguity resolution and the dispatcher must not replay it. The repair +/// epoch advances so redelivery carries an identity Restate cannot memoize. +/// +/// That fully repairs the two cancellation kinds and `external_cancel`, whose targets are +/// addressed by idempotency key alone. The two attempt kinds split by what Restate still +/// holds, because a workflow `run` handler is retained under its *workflow key* — the bare +/// dispatch UID, which `require_dispatch_key` pins — independently of any idempotency key: +/// +/// - Total Restate state loss, the case this grace window exists for, leaves the workflow +/// key uninvoked, so redelivery starts the attempt normally. +/// - A key for which Restate still holds a completed `run` cannot be restarted under any +/// delivery identity. That attempt stays parked until its `attempt_deadline_at` watchdog +/// settles it — the same backstop a lost running attempt already relies on. pub(super) async fn requeue_current_accepted_dispatches_in_conn( conn: &mut PgConnection, batch_size: u32, @@ -1251,6 +1679,7 @@ pub(super) async fn requeue_current_accepted_dispatches_in_conn( ) UPDATE moa.execution_dispatch_outbox AS dispatch SET state='pending', delivered_at=NULL, delivery_attempts=0, + repair_epoch=dispatch.repair_epoch + 1, claim_owner=NULL, claimed_at=NULL, claim_expires_at=NULL, last_error=NULL, updated_at=now() FROM candidates @@ -1561,6 +1990,10 @@ fn dispatch_from_row(row: &sqlx::postgres::PgRow) -> Result>(), + ExecutionAdmissionResourceDimension::ALL.to_vec(), + "every bounded dimension must be reported on every snapshot" + ); + let active_tasks = samples + .iter() + .find(|sample| sample.resource == ExecutionAdmissionResourceDimension::ActiveTasks) + .expect("observed dimension is retained"); + assert!((active_tasks.fleet_ratio - 0.25).abs() < f64::EPSILON); + assert!((active_tasks.tenant_peak_ratio - 0.75).abs() < f64::EPSILON); + assert!((active_tasks.tenant_max_share_ratio - 0.6).abs() < f64::EPSILON); + for idle in samples.iter().filter(|sample| { + !matches!( + sample.resource, + ExecutionAdmissionResourceDimension::ActiveTasks + | ExecutionAdmissionResourceDimension::ExternalJobs + ) + }) { + assert_eq!(idle.fleet_ratio, 0.0, "{:?}", idle.resource); + assert_eq!(idle.tenant_peak_ratio, 0.0, "{:?}", idle.resource); + assert_eq!(idle.tenant_max_share_ratio, 0.0, "{:?}", idle.resource); + } + + assert!( + admission_utilization_samples(&[("active_sandboxes".to_string(), 1.0, 1.0, 1.0)]) + .is_err(), + "an unmodelled durable dimension must fail closed, not vanish from the report" + ); + } + + #[test] + fn idle_run_phases_still_report_their_healthy_zero_offline() { + // Pins: `GROUP BY status` returns no row for a phase holding no runs. Every bounded + // phase must still be reported, or its gauge would be absent on a quiet fleet and its + // `absent()` alert would page. An unmodelled durable status must fail the snapshot + // rather than silently drop runs out of a census whose sum is the live fleet. + let observed = vec![("running".to_string(), 7_i64), ("paused".to_string(), 4)]; + let samples = run_phase_samples(&observed).expect("known statuses must decode"); + + assert_eq!( + samples + .iter() + .map(|sample| sample.phase) + .collect::>(), + ExecutionRunPhaseDimension::ALL.to_vec(), + "every bounded phase must be reported on every snapshot" + ); + let count_of = |phase: ExecutionRunPhaseDimension| { + samples + .iter() + .find(|sample| sample.phase == phase) + .map(|sample| sample.run_count) + .expect("every phase is reported") + }; + assert_eq!(count_of(ExecutionRunPhaseDimension::Running), 7); + assert_eq!(count_of(ExecutionRunPhaseDimension::Paused), 4); + for idle in samples.iter().filter(|sample| { + !matches!( + sample.phase, + ExecutionRunPhaseDimension::Running | ExecutionRunPhaseDimension::Paused + ) + }) { + assert_eq!(idle.run_count, 0, "{:?}", idle.phase); + } + assert_eq!( + samples.iter().map(|sample| sample.run_count).sum::(), + 11, + "the census must sum to exactly the observed live fleet" + ); + + assert!( + run_phase_samples(&[("completed".to_string(), 1)]).is_err(), + "a terminal status is not a live phase and must fail closed" + ); + assert!( + run_phase_samples(&[("quarantined".to_string(), 1)]).is_err(), + "an unmodelled durable status must fail closed, not vanish from the census" + ); + } + + #[test] + fn run_phase_census_query_covers_exactly_the_bounded_phase_set_offline() { + // Pins: the census status list is spelled out in SQL so it matches the partial index + // predicate literally and keeps the index-only scan. That literal and the exported + // label set can drift apart silently, which would drop a live phase from the census + // while every gauge still looked healthy. + let (_, after) = RUN_PHASE_CENSUS_SQL + .split_once("status IN (") + .expect("census query filters on the nonterminal status list"); + let (list, _) = after + .split_once(')') + .expect("the status list is parenthesized"); + let mut queried = list + .split(',') + .map(|status| status.trim().trim_matches('\'').to_string()) + .collect::>(); + queried.sort(); + let mut bounded = ExecutionRunPhaseDimension::ALL + .iter() + .map(|phase| phase.as_str().to_string()) + .collect::>(); + bounded.sort(); + assert_eq!( + queried, bounded, + "the census query and the bounded phase set must name the same statuses" + ); + } + #[test] fn correctness_dispatches_never_terminally_dead_letter_offline() { // Pins: correctness work remains behind capped sparse retries; only never-started diff --git a/crates/moa-execution/src/repository/ready.rs b/crates/moa-execution/src/repository/ready.rs index 62d2b71fc..fb68eb0c4 100644 --- a/crates/moa-execution/src/repository/ready.rs +++ b/crates/moa-execution/src/repository/ready.rs @@ -2,16 +2,21 @@ use std::collections::{BTreeMap, BTreeSet}; -use moa_artifacts::execution_plan::{ExecutionTemporalTarget, ExecutionWaitPolicy, InputAudience}; +use moa_artifacts::execution_plan::{ + ExecutionFailureClass, ExecutionTemporalTarget, ExecutionUsage, ExecutionWaitPolicy, + InputAudience, +}; use moa_config::ExecutionConfig; use sqlx::{Row, postgres::PgRow}; use crate::capability::node_output_hash; +use crate::interpreter::TemporalTargetResolution; use crate::schema::validate_instance; use super::*; use super::{ materialize::{ensure_materialization_replay_matches, prepare_task_materialization_batch}, + outcome_support::outcome_projection_fields, rows::*, sql::*, trigger::{ExecutionTriggerKind, NewExecutionTrigger, create_trigger_with_dispatch_in_conn}, @@ -194,6 +199,8 @@ pub struct ReadyMaterializationRequest { pub source_exhausted: bool, /// Aggregate output when the source completes without creating a logical task. pub terminal_output: Option, + /// Whether the node's declared condition evaluated false and no work may exist. + pub condition_skipped: bool, /// Bounded deterministic logical tasks in source order. pub tasks: Vec, } @@ -915,10 +922,28 @@ impl ExecutionRepository { reduce_cursor, source_exhausted, terminal_output, + condition_skipped, tasks, } = request; + // A condition skip is the only page that legitimately carries neither tasks nor a + // terminal output, and it can only ever be the node's first page: the interpreter + // evaluates `when` at cursor zero and declares the source exhausted there. + if condition_skipped + && !(tasks.is_empty() + && source_exhausted + && terminal_output.is_none() + && reduce_cursor.is_none() + && expected_cursor == 0) + { + return Err(Error::InvalidRepositoryInput { + message: "a condition-skipped page must be the empty exhausted first page" + .to_string(), + }); + } if tasks.len() > MAX_READY_PAGE_SIZE_USIZE - || (tasks.is_empty() && (!source_exhausted || terminal_output.is_none())) + || (!condition_skipped + && tasks.is_empty() + && (!source_exhausted || terminal_output.is_none())) || (!tasks.is_empty() && terminal_output.is_some()) { return Err(Error::InvalidRepositoryInput { @@ -973,7 +998,12 @@ impl ExecutionRepository { .fetch_one(conn.as_mut()) .await .map_err(sqlx_error)?; - let storage_wait = storage_wait_for_tasks(&tasks, &run, wait_entered_at)?; + let (storage_wait, wait_deadline_failure) = + match storage_wait_for_tasks(&tasks, &run, wait_entered_at)? { + Some(StorageWaitPlan::Enter(wait)) => (Some(*wait), None), + Some(StorageWaitPlan::DeadlineExceeded(message)) => (None, Some(message)), + None => (None, None), + }; let plan_node = run .active_plan .definition @@ -983,6 +1013,19 @@ impl ExecutionRepository { .ok_or_else(|| Error::InvalidRepositoryData { message: format!("active plan is missing materialized node `{node_id}`"), })?; + // A skipped branch does NOT reuse the `terminal_output` path: that path validates + // its output against the node's `output_schema` (which a null branch output cannot + // satisfy) and lands the node in `completed`. A skip must land in `skipped`, whose + // null aggregate is what `load_activation_projection` already materializes for + // dependents and what completion accounting already excludes from requirements. + if condition_skipped { + if plan_node.when.is_none() { + return Err(Error::InvalidRepositoryInput { + message: format!("node `{node_id}` was skipped without declaring a condition"), + }); + } + return commit_condition_skip(conn, &run, &node_id, expected_cursor).await; + } let reduce_batch_size = match (&plan_node.operation, reduce_cursor) { (ExecutionOperation::Reduce { batch_size, .. }, Some(cursor)) => { let minimum_inputs = if tasks.is_empty() { 1 } else { 2 }; @@ -1236,14 +1279,31 @@ impl ExecutionRepository { .iter() .map(|task| task.task_id.as_uuid()) .collect::>(); - let (task_status, attempt_state, waiting_since, ready_at) = storage_wait - .as_ref() - .map_or(("ready", "idle", None, Some(wait_entered_at)), |wait| { - (wait.task_status, "waiting", Some(wait_entered_at), None) - }); + let wait_deadline_outcome = wait_deadline_failure.map(|message| { + crate::state::failed_task_outcome( + ExecutionFailureClass::DeadlineExceeded, + message, + zero_usage(), + ) + }); + let (task_status, attempt_state, waiting_since, ready_at) = + match (storage_wait.as_ref(), wait_deadline_outcome.as_ref()) { + (_, Some(_)) => ("failed", "terminal", None, None), + (Some(wait), None) => (wait.task_status, "waiting", Some(wait_entered_at), None), + (None, None) => ("ready", "idle", None, Some(wait_entered_at)), + }; + let (current_outcome, current_error) = match wait_deadline_outcome.as_ref() { + Some(outcome) => { + let (_, error, _) = outcome_projection_fields(outcome)?; + (Some(serde_json::to_value(outcome)?), error) + } + None => (None, None), + }; let transitioned = sqlx::query( "UPDATE moa.execution_task SET status = $3, attempt_state = $4, \ waiting_since = $5, ready_at = $6, \ + current_outcome = $7, error = $8, \ + completed_at = CASE WHEN $7::JSONB IS NULL THEN completed_at ELSE NOW() END, \ last_progress_at = NOW(), updated_at = NOW() \ WHERE run_uid = $1 AND task_id = ANY($2::UUID[]) AND status = 'pending'", ) @@ -1253,6 +1313,8 @@ impl ExecutionRepository { .bind(attempt_state) .bind(waiting_since) .bind(ready_at) + .bind(current_outcome) + .bind(current_error) .execute(conn.as_mut()) .await .map_err(sqlx_error)?; @@ -1261,7 +1323,7 @@ impl ExecutionRepository { message: "inserted ready page did not transition every task".to_string(), }); } - let ready_delta = if storage_wait.is_some() { + let ready_delta = if storage_wait.is_some() || wait_deadline_outcome.is_some() { 0 } else { page_count @@ -1271,10 +1333,12 @@ impl ExecutionRepository { } else { 0 }; - let node_status = if storage_wait.is_some() { - "waiting" - } else { - "ready" + // A deadline-failed wait keeps the pre-terminal node status here; the counter + // transition below flips it to `failed` so dependent cancellation runs once. + let node_status = match (storage_wait.as_ref(), wait_deadline_outcome.as_ref()) { + (_, Some(_)) => "pending", + (Some(_), None) => "waiting", + (None, None) => "ready", }; let (reduce_round, reduce_batch_cursor, reduce_input_count, reduce_task_delta) = next_reduce_cursor.map_or((None, None, None, 0), |(cursor, next_batch)| { @@ -1387,6 +1451,7 @@ impl ExecutionRepository { waiting_review_task_count = waiting_review_task_count + $10, \ waiting_signal_task_count = waiting_signal_task_count + $11, \ waiting_timer_task_count = waiting_timer_task_count + $12, \ + progress_failed_tasks = progress_failed_tasks + $13, \ last_progress_at = NOW(), updated_at = NOW() \ WHERE run_uid = $1", ) @@ -1402,9 +1467,25 @@ impl ExecutionRepository { .bind(run_waiting.review) .bind(run_waiting.signal) .bind(run_waiting.timer) + .bind(to_i64( + u64::from(wait_deadline_outcome.is_some()), + "wait deadline failure count", + )?) .execute(conn.as_mut()) .await .map_err(sqlx_error)?; + if wait_deadline_outcome.is_some() { + let task = &tasks[0]; + transition_node_counters_in_tx( + &mut conn, + run_uid, + &node_id, + &task.item_key, + ExecutionTaskStatus::Pending, + ExecutionTaskStatus::Failed, + ) + .await?; + } let mut triggers = Vec::new(); if let Some(wait) = storage_wait { let task = &tasks[0]; @@ -1683,11 +1764,28 @@ struct StorageWaitMaterialization { reason: WaitingReason, } +const fn zero_usage() -> ExecutionUsage { + ExecutionUsage { + cost_microusd: 0, + tokens: 0, + tool_calls: 0, + retrieved_bytes: 0, + } +} + +/// How one storage-only wait node materializes at wait entry. +enum StorageWaitPlan { + /// The wait is enterable and parks the task on its durable trigger. + Enter(Box), + /// The wait cannot finish before the run deadline and fails the task instead. + DeadlineExceeded(String), +} + fn storage_wait_for_tasks( tasks: &[LogicalTask], run: &ExecutionRunRecord, wait_entered_at: DateTime, -) -> Result> { +) -> Result> { let Some(first) = tasks.first() else { return Ok(None); }; @@ -1719,8 +1817,23 @@ fn storage_wait_for_tasks( .ok_or_else(|| Error::InvalidRepositoryInput { message: "storage-only waits require an absolute run deadline".to_string(), })?; - let due_at = - crate::interpreter::resolve_temporal_target(target, wait_entered_at, run_deadline_at)?; + let due_at = match crate::interpreter::resolve_temporal_target_within_deadline( + target, + wait_entered_at, + run_deadline_at, + )? { + TemporalTargetResolution::Due(due_at) => due_at, + TemporalTargetResolution::DeadlineExceeded { + due_at, + run_deadline_at, + } => { + return Ok(Some(StorageWaitPlan::DeadlineExceeded(format!( + "wait on node `{}` entered at {wait_entered_at} resolves at {due_at}, \ + at or after the run deadline {run_deadline_at}", + first.node_id + )))); + } + }; let exact_target = ExecutionTemporalTarget::At { at: due_at }; let reason = match &first.kind { LogicalTaskKind::Review { @@ -1758,12 +1871,14 @@ fn storage_wait_for_tasks( }); } }; - Ok(Some(StorageWaitMaterialization { - task_status, - trigger_kind, - due_at, - reason, - })) + Ok(Some(StorageWaitPlan::Enter(Box::new( + StorageWaitMaterialization { + task_status, + trigger_kind, + due_at, + reason, + }, + )))) } fn waiting_reason_task_id(reason: &WaitingReason) -> Option { @@ -1902,6 +2017,109 @@ fn waiting_run_status_after( } } +/// Commits the one durable effect of a false node condition: a `skipped` node aggregate. +/// +/// The node keeps zero logical tasks forever, so its aggregate output is JSON `null` +/// with a verified hash and `aggregate_complete`, exactly the shape dependents already +/// load. Dependents are then released so ordering-only successors of a skipped branch +/// still run. +async fn commit_condition_skip( + mut conn: ScopedConn<'_>, + run: &ExecutionRunRecord, + node_id: &str, + expected_cursor: u64, +) -> Result { + let Some(row) = sqlx::query( + "SELECT node_status, materialization_cursor, materialization_complete, \ + total_task_count, aggregate_complete \ + FROM moa.execution_node_state WHERE run_uid = $1 AND node_id = $2 FOR UPDATE", + ) + .bind(run.run_uid) + .bind(node_id) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + conn.commit().await.map_err(storage_error)?; + return Ok(ReadyMaterializationOutcome::Conflict); + }; + let status: String = row.try_get("node_status").map_err(row_error)?; + let cursor = required_u64(&row, "materialization_cursor")?; + let total_task_count = required_u64(&row, "total_task_count")?; + let materialization_complete: bool = + row.try_get("materialization_complete").map_err(row_error)?; + let aggregate_complete: bool = row.try_get("aggregate_complete").map_err(row_error)?; + // `skipped` is also reachable for a node whose tasks all settled without succeeding, + // so the zero-task shape is what distinguishes an already-committed condition skip + // from that unrelated aggregate. + if status == "skipped" + && total_task_count == 0 + && materialization_complete + && aggregate_complete + { + conn.commit().await.map_err(storage_error)?; + return Ok(ReadyMaterializationOutcome::Replayed { + tasks: Vec::new(), + next_cursor: expected_cursor, + triggers: Vec::new(), + }); + } + if cursor != expected_cursor + || materialization_complete + || total_task_count != 0 + || status != "pending" + { + conn.commit().await.map_err(storage_error)?; + return Ok(ReadyMaterializationOutcome::Conflict); + } + let output_hash = node_output_hash(&Value::Null)?.to_string(); + let updated = sqlx::query( + "UPDATE moa.execution_node_state SET node_status = 'skipped', \ + materialization_complete = TRUE, aggregate_output = 'null'::JSONB, \ + aggregate_output_hash = $4, aggregate_complete = TRUE, updated_at = NOW() \ + WHERE run_uid = $1 AND node_id = $2 AND materialization_cursor = $3 \ + AND NOT materialization_complete AND node_status = 'pending' \ + AND total_task_count = 0", + ) + .bind(run.run_uid) + .bind(node_id) + .bind(to_i64( + expected_cursor, + "expected node materialization cursor", + )?) + .bind(&output_hash) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if updated.rows_affected() != 1 { + conn.rollback().await.map_err(storage_error)?; + return Ok(ReadyMaterializationOutcome::Conflict); + } + release_node_dependents_in_tx(&mut conn, run, node_id).await?; + sqlx::query( + "UPDATE moa.execution_run SET last_progress_at = NOW(), updated_at = NOW() \ + WHERE run_uid = $1", + ) + .bind(run.run_uid) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + conn.commit().await.map_err(storage_error)?; + Ok(ReadyMaterializationOutcome::Applied { + tasks: Vec::new(), + next_cursor: expected_cursor, + triggers: Vec::new(), + }) +} + +/// Releases one dependency edge on every direct dependent of a settled node. +/// +/// A dependent whose counter cannot be decremented is normally a corrupted projection, +/// with one legitimate exception: a *sibling* dependency may have failed first and +/// cancelled this dependent through `cancel_unmaterialized_dependents_in_tx`, which +/// zeroes its counter. That is reachable whenever one branch of a fan-in settles +/// without tasks — an empty map, or a node whose condition evaluated false — after a +/// sibling has already failed. async fn release_node_dependents_in_tx( conn: &mut ScopedConn<'_>, run: &ExecutionRunRecord, @@ -1924,6 +2142,20 @@ async fn release_node_dependents_in_tx( .await .map_err(sqlx_error)?; if released.rows_affected() != 1 { + let already_cancelled = sqlx::query_scalar::<_, bool>( + "SELECT EXISTS (SELECT 1 FROM moa.execution_node_state \ + WHERE run_uid = $1 AND node_id = $2 AND node_status = 'cancelled' \ + AND total_task_count = 0 AND remaining_dependency_count = 0 \ + AND materialization_complete AND aggregate_complete)", + ) + .bind(run.run_uid) + .bind(&dependent.id) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if already_cancelled { + continue; + } return Err(Error::InvalidRepositoryData { message: format!( "dependent node `{}` lost its dependency counter", @@ -2218,6 +2450,10 @@ async fn transition_node_counters_inner( updated_node_status = "pending".to_string(); } } + // `waiting_reasons_truncated` is derived from the post-update counters rather than + // maintained separately. Not every durable wait has a sampleable reason — a WaitingReplan + // task has no `WaitingReason` variant at all — so a wait that raises `waiting_task_count` + // without appending a sample must still leave the row readable by `run_from_row`. let run_updated = sqlx::query( "UPDATE moa.execution_run SET ready_task_count = ready_task_count + $2, \ active_task_count = active_task_count + $3, \ @@ -2232,6 +2468,8 @@ async fn transition_node_counters_inner( waiting_input_tenant_admin_task_count = \ waiting_input_tenant_admin_task_count + $12, \ waiting_input_external_task_count = waiting_input_external_task_count + $13, \ + waiting_reasons_truncated = \ + jsonb_array_length(waiting_reasons) < waiting_task_count + $4, \ last_progress_at = GREATEST(last_progress_at, NOW()), \ wake_epoch = wake_epoch + 1, updated_at = NOW() \ WHERE run_uid = $1 AND ready_task_count + $2 >= 0 AND active_task_count + $3 >= 0 \ diff --git a/crates/moa-execution/src/repository/run.rs b/crates/moa-execution/src/repository/run.rs index 1beb16e28..dcd544b2a 100644 --- a/crates/moa-execution/src/repository/run.rs +++ b/crates/moa-execution/src/repository/run.rs @@ -50,6 +50,60 @@ const LOAD_RUN_BY_IDEMPOTENCY_FOR_SESSION_SQL: &str = r#" AND session_id = $4 "#; +/// Recovery request for one crashed controller activation of an exact wake. +#[derive(Clone, Debug, PartialEq)] +pub struct ResumedControllerRecoveryRequest { + /// Exact controller generation claimed by the crashed activation. + pub controller_generation: u64, + /// Exact wake epoch claimed by the crashed activation. + pub wake_epoch: u64, + /// Durable checkpoint installed while the replacement activation is enqueued. + pub checkpoint: ExecutionRunActivationCheckpoint, + /// Structured activation payload carried by the replacement activation. + pub continuation_payload: Value, + /// Earliest time at which the replacement activation may be dispatched. + pub continuation_not_before_at: DateTime, + /// Consecutive crashed activations tolerated before the run must fail instead. + pub maximum_consecutive_failures: u64, +} + +/// Bounded outcome of one resumed-activation recovery. +#[derive(Clone, Debug, PartialEq)] +pub enum ResumedControllerRecoveryOutcome { + /// The wake was acknowledged and exactly one replacement activation enqueued. + Recovered { + /// Current run after the commit. + run: Box, + /// Exactly one replacement activation committed with the checkpoint. + continuation: Box, + /// Consecutive crashed activations recorded by this recovery. + consecutive_failures: u64, + }, + /// The recovery budget is spent; neither the wake nor the failure count was mutated. + BudgetExhausted { + /// Consecutive crashed activations that would have been recorded. + consecutive_failures: u64, + }, + /// The claimed wake had already been acknowledged. + Replayed(Box), + /// A newer controller generation owns the run. + StaleGeneration { + /// Controller generation currently owning the run. + current_generation: u64, + }, + /// A newer wake epoch owns the run. + StaleWake { + /// Wake epoch currently owning the run. + current_wake_epoch: u64, + /// Last wake epoch acknowledged by compare-and-set. + processed_wake_epoch: u64, + }, + /// The run no longer exists. + NotFound, + /// The run is not in a state that can acknowledge the claimed wake. + InvalidState, +} + /// Atomic run-admission result, including durable idempotency replay and capacity deferral. #[derive(Clone, Debug, PartialEq)] pub enum RunAdmissionOutcome { @@ -715,6 +769,144 @@ impl ExecutionRepository { Ok(outcome) } + /// Recovers one crashed controller activation under a bounded consecutive-failure budget. + /// + /// A resumed claim proves that a prior activation of this exact wake never acknowledged it. + /// While the budget holds, the wake is acknowledged and exactly one replacement activation is + /// enqueued in the same transaction. Once the budget is spent the wake is deliberately left + /// unacknowledged and [`ResumedControllerRecoveryOutcome::BudgetExhausted`] is returned, so the + /// caller can commit an explicit terminal intent against the same still-current wake instead of + /// re-enqueueing a continuation that can only crash again. + pub async fn recover_resumed_controller_wake( + &self, + scope: ExecutionScope, + config: &ExecutionConfig, + run_uid: Uuid, + request: ResumedControllerRecoveryRequest, + ) -> Result { + validate_activation_checkpoint(&request.checkpoint)?; + if !request.continuation_payload.is_object() { + return Err(Error::InvalidRepositoryInput { + message: "controller continuation payload must be a JSON object".to_string(), + }); + } + if request.maximum_consecutive_failures == 0 { + return Err(Error::InvalidRepositoryInput { + message: "resumed activation recovery budget must be greater than zero".to_string(), + }); + } + let mut conn = scope.begin(&self.pool).await?; + let tenant_id = sqlx::query_scalar::<_, Uuid>( + "SELECT tenant_id FROM moa.execution_run WHERE run_uid=$1", + ) + .bind(run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(tenant_id) = tenant_id else { + conn.commit().await.map_err(storage_error)?; + return Ok(ResumedControllerRecoveryOutcome::NotFound); + }; + prelock_capacity_dimensions_in_tx( + conn.as_mut(), + config, + TenantId(tenant_id), + &[ + ExecutionCapacityDimension::ActiveRuns, + ExecutionCapacityDimension::ParkedRuns, + ], + ) + .await?; + let observed = sqlx::query_scalar::<_, i64>( + "SELECT activation_failure_count FROM moa.execution_run WHERE run_uid=$1 FOR UPDATE", + ) + .bind(run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(observed) = observed else { + conn.commit().await.map_err(storage_error)?; + return Ok(ResumedControllerRecoveryOutcome::NotFound); + }; + let consecutive_failures = to_u64(observed, "activation failure count")? + .checked_add(1) + .ok_or_else(|| Error::ArithmeticOverflow { + context: "controller activation failure count".to_string(), + })?; + if consecutive_failures > request.maximum_consecutive_failures { + conn.commit().await.map_err(storage_error)?; + return Ok(ResumedControllerRecoveryOutcome::BudgetExhausted { + consecutive_failures, + }); + } + let checkpoint = ExecutionRunActivationCheckpoint { + activation_state: ExecutionActivationState::Idle, + ..request.checkpoint + }; + let completion = complete_controller_wake_in_conn( + &mut conn, + run_uid, + request.controller_generation, + request.wake_epoch, + checkpoint, + ) + .await?; + let outcome = match completion { + RunControllerCompletionOutcome::Applied { .. } => { + record_activation_failure_count_in_conn( + conn.as_mut(), + run_uid, + consecutive_failures, + ) + .await?; + let continuation = enqueue_run_activation_in_conn( + conn.as_mut(), + TenantId(tenant_id), + run_uid, + request.controller_generation, + request.continuation_not_before_at, + request.continuation_payload, + ) + .await?; + let row = sqlx::query(LOAD_RUN_SQL) + .bind(run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + ResumedControllerRecoveryOutcome::Recovered { + run: Box::new(run_from_row(&row)?), + continuation: Box::new(continuation), + consecutive_failures, + } + } + RunControllerCompletionOutcome::Replayed(run) => { + ResumedControllerRecoveryOutcome::Replayed(run) + } + RunControllerCompletionOutcome::StaleGeneration { current_generation } => { + ResumedControllerRecoveryOutcome::StaleGeneration { current_generation } + } + RunControllerCompletionOutcome::StaleWake { + current_wake_epoch, + processed_wake_epoch, + } => ResumedControllerRecoveryOutcome::StaleWake { + current_wake_epoch, + processed_wake_epoch, + }, + RunControllerCompletionOutcome::NotFound => ResumedControllerRecoveryOutcome::NotFound, + RunControllerCompletionOutcome::InvalidState => { + ResumedControllerRecoveryOutcome::InvalidState + } + RunControllerCompletionOutcome::CapacitySaturated { dimension } => { + conn.rollback().await.map_err(storage_error)?; + return Err(Error::CapacitySaturated { + dimension: dimension.as_str(), + }); + } + }; + conn.commit().await.map_err(storage_error)?; + Ok(outcome) + } + /// Arms or idempotently replaces the exact deadline trigger for a run generation. pub async fn arm_run_deadline( &self, @@ -796,7 +988,7 @@ pub(super) async fn arm_run_deadline_in_conn( "SELECT trigger_uid, controller_generation \ FROM moa.execution_trigger \ WHERE run_uid = $1 AND trigger_kind = 'run_deadline' \ - AND state IN ('pending', 'dispatching') AND trigger_uid <> $2 \ + AND state = 'pending' AND trigger_uid <> $2 \ ORDER BY controller_generation, trigger_uid \ LIMIT 2 FOR UPDATE", ) @@ -992,7 +1184,8 @@ pub async fn complete_controller_wake_in_conn( let updated = sqlx::query( "UPDATE moa.execution_run SET status = $4, activation_state = $5, \ next_wake_at = $6, waiting_since = $7, ready_task_count = $8, \ - active_task_count = $9, processed_wake_epoch = $3, updated_at = NOW() \ + active_task_count = $9, processed_wake_epoch = $3, \ + activation_failure_count = 0, updated_at = NOW() \ WHERE run_uid = $1 AND controller_generation = $2 \ AND wake_epoch = $3 AND processed_wake_epoch < $3 RETURNING *", ) @@ -1017,6 +1210,27 @@ pub async fn complete_controller_wake_in_conn( }) } +/// Records consecutive crashed controller activations inside the caller's transaction. +/// +/// [`complete_controller_wake_in_conn`] resets this counter for every acknowledged wake, so the +/// resumed-recovery path must restore its incremented value after acknowledging the crashed wake. +async fn record_activation_failure_count_in_conn( + conn: &mut PgConnection, + run_uid: Uuid, + consecutive_failures: u64, +) -> Result<()> { + sqlx::query( + "UPDATE moa.execution_run SET activation_failure_count = $2, updated_at = NOW() \ + WHERE run_uid = $1", + ) + .bind(run_uid) + .bind(to_i64(consecutive_failures, "activation failure count")?) + .execute(&mut *conn) + .await + .map_err(sqlx_error)?; + Ok(()) +} + fn run_activation_dispatch_uid(run_uid: Uuid, controller_generation: u64, wake_epoch: u64) -> Uuid { let name = format!("{run_uid}:{controller_generation}:{wake_epoch}"); Uuid::new_v5(&RUN_ACTIVATION_DISPATCH_NAMESPACE, name.as_bytes()) diff --git a/crates/moa-execution/src/repository/schedule.rs b/crates/moa-execution/src/repository/schedule.rs index 7a080ddd2..9a520c4f5 100644 --- a/crates/moa-execution/src/repository/schedule.rs +++ b/crates/moa-execution/src/repository/schedule.rs @@ -1026,7 +1026,7 @@ async fn cancel_armed_occurrences( let trigger_uids = sqlx::query_scalar::<_, Uuid>( "SELECT trigger_uid FROM moa.execution_trigger \ WHERE schedule_uid=$1 AND schedule_incarnation=$2 \ - AND trigger_kind='schedule_occurrence' AND state IN ('pending','dispatching') \ + AND trigger_kind='schedule_occurrence' AND state = 'pending' \ ORDER BY trigger_uid LIMIT 2 FOR UPDATE", ) .bind(schedule.schedule_uid) diff --git a/crates/moa-execution/src/repository/task.rs b/crates/moa-execution/src/repository/task.rs index 93d242a6c..160cb9aa3 100644 --- a/crates/moa-execution/src/repository/task.rs +++ b/crates/moa-execution/src/repository/task.rs @@ -2132,6 +2132,35 @@ impl ExecutionRepository { return Ok(TaskAttemptSettlementOutcome::NotFound); }; let task = task_from_row(&task_row)?; + // A relative input-wait expiry resolves against wait entry, not compile time, so a delay + // that was legal when the plan compiled can land past the run deadline here. That is a + // product outcome for a long-horizon run, so the task fails terminally with a typed + // deadline failure instead of parking on a wait that can never settle in time. + let needs_input = matches!(outcome.result, ExecutionTaskResult::NeedsInput { .. }); + let outcome = match run.approved_budget.deadline_at { + Some(run_deadline_at) if needs_input => { + match crate::interpreter::resolve_temporal_target_within_deadline( + &run.active_plan.definition.input_wait_policy.expiry, + settled_at, + run_deadline_at, + )? { + crate::interpreter::TemporalTargetResolution::Due(_) => outcome, + crate::interpreter::TemporalTargetResolution::DeadlineExceeded { + due_at, + run_deadline_at, + } => failed_task_outcome( + moa_artifacts::execution_plan::ExecutionFailureClass::DeadlineExceeded, + format!( + "input wait on node `{}` entered at {settled_at} resolves at \ + {due_at}, at or after the run deadline {run_deadline_at}", + task.node_id + ), + outcome.usage.clone(), + ), + } + } + _ => outcome, + }; if capacity == CapacityReleaseOutcome::AlreadyReleased { let replay = task_attempt_settlement_replayed(&task, &fence, &outcome); if replay { @@ -4124,7 +4153,7 @@ async fn task_attempt_resources_match( SELECT 1 FROM moa.execution_trigger \ WHERE trigger_uid = $7 AND tenant_id = $2 AND run_uid = $3 AND task_id = $4 \ AND trigger_kind = 'task_watchdog' AND controller_generation = $5 \ - AND attempt_generation = $6 AND state IN ('pending', 'dispatching') \ + AND attempt_generation = $6 AND state = 'pending' \ )", ) .bind(fence.capacity_reservation_uid) @@ -4589,7 +4618,7 @@ impl ExecutionRepository { let trigger_uids = sqlx::query_scalar::<_, Uuid>( "SELECT trigger_uid FROM moa.execution_trigger \ WHERE run_uid=$1 AND task_id=$2 AND trigger_kind='wait_expiry' \ - AND state IN ('pending','dispatching') \ + AND state = 'pending' \ ORDER BY trigger_uid LIMIT 2 FOR UPDATE", ) .bind(run_uid) @@ -4981,7 +5010,7 @@ impl ExecutionRepository { let trigger_uids = sqlx::query_scalar::<_, Uuid>( "SELECT trigger_uid FROM moa.execution_trigger \ WHERE run_uid=$1 AND task_id=$2 AND trigger_kind='wait_expiry' \ - AND state IN ('pending','dispatching') \ + AND state = 'pending' \ ORDER BY trigger_uid LIMIT 2 FOR UPDATE", ) .bind(run_uid) diff --git a/crates/moa-execution/src/repository/terminal.rs b/crates/moa-execution/src/repository/terminal.rs index 7e84d58bb..21176e502 100644 --- a/crates/moa-execution/src/repository/terminal.rs +++ b/crates/moa-execution/src/repository/terminal.rs @@ -485,7 +485,7 @@ impl ExecutionRepository { let has_pending_trigger_work: bool = sqlx::query_scalar( "SELECT \ EXISTS (SELECT 1 FROM moa.execution_trigger \ - WHERE run_uid = $1 AND state IN ('pending', 'dispatching')) \ + WHERE run_uid = $1 AND state = 'pending') \ OR EXISTS (SELECT 1 FROM moa.execution_dispatch_outbox \ WHERE run_uid = $1 AND trigger_uid IS NOT NULL \ AND state IN ('pending', 'dispatching')) \ @@ -571,7 +571,7 @@ pub(super) async fn drain_run_triggers_page_in_conn( "SELECT trigger_uid, trigger_kind, controller_generation, attempt_generation, \ compensation_generation, compensation_attempt_generation \ FROM moa.execution_trigger \ - WHERE run_uid = $1 AND state IN ('pending', 'dispatching') \ + WHERE run_uid = $1 AND state = 'pending' \ ORDER BY trigger_kind, trigger_uid LIMIT $2 FOR UPDATE", ) .bind(run.run_uid) @@ -615,7 +615,7 @@ pub(super) async fn drain_run_triggers_page_in_conn( } let next_wake_at: Option> = sqlx::query_scalar( "SELECT MIN(due_at) FROM moa.execution_trigger \ - WHERE run_uid = $1 AND state IN ('pending', 'dispatching')", + WHERE run_uid = $1 AND state = 'pending'", ) .bind(run.run_uid) .fetch_one(conn.as_mut()) diff --git a/crates/moa-execution/src/repository/transition.rs b/crates/moa-execution/src/repository/transition.rs index 2b855f0b4..bf7d90f42 100644 --- a/crates/moa-execution/src/repository/transition.rs +++ b/crates/moa-execution/src/repository/transition.rs @@ -309,56 +309,6 @@ impl ExecutionRepository { Ok(TransitionOutcome::RunApplied(updated)) } - /// Atomically settles one due storage-only wait under its run, task, and wait-entry fences. - pub async fn settle_wait( - &self, - scope: ExecutionScope, - run_uid: Uuid, - expected_task_generation: u64, - expected_waiting_since: DateTime, - settlement: WaitSettlement, - settled_at: DateTime, - ) -> Result { - let task_id = match &settlement { - WaitSettlement::TimerElapsed { task_id, .. } - | WaitSettlement::WaitExpired { task_id, .. } => *task_id, - }; - let mut conn = scope.begin(&self.pool).await?; - let Some(run_row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) - .bind(run_uid) - .fetch_optional(conn.as_mut()) - .await - .map_err(sqlx_error)? - else { - conn.commit().await.map_err(storage_error)?; - return Ok(TransitionOutcome::NotFound); - }; - let run = run_from_row(&run_row)?; - let Some(task_row) = sqlx::query(LOAD_TASK_FOR_UPDATE_SQL) - .bind(run_uid) - .bind(task_id.as_uuid()) - .fetch_optional(conn.as_mut()) - .await - .map_err(sqlx_error)? - else { - conn.commit().await.map_err(storage_error)?; - return Ok(TransitionOutcome::NotFound); - }; - let task = super::rows::task_from_row(&task_row)?; - let outcome = settle_wait_locked_in_conn( - &mut conn, - &run, - &task, - expected_task_generation, - expected_waiting_since, - settlement, - settled_at, - ) - .await?; - conn.commit().await.map_err(storage_error)?; - Ok(outcome) - } - /// Delivers and settles one exact due task wait, activating only a non-paused run. pub async fn fire_wait_trigger( &self, @@ -684,7 +634,7 @@ pub(super) async fn refresh_run_after_wait_settlement_in_conn( ), next_trigger AS ( SELECT ( SELECT due_at FROM moa.execution_trigger - WHERE run_uid = $1 AND state IN ('pending', 'dispatching') + WHERE run_uid = $1 AND state = 'pending' ORDER BY due_at, trigger_uid LIMIT 1 ) AS due_at ), remaining_wait AS ( @@ -873,11 +823,6 @@ fn wait_settlement_outcome( "storage wait expired with fail_task policy".to_string(), usage, )), - ExecutionWaitExpiryAction::FailRun => Ok(failed_task_outcome( - ExecutionFailureClass::Terminal, - "storage wait expired with fail_run policy".to_string(), - usage, - )), } } } @@ -943,7 +888,7 @@ async fn enqueue_pause_cancellations( AND trigger.attempt_generation=task.attempt_generation \ AND trigger.controller_generation=$2 \ AND trigger.trigger_kind='task_watchdog' \ - AND trigger.state IN ('pending','dispatching') \ + AND trigger.state = 'pending' \ WHERE task.run_uid=$1 AND task.active_dispatch_uid IS NOT NULL \ AND task.attempt_state IN ('dispatching','running') \ ORDER BY task.task_id LIMIT $3", @@ -990,7 +935,7 @@ async fn enqueue_pause_cancellations( AND trigger.compensation_attempt_generation=compensation.attempt_generation \ AND trigger.controller_generation=$2 \ AND trigger.trigger_kind='compensation_watchdog' \ - AND trigger.state IN ('pending','dispatching') \ + AND trigger.state = 'pending' \ WHERE compensation.run_uid=$1 AND compensation.active_dispatch_uid IS NOT NULL \ AND compensation.attempt_state IN ('dispatching','running') \ ORDER BY compensation.compensation_id LIMIT $3", diff --git a/crates/moa-execution/src/repository/trigger.rs b/crates/moa-execution/src/repository/trigger.rs index 236dca574..61f8634ed 100644 --- a/crates/moa-execution/src/repository/trigger.rs +++ b/crates/moa-execution/src/repository/trigger.rs @@ -1040,8 +1040,7 @@ impl ExecutionRepository { rearmed_trigger_delivery_dispatch_uid(request.trigger_uid, retry_at); sqlx::query( "UPDATE moa.execution_trigger SET state='pending', due_at=$2, \ - claim_owner=NULL,claimed_at=NULL,claim_expires_at=NULL,delivered_at=NULL, \ - last_error=$3,updated_at=now() WHERE trigger_uid=$1", + delivered_at=NULL, last_error=$3, updated_at=now() WHERE trigger_uid=$1", ) .bind(request.trigger_uid) .bind(retry_at) @@ -1440,11 +1439,9 @@ pub async fn create_trigger_with_dispatch_in_conn( request: &NewExecutionTrigger, ) -> Result { let trigger = create_trigger_in_conn(conn, request).await?; - if matches!( - trigger.state, - ExecutionDeliveryState::Pending | ExecutionDeliveryState::Dispatching - ) && reserve_capacity_in_tx(conn, config, trigger_capacity_request(&trigger)).await? - == CapacityReserveOutcome::Saturated + if trigger.state == ExecutionDeliveryState::Pending + && reserve_capacity_in_tx(conn, config, trigger_capacity_request(&trigger)).await? + == CapacityReserveOutcome::Saturated { return Err(Error::CapacitySaturated { dimension: ExecutionCapacityDimension::ScheduledTriggers.as_str(), @@ -1516,9 +1513,8 @@ pub async fn fire_trigger_in_conn( sqlx::query( r#" UPDATE moa.execution_trigger - SET state = 'superseded', claim_owner = NULL, claimed_at = NULL, - claim_expires_at = NULL, updated_at = now() - WHERE trigger_uid = $1 AND state IN ('pending', 'dispatching') + SET state = 'superseded', updated_at = now() + WHERE trigger_uid = $1 AND state = 'pending' "#, ) .bind(trigger_uid) @@ -1535,10 +1531,9 @@ pub async fn fire_trigger_in_conn( sqlx::query( r#" UPDATE moa.execution_trigger - SET state = 'delivered', delivered_at = now(), claim_owner = NULL, - claimed_at = NULL, claim_expires_at = NULL, last_error = NULL, + SET state = 'delivered', delivered_at = now(), last_error = NULL, updated_at = now() - WHERE trigger_uid = $1 AND state IN ('pending', 'dispatching') + WHERE trigger_uid = $1 AND state = 'pending' "#, ) .bind(trigger_uid) @@ -1637,9 +1632,8 @@ pub(super) async fn deliver_wait_trigger_in_conn( } if !trigger_is_current(conn, &trigger).await? { sqlx::query( - "UPDATE moa.execution_trigger SET state='superseded', claim_owner=NULL, \ - claimed_at=NULL, claim_expires_at=NULL, updated_at=now() \ - WHERE trigger_uid=$1 AND state IN ('pending','dispatching')", + "UPDATE moa.execution_trigger SET state='superseded', updated_at=now() \ + WHERE trigger_uid=$1 AND state='pending'", ) .bind(trigger_uid) .execute(&mut *conn) @@ -1656,8 +1650,8 @@ pub(super) async fn deliver_wait_trigger_in_conn( .map_err(super::row_error)?; sqlx::query( "UPDATE moa.execution_trigger SET state='delivered', delivered_at=$2, \ - claim_owner=NULL, claimed_at=NULL, claim_expires_at=NULL, last_error=NULL, \ - updated_at=now() WHERE trigger_uid=$1 AND state IN ('pending','dispatching')", + last_error=NULL, updated_at=now() \ + WHERE trigger_uid=$1 AND state='pending'", ) .bind(trigger_uid) .bind(observed_at) @@ -1727,8 +1721,7 @@ pub async fn supersede_trigger_in_conn( sqlx::query( r#" UPDATE moa.execution_trigger - SET state = 'superseded', claim_owner = NULL, claimed_at = NULL, - claim_expires_at = NULL, updated_at = now() + SET state = 'superseded', updated_at = now() WHERE trigger_uid = $1 "#, ) @@ -2371,7 +2364,7 @@ fn trigger_matches_request(record: &ExecutionTriggerRecord, request: &NewExecuti && record.payload == request.payload } -fn trigger_from_row(row: &sqlx::postgres::PgRow) -> Result { +pub(super) fn trigger_from_row(row: &sqlx::postgres::PgRow) -> Result { let controller_generation = row .try_get::, _>("controller_generation") .map_err(super::row_error)?; diff --git a/crates/moa-execution/src/state.rs b/crates/moa-execution/src/state.rs index a532eda05..eed42a9ef 100644 --- a/crates/moa-execution/src/state.rs +++ b/crates/moa-execution/src/state.rs @@ -563,8 +563,6 @@ pub enum ExecutionTerminalCause { /// Exact exhausted limit, with deadline taking precedence over budget. reason: ExecutionLimitStop, }, - /// Pending work existed but the scheduler could neither dispatch nor wait. - SchedulerNoProgress, /// Deterministic replan stop policy ended the run. ReplanStop { /// Exact closed replan stop reason. @@ -601,7 +599,6 @@ enum StrictExecutionTerminalCause { LimitStop { reason: ExecutionLimitStop, }, - SchedulerNoProgress {}, ReplanStop { reason: ReplanStopReason, }, @@ -628,7 +625,6 @@ impl<'de> Deserialize<'de> for ExecutionTerminalCause { } StrictExecutionTerminalCause::TaskFailure { class } => Self::TaskFailure { class }, StrictExecutionTerminalCause::LimitStop { reason } => Self::LimitStop { reason }, - StrictExecutionTerminalCause::SchedulerNoProgress {} => Self::SchedulerNoProgress, StrictExecutionTerminalCause::ReplanStop { reason } => Self::ReplanStop { reason }, StrictExecutionTerminalCause::Cancellation {} => Self::Cancellation, StrictExecutionTerminalCause::InternalFailure {} => Self::InternalFailure, @@ -921,26 +917,7 @@ impl LogicalTaskKind { } } -/// Pure scheduler decision. -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -#[serde(rename_all = "snake_case")] -pub enum ScheduleDecision { - /// Newly ready logical tasks. - Ready(Vec), - /// One storage-only wait reached its deterministic settlement time. - SettleWait(WaitSettlement), - /// Durable work is waiting on execution or an external condition. - Waiting(Vec), - /// The run has a terminal projection. - Terminal(TerminalProjection), - /// Unfinished nodes exist but no work or wait can advance them. - NoProgress { - /// Stable pending node IDs, sorted and duplicate-free. - pending_node_ids: Vec, - }, -} - -/// One deterministic storage-only wait transition selected by the scheduler. +/// One deterministic storage-only wait transition applied by the repository. #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] pub enum WaitSettlement { @@ -1122,26 +1099,6 @@ pub struct FailureFingerprintInput { pub message: String, } -/// Compact task summary supplied to a completion verifier. -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -#[serde(deny_unknown_fields)] -pub struct VerifierTaskSummary { - /// Stable task ID. - pub task_id: ExecutionTaskId, - /// Stable node ID. - pub node_id: String, - /// Stable task item key. - pub item_key: String, - /// Current terminal task status. - pub status: ExecutionTaskStatus, - /// Canonical structured-output hash when output exists. - pub output_hash: Option, - /// Typed failure when present. - pub failure: Option, - /// Sorted unique citation source IDs. - pub citation_source_ids: Vec, -} - /// Derives the durable task status implied by one persisted outcome. #[must_use] pub fn task_status_from_outcome( @@ -1411,12 +1368,14 @@ fn append_frame(output: &mut Vec, value: &[u8]) -> Result<()> { #[cfg(test)] mod tests { - use moa_artifacts::execution_plan::{ExecutionFailureClass, ExecutionTaskResult, RetryPolicy}; + use moa_artifacts::execution_plan::{ + ExecutionFailureClass, ExecutionTaskResult, ExecutionUsage, RetryPolicy, + }; use serde_json::json; use super::{ - CompensationId, CompensationStatus, ExecutionLimitStop, ExecutionRunStatus, - ExecutionTaskId, ExecutionTerminalCause, ExecutionTerminalEvidence, + CompensationId, CompensationStatus, ExecutionCompensationOutcome, ExecutionLimitStop, + ExecutionRunStatus, ExecutionTaskId, ExecutionTerminalCause, ExecutionTerminalEvidence, ExecutionTerminalReason, PendingExecutionTerminal, exhaust_retry_outcome, failed_task_outcome, retry_delay_ms, }; @@ -1447,10 +1406,6 @@ mod tests { }, json!({"kind":"limit_stop","reason":"budget_exceeded"}), ), - ( - ExecutionTerminalCause::SchedulerNoProgress, - json!({"kind":"scheduler_no_progress"}), - ), ( ExecutionTerminalCause::ReplanStop { reason: ReplanStopReason::DuplicateAmendment, @@ -1465,6 +1420,37 @@ mod tests { ExecutionTerminalCause::InternalFailure, json!({"kind":"internal_failure"}), ), + ( + ExecutionTerminalCause::CompensationFailure { + original_status: ExecutionRunStatus::Cancelled, + original_reason: ExecutionTerminalReason::Cancelled, + original_cause: Box::new(ExecutionTerminalCause::Cancellation), + compensation_id: compensation_id(), + outcome: ExecutionCompensationOutcome::Failed { + message: "undo rejected".to_string(), + retryable: false, + usage: usage(), + }, + }, + json!({ + "kind":"compensation_failure", + "original_status":"cancelled", + "original_reason":"cancelled", + "original_cause":{"kind":"cancellation"}, + "compensation_id": compensation_id().as_uuid(), + "outcome":{ + "kind":"failed", + "message":"undo rejected", + "retryable":false, + "usage":{ + "cost_microusd":7, + "tokens":11, + "tool_calls":1, + "retrieved_bytes":0, + }, + }, + }), + ), ]; for (cause, expected) in cases { assert_eq!( @@ -1489,6 +1475,65 @@ mod tests { ) .is_err() ); + assert!( + serde_json::from_value::(json!({ + "kind":"compensation_failure", + "original_status":"cancelled", + "original_reason":"cancelled", + "original_cause":{"kind":"cancellation","message":"not schema"}, + "compensation_id": compensation_id().as_uuid(), + "outcome":{ + "kind":"failed", + "message":"undo rejected", + "retryable":false, + "usage":{ + "cost_microusd":7, + "tokens":11, + "tool_calls":1, + "retrieved_bytes":0, + }, + }, + })) + .is_err(), + "the boxed original cause must stay closed to unknown fields" + ); + assert!( + serde_json::from_value::(json!({ + "kind":"compensation_failure", + "original_status":"cancelled", + "original_reason":"cancelled", + "compensation_id": compensation_id().as_uuid(), + "outcome":{ + "kind":"failed", + "message":"undo rejected", + "retryable":false, + "usage":{ + "cost_microusd":7, + "tokens":11, + "tool_calls":1, + "retrieved_bytes":0, + }, + }, + })) + .is_err(), + "a compensation failure must never default away the terminal decision it superseded" + ); + } + + fn compensation_id() -> CompensationId { + CompensationId::derive(ExecutionTaskId::from_uuid( + uuid::Uuid::parse_str("019c2222-3333-7444-8555-666666666666") + .expect("valid compensation forward task UUID"), + )) + } + + fn usage() -> ExecutionUsage { + ExecutionUsage { + cost_microusd: 7, + tokens: 11, + tool_calls: 1, + retrieved_bytes: 0, + } } #[test] diff --git a/crates/moa-execution/tests/compiler.rs b/crates/moa-execution/tests/compiler.rs index 8e0bcdd43..89f9a2f14 100644 --- a/crates/moa-execution/tests/compiler.rs +++ b/crates/moa-execution/tests/compiler.rs @@ -1152,15 +1152,8 @@ fn compile_rejects_unknown_dependency_output_reference_path_before_persistence() } #[test] -fn compile_checks_reference_paths_in_conditions_map_and_reduce_items() { - // Pins: every condition and operation-level collection binding receives the same schema-aware path check. - let mut condition = valid_request(); - condition.plan.nodes[0].when = Some(ExecutionCondition::Exists { - reference: ExecutionReference { - path: "$.input.missing".to_string(), - }, - }); - +fn compile_checks_reference_paths_in_map_and_reduce_items() { + // Pins: every operation-level collection binding receives the same schema-aware path check. let mut map = valid_request(); map.plan.nodes[0].operation = ExecutionOperation::Map { items: json!({ "$ref": "$.input.missing" }), @@ -1183,11 +1176,6 @@ fn compile_checks_reference_paths_in_conditions_map_and_reduce_items() { }; let outcomes = [ - ( - "condition", - compile(condition), - "plan.nodes[0].when.reference.$ref", - ), ("map items", compile(map), "plan.nodes[0].operation.items"), ( "reduce items", @@ -1208,6 +1196,235 @@ fn compile_checks_reference_paths_in_conditions_map_and_reduce_items() { } } +/// Builds the accepted three-node shape: one conditional effectful leaf nothing reads. +fn conditional_request() -> CompileExecutionRequest { + let mut request = valid_request(); + let reference = request.authorization.capability_refs[0].clone(); + let notify = ExecutionNode { + id: "notify".to_string(), + requirement_ids: vec!["req_one".to_string()], + depends_on: vec!["lookup".to_string()], + when: Some(ExecutionCondition::Equals { + reference: ExecutionReference { + path: "$.input.order_id".to_string(), + }, + value: json!("ord-1"), + }), + input: json!({}), + output_schema: json!({ "type": "object" }), + operation: ExecutionOperation::Capability { reference }, + compensation: None, + retry: retry(1), + budget: None, + }; + request.plan.nodes.insert(1, notify); + request.plan.nodes[2].depends_on.push("notify".to_string()); + request +} + +#[test] +fn compile_accepts_a_conditional_branch_whose_output_nothing_reads() { + // Pins: a conditional node is legal as an effectful leaf — depended on for ordering, + // never read — so authors can express a branch without any downstream null handling. + let outcome = compile(conditional_request()); + + assert!(outcome.compiled.is_some(), "{:?}", outcome.report.issues); + assert!(outcome.report.issues.is_empty()); +} + +#[test] +fn compile_rejects_every_way_a_skipped_branch_could_be_read_or_required() { + // Pins: each plan whose meaning depends on a conditional node having run. A skipped + // branch has a null output and counts as failed everywhere completion is measured, so + // accepting any of these would trade the old silent double-execution for a silent + // partial run. + let mut read_output = conditional_request(); + read_output.plan.nodes[2].operation = ExecutionOperation::Output { + value: json!({ "escalation": { "$ref": "$.nodes.notify.output" } }), + }; + + let mut required_node = conditional_request(); + required_node.goal.completion_checks.push(CompletionCheck { + id: "notify_required".to_string(), + description: "the branch ran".to_string(), + requirement_ids: vec!["req_one".to_string()], + constraint_ids: vec![], + kind: CompletionCheckKind::RequiredNodes { + node_ids: vec!["notify".to_string()], + }, + }); + + let mut conditional_output = conditional_request(); + conditional_output.plan.nodes[2].when = Some(ExecutionCondition::Exists { + reference: ExecutionReference { + path: "$.input.order_id".to_string(), + }, + }); + + let mut only_conditional = conditional_request(); + only_conditional + .goal + .requirements + .push(ExecutionRequirement { + id: "req_branch".to_string(), + description: "Notify only when the order matches".to_string(), + }); + only_conditional.goal.completion_checks[0] + .requirement_ids + .push("req_branch".to_string()); + only_conditional.plan.nodes[1].requirement_ids = vec!["req_branch".to_string()]; + + let mut hidden_reference = conditional_request(); + hidden_reference.plan.nodes[1].when = Some(ExecutionCondition::Exists { + reference: ExecutionReference { + path: "$.nodes.output.output".to_string(), + }, + }); + + // Path and message are asserted, not just the code: a rejection has to name the exact + // value an author must change, and the required-node rejection has to name the completion + // check that made the node required. A service-e2e scenario asserts the same strings + // against a live planning context. + let cases = [ + ( + "read a skipped output", + read_output, + "conditional_output_read", + "plan.nodes[2].operation.value.escalation", + "cannot be read", + ), + ( + "require a skippable node", + required_node, + "conditional_required_node", + "plan.nodes[1].when", + "notify_required", + ), + ( + "make the terminal output conditional", + conditional_output, + "conditional_output_node", + "plan.nodes[2].when", + "terminal output node", + ), + ( + "serve a requirement only conditionally", + only_conditional, + "requirement_only_conditional", + "plan.nodes.notify.requirement_ids", + "req_branch", + ), + ( + "read an undeclared dependency", + hidden_reference, + "condition_reference_not_visible", + "plan.nodes[1].when.reference.$ref", + "declared dependency output", + ), + ]; + for (case, request, expected_code, expected_path, expected_message) in cases { + let outcome = compile(request); + assert!( + outcome.compiled.is_none(), + "compiled a plan that would {case}" + ); + let issue = outcome + .report + .issues + .iter() + .find(|issue| issue.code == expected_code) + .unwrap_or_else(|| { + panic!( + "expected {expected_code} for `{case}`, got {:?}", + outcome.report.issues + ) + }); + assert_eq!(issue.path, expected_path, "wrong path for `{case}`"); + assert!( + issue.message.contains(expected_message), + "message for `{case}` must name {expected_message}: {}", + issue.message + ); + } +} + +#[test] +fn amendment_enforces_conditional_scope_and_cannot_relitigate_a_taken_skip() { + // Pins: `when` cannot arrive through the amendment path either, and a branch whose skip + // is already committed is immutable — the decision was durable when it was taken. + let request = conditional_request(); + let compiled = compile(request.clone()) + .compiled + .expect("compile conditional amendment fixture"); + let mut readable = compiled.plan.definition.nodes[2].clone(); + readable.id = "replacement_output".to_string(); + readable.operation = ExecutionOperation::Output { + value: json!({ "escalation": { "$ref": "$.nodes.notify.output" } }), + }; + + let smuggled = validate_amendment(ValidateAmendmentRequest { + goal: compiled.goal.clone(), + active_plan: compiled.plan.clone(), + amendment: PlanAmendment { + base_plan_revision: 4, + reason: "Read the conditional branch output".to_string(), + evidence: json!({}), + operations: vec![PlanAmendmentOperation::ReplacePendingNode { + node_id: "output".to_string(), + node: readable, + }], + }, + projection: amendment_projection(ExecutionProjection { + plan_revision: 4, + node_statuses: BTreeMap::new(), + tasks: vec![], + }), + catalog: request.catalog.clone(), + authorization: request.authorization.clone(), + remaining_budget: generous_budget(), + config: ExecutionConfig::default(), + now: now(), + }); + assert!(smuggled.plan.is_none()); + assert!( + smuggled + .report + .issues + .iter() + .any(|issue| issue.code == "conditional_output_read"), + "{:?}", + smuggled.report.issues + ); + + let taken = validate_amendment(ValidateAmendmentRequest { + goal: compiled.goal, + active_plan: compiled.plan, + amendment: PlanAmendment { + base_plan_revision: 4, + reason: "Undo a branch that was already skipped".to_string(), + evidence: json!({}), + operations: vec![PlanAmendmentOperation::RemovePendingNode { + node_id: "notify".to_string(), + }], + }, + projection: amendment_projection(ExecutionProjection { + plan_revision: 4, + node_statuses: BTreeMap::from([("notify".to_string(), ExecutionNodeStatus::Skipped)]), + tasks: vec![], + }), + catalog: request.catalog, + authorization: request.authorization, + remaining_budget: generous_budget(), + config: ExecutionConfig::default(), + now: now(), + }); + assert!( + taken.plan.is_none(), + "a committed skip must not be amendable: {:?}", + taken.report.issues + ); +} + #[test] fn compile_accepts_declared_nested_reference_paths() { // Pins: local schema references and allOf composition preserve declared nested input and dependency-output paths. @@ -1259,11 +1476,6 @@ fn compile_accepts_declared_nested_reference_paths() { }, "allOf": [{ "$ref": "#/$defs/LookupOutput" }] }); - request.plan.nodes[1].when = Some(ExecutionCondition::Exists { - reference: ExecutionReference { - path: "$.nodes.lookup.output.result.order.id".to_string(), - }, - }); request.plan.nodes[1].output_schema = json!({ "type": "string" }); request.plan.nodes[1].operation = ExecutionOperation::Output { value: json!({ "$ref": "$.nodes.lookup.output.result.order.id" }), diff --git a/crates/moa-execution/tests/execution_db.rs b/crates/moa-execution/tests/execution_db.rs index 31278b91f..ff037ce1c 100644 --- a/crates/moa-execution/tests/execution_db.rs +++ b/crates/moa-execution/tests/execution_db.rs @@ -12,6 +12,10 @@ mod compensation_attempts_db; mod compensation_db; #[path = "execution_db/completion_projection_db.rs"] mod completion_projection_db; +#[path = "execution_db/conditional_execution_db.rs"] +mod conditional_execution_db; +#[path = "execution_db/controller_wake_recovery_db.rs"] +mod controller_wake_recovery_db; #[path = "execution_db/execution_capacity_db.rs"] mod execution_capacity_db; #[path = "execution_db/incremental_scheduler_db.rs"] @@ -30,3 +34,5 @@ mod scope_and_lifecycle_db; mod support; #[path = "execution_db/trigger_outbox_db.rs"] mod trigger_outbox_db; +#[path = "execution_db/wait_entry_deadline_db.rs"] +mod wait_entry_deadline_db; diff --git a/crates/moa-execution/tests/execution_db/active_run_capacity_db.rs b/crates/moa-execution/tests/execution_db/active_run_capacity_db.rs index 7eaacdfc1..74012e9f5 100644 --- a/crates/moa-execution/tests/execution_db/active_run_capacity_db.rs +++ b/crates/moa-execution/tests/execution_db/active_run_capacity_db.rs @@ -737,7 +737,7 @@ async fn concurrent_deadline_arm_and_terminal_finalization_use_scheduled_before_ let active_trigger_count: i64 = sqlx::query_scalar( "SELECT count(*) FROM moa.execution_trigger \ WHERE run_uid=$1 AND trigger_kind='run_deadline' \ - AND state IN ('pending','dispatching')", + AND state = 'pending'", ) .bind(running.run_uid) .fetch_one(&pool) @@ -1304,7 +1304,7 @@ async fn high_fanout_terminal_trigger_cleanup_is_strictly_activation_bounded_db( run = commit.run; let active: i64 = sqlx::query_scalar( "SELECT count(*) FROM moa.execution_trigger \ - WHERE run_uid = $1 AND state IN ('pending', 'dispatching')", + WHERE run_uid = $1 AND state = 'pending'", ) .bind(run.run_uid) .fetch_one(&pool) diff --git a/crates/moa-execution/tests/execution_db/amendment_projection_db.rs b/crates/moa-execution/tests/execution_db/amendment_projection_db.rs index db9387c2c..5425af11a 100644 --- a/crates/moa-execution/tests/execution_db/amendment_projection_db.rs +++ b/crates/moa-execution/tests/execution_db/amendment_projection_db.rs @@ -68,6 +68,7 @@ async fn amendment_projection_counts_twenty_five_hundred_prior_failures_in_one_b reduce_cursor: None, source_exhausted: page_index == 2, terminal_output: None, + condition_skipped: false, tasks: page.to_vec(), }, ) diff --git a/crates/moa-execution/tests/execution_db/compensation_attempts_db.rs b/crates/moa-execution/tests/execution_db/compensation_attempts_db.rs index f93548ef8..93208cf1d 100644 --- a/crates/moa-execution/tests/execution_db/compensation_attempts_db.rs +++ b/crates/moa-execution/tests/execution_db/compensation_attempts_db.rs @@ -27,9 +27,9 @@ use moa_execution::{ ExecutionExternalJobStartRecoveryAdoptionOutcome, NewExecutionExternalJobIntent, }, external_job::{ExecutionExternalJobOwner, ExecutionExternalJobState}, - terminal::PendingTerminalAdvanceOutcome, + terminal::{PendingTerminalAdvanceOutcome, PendingTerminalAdvanceStage}, }, - state::{ExecutionCompensationOutcome, ExecutionTerminalEvidence}, + state::{CompensationStatus, ExecutionCompensationOutcome, ExecutionTerminalEvidence}, wire::{ ExecutionCompensationAttemptCancelRequest, ExecutionCompensationReleaseIntent, ExecutionExternalJobStartRecoveryOwner, ExecutionExternalJobStartRecoveryRequest, @@ -593,7 +593,7 @@ async fn dispatch_delivery_loss_releases_never_started_compensation_for_retry_db assert_eq!(reservation_state, "released"); let active_watchdogs: i64 = sqlx::query_scalar( "SELECT COUNT(*) FROM moa.execution_trigger WHERE trigger_uid=$1 \ - AND state IN ('pending','dispatching')", + AND state = 'pending'", ) .bind(admission.watchdog.trigger.trigger_uid) .fetch_one(test_db.store().pool()) @@ -1458,6 +1458,256 @@ async fn direct_external_callback_waits_for_compensation_hand_release_db() -> Te Ok(()) } +#[tokio::test] +async fn terminal_compensation_failure_finalizes_manual_repair_with_nested_cause_db() -> TestResult +{ + // Pins: a non-retryable compensation failure terminalizes the run as CompensationFailed and + // replaces the held terminal cause with CompensationFailure carrying the original terminal + // intent, the exact compensation identity, and the verbatim undo outcome, under manual repair. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let repository = ExecutionRepository::new(test_db.store().pool().clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let config = ExecutionConfig::default(); + let undo_failure = ExecutionCompensationOutcome::Failed { + message: "undo was rejected permanently".to_string(), + retryable: false, + usage: usage(1), + }; + let (compensation_id, current) = settled_compensation_run( + &repository, + test_db.store().pool(), + scope, + tenant_id, + "undoable", + undo_failure.clone(), + ) + .await?; + + let PendingTerminalAdvanceOutcome::Applied(commit) = repository + .advance_pending_terminal_settlement( + &config, + scope, + current.run_uid, + current.controller_generation, + current.wake_epoch, + moa_test_support::fixtures::pg_now(), + 1, + ) + .await? + else { + panic!("a failed compensation must advance its held terminal to manual repair"); + }; + assert_eq!( + commit.stage, + PendingTerminalAdvanceStage::ManualRepairRequired + ); + assert!(!commit.work_remaining); + assert!(commit.continuation.is_none()); + assert!(commit.compensation_admission.is_none()); + + let expected_cause = ExecutionTerminalCause::CompensationFailure { + original_status: ExecutionRunStatus::Failed, + original_reason: ExecutionTerminalReason::InternalFailure, + original_cause: Box::new(ExecutionTerminalCause::InternalFailure), + compensation_id, + outcome: undo_failure, + }; + for observed in [ + commit.run.clone(), + repository + .load_run(scope, current.run_uid) + .await? + .expect("manual-repair run stays visible after finalization"), + ] { + assert_eq!(observed.status, ExecutionRunStatus::Failed); + assert_eq!( + observed.terminal_reason, + Some(ExecutionTerminalReason::CompensationFailed) + ); + assert!(observed.manual_repair_required); + assert!(observed.pending_terminal.is_none()); + assert_eq!( + observed + .terminal_evidence + .as_ref() + .map(|evidence| &evidence.cause), + Some(&expected_cause) + ); + } + assert_eq!( + held_non_lifetime_capacity(test_db.store().pool(), current.run_uid).await?, + 0, + "a manual-repair terminal must leave no non-lifetime capacity receipt held" + ); + Ok(()) +} + +#[tokio::test] +async fn successful_compensation_finalizes_the_original_held_terminal_db() -> TestResult { + // Pins: when every undo succeeds, the bounded drain installs the terminal intent the run was + // already holding — unchanged and without manual repair — instead of tripping the + // "completed compensation retained non-lifetime capacity" guard. Both terminal-drain branches + // probe for held reservations, so a receipt leaked by compensation settlement wedges the + // success path exactly like the failure path. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let repository = ExecutionRepository::new(test_db.store().pool().clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let config = ExecutionConfig::default(); + let (_, current) = settled_compensation_run( + &repository, + test_db.store().pool(), + scope, + tenant_id, + "undone", + ExecutionCompensationOutcome::Completed { + output: json!({"tokens": 0}), + usage: usage(1), + }, + ) + .await?; + + let PendingTerminalAdvanceOutcome::Applied(commit) = repository + .advance_pending_terminal_settlement( + &config, + scope, + current.run_uid, + current.controller_generation, + current.wake_epoch, + moa_test_support::fixtures::pg_now(), + 1, + ) + .await? + else { + panic!("a fully compensated run must advance its held terminal to finalization"); + }; + assert_eq!(commit.stage, PendingTerminalAdvanceStage::Finalized); + assert!(!commit.work_remaining); + assert!(commit.continuation.is_none()); + assert!(commit.compensation_admission.is_none()); + + for observed in [ + commit.run.clone(), + repository + .load_run(scope, current.run_uid) + .await? + .expect("finalized run stays visible after successful compensation"), + ] { + assert_eq!(observed.status, ExecutionRunStatus::Failed); + assert_eq!( + observed.terminal_reason, + Some(ExecutionTerminalReason::InternalFailure), + "a successful undo must not rewrite the reason the run was failing for" + ); + assert!( + !observed.manual_repair_required, + "a successful undo needs no operator repair" + ); + assert!(observed.pending_terminal.is_none()); + assert_eq!( + observed + .terminal_evidence + .as_ref() + .map(|evidence| &evidence.cause), + Some(&ExecutionTerminalCause::InternalFailure), + "the held cause must survive compensation verbatim" + ); + } + assert_eq!( + held_non_lifetime_capacity(test_db.store().pool(), current.run_uid).await?, + 0, + "a finalized terminal must leave no non-lifetime capacity receipt held" + ); + Ok(()) +} + +/// Drives a compensating run's single reverse-order slice to one exact terminal undo outcome. +/// +/// Returns the settled compensation identity and the run reloaded afterwards, which is the state +/// the next controller activation observes before it advances the run's held terminal intent. +async fn settled_compensation_run( + repository: &ExecutionRepository, + pool: &sqlx::PgPool, + scope: ExecutionScope, + tenant_id: TenantId, + node_id: &str, + outcome: ExecutionCompensationOutcome, +) -> Result< + (moa_execution::state::CompensationId, ExecutionRunRecord), + Box, +> { + let expected_status = match outcome { + ExecutionCompensationOutcome::Completed { .. } => CompensationStatus::Completed, + ExecutionCompensationOutcome::Failed { .. } => CompensationStatus::Failed, + ExecutionCompensationOutcome::UnknownOutcome { .. } => CompensationStatus::UnknownOutcome, + }; + let (run, _) = compensating_run(repository, scope, tenant_id, &[node_id]).await?; + let now = moa_test_support::fixtures::pg_now(); + let config = ExecutionConfig::default(); + let admission = + active_compensation_admission(repository, scope, &config, run.run_uid, now).await?; + let compensation_id = admission.attempt.registration.compensation_id; + let fence = fence(&admission); + assert!(matches!( + repository + .start_compensation_attempt(scope, fence, now + Duration::milliseconds(1)) + .await?, + CompensationAttemptWriteOutcome::Applied(_) + )); + let request = cancel_request( + &admission, + tenant_id, + ExecutionCompensationReleaseIntent::Outcome, + ); + assert!(matches!( + repository + .begin_compensation_attempt_release(&request, now + Duration::milliseconds(2)) + .await?, + CompensationAttemptReleaseClaimOutcome::Applied(_) + )); + let release_receipt = + persist_compensation_release_receipt(pool, &request, now + Duration::milliseconds(3)) + .await?; + let CompensationAttemptWriteOutcome::Applied(settled) = repository + .settle_released_compensation_attempt( + &request, + outcome, + now + Duration::milliseconds(3), + Some(release_receipt), + ) + .await? + else { + panic!("a verified release must settle the compensation attempt"); + }; + assert_eq!(settled.registration.status, expected_status); + assert_eq!(settled.attempt_state, CompensationAttemptState::Terminal); + let current = repository + .load_run(scope, run.run_uid) + .await? + .expect("settled-compensation run stays visible before finalization"); + assert_eq!(current.status, ExecutionRunStatus::Compensating); + Ok((compensation_id, current)) +} + +/// Counts capacity receipts a terminal run must never still hold. +/// +/// `active_runs` and `parked_runs` are lifetime receipts released by finalization itself; the +/// other three dimensions are what both terminal-drain branches refuse to finalize against. +async fn held_non_lifetime_capacity( + pool: &sqlx::PgPool, + run_uid: Uuid, +) -> Result { + sqlx::query_scalar( + "SELECT COUNT(*) FROM moa.execution_capacity_reservation WHERE run_uid=$1 \ + AND resource_dimension IN ('active_tasks','scheduled_triggers','external_jobs') \ + AND state IN ('reserved','reconciling')", + ) + .bind(run_uid) + .fetch_one(pool) + .await +} + #[tokio::test] async fn compensation_cancel_releases_capacity_only_after_verified_finalize_db() -> TestResult { // Pins: claiming compensation teardown makes the attempt non-dispatchable but preserves its diff --git a/crates/moa-execution/tests/execution_db/completion_projection_db.rs b/crates/moa-execution/tests/execution_db/completion_projection_db.rs index fb60f3a7b..9d29a3576 100644 --- a/crates/moa-execution/tests/execution_db/completion_projection_db.rs +++ b/crates/moa-execution/tests/execution_db/completion_projection_db.rs @@ -13,7 +13,9 @@ use moa_execution::{ TaskAttemptFence, TaskAttemptReleaseClaimOutcome, TaskAttemptSettlementOutcome, TaskAttemptStartOutcome, }, + terminal::{PendingTerminalAdvanceOutcome, PendingTerminalAdvanceStage}, }, + state::{ExecutionLimitStop, ExecutionTerminalEvidence}, }; use super::support::*; @@ -114,6 +116,7 @@ async fn failed_and_unknown_outcome_tasks_cancel_transitive_unmaterialized_depen reduce_cursor: None, source_exhausted: true, terminal_output: None, + condition_skipped: false, tasks: vec![source], }, ) @@ -280,6 +283,7 @@ async fn completion_projection_pages_twenty_five_hundred_tasks_and_nodes_db() -> reduce_cursor: None, source_exhausted, terminal_output: None, + condition_skipped: false, tasks: page.to_vec(), }, ) @@ -489,6 +493,7 @@ async fn replan_stop_completion_pages_rebind_exact_wake_without_duplicate_verifi reduce_cursor: None, source_exhausted: true, terminal_output: None, + condition_skipped: false, tasks, }, ) @@ -663,3 +668,151 @@ async fn replan_stop_completion_pages_rebind_exact_wake_without_duplicate_verifi assert_eq!(verifier_count, 0); Ok(()) } + +#[tokio::test] +async fn every_terminal_cause_cohort_survives_finalization_and_reload_db() -> TestResult { + // Pins: the limit-stopped-completion, budget limit-stop, deadline limit-stop, and + // internal-failure terminal causes each finalize a run through the bounded terminal drain and + // decode back into the identical closed cause and normalized reason on reload. Only four of + // the eight causes reach a persisted run anywhere else, so a column, encoding, or + // strict-decode regression on the other cohorts is otherwise silent. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let repository = ExecutionRepository::new(test_db.store().pool().clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let config = ExecutionConfig::default(); + + let cohorts = [ + ( + "limit-stopped-completion", + ExecutionRunStatus::Failed, + ExecutionTerminalReason::BudgetExceeded, + ExecutionTerminalCause::Completion { + limit_stop: Some(ExecutionLimitStop::BudgetExceeded), + }, + ), + ( + "budget-limit-stop", + ExecutionRunStatus::Failed, + ExecutionTerminalReason::BudgetExceeded, + ExecutionTerminalCause::LimitStop { + reason: ExecutionLimitStop::BudgetExceeded, + }, + ), + ( + "internal-failure", + ExecutionRunStatus::Failed, + ExecutionTerminalReason::InternalFailure, + ExecutionTerminalCause::InternalFailure, + ), + ]; + for (key, status, reason, cause) in cohorts { + let run = create_run( + &repository, + scope, + new_run( + tenant_id, + None, + &format!("terminal-cause-{key}-{}", Uuid::now_v7()), + ExecutionRunStatus::Queued, + budget(4), + ), + ) + .await?; + let PendingTerminalAdvanceOutcome::Applied(commit) = repository + .fence_completion_terminal_and_enqueue_settlement( + &config, + scope, + run.run_uid, + run.controller_generation, + run.wake_epoch, + PendingExecutionTerminal { + status, + reason, + terminal_evidence: ExecutionTerminalEvidence { + cause: cause.clone(), + satisfied_requirement_count: 0, + requirement_count: 0, + }, + completion_check_results: Vec::new(), + terminal_gaps: vec![format!("{key} gap")], + output: None, + cancellation_reason: None, + }, + moa_test_support::fixtures::pg_now(), + 1, + ) + .await? + else { + panic!("a task-free {key} terminal must finalize on its first bounded page"); + }; + assert_eq!( + commit.stage, + PendingTerminalAdvanceStage::Finalized, + "{key} must finalize rather than defer" + ); + let reloaded = repository + .load_run(scope, run.run_uid) + .await? + .expect("finalized run stays visible"); + assert_eq!(reloaded.status, status, "{key} status"); + assert_eq!(reloaded.terminal_reason, Some(reason), "{key} reason"); + assert_eq!( + reloaded + .terminal_evidence + .as_ref() + .map(|evidence| &evidence.cause), + Some(&cause), + "{key} cause must decode back verbatim" + ); + assert!(!reloaded.manual_repair_required, "{key} needs no repair"); + assert!(reloaded.pending_terminal.is_none(), "{key} intent consumed"); + } + + // The deadline cohort has a real producer, so drive that instead of asserting a hand-built + // intent: an already-elapsed approved deadline must fence LimitStop { DeadlineExceeded }. + let mut elapsed = new_run( + tenant_id, + None, + &format!("terminal-cause-deadline-{}", Uuid::now_v7()), + ExecutionRunStatus::Queued, + budget(4), + ); + elapsed.approved_budget.deadline_at = Some(pg_deadline(Duration::seconds(-30))); + let run = create_run(&repository, scope, elapsed).await?; + let PendingTerminalAdvanceOutcome::Applied(commit) = repository + .fence_deadline_and_enqueue_settlement( + &config, + scope, + run.run_uid, + run.controller_generation, + run.wake_epoch, + moa_test_support::fixtures::pg_now(), + 1, + ) + .await? + else { + panic!("an elapsed approved deadline must fence and finalize its own terminal"); + }; + assert_eq!(commit.stage, PendingTerminalAdvanceStage::Finalized); + let reloaded = repository + .load_run(scope, run.run_uid) + .await? + .expect("deadline-terminal run stays visible"); + assert_eq!(reloaded.status, ExecutionRunStatus::Failed); + assert_eq!( + reloaded.terminal_reason, + Some(ExecutionTerminalReason::DeadlineExceeded) + ); + assert_eq!( + reloaded + .terminal_evidence + .as_ref() + .map(|evidence| &evidence.cause), + Some(&ExecutionTerminalCause::LimitStop { + reason: ExecutionLimitStop::DeadlineExceeded, + }), + "the deadline fence must record the limit stop itself, not only the normalized reason" + ); + Ok(()) +} diff --git a/crates/moa-execution/tests/execution_db/conditional_execution_db.rs b/crates/moa-execution/tests/execution_db/conditional_execution_db.rs new file mode 100644 index 000000000..53168c4a6 --- /dev/null +++ b/crates/moa-execution/tests/execution_db/conditional_execution_db.rs @@ -0,0 +1,560 @@ +//! Durable conditional-branch contracts for `ExecutionNode.when`. + +use moa_artifacts::execution_plan::{ + CapabilityReference, ExecutionCondition, ExecutionFailureClass, ExecutionNode, + ExecutionOperation, ExecutionReducer, ExecutionReference, MapTask, +}; +use moa_config::ExecutionConfig; +use moa_execution::budget::BudgetLedger; +use moa_execution::interpreter::{ + NodeMaterializationPage, ReduceMaterializationPageInput, ScheduleRequest, materialize_node_page, +}; +use moa_execution::repository::ready::{ + ExecutionReduceMaterializationCursor, ReadyMaterializationOutcome, ReadyMaterializationRequest, +}; +use moa_execution::repository::task::{ + TaskAttemptFence, TaskAttemptReleaseClaimOutcome, TaskAttemptSettlementOutcome, + TaskAttemptStartOutcome, +}; +use moa_execution::state::{ExecutionProjection, failed_task_outcome}; +use serde_json::Value; +use std::collections::BTreeMap; + +use super::support::*; + +fn branch_node(id: &str, when: Option) -> ExecutionNode { + ExecutionNode { + id: id.to_string(), + requirement_ids: vec!["req".to_string()], + depends_on: Vec::new(), + when, + input: json!({}), + output_schema: json!({ "type": "object" }), + operation: ExecutionOperation::Output { + value: json!({ "branch": id }), + }, + compensation: None, + retry: RetryPolicy { + max_attempts: 1, + initial_backoff_ms: 1, + max_backoff_ms: 1, + }, + budget: None, + } +} + +fn input_equals(field: &str, value: Value) -> Option { + Some(ExecutionCondition::Equals { + reference: ExecutionReference { + path: format!("$.input.{field}"), + }, + value, + }) +} + +/// Drives one bounded controller pass over every dependency-ready node. +/// +/// This is the exact production pairing the controller performs — pure +/// `materialize_node_page` feeding the durable `materialize_ready_page` transaction — +/// so a condition that is never consulted, or a skip that is never committed, shows up +/// here as materialized tasks rather than as a passing assertion. +async fn advance_ready_nodes( + repository: &ExecutionRepository, + scope: ExecutionScope, + run_uid: Uuid, + only: Option<&str>, +) -> Result, Box> { + let config = ExecutionConfig::default(); + let projection = repository + .load_activation_projection(scope, run_uid, 64) + .await? + .expect("run must remain visible"); + let run = &projection.run; + let mut outcomes = Vec::new(); + for node in &projection.nodes { + if only.is_some_and(|wanted| wanted != node.node_id) { + continue; + } + let plan_node = run + .active_plan + .definition + .nodes + .iter() + .find(|plan_node| plan_node.id == node.node_id) + .expect("activation node must exist in the active plan"); + let schedule = ScheduleRequest { + run_uid, + goal: run.goal.clone(), + plan: run.active_plan.clone(), + catalog: run.catalog.clone(), + run_input: run.input.clone(), + projection: ExecutionProjection { + plan_revision: run.plan_revision, + node_statuses: BTreeMap::new(), + tasks: Vec::new(), + }, + config: config.clone(), + budget_ledger: BudgetLedger { + limit: run.approved_budget.clone(), + reserved: run.reserved, + consumed: run.consumed, + overrun: run.budget_overrun, + }, + now: Utc::now(), + }; + let reduce_input = + matches!(plan_node.operation, ExecutionOperation::Reduce { .. }).then(|| { + ReduceMaterializationPageInput { + round: node.reduce_round, + batch_cursor: node.reduce_batch_cursor, + round_input_count: node.reduce_round_input_count, + page_inputs: Vec::new(), + } + }); + let NodeMaterializationPage { + tasks, + source_exhausted, + reduce_cursor, + terminal_output, + condition_skipped, + .. + } = materialize_node_page( + &schedule, + &node.node_id, + &projection.referenced_outputs, + node.materialization_cursor, + 64, + reduce_input.as_ref(), + )?; + let outcome = repository + .materialize_ready_page( + scope, + &config, + ReadyMaterializationRequest { + run_uid, + plan_revision: run.plan_revision, + node_id: node.node_id.clone(), + expected_cursor: node.materialization_cursor, + reduce_cursor: reduce_cursor.map(|cursor| { + ExecutionReduceMaterializationCursor { + round: cursor.round, + batch_cursor: cursor.batch_cursor, + round_input_count: cursor.round_input_count, + } + }), + source_exhausted, + terminal_output, + condition_skipped, + tasks, + }, + ) + .await?; + outcomes.push((node.node_id.clone(), outcome)); + } + Ok(outcomes) +} + +/// Fails the run's single admitted attempt terminally through the real settlement path. +async fn fail_worker( + repository: &ExecutionRepository, + config: &ExecutionConfig, +) -> Result<(), Box> { + let admission = repository + .admit_ready_attempts(config, 1, Utc::now()) + .await? + .admitted + .into_iter() + .next() + .expect("the unconditional branch must be admissible"); + let fence = TaskAttemptFence { + tenant_id: admission.tenant_id, + run_uid: admission.run_uid, + task_id: admission.task_id, + controller_generation: admission.controller_generation, + attempt_generation: admission.attempt_generation, + dispatch_uid: admission.dispatch_uid, + capacity_reservation_uid: admission.capacity_reservation_uid, + watchdog_trigger_uid: admission.watchdog_trigger_uid, + attempt_deadline_at: admission.attempt_deadline_at, + }; + let TaskAttemptStartOutcome::Started(started) = repository.start_task_attempt(fence).await? + else { + panic!("the admitted attempt must start"); + }; + let settled_at = Utc::now(); + assert!(matches!( + repository + .begin_task_attempt_release(fence, started.task.generation, "terminal", settled_at) + .await?, + TaskAttemptReleaseClaimOutcome::Applied(_) + )); + let outcome = failed_task_outcome( + ExecutionFailureClass::Terminal, + "worker failed terminally".to_string(), + usage(1), + ); + assert!(matches!( + repository + .settle_released_task_attempt(config, fence, outcome, None, settled_at, None) + .await?, + TaskAttemptSettlementOutcome::Applied { .. } + )); + Ok(()) +} + +/// Exact durable scheduler aggregate for one node, read straight from its row. +struct NodeState { + status: String, + remaining_dependency_count: i64, + total_task_count: i64, + materialization_complete: bool, + aggregate_complete: bool, + aggregate_output: Option, + aggregate_output_hash: Option, + reduce_round: i64, +} + +async fn node_state( + pool: &sqlx::PgPool, + run_uid: Uuid, + node_id: &str, +) -> Result> { + let row = sqlx::query_as::< + _, + ( + String, + i64, + i64, + bool, + bool, + Option, + Option, + i64, + ), + >( + "SELECT node_status, remaining_dependency_count, total_task_count, \ + materialization_complete, aggregate_complete, aggregate_output, \ + aggregate_output_hash, reduce_round \ + FROM moa.execution_node_state WHERE run_uid=$1 AND node_id=$2", + ) + .bind(run_uid) + .bind(node_id) + .fetch_one(pool) + .await?; + Ok(NodeState { + status: row.0, + remaining_dependency_count: row.1, + total_task_count: row.2, + materialization_complete: row.3, + aggregate_complete: row.4, + aggregate_output: row.5, + aggregate_output_hash: row.6, + reduce_round: row.7, + }) +} + +#[tokio::test] +async fn a_false_node_condition_skips_only_its_own_branch_db() -> TestResult { + // Pins: exactly one branch of a two-branch plan materializes work; the branch whose + // condition is false commits `skipped` with a null aggregate and never creates a task. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let mut candidate = new_run( + tenant_id, + None, + "conditional-branches", + ExecutionRunStatus::Queued, + budget(64), + ); + candidate.input = json!({ "route": "escalate" }); + let mut join = branch_node("join", None); + join.depends_on = vec!["escalate".to_string(), "standard".to_string()]; + candidate.plan.definition.nodes = vec![ + branch_node("escalate", input_equals("route", json!("escalate"))), + branch_node("standard", input_equals("route", json!("standard"))), + join, + ]; + let run = create_run(&repository, scope, candidate).await?; + + let outcomes = advance_ready_nodes(&repository, scope, run.run_uid, None).await?; + assert_eq!( + outcomes + .iter() + .map(|(id, _)| id.as_str()) + .collect::>(), + vec!["escalate", "standard"], + "only dependency-ready nodes may be advanced" + ); + let escalate_tasks = match &outcomes[0].1 { + ReadyMaterializationOutcome::Applied { tasks, .. } => tasks.len(), + outcome => panic!("the true branch must apply one page: {outcome:?}"), + }; + assert_eq!( + escalate_tasks, 1, + "the true branch must materialize its task" + ); + let standard_tasks = match &outcomes[1].1 { + ReadyMaterializationOutcome::Applied { tasks, .. } => tasks.len(), + outcome => panic!("the false branch must apply its skip: {outcome:?}"), + }; + assert_eq!( + standard_tasks, 0, + "a false condition must not materialize any logical task" + ); + + let skipped = node_state(&pool, run.run_uid, "standard").await?; + assert_eq!(skipped.status, "skipped"); + assert_eq!(skipped.total_task_count, 0); + assert!(skipped.materialization_complete); + assert!(skipped.aggregate_complete); + assert_eq!(skipped.aggregate_output, Some(Value::Null)); + assert_eq!( + skipped.aggregate_output_hash, + Some(moa_execution::capability::node_output_hash(&Value::Null)?.to_string()), + "a skipped aggregate must carry the verified hash dependents check on load" + ); + + let taken = node_state(&pool, run.run_uid, "escalate").await?; + assert_eq!(taken.status, "ready"); + assert_eq!(taken.total_task_count, 1); + + let listed = repository + .list_tasks(scope, run.run_uid, ExecutionTaskPageRequest::default()) + .await?; + assert_eq!( + listed + .tasks + .iter() + .map(|task| task.node_id.as_str()) + .collect::>(), + vec!["escalate"], + "the skipped branch must own no task row at all" + ); + Ok(()) +} + +#[tokio::test] +async fn a_fan_in_of_two_skipped_branches_releases_its_dependent_exactly_once_db() -> TestResult { + // Pins: each skip decrements the fan-in counter once, and replaying either skip page + // returns Replayed without decrementing again. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let mut candidate = new_run( + tenant_id, + None, + "conditional-fan-in", + ExecutionRunStatus::Queued, + budget(64), + ); + candidate.input = json!({ "route": "neither" }); + let mut join = branch_node("join", None); + join.depends_on = vec!["escalate".to_string(), "standard".to_string()]; + candidate.plan.definition.nodes = vec![ + branch_node("escalate", input_equals("route", json!("escalate"))), + branch_node("standard", input_equals("route", json!("standard"))), + join, + ]; + let run = create_run(&repository, scope, candidate).await?; + + assert_eq!( + node_state(&pool, run.run_uid, "join") + .await? + .remaining_dependency_count, + 2 + ); + advance_ready_nodes(&repository, scope, run.run_uid, None).await?; + let released = node_state(&pool, run.run_uid, "join").await?; + assert_eq!( + released.remaining_dependency_count, 0, + "both skipped branches must release the fan-in exactly once each" + ); + assert_eq!(released.status, "pending"); + + for node_id in ["escalate", "standard"] { + let replayed = repository + .materialize_ready_page( + scope, + &ExecutionConfig::default(), + ReadyMaterializationRequest { + run_uid: run.run_uid, + plan_revision: run.plan_revision, + node_id: node_id.to_string(), + expected_cursor: 0, + reduce_cursor: None, + source_exhausted: true, + terminal_output: None, + condition_skipped: true, + tasks: Vec::new(), + }, + ) + .await?; + assert!( + matches!(replayed, ReadyMaterializationOutcome::Replayed { .. }), + "a committed skip must replay rather than reapply: {replayed:?}" + ); + } + let after_replay = node_state(&pool, run.run_uid, "join").await?; + assert_eq!( + after_replay.remaining_dependency_count, 0, + "replaying a skip must not decrement the fan-in a second time" + ); + let skipped = node_state(&pool, run.run_uid, "escalate").await?; + assert_eq!(skipped.status, "skipped"); + assert_eq!(skipped.total_task_count, 0); + Ok(()) +} + +#[tokio::test] +async fn a_dependent_of_a_skipped_and_a_failed_branch_cancels_without_raising_db() -> TestResult { + // Pins: the diamond where one dependency is skipped and a sibling fails terminally, in + // both settle orders. The cancellation cascade asserts every descendant is pending with + // zero tasks, and a released-then-cancelled dependent must not lose its counter. + for skip_first in [true, false] { + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let mut candidate = new_run( + tenant_id, + None, + "conditional-diamond", + ExecutionRunStatus::Queued, + budget(64), + ); + candidate.input = json!({ "route": "neither" }); + let mut join = branch_node("join", None); + join.depends_on = vec!["worker".to_string(), "conditional".to_string()]; + candidate.plan.definition.nodes = vec![ + branch_node("worker", None), + branch_node("conditional", input_equals("route", json!("taken"))), + join, + ]; + let run = create_run(&repository, scope, candidate).await?; + if skip_first { + advance_ready_nodes(&repository, scope, run.run_uid, Some("conditional")).await?; + advance_ready_nodes(&repository, scope, run.run_uid, Some("worker")).await?; + fail_worker(&repository, &ExecutionConfig::default()).await?; + } else { + advance_ready_nodes(&repository, scope, run.run_uid, Some("worker")).await?; + fail_worker(&repository, &ExecutionConfig::default()).await?; + advance_ready_nodes(&repository, scope, run.run_uid, Some("conditional")).await?; + } + + let join_state = node_state(&pool, run.run_uid, "join").await?; + assert_eq!( + join_state.status, "cancelled", + "a dependent of a terminally failed branch must be cancelled" + ); + assert_eq!(join_state.remaining_dependency_count, 0); + assert_eq!(join_state.total_task_count, 0); + assert_eq!( + node_state(&pool, run.run_uid, "conditional").await?.status, + "skipped" + ); + } + Ok(()) +} + +#[tokio::test] +async fn a_false_condition_skips_map_and_reduce_nodes_before_any_paging_db() -> TestResult { + // Pins: an aggregate node whose condition is false never opens a map page or a reduce + // round, and its completed aggregate keeps it out of the pending map-aggregate queue. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let mut candidate = new_run( + tenant_id, + None, + "conditional-aggregates", + ExecutionRunStatus::Queued, + budget(64), + ); + candidate.input = json!({ "route": "neither" }); + let mut map = branch_node("map", input_equals("route", json!("taken"))); + map.operation = ExecutionOperation::Map { + items: json!([1, 2, 3]), + item_key: String::new(), + max_items: 3, + item_output_schema: json!({}), + task: MapTask::Capability { + reference: CapabilityReference { + name: "test.map".to_string(), + version: "v1".to_string(), + }, + }, + }; + let mut reduce = branch_node("reduce", input_equals("route", json!("taken"))); + reduce.operation = ExecutionOperation::Reduce { + items: json!([1, 2, 3, 4]), + max_items: 4, + reducer: ExecutionReducer::Capability { + reference: CapabilityReference { + name: "test.reduce".to_string(), + version: "v1".to_string(), + }, + }, + batch_size: 2, + }; + candidate.plan.definition.nodes = vec![map, reduce]; + let run = create_run(&repository, scope, candidate).await?; + + advance_ready_nodes(&repository, scope, run.run_uid, None).await?; + + for node_id in ["map", "reduce"] { + let state = node_state(&pool, run.run_uid, node_id).await?; + assert_eq!( + state.status, "skipped", + "conditional {node_id} node must skip" + ); + assert_eq!(state.total_task_count, 0); + assert!(state.materialization_complete); + assert!( + state.aggregate_complete, + "a skipped aggregate must be final so nothing re-aggregates it" + ); + assert_eq!(state.aggregate_output, Some(Value::Null)); + } + assert_eq!( + node_state(&pool, run.run_uid, "reduce").await?.reduce_round, + 1, + "a skipped reduce must never open a later round" + ); + + let current = repository + .load_run(scope, run.run_uid) + .await? + .expect("conditional aggregate run"); + assert!(matches!( + repository + .claim_controller_wake( + scope, + run.run_uid, + current.controller_generation, + current.wake_epoch, + ) + .await?, + RunControllerClaimOutcome::Claimed(_) + )); + assert!( + repository + .load_map_aggregate_candidate( + scope, + run.run_uid, + current.controller_generation, + current.wake_epoch, + ) + .await? + .is_none(), + "a skipped map must not enter the pending map-aggregate queue" + ); + Ok(()) +} diff --git a/crates/moa-execution/tests/execution_db/controller_wake_recovery_db.rs b/crates/moa-execution/tests/execution_db/controller_wake_recovery_db.rs new file mode 100644 index 000000000..0aa5d27ab --- /dev/null +++ b/crates/moa-execution/tests/execution_db/controller_wake_recovery_db.rs @@ -0,0 +1,601 @@ +//! Durable controller wake claim/complete compare-and-set and crashed-activation recovery. + +use moa_execution::{ + repository::run::{ResumedControllerRecoveryOutcome, ResumedControllerRecoveryRequest}, + repository::terminal::{PendingTerminalAdvanceOutcome, PendingTerminalAdvanceStage}, + state::ExecutionTerminalEvidence, +}; + +use super::support::*; + +/// Admits one queued run whose current wake is claimable. +async fn queued_run( + repository: &ExecutionRepository, + scope: ExecutionScope, + tenant_id: TenantId, + key: &str, +) -> Result> { + Ok(create_run( + repository, + scope, + new_run(tenant_id, None, key, ExecutionRunStatus::Queued, budget(10)), + ) + .await?) +} + +/// Builds the bounded continuation checkpoint the controller commits for a crashed activation. +fn continuation_checkpoint( + run: &ExecutionRunRecord, + status: ExecutionRunStatus, +) -> ExecutionRunActivationCheckpoint { + ExecutionRunActivationCheckpoint { + status, + activation_state: ExecutionActivationState::Queued, + next_wake_at: run.next_wake_at, + waiting_since: run.waiting_since, + ready_task_count: run.ready_task_count, + active_task_count: run.active_task_count, + } +} + +/// Builds the recovery request the controller issues for one resumed wake. +fn recovery_request( + run: &ExecutionRunRecord, + status: ExecutionRunStatus, + maximum_consecutive_failures: u64, +) -> ResumedControllerRecoveryRequest { + ResumedControllerRecoveryRequest { + controller_generation: run.controller_generation, + wake_epoch: run.wake_epoch, + checkpoint: continuation_checkpoint(run, status), + continuation_payload: json!({"cause": "resumed_activation_recovery"}), + continuation_not_before_at: Utc::now(), + maximum_consecutive_failures, + } +} + +/// Reads the durable activation-failure budget consumed by one run. +async fn activation_failure_count(pool: &sqlx::PgPool, run_uid: Uuid) -> Result { + sqlx::query_scalar("SELECT activation_failure_count FROM moa.execution_run WHERE run_uid=$1") + .bind(run_uid) + .fetch_one(pool) + .await +} + +/// Counts the run-activation outbox rows owning one exact wake epoch. +async fn run_activation_rows( + pool: &sqlx::PgPool, + run_uid: Uuid, + wake_epoch: i64, +) -> Result { + sqlx::query_scalar( + "SELECT COUNT(*) FROM moa.execution_dispatch_outbox \ + WHERE run_uid=$1 AND dispatch_kind='run_activation' AND wake_epoch=$2", + ) + .bind(run_uid) + .bind(wake_epoch) + .fetch_one(pool) + .await +} + +#[tokio::test] +async fn claim_controller_wake_replays_an_already_processed_wake_db() -> TestResult { + // Pins: the acknowledgement fence, not the caller, decides whether a redelivered activation + // may run again. A wake whose epoch is already acknowledged must resolve as a replay and must + // leave the durable activation state untouched, so a duplicate delivery of the same dispatch + // can never re-enter the bounded scheduler work that wake already committed. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let run = queued_run(&repository, scope, tenant_id, "controller-replayed-wake").await?; + + assert_eq!( + sqlx::query( + "UPDATE moa.execution_run SET processed_wake_epoch = wake_epoch WHERE run_uid=$1" + ) + .bind(run.run_uid) + .execute(&pool) + .await? + .rows_affected(), + 1 + ); + + let outcome = repository + .claim_controller_wake( + scope, + run.run_uid, + run.controller_generation, + run.wake_epoch, + ) + .await?; + + assert!( + matches!(outcome, RunControllerClaimOutcome::Replayed(_)), + "an acknowledged wake must replay, got {outcome:?}" + ); + let activation_state: String = + sqlx::query_scalar("SELECT activation_state FROM moa.execution_run WHERE run_uid=$1") + .bind(run.run_uid) + .fetch_one(&pool) + .await?; + assert_eq!( + activation_state, "queued", + "a replayed claim must not advance the durable activation state" + ); + Ok(()) +} + +#[tokio::test] +async fn concurrent_controller_wake_claims_admit_exactly_one_activation_db() -> TestResult { + // Pins: the claim is a real compare-and-set under the run row lock, not a read-then-write. + // Two deliveries racing for the same exact wake must produce exactly one Claimed activation; + // the loser must observe Resumed, which is the signal that a prior activation already owns the + // wake and that recovery — not a second bounded page — is the only legal continuation. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let repository = ExecutionRepository::new(test_db.store().pool().clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let run = queued_run(&repository, scope, tenant_id, "controller-claim-race").await?; + + let (left, right) = tokio::join!( + repository.claim_controller_wake( + scope, + run.run_uid, + run.controller_generation, + run.wake_epoch, + ), + repository.claim_controller_wake( + scope, + run.run_uid, + run.controller_generation, + run.wake_epoch, + ), + ); + let outcomes = [left?, right?]; + + assert_eq!( + outcomes + .iter() + .filter(|outcome| matches!(outcome, RunControllerClaimOutcome::Claimed(_))) + .count(), + 1, + "exactly one racing delivery may claim the wake, got {outcomes:?}" + ); + assert_eq!( + outcomes + .iter() + .filter(|outcome| matches!(outcome, RunControllerClaimOutcome::Resumed(_))) + .count(), + 1, + "the losing delivery must observe the in-flight activation, got {outcomes:?}" + ); + Ok(()) +} + +#[tokio::test] +async fn claim_controller_wake_refuses_an_unqueued_activation_state_db() -> TestResult { + // Pins: the claim predicate requires an actually queued activation. A pending wake epoch on a + // run whose activation state says no activation is queued is inconsistent durable state, and + // claiming it would start bounded work the scheduler never admitted. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let run = queued_run(&repository, scope, tenant_id, "controller-unqueued-claim").await?; + + assert_eq!( + sqlx::query( + "UPDATE moa.execution_run SET activation_state='idle', wake_epoch = wake_epoch + 1 \ + WHERE run_uid=$1", + ) + .bind(run.run_uid) + .execute(&pool) + .await? + .rows_affected(), + 1 + ); + + assert_eq!( + repository + .claim_controller_wake( + scope, + run.run_uid, + run.controller_generation, + run.wake_epoch + 1, + ) + .await?, + RunControllerClaimOutcome::InvalidState + ); + Ok(()) +} + +#[tokio::test] +async fn resumed_recovery_enqueues_exactly_one_replacement_activation_db() -> TestResult { + // Pins: recovering a crashed activation acknowledges the claimed wake exactly once, mints + // exactly one successor wake, and charges exactly one unit of the bounded failure budget. + // A recovery that enqueued zero successors would strand the run; one that enqueued two, or + // that skipped an epoch, would let concurrent activations race the same durable scheduler + // state; one that forgot to charge the budget would restore the unbounded stall this exists + // to end. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let config = ExecutionConfig::default(); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let run = queued_run( + &repository, + scope, + tenant_id, + "controller-recovery-successor", + ) + .await?; + let RunControllerClaimOutcome::Claimed(claimed) = repository + .claim_controller_wake( + scope, + run.run_uid, + run.controller_generation, + run.wake_epoch, + ) + .await? + else { + panic!("the admitted queued wake must be claimable"); + }; + + let outcome = repository + .recover_resumed_controller_wake( + scope, + &config, + claimed.run_uid, + recovery_request(&claimed, ExecutionRunStatus::Running, 5), + ) + .await?; + + let ResumedControllerRecoveryOutcome::Recovered { + run: recovered, + continuation, + consecutive_failures, + } = outcome + else { + panic!("a first crashed activation must recover, got {outcome:?}"); + }; + assert_eq!(consecutive_failures, 1); + assert_eq!(recovered.wake_epoch, claimed.wake_epoch + 1); + assert_eq!(recovered.processed_wake_epoch, claimed.wake_epoch); + assert_eq!( + recovered.controller_generation, + claimed.controller_generation + ); + assert_eq!(recovered.activation_state, ExecutionActivationState::Queued); + assert_eq!(continuation.wake_epoch, Some(recovered.wake_epoch)); + assert_eq!(continuation.run_uid, Some(recovered.run_uid)); + assert_eq!( + run_activation_rows(&pool, run.run_uid, i64::try_from(recovered.wake_epoch)?).await?, + 1, + "recovery must own exactly one successor activation" + ); + assert_eq!(activation_failure_count(&pool, run.run_uid).await?, 1); + Ok(()) +} + +#[tokio::test] +async fn resumed_recovery_fails_the_run_once_its_budget_is_exhausted_db() -> TestResult { + // Pins: a deterministically failing activation stops being re-enqueued. On exhaustion the + // recovery must acknowledge nothing — the wake stays current and unacknowledged — precisely so + // the caller can fence an explicit terminal intent against that same wake in one transaction. + // If exhaustion acknowledged the wake, the terminal fence would see a replayed epoch, decline + // to install the intent, and the run would park forever with no product failure: the exact + // permanent stall this budget exists to end. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let config = ExecutionConfig::default(); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let run = queued_run( + &repository, + scope, + tenant_id, + "controller-recovery-exhausted", + ) + .await?; + let RunControllerClaimOutcome::Claimed(claimed) = repository + .claim_controller_wake( + scope, + run.run_uid, + run.controller_generation, + run.wake_epoch, + ) + .await? + else { + panic!("the admitted queued wake must be claimable"); + }; + assert_eq!( + sqlx::query("UPDATE moa.execution_run SET activation_failure_count = 5 WHERE run_uid=$1") + .bind(run.run_uid) + .execute(&pool) + .await? + .rows_affected(), + 1 + ); + + let outcome = repository + .recover_resumed_controller_wake( + scope, + &config, + claimed.run_uid, + recovery_request(&claimed, ExecutionRunStatus::Running, 5), + ) + .await?; + + assert_eq!( + outcome, + ResumedControllerRecoveryOutcome::BudgetExhausted { + consecutive_failures: 6 + } + ); + let stalled = repository + .load_run(scope, run.run_uid) + .await? + .expect("the exhausted run must remain visible"); + assert_eq!( + stalled.processed_wake_epoch, claimed.processed_wake_epoch, + "exhaustion must not acknowledge the claimed wake" + ); + assert_eq!(stalled.wake_epoch, claimed.wake_epoch); + assert_eq!( + stalled.activation_state, + ExecutionActivationState::Advancing + ); + assert_eq!( + run_activation_rows(&pool, run.run_uid, i64::try_from(claimed.wake_epoch + 1)?).await?, + 0, + "exhaustion must not enqueue another activation" + ); + + // The unacknowledged wake is exactly what lets the caller commit its explicit terminal intent. + let fenced = repository + .fence_completion_terminal_and_enqueue_settlement( + &config, + scope, + run.run_uid, + claimed.controller_generation, + claimed.wake_epoch, + PendingExecutionTerminal { + status: ExecutionRunStatus::Failed, + reason: ExecutionTerminalReason::InternalFailure, + terminal_evidence: ExecutionTerminalEvidence { + cause: ExecutionTerminalCause::InternalFailure, + satisfied_requirement_count: 0, + requirement_count: 0, + }, + completion_check_results: Vec::new(), + terminal_gaps: vec![ + "controller activation failed 6 consecutive times and requires manual repair" + .to_string(), + ], + output: None, + cancellation_reason: None, + }, + moa_test_support::fixtures::pg_now(), + 1, + ) + .await?; + + let PendingTerminalAdvanceOutcome::Applied(commit) = fenced else { + panic!("the terminal intent must commit against the still-current wake, got {fenced:?}"); + }; + assert_eq!( + commit.stage, + PendingTerminalAdvanceStage::Finalized, + "a wedged run with no outstanding work must finalize in its first bounded page" + ); + assert_eq!(commit.run.status, ExecutionRunStatus::Failed); + assert_eq!( + commit.run.terminal_reason, + Some(ExecutionTerminalReason::InternalFailure) + ); + let (status, terminal_reason): (String, Option) = + sqlx::query_as("SELECT status, terminal_reason FROM moa.execution_run WHERE run_uid=$1") + .bind(run.run_uid) + .fetch_one(&pool) + .await?; + assert_eq!(status, "failed"); + assert_eq!( + terminal_reason.as_deref(), + Some("internal_failure"), + "the exhausted activation must surface an explicit durable product failure" + ); + Ok(()) +} + +#[tokio::test] +async fn acknowledged_wake_resets_the_activation_failure_budget_db() -> TestResult { + // Pins: the budget counts *consecutive* crashes. One activation that reaches its checkpoint + // proves the run is not deterministically wedged, so the budget must return to zero. A counter + // that only ever accumulated would eventually fail a healthy long-running run for repair. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let config = ExecutionConfig::default(); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let run = queued_run(&repository, scope, tenant_id, "controller-budget-reset").await?; + let RunControllerClaimOutcome::Claimed(claimed) = repository + .claim_controller_wake( + scope, + run.run_uid, + run.controller_generation, + run.wake_epoch, + ) + .await? + else { + panic!("the admitted queued wake must be claimable"); + }; + let ResumedControllerRecoveryOutcome::Recovered { run: recovered, .. } = repository + .recover_resumed_controller_wake( + scope, + &config, + claimed.run_uid, + recovery_request(&claimed, ExecutionRunStatus::Running, 5), + ) + .await? + else { + panic!("a first crashed activation must recover"); + }; + assert_eq!(activation_failure_count(&pool, run.run_uid).await?, 1); + let RunControllerClaimOutcome::Claimed(successor) = repository + .claim_controller_wake( + scope, + recovered.run_uid, + recovered.controller_generation, + recovered.wake_epoch, + ) + .await? + else { + panic!("the replacement wake must be claimable"); + }; + + let completed = repository + .complete_controller_wake( + scope, + &config, + successor.run_uid, + RunControllerCompletionRequest { + controller_generation: successor.controller_generation, + wake_epoch: successor.wake_epoch, + checkpoint: ExecutionRunActivationCheckpoint { + status: ExecutionRunStatus::Running, + activation_state: ExecutionActivationState::Idle, + next_wake_at: successor.next_wake_at, + waiting_since: None, + ready_task_count: 0, + active_task_count: 0, + }, + continuation_payload: None, + continuation_not_before_at: Utc::now(), + }, + ) + .await?; + + assert!( + matches!(completed, RunControllerCompletionOutcome::Applied { .. }), + "the replacement activation must reach its checkpoint, got {completed:?}" + ); + assert_eq!( + activation_failure_count(&pool, run.run_uid).await?, + 0, + "an acknowledged wake must clear the consecutive-failure budget" + ); + Ok(()) +} + +#[tokio::test] +async fn resumed_recovery_keeps_a_compensating_run_compensating_db() -> TestResult { + // Pins: a controller crash while draining a compensating run must preserve the run phase. + // `trigger_is_current` resolves a CompensationWatchdog only while its run row still satisfies + // `status='compensating' AND controller_generation=`; a continuation that + // rewrote either conjunct would make the in-flight watchdog non-current and let + // `prepare_watchdog_trigger` supersede it, permanently disarming ambiguity resolution for that + // compensation attempt. The second half pins why preserving it is mandatory rather than + // cosmetic: the durable transition table has no `compensating -> running` edge at all, so the + // rewrite does not quietly degrade the phase — it aborts the whole recovery transaction. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let config = ExecutionConfig::default(); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let run = queued_run( + &repository, + scope, + tenant_id, + "controller-recovery-compensating", + ) + .await?; + set_run_status_path(&pool, run.run_uid, &["running", "compensating"]).await?; + let compensating = repository + .load_run(scope, run.run_uid) + .await? + .expect("the compensating fixture run must be visible"); + assert_eq!(compensating.status, ExecutionRunStatus::Compensating); + assert!(compensating.pending_terminal.is_some()); + let RunControllerClaimOutcome::Claimed(claimed) = repository + .claim_controller_wake( + scope, + compensating.run_uid, + compensating.controller_generation, + compensating.wake_epoch, + ) + .await? + else { + panic!("a compensating run's queued wake must be claimable"); + }; + + let outcome = repository + .recover_resumed_controller_wake( + scope, + &config, + claimed.run_uid, + recovery_request(&claimed, ExecutionRunStatus::Compensating, 5), + ) + .await?; + + let ResumedControllerRecoveryOutcome::Recovered { run: recovered, .. } = outcome else { + panic!("recovering a compensating run must commit, got {outcome:?}"); + }; + assert_eq!( + recovered.status, + ExecutionRunStatus::Compensating, + "recovery must not rewrite the compensation phase the watchdog is fenced against" + ); + assert_eq!( + recovered.controller_generation, claimed.controller_generation, + "recovery must not rewrite the controller generation the watchdog is fenced against" + ); + + let rewritten = queued_run( + &repository, + scope, + tenant_id, + "controller-recovery-compensating-rewrite", + ) + .await?; + set_run_status_path(&pool, rewritten.run_uid, &["running", "compensating"]).await?; + let rewritten = repository + .load_run(scope, rewritten.run_uid) + .await? + .expect("the second compensating fixture run must be visible"); + let RunControllerClaimOutcome::Claimed(rewritten_claim) = repository + .claim_controller_wake( + scope, + rewritten.run_uid, + rewritten.controller_generation, + rewritten.wake_epoch, + ) + .await? + else { + panic!("the second compensating run's queued wake must be claimable"); + }; + + let error = repository + .recover_resumed_controller_wake( + scope, + &config, + rewritten_claim.run_uid, + recovery_request(&rewritten_claim, ExecutionRunStatus::Running, 5), + ) + .await + .expect_err("rewriting a compensating run to running must be rejected durably"); + + assert!( + error + .to_string() + .contains("invalid execution run status transition: compensating -> running"), + "expected the durable transition guard to reject the rewrite, got `{error}`" + ); + Ok(()) +} diff --git a/crates/moa-execution/tests/execution_db/execution_capacity_db.rs b/crates/moa-execution/tests/execution_db/execution_capacity_db.rs index 78c75d21d..a0e22c6a5 100644 --- a/crates/moa-execution/tests/execution_db/execution_capacity_db.rs +++ b/crates/moa-execution/tests/execution_db/execution_capacity_db.rs @@ -65,6 +65,7 @@ async fn ready_run( reduce_cursor: None, source_exhausted: true, terminal_output: None, + condition_skipped: false, tasks, }, ) diff --git a/crates/moa-execution/tests/execution_db/incremental_scheduler_db.rs b/crates/moa-execution/tests/execution_db/incremental_scheduler_db.rs index 4b2a78ebd..c29583c5a 100644 --- a/crates/moa-execution/tests/execution_db/incremental_scheduler_db.rs +++ b/crates/moa-execution/tests/execution_db/incremental_scheduler_db.rs @@ -1,6 +1,5 @@ //! Bounded incremental scheduler projection and materialization contracts. -use chrono::DateTime; use moa_artifacts::execution_plan::{ CapabilityReference, ExecutionNode, ExecutionOperation, ExecutionReducer, MapTask, }; @@ -12,7 +11,7 @@ use moa_execution::repository::ready::{ }; use moa_execution::repository::task::{TaskAttemptFence, TaskAttemptStartOutcome}; use moa_execution::repository::{TransitionOutcome, TransitionRejection}; -use moa_execution::state::{WaitSettlement, completed_task_outcome}; +use moa_execution::state::completed_task_outcome; use serde_json::Value; use super::support::*; @@ -181,6 +180,7 @@ async fn ten_thousand_tasks_materialize_in_cursor_fenced_pages_db() -> TestResul reduce_cursor: None, source_exhausted: false, terminal_output: None, + condition_skipped: false, tasks, }, ) @@ -259,6 +259,7 @@ async fn ten_thousand_map_outputs_aggregate_in_sixteen_row_crash_replay_pages_db reduce_cursor: None, source_exhausted: page == 9, terminal_output: None, + condition_skipped: false, tasks, }, ) @@ -418,6 +419,7 @@ async fn seventeenth_near_sixty_four_kib_map_output_fails_before_unbounded_aggre reduce_cursor: None, source_exhausted: true, terminal_output: None, + condition_skipped: false, tasks, }, ) @@ -549,6 +551,7 @@ async fn twenty_five_hundred_reduce_batches_persist_exact_round_cursor_db() -> T }), source_exhausted: batch_cursor + page_count == 2_501, terminal_output: None, + condition_skipped: false, tasks, }; let ReadyMaterializationOutcome::Applied { next_cursor, .. } = repository @@ -587,6 +590,7 @@ async fn twenty_five_hundred_reduce_batches_persist_exact_round_cursor_db() -> T }), source_exhausted: true, terminal_output: None, + condition_skipped: false, tasks: vec![logical_task(run.run_uid, "reduce", "r1:b2500", estimate(1))], }; assert_eq!( @@ -687,6 +691,7 @@ async fn empty_map_completion_cas_releases_its_dependent_without_repeating_db() reduce_cursor: None, source_exhausted: true, terminal_output: Some(json!({ "items": [] })), + condition_skipped: false, tasks: Vec::new(), }; assert!(matches!( @@ -761,6 +766,7 @@ async fn terminal_partial_map_page_cannot_complete_node_before_source_exhaustion reduce_cursor: None, source_exhausted: false, terminal_output: None, + condition_skipped: false, tasks: vec![first], }, ) @@ -794,6 +800,7 @@ async fn terminal_partial_map_page_cannot_complete_node_before_source_exhaustion reduce_cursor: None, source_exhausted: true, terminal_output: None, + condition_skipped: false, tasks: vec![second], }, ) @@ -903,6 +910,7 @@ async fn failed_root_cancels_join_without_rolling_back_later_sibling_settlement_ reduce_cursor: None, source_exhausted: true, terminal_output: None, + condition_skipped: false, tasks: vec![task.clone()], }, ) @@ -1044,6 +1052,7 @@ async fn external_signal_settlement_preserves_newer_task_progress_db() -> TestRe reduce_cursor: None, source_exhausted: true, terminal_output: None, + condition_skipped: false, tasks: vec![signal], }, ) @@ -1147,6 +1156,7 @@ async fn relative_timer_is_parked_once_and_stale_delivery_is_fenced_db() -> Test reduce_cursor: None, source_exhausted: true, terminal_output: None, + condition_skipped: false, tasks: vec![task], }, ) @@ -1204,6 +1214,7 @@ async fn relative_timer_is_parked_once_and_stale_delivery_is_fenced_db() -> Test reduce_cursor: None, source_exhausted: true, terminal_output: None, + condition_skipped: false, tasks: vec![task], }, ) @@ -1280,99 +1291,3 @@ async fn relative_timer_is_parked_once_and_stale_delivery_is_fenced_db() -> Test assert!(activation.is_none()); Ok(()) } - -#[tokio::test] -async fn storage_wait_settlement_preserves_newer_database_progress_db() -> TestResult { - // Pins: a process-observed settlement may predate progress already committed by Postgres; - // settlement preserves that newer task clock while retaining the observation in audit history. - let test_db = moa_test_support::postgres::bootstrap_test_db().await?; - let pool = test_db.store().pool().clone(); - let repository = ExecutionRepository::new(pool.clone()); - let tenant_id = TenantId::new(); - let scope = ExecutionScope::Tenant { tenant_id }; - let mut candidate = new_run( - tenant_id, - None, - "stale-process-wait-settlement", - ExecutionRunStatus::Queued, - budget(1), - ); - candidate.plan.definition.nodes = vec![output_node("timer")]; - let run = create_run(&repository, scope, candidate).await?; - repository - .initialize_scheduler_state(scope, run.run_uid) - .await?; - - let settled_at = moa_test_support::fixtures::pg_now() - Duration::seconds(30); - let mut task = logical_task(run.run_uid, "timer", "", estimate(1)); - task.kind = LogicalTaskKind::WaitUntil { - wake: ExecutionTemporalTarget::At { - at: settled_at - Duration::seconds(1), - }, - result: json!({ "elapsed": true }), - }; - let ReadyMaterializationOutcome::Applied { tasks, .. } = repository - .materialize_ready_page( - scope, - &ExecutionConfig::default(), - ReadyMaterializationRequest { - run_uid: run.run_uid, - plan_revision: 1, - node_id: "timer".to_string(), - expected_cursor: 0, - reduce_cursor: None, - source_exhausted: true, - terminal_output: None, - tasks: vec![task], - }, - ) - .await? - else { - panic!("fresh timer wait must materialize"); - }; - let waiting_task = tasks - .into_iter() - .next() - .expect("timer materialization must return exactly one task"); - let waiting_since = waiting_task - .waiting_since - .expect("timer materialization must persist its wait anchor"); - let database_progress_at: DateTime = sqlx::query_scalar( - "UPDATE moa.execution_task SET last_progress_at=NOW(), updated_at=NOW() \ - WHERE run_uid=$1 AND task_id=$2 RETURNING last_progress_at", - ) - .bind(run.run_uid) - .bind(waiting_task.task_id.as_uuid()) - .fetch_one(&pool) - .await?; - assert!( - database_progress_at > settled_at, - "fixture must establish database progress newer than the supplied observation" - ); - - let outcome = repository - .settle_wait( - scope, - run.run_uid, - waiting_task.generation, - waiting_since, - WaitSettlement::TimerElapsed { - task_id: waiting_task.task_id, - output: json!({ "elapsed": true }), - }, - settled_at, - ) - .await?; - let TransitionOutcome::Applied(settled_task) = outcome else { - panic!("due timer settlement must apply: {outcome:?}"); - }; - assert!( - settled_task.last_progress_at >= database_progress_at, - "settlement must not move canonical task progress backward" - ); - assert!(settled_task.generation_history.iter().any(|entry| { - entry.get("kind").and_then(Value::as_str) == Some("storage_wait_settlement") - && entry.get("settled_at") == Some(&json!(settled_at)) - })); - Ok(()) -} diff --git a/crates/moa-execution/tests/execution_db/support.rs b/crates/moa-execution/tests/execution_db/support.rs index 2522a2811..64d6451d3 100644 --- a/crates/moa-execution/tests/execution_db/support.rs +++ b/crates/moa-execution/tests/execution_db/support.rs @@ -292,7 +292,7 @@ pub(crate) async fn set_run_status_path( for status in path { let terminal_cause = match *status { "completed" | "partial" => Some(json!({"kind":"completion","limit_stop":null})), - "blocked" => Some(json!({"kind":"scheduler_no_progress"})), + "blocked" => Some(json!({"kind":"completion","limit_stop":null})), "unsupported" => Some(json!({"kind":"task_failure","class":"unsupported"})), "failed" => Some(json!({"kind":"internal_failure"})), "cancelled" => Some(json!({"kind":"cancellation"})), @@ -302,7 +302,7 @@ pub(crate) async fn set_run_status_path( let terminal_reason = match *status { "completed" => Some("completed"), "partial" => Some("goal_incomplete"), - "blocked" => Some("no_progress"), + "blocked" => Some("blocked"), "unsupported" => Some("unsupported_plan"), "failed" => Some("internal_failure"), "cancelled" => Some("cancelled"), diff --git a/crates/moa-execution/tests/execution_db/trigger_outbox_db.rs b/crates/moa-execution/tests/execution_db/trigger_outbox_db.rs index 56715dad0..b61e91ad6 100644 --- a/crates/moa-execution/tests/execution_db/trigger_outbox_db.rs +++ b/crates/moa-execution/tests/execution_db/trigger_outbox_db.rs @@ -1229,6 +1229,7 @@ async fn task_watchdog_preparation_is_due_fenced_and_exact_owner_replay_safe_db( reduce_cursor: None, source_exhausted: true, terminal_output: None, + condition_skipped: false, tasks: vec![logical_task( run.run_uid, "watchdog-work", @@ -1342,6 +1343,124 @@ async fn task_watchdog_preparation_is_due_fenced_and_exact_owner_replay_safe_db( Ok(()) } +#[tokio::test] +async fn one_attempt_generation_admits_exactly_one_armed_watchdog_db() -> TestResult { + // Pins: `execution_trigger_current_run_generation_uidx` still keys the armed-trigger + // uniqueness on the single `pending` state after the trigger claim apparatus was + // deleted. A second watchdog for the same attempt generation must be rejected by the + // index, and rearming after the first one settles must be admitted again — the second + // half fails if the partial predicate is widened past `pending`. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let config = execution_capacity_config(); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let mut candidate = new_run( + tenant_id, + None, + "watchdog-generation-uniqueness", + ExecutionRunStatus::Queued, + budget(10), + ); + candidate.plan.definition.nodes = vec![watchdog_output_node()]; + let run = create_run(&repository, scope, candidate).await?; + assert!( + repository + .initialize_scheduler_state(scope, run.run_uid) + .await? + ); + assert!(matches!( + repository + .materialize_ready_page( + scope, + &config, + ReadyMaterializationRequest { + run_uid: run.run_uid, + plan_revision: 1, + node_id: "watchdog-work".to_string(), + expected_cursor: 0, + reduce_cursor: None, + source_exhausted: true, + terminal_output: None, + condition_skipped: false, + tasks: vec![logical_task( + run.run_uid, + "watchdog-work", + "one", + estimate(1), + )], + }, + ) + .await?, + ReadyMaterializationOutcome::Applied { .. } + )); + let admitted = repository + .admit_ready_attempts(&config, 1, Utc::now()) + .await? + .admitted + .into_iter() + .next() + .expect("one task must be admitted"); + let armed_state: String = + sqlx::query_scalar("SELECT state FROM moa.execution_trigger WHERE trigger_uid=$1") + .bind(admitted.watchdog_trigger_uid) + .fetch_one(&pool) + .await?; + assert_eq!( + armed_state, "pending", + "an armed watchdog is never claimed; claiming lives on the dispatch outbox" + ); + + let duplicate = task_watchdog( + Uuid::now_v7(), + tenant_id, + admitted.run_uid, + admitted.task_id.as_uuid(), + admitted.controller_generation, + admitted.attempt_generation, + pg_deadline(Duration::minutes(5)), + ); + let mut transaction = pool.begin().await?; + let conflict = create_trigger_with_dispatch_in_conn(&mut transaction, &config, &duplicate) + .await + .expect_err("one attempt generation must never arm two watchdogs"); + transaction.rollback().await?; + let moa_execution::Error::Database { source } = conflict else { + panic!("a duplicate armed watchdog must surface the unique-index violation"); + }; + let violation = source + .as_database_error() + .expect("the unique violation must carry its PostgreSQL provenance"); + assert_eq!(violation.code().as_deref(), Some("23505")); + assert_eq!( + violation.constraint(), + Some("execution_trigger_current_run_generation_uidx") + ); + + assert_eq!( + repository + .settle_watchdog_trigger(scope, admitted.watchdog_trigger_uid) + .await?, + ExecutionTriggerSupersedeOutcome::Superseded + ); + let mut transaction = pool.begin().await?; + let rearmed = + create_trigger_with_dispatch_in_conn(&mut transaction, &config, &duplicate).await?; + transaction.commit().await?; + assert_eq!(rearmed.trigger.trigger_uid, duplicate.trigger_uid); + assert_eq!(rearmed.trigger.state, ExecutionDeliveryState::Pending); + let armed_uids: Vec = sqlx::query_scalar( + "SELECT trigger_uid FROM moa.execution_trigger \ + WHERE run_uid=$1 AND trigger_kind='task_watchdog' AND state='pending'", + ) + .bind(run.run_uid) + .fetch_all(&pool) + .await?; + assert_eq!(armed_uids, vec![duplicate.trigger_uid]); + Ok(()) +} + #[tokio::test] async fn task_external_start_recovery_adopts_started_not_started_and_replay_atomically_db() -> TestResult { @@ -1382,6 +1501,7 @@ async fn task_external_start_recovery_adopts_started_not_started_and_replay_atom reduce_cursor: None, source_exhausted: true, terminal_output: None, + condition_skipped: false, tasks: ["not-started", "started", "missing-checkpoint"] .into_iter() .map(|item| { @@ -2457,6 +2577,7 @@ async fn terminal_callback_before_task_release_commits_then_settles_once_db() -> reduce_cursor: None, source_exhausted: true, terminal_output: None, + condition_skipped: false, tasks: vec![logical_task( run.run_uid, "watchdog-work", @@ -2635,6 +2756,7 @@ async fn retry_settlement_preserves_cancelling_until_ready_transition_db() -> Te reduce_cursor: None, source_exhausted: true, terminal_output: None, + condition_skipped: false, tasks: vec![logical_task( run.run_uid, "watchdog-work", @@ -2766,6 +2888,7 @@ async fn durable_task_release_receipt_splits_capacity_from_outcome_settlement_db reduce_cursor: None, source_exhausted: true, terminal_output: None, + condition_skipped: false, tasks: vec![logical_task( run.run_uid, "watchdog-work", @@ -3203,6 +3326,7 @@ async fn assert_paused_task_review_resolution( reduce_cursor: None, source_exhausted: true, terminal_output: None, + condition_skipped: false, tasks: vec![logical_task( run.run_uid, "watchdog-work", @@ -3399,6 +3523,34 @@ fn run_deadline( } } +fn task_watchdog( + trigger_uid: Uuid, + tenant_id: TenantId, + run_uid: Uuid, + task_id: Uuid, + controller_generation: u64, + attempt_generation: u64, + due_at: chrono::DateTime, +) -> NewExecutionTrigger { + NewExecutionTrigger { + trigger_uid, + tenant_id, + run_uid: Some(run_uid), + task_id: Some(task_id), + compensation_id: None, + schedule_uid: None, + schedule_incarnation: None, + kind: ExecutionTriggerKind::TaskWatchdog, + controller_generation: Some(controller_generation), + attempt_generation: Some(attempt_generation), + compensation_generation: None, + compensation_attempt_generation: None, + occurrence_sequence: None, + due_at, + payload: json!({}), + } +} + fn run_activation( tenant_id: TenantId, run_uid: Uuid, diff --git a/crates/moa-execution/tests/execution_db/wait_entry_deadline_db.rs b/crates/moa-execution/tests/execution_db/wait_entry_deadline_db.rs new file mode 100644 index 000000000..109fc2523 --- /dev/null +++ b/crates/moa-execution/tests/execution_db/wait_entry_deadline_db.rs @@ -0,0 +1,282 @@ +//! Wait entry that cannot finish before the run deadline projects a typed task failure. + +use moa_artifacts::execution_plan::{ExecutionNode, ExecutionOperation}; +use moa_execution::repository::ready::{ReadyMaterializationOutcome, ReadyMaterializationRequest}; +use moa_execution::repository::task::{ + TaskAttemptFence, TaskAttemptSettlementOutcome, TaskAttemptStartOutcome, +}; + +use super::support::*; + +fn output_node(id: &str, depends_on: &[&str]) -> ExecutionNode { + ExecutionNode { + id: id.to_string(), + requirement_ids: vec!["req".to_string()], + depends_on: depends_on.iter().map(|id| (*id).to_string()).collect(), + when: None, + input: json!({}), + output_schema: json!({ "type": "object" }), + operation: ExecutionOperation::Output { value: json!({}) }, + compensation: None, + retry: RetryPolicy { + max_attempts: 1, + initial_backoff_ms: 1, + max_backoff_ms: 1, + }, + budget: None, + } +} + +fn failure_class(task: &ExecutionTaskRecord) -> Option<(ExecutionFailureClass, String)> { + match task.current_outcome.as_ref().map(|outcome| &outcome.result) { + Some(ExecutionTaskResult::Failed { class, message }) => { + Some((class.clone(), message.clone())) + } + _ => None, + } +} + +#[tokio::test] +async fn storage_wait_past_run_deadline_fails_its_node_instead_of_erroring_db() -> TestResult { + // Pins: a relative storage wait that was legal at compile time but resolves at or after the + // run deadline once wait entry is reached materializes as a terminal DeadlineExceeded task + // failure naming its node, cascades to its unmaterialized dependent, and parks no trigger — + // instead of aborting materialization with an infra-shaped error. A wait that still fits + // inside the deadline keeps parking normally. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let config = ExecutionConfig::default(); + let mut candidate = new_run( + tenant_id, + None, + "storage-wait-past-deadline", + ExecutionRunStatus::Queued, + budget(10), + ); + candidate.plan.definition.nodes = vec![ + output_node("late", &[]), + output_node("after", &["late"]), + output_node("early", &[]), + ]; + let run = create_run(&repository, scope, candidate).await?; + repository + .initialize_scheduler_state(scope, run.run_uid) + .await?; + + // The approved budget expires in one hour, so a two-hour wait can never settle in time. + let mut late = logical_task(run.run_uid, "late", "", estimate(1)); + late.kind = LogicalTaskKind::WaitUntil { + wake: ExecutionTemporalTarget::After { + delay_seconds: 7_200, + }, + result: json!({ "elapsed": true }), + }; + let ReadyMaterializationOutcome::Applied { + tasks, + triggers, + next_cursor, + } = repository + .materialize_ready_page( + scope, + &config, + ReadyMaterializationRequest { + run_uid: run.run_uid, + plan_revision: 1, + node_id: "late".to_string(), + expected_cursor: 0, + reduce_cursor: None, + source_exhausted: true, + terminal_output: None, + condition_skipped: false, + tasks: vec![late], + }, + ) + .await? + else { + panic!("a wait past the run deadline must still apply as a typed failure"); + }; + assert_eq!(next_cursor, 1); + assert_eq!(tasks.len(), 1); + assert_eq!(tasks[0].status, ExecutionTaskStatus::Failed); + assert!( + triggers.is_empty(), + "an unenterable wait must not park a durable trigger" + ); + let (class, message) = + failure_class(&tasks[0]).expect("the failed wait must carry a typed failure outcome"); + assert_eq!(class, ExecutionFailureClass::DeadlineExceeded); + assert!( + message.contains("`late`"), + "the failure must name its node, got `{message}`" + ); + assert!(tasks[0].completed_at.is_some()); + + let node_projection = sqlx::query_as::<_, (String, i64, i64, i64)>( + "SELECT node_status, total_task_count, terminal_task_count, failed_task_count \ + FROM moa.execution_node_state WHERE run_uid=$1 AND node_id='late'", + ) + .bind(run.run_uid) + .fetch_one(&pool) + .await?; + assert_eq!(node_projection, ("failed".to_string(), 1, 1, 1)); + let dependent_status = sqlx::query_scalar::<_, String>( + "SELECT node_status FROM moa.execution_node_state WHERE run_uid=$1 AND node_id='after'", + ) + .bind(run.run_uid) + .fetch_one(&pool) + .await?; + assert_eq!( + dependent_status, "cancelled", + "the failed wait must cascade through the normal terminal projection" + ); + + let failed_run = repository + .load_run(scope, run.run_uid) + .await? + .expect("run must remain visible"); + assert_eq!(failed_run.progress_failed_tasks, 1); + assert_eq!(failed_run.waiting_task_count, 0); + assert_eq!(failed_run.ready_task_count, 0); + assert_eq!(failed_run.next_wake_at, None); + assert!(failed_run.waiting_reasons.is_empty()); + + // A wait that still fits inside the same deadline keeps its ordinary parked projection. + let mut early = logical_task(run.run_uid, "early", "", estimate(1)); + early.kind = LogicalTaskKind::WaitUntil { + wake: ExecutionTemporalTarget::After { delay_seconds: 60 }, + result: json!({ "elapsed": true }), + }; + let ReadyMaterializationOutcome::Applied { + tasks, triggers, .. + } = repository + .materialize_ready_page( + scope, + &config, + ReadyMaterializationRequest { + run_uid: run.run_uid, + plan_revision: 1, + node_id: "early".to_string(), + expected_cursor: 0, + reduce_cursor: None, + source_exhausted: true, + terminal_output: None, + condition_skipped: false, + tasks: vec![early], + }, + ) + .await? + else { + panic!("a wait inside the run deadline must park normally"); + }; + assert_eq!(tasks[0].status, ExecutionTaskStatus::WaitingTimer); + assert_eq!(triggers.len(), 1); + Ok(()) +} + +#[tokio::test] +async fn input_wait_past_run_deadline_fails_its_task_instead_of_erroring_db() -> TestResult { + // Pins: settling a NeedsInput attempt whose plan-level input-wait expiry resolves at or after + // the run deadline terminates the task with a typed DeadlineExceeded failure naming its node + // rather than aborting settlement; an expiry that still fits parks the ordinary input wait. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let config = ExecutionConfig::default(); + + for (key, expiry_seconds, expected_status) in [ + ( + "input-wait-past-deadline", + 86_400_u64, + ExecutionTaskStatus::Failed, + ), + ( + "input-wait-inside-deadline", + 60, + ExecutionTaskStatus::WaitingInput, + ), + ] { + let mut candidate = new_run(tenant_id, None, key, ExecutionRunStatus::Queued, budget(10)); + candidate.plan.definition.nodes = vec![output_node("ask", &[])]; + candidate.plan.definition.input_wait_policy = ExecutionWaitPolicy { + expiry: ExecutionTemporalTarget::After { + delay_seconds: expiry_seconds, + }, + on_expiry: ExecutionWaitExpiryAction::FailTask, + }; + let run = create_run(&repository, scope, candidate).await?; + repository + .initialize_scheduler_state(scope, run.run_uid) + .await?; + assert!(matches!( + repository + .materialize_ready_page( + scope, + &config, + ReadyMaterializationRequest { + run_uid: run.run_uid, + plan_revision: 1, + node_id: "ask".to_string(), + expected_cursor: 0, + reduce_cursor: None, + source_exhausted: true, + terminal_output: None, + condition_skipped: false, + tasks: vec![logical_task(run.run_uid, "ask", "", estimate(1))], + }, + ) + .await?, + ReadyMaterializationOutcome::Applied { .. } + )); + let admission = repository + .admit_ready_attempts(&config, 1, Utc::now()) + .await?; + let admitted = admission + .admitted + .into_iter() + .find(|item| item.run_uid == run.run_uid) + .expect("the only ready task must be admitted"); + let fence = TaskAttemptFence { + tenant_id: admitted.tenant_id, + run_uid: admitted.run_uid, + task_id: admitted.task_id, + controller_generation: admitted.controller_generation, + attempt_generation: admitted.attempt_generation, + dispatch_uid: admitted.dispatch_uid, + capacity_reservation_uid: admitted.capacity_reservation_uid, + watchdog_trigger_uid: admitted.watchdog_trigger_uid, + attempt_deadline_at: admitted.attempt_deadline_at, + }; + assert!(matches!( + repository.start_task_attempt(fence).await?, + TaskAttemptStartOutcome::Started(_) + )); + let TaskAttemptSettlementOutcome::Applied { task, .. } = repository + .settle_task_attempt(&config, fence, needs_input(1), None, Utc::now()) + .await? + else { + panic!("{key} settlement must apply"); + }; + assert_eq!(task.status, expected_status, "{key}"); + if expected_status == ExecutionTaskStatus::Failed { + let (class, message) = + failure_class(&task).expect("the failed input wait must carry a typed outcome"); + assert_eq!(class, ExecutionFailureClass::DeadlineExceeded); + assert!( + message.contains("`ask`"), + "the failure must name its node, got `{message}`" + ); + let settled_run = repository + .load_run(scope, run.run_uid) + .await? + .expect("run must remain visible"); + assert_eq!(settled_run.waiting_input_task_count, 0); + assert_eq!(settled_run.progress_failed_tasks, 1); + } + } + Ok(()) +} diff --git a/crates/moa-execution/tests/interpreter.rs b/crates/moa-execution/tests/interpreter.rs index f22b82e60..f664da22d 100644 --- a/crates/moa-execution/tests/interpreter.rs +++ b/crates/moa-execution/tests/interpreter.rs @@ -7,9 +7,9 @@ use chrono::{TimeZone, Utc}; use moa_artifacts::execution_plan::{ CapabilityReference, CompletionCheck, CompletionCheckKind, ExecutionBudgetLimit, ExecutionCancelPolicy, ExecutionCondition, ExecutionGoalContract, ExecutionNode, - ExecutionOperation, ExecutionPlanDefinition, ExecutionReference, ExecutionRequirement, - ExecutionTaskOutcome, ExecutionTaskResult, ExecutionTemporalTarget, ExecutionUsage, - ExecutionWaitExpiryAction, ExecutionWaitPolicy, MapTask, RetryPolicy, + ExecutionOperation, ExecutionPlanDefinition, ExecutionReducer, ExecutionReference, + ExecutionRequirement, ExecutionTaskOutcome, ExecutionTaskResult, ExecutionTemporalTarget, + ExecutionUsage, ExecutionWaitExpiryAction, ExecutionWaitPolicy, MapTask, RetryPolicy, }; use moa_config::ExecutionConfig; use moa_core::types::{ @@ -23,94 +23,24 @@ use moa_execution::{ ExecutionClass, ExecutionEstimate, ExecutionHash, catalog_hash, }, compiler::{CanonicalExecutionPlan, ExecutionValidationReport}, - completion::{CompletionEvaluationRequest, CompletionStatus, evaluate_completion}, interpreter::{ ReduceMaterializationPageInput, ScheduleRequest, materialize_node_page, - resolve_temporal_target, schedule as schedule_outcome, + resolve_temporal_target, }, state::{ ExecutionNodeStatus, ExecutionProjection, ExecutionRunStatus, ExecutionTaskId, - ExecutionTaskProjection, ExecutionTaskStatus, LogicalTaskKind, ScheduleDecision, - TerminalProjection, WaitSettlement, WaitingReason, input_resume_counters, + ExecutionTaskProjection, ExecutionTaskStatus, input_resume_counters, retry_dispatch_counters, run_status_after_task_outcome, supersede_waiting_replan, task_status_from_outcome, validate_outcome_generation, }, }; -use proptest::{ - prelude::*, - test_runner::{Config as ProptestConfig, FileFailurePersistence}, -}; use serde_json::{Value, json}; use uuid::Uuid; -fn schedule(request: ScheduleRequest) -> Result { - schedule_outcome(request).map(|outcome| outcome.decision) -} - -proptest! { - #![proptest_config(property_config())] - - #[test] - fn property_scheduler_is_idempotent_for_unchanged_projection( - item_count in 0_u64..=12, - completed_seed in 0_u64..=12, - run_seed in 1_u128..=u128::MAX, - ) { - // Pins: replaying an unchanged durable projection yields the exact same decision and projection. - let completed_count = completed_seed.min(item_count); - let items = (0..item_count).map(|item| json!(item)).collect::>(); - let map = node( - "inspect", - &[], - ExecutionOperation::Map { - items: Value::Array(items), - item_key: "".to_string(), - max_items: item_count, - item_output_schema: json!({ "type": "object" }), - task: MapTask::Capability { - reference: capability(), - }, - }, - ); - let plan = canonical(vec![map, output_node("inspect")]); - let run_uid = Uuid::from_u128(run_seed); - let statuses = if completed_count == 0 { - BTreeMap::new() - } else { - BTreeMap::from([("inspect".to_string(), ExecutionNodeStatus::Running)]) - }; - let tasks = (0..completed_count) - .map(|item| { - completed_item_task( - run_uid, - "inspect", - &format!("number:{item}"), - json!({ "item": item }), - json!({ "ok": true }), - ) - }) - .collect::>(); - let request = request(run_uid, plan, statuses, tasks); - - let first = schedule_outcome(request.clone()).expect("generated projection schedules"); - let second = schedule_outcome(request).expect("unchanged generated projection schedules"); - prop_assert_eq!(first, second); - } -} - -fn property_config() -> ProptestConfig { - ProptestConfig { - cases: 256, - failure_persistence: Some(Box::new(FileFailurePersistence::Direct( - "proptest-regressions/properties.txt", - ))), - ..ProptestConfig::default() - } -} - #[test] -fn scheduler_materializes_every_ready_map_item_with_stable_typed_keys() { - // Pins: max_items is accounting, not a hidden active-worker cap. +fn controller_materializes_every_map_item_with_stable_typed_keys() { + // Pins: max_items is accounting, not a hidden active-worker cap, and one page keeps the + // plan's item order so the persisted cursor addresses the same item on every replay. let run_uid = Uuid::from_u128(11); let plan = canonical(vec![ ExecutionNode { @@ -136,19 +66,25 @@ fn scheduler_materializes_every_ready_map_item_with_stable_typed_keys() { output_node("inspect"), ]); - let decision = schedule(request(run_uid, plan, BTreeMap::new(), vec![])).expect("schedule map"); - let ScheduleDecision::Ready(tasks) = decision else { - panic!("expected ready tasks, got {decision:?}"); - }; - assert_eq!(tasks.len(), 3); + let page = materialize_node_page( + &request(run_uid, plan, BTreeMap::new(), vec![]), + "inspect", + &BTreeMap::new(), + 0, + 3, + None, + ) + .expect("materialize map page"); + assert!(page.source_exhausted); + assert_eq!(page.next_cursor, 3); assert_eq!( - tasks + page.tasks .iter() .map(|task| task.item_key.as_str()) .collect::>(), - ["number:1", "object:{\"id\":1}", "string:\"1\""] + ["number:1", "string:\"1\"", "object:{\"id\":1}"] ); - for task in tasks { + for task in page.tasks { assert_eq!( task.task_id, ExecutionTaskId::derive(run_uid, "inspect", &task.item_key).expect("stable id") @@ -358,8 +294,9 @@ fn controller_pages_more_than_twenty_five_hundred_reduce_batches_across_rounds() } #[test] -fn scheduler_rejects_duplicate_dynamic_map_keys() { - // Pins: duplicate item identities fail materialization before any task can be returned. +fn controller_rejects_duplicate_dynamic_map_keys() { + // Pins: duplicate item identities fail materialization before any task can be returned, + // because two items sharing an item key would collide on one derived task ID. let plan = canonical(vec![ ExecutionNode { id: "inspect".to_string(), @@ -383,53 +320,109 @@ fn scheduler_rejects_duplicate_dynamic_map_keys() { }, output_node("inspect"), ]); - assert!(schedule(request(Uuid::from_u128(12), plan, BTreeMap::new(), vec![])).is_err()); + let error = materialize_node_page( + &request(Uuid::from_u128(12), plan, BTreeMap::new(), vec![]), + "inspect", + &BTreeMap::new(), + 0, + 2, + None, + ) + .expect_err("duplicate item keys must fail the page"); + assert!( + error.to_string().contains("duplicate item key"), + "unexpected error: {error}" + ); } #[test] -fn scheduler_does_not_validate_a_partial_map_as_its_terminal_aggregate() { - // Pins: an in-flight map is not validated against its completed aggregate schema. - let run_uid = Uuid::from_u128(120); - let mut map = node( - "inspect", - &[], - ExecutionOperation::Map { - items: json!(["one", "two"]), - item_key: "".to_string(), - max_items: 2, - item_output_schema: json!({ "type": "object" }), - task: MapTask::Capability { - reference: capability(), +fn a_false_condition_short_circuits_before_map_or_reduce_paging() { + // Pins: `when` is evaluated in the wrapper, so a false branch never resolves map items + // and never opens a reduce round. Both sources below are deliberately unmaterializable — + // the map exceeds its own `max_items` and the reduce is handed no round cursor — so a + // condition consulted anywhere later than the wrapper surfaces as a hard error here. + let condition = |value: &str| { + Some(ExecutionCondition::Equals { + reference: ExecutionReference { + path: "$.input.route".to_string(), }, - }, - ); - map.output_schema = json!({ - "type": "object", - "required": ["items"], - "properties": { "items": { "type": "array", "minItems": 2 } } - }); - let plan = canonical(vec![map, output_node("inspect")]); - let statuses = BTreeMap::from([("inspect".to_string(), ExecutionNodeStatus::Running)]); - let tasks = vec![completed_item_task( - run_uid, - "inspect", - "string:\"one\"", - json!({}), - json!({ "ok": true }), - )]; + value: json!(value), + }) + }; + let build = |when, is_reduce: bool| { + let mut node = ExecutionNode { + id: "branch".to_string(), + requirement_ids: vec!["req_one".to_string()], + depends_on: vec![], + when, + input: json!({ "$item": true }), + output_schema: json!({ "type": "object" }), + operation: ExecutionOperation::Map { + items: json!([{ "id": "a" }, { "id": "b" }, { "id": "c" }]), + item_key: "/id".to_string(), + max_items: 1, + item_output_schema: json!({ "type": "object" }), + task: MapTask::Capability { + reference: capability(), + }, + }, + compensation: None, + retry: retry(), + budget: None, + }; + if is_reduce { + node.input = json!({}); + node.operation = ExecutionOperation::Reduce { + items: json!([1, 2, 3, 4]), + max_items: 4, + reducer: ExecutionReducer::Capability { + reference: capability(), + }, + batch_size: 2, + }; + } + node + }; - let decision = schedule(request(run_uid, plan, statuses, tasks)) - .expect("partial map must not be validated as a terminal aggregate"); - assert_eq!( - decision, - ScheduleDecision::Waiting(vec![moa_execution::state::WaitingReason::Dependencies { - node_ids: vec!["output".to_string()] - }]) - ); + for is_reduce in [false, true] { + let mut skipped = request( + Uuid::from_u128(41), + canonical(vec![ + build(condition("taken"), is_reduce), + output_node("branch"), + ]), + BTreeMap::new(), + vec![], + ); + skipped.run_input = json!({ "route": "not-taken" }); + let page = materialize_node_page(&skipped, "branch", &BTreeMap::new(), 0, 8, None) + .expect("a false condition must page without touching the node source"); + assert!(page.condition_skipped); + assert!(page.tasks.is_empty()); + assert!(page.source_exhausted); + assert_eq!(page.next_cursor, 0); + assert!(page.terminal_output.is_none()); + assert!(page.reduce_cursor.is_none()); + + let mut taken = request( + Uuid::from_u128(42), + canonical(vec![ + build(condition("taken"), is_reduce), + output_node("branch"), + ]), + BTreeMap::new(), + vec![], + ); + taken.run_input = json!({ "route": "taken" }); + assert!( + materialize_node_page(&taken, "branch", &BTreeMap::new(), 0, 8, None).is_err(), + "the same source must still be reached when the condition holds" + ); + } } #[test] -fn scheduler_builds_exact_hierarchical_reducer_batch_inputs() { +fn controller_builds_exact_hierarchical_reducer_batch_inputs() { // Pins: reducer tasks use r{round}:b{batch} keys and exact structured batch input. let mut reduce = node( "reduce", @@ -445,124 +438,42 @@ fn scheduler_builds_exact_hierarchical_reducer_batch_inputs() { ); reduce.output_schema = json!({}); let plan = canonical(vec![reduce, output_node("reduce")]); - let decision = schedule(request(Uuid::from_u128(13), plan, BTreeMap::new(), vec![])) - .expect("schedule reduce"); - let ScheduleDecision::Ready(tasks) = decision else { - panic!("expected reducer tasks, got {decision:?}"); - }; + let page = materialize_node_page( + &request(Uuid::from_u128(13), plan, BTreeMap::new(), vec![]), + "reduce", + &BTreeMap::new(), + 0, + 1_000, + Some(&ReduceMaterializationPageInput { + round: 1, + batch_cursor: 0, + round_input_count: None, + page_inputs: Vec::new(), + }), + ) + .expect("materialize first reduce round"); assert_eq!( - tasks + page.tasks .iter() .map(|task| task.item_key.as_str()) .collect::>(), ["r1:b0", "r1:b1", "r1:b2"] ); assert_eq!( - tasks[0].input, + page.tasks[0].input, json!({ "round": 1, "batch_index": 0, "items": [1, 2] }) ); -} - -#[test] -fn scheduler_propagates_fixed_dependency_failed_terminal() { - // Pins: terminal predecessor failure uses dependency_failed without configurable policy. - let plan = canonical(vec![node( - "output", - &["lookup"], - ExecutionOperation::Output { value: json!({}) }, - )]); - let statuses = BTreeMap::from([ - ("lookup".to_string(), ExecutionNodeStatus::Failed), - ("output".to_string(), ExecutionNodeStatus::Pending), - ]); - let decision = schedule(request(Uuid::from_u128(14), plan, statuses, vec![])) - .expect("schedule dependency failure"); - let ScheduleDecision::Terminal(TerminalProjection::Failed { failure }) = decision else { - panic!("expected failed terminal, got {decision:?}"); - }; assert_eq!( - failure.class, - moa_artifacts::execution_plan::ExecutionFailureClass::DependencyFailed + page.tasks[2].input, + json!({ "round": 1, "batch_index": 2, "items": [5] }) ); } #[test] -fn scheduler_materializes_completion_verifier_after_ordinary_nodes_finish() { - // Pins: verifier is one synthetic stable task with summaries but no embedded raw outputs. - let run_uid = Uuid::from_u128(15); - let plan = canonical(vec![ - node( - "lookup", - &[], - ExecutionOperation::Capability { - reference: capability(), - }, - ), - output_node("lookup"), - ]); - let statuses = BTreeMap::from([ - ("lookup".to_string(), ExecutionNodeStatus::Completed), - ("output".to_string(), ExecutionNodeStatus::Completed), - ]); - let tasks = vec![ - completed_task(run_uid, "lookup", json!({ "secret": "raw" })), - completed_task(run_uid, "output", json!({ "ok": true })), - ]; - let decision = schedule(request(run_uid, plan, statuses, tasks)).expect("schedule verifier"); - let ScheduleDecision::Ready(tasks) = decision else { - panic!("expected verifier task, got {decision:?}"); - }; - assert_eq!(tasks.len(), 1); - let verifier = &tasks[0]; - assert_eq!(verifier.node_id, "@check/semantic"); - assert_eq!(verifier.item_key, "check:semantic"); - assert_eq!(verifier.retry.max_attempts, 1); - assert_eq!( - verifier.reservation, - ExecutionEstimate { - cost_microusd: 400_000, - tokens: 32_000, - tool_calls: 8, - retrieved_bytes: 2_000_000, - tasks: 1, - } - ); - assert!(matches!( - verifier.kind, - LogicalTaskKind::CompletionVerifier { .. } - )); - let object = verifier - .input - .as_object() - .expect("verifier input should be an object"); - assert_eq!( - object.keys().map(String::as_str).collect::>(), - BTreeSet::from([ - "check_id", - "description", - "goal", - "task_summaries", - "terminal_output" - ]) - ); - assert_eq!(object.get("check_id"), Some(&json!("semantic"))); - assert_eq!(object.get("terminal_output"), Some(&json!({ "ok": true }))); - let summaries = object - .get("task_summaries") - .and_then(Value::as_array) - .expect("verifier summaries should be an array"); - assert_eq!(summaries.len(), 2); - assert!( - summaries - .iter() - .all(|summary| summary.get("output_hash").is_some()) - ); - assert!(!verifier.input.to_string().contains("secret")); -} - -#[test] -fn scheduler_rejects_catalog_drift_and_validates_capability_input() { - // Pins: a run revision uses only the catalog snapshot whose canonical hash is pinned by the plan. +fn controller_validates_capability_input_and_reservation_against_the_pinned_catalog() { + // Pins: materialization resolves a capability task's reservation and input schema from the + // catalog snapshot pinned to the run, and refuses a catalog whose estimates would let one + // logical task consume more than one task budget unit. let run_uid = Uuid::from_u128(16); let mut capability_node = node( "lookup", @@ -574,13 +485,17 @@ fn scheduler_rejects_catalog_drift_and_validates_capability_input() { capability_node.input = json!({ "order_id": "ord-1" }); capability_node.retry.max_attempts = 2; let plan = canonical(vec![capability_node, output_node("lookup")]); - let decision = schedule(request(run_uid, plan.clone(), BTreeMap::new(), vec![])) - .expect("matching catalog should schedule"); - let ScheduleDecision::Ready(tasks) = decision else { - panic!("expected ready capability task, got {decision:?}"); - }; + let page = materialize_node_page( + &request(run_uid, plan.clone(), BTreeMap::new(), vec![]), + "lookup", + &BTreeMap::new(), + 0, + 1, + None, + ) + .expect("matching catalog should materialize"); assert_eq!( - tasks[0].reservation, + page.tasks[0].reservation, ExecutionEstimate { cost_microusd: 14, tokens: 22, @@ -590,16 +505,6 @@ fn scheduler_rejects_catalog_drift_and_validates_capability_input() { } ); - let mut drifted = request(run_uid, plan.clone(), BTreeMap::new(), vec![]); - drifted.catalog.capabilities[0].estimate.tokens = 12; - drifted.catalog.catalog_hash = - catalog_hash(&drifted.catalog.capabilities).expect("drifted catalog should hash"); - let error = schedule(drifted).expect_err("catalog content drift must be rejected"); - assert_eq!( - error.to_string(), - "invalid execution projection: scheduler capability catalog hash does not match the canonical plan" - ); - let mut invalid_input = request(run_uid, plan.clone(), BTreeMap::new(), vec![]); invalid_input.catalog.capabilities[0].input_schema = json!({ "type": "object", @@ -608,7 +513,8 @@ fn scheduler_rejects_catalog_drift_and_validates_capability_input() { invalid_input.catalog.catalog_hash = catalog_hash(&invalid_input.catalog.capabilities).expect("catalog should hash"); invalid_input.plan.catalog_hash = invalid_input.catalog.catalog_hash; - let error = schedule(invalid_input).expect_err("resolved capability input must validate"); + let error = materialize_node_page(&invalid_input, "lookup", &BTreeMap::new(), 0, 1, None) + .expect_err("resolved capability input must validate"); assert!(matches!(error, moa_execution::Error::Schema { .. })); let mut invalid_task_count = request(run_uid, plan, BTreeMap::new(), vec![]); @@ -616,348 +522,15 @@ fn scheduler_rejects_catalog_drift_and_validates_capability_input() { invalid_task_count.catalog.catalog_hash = catalog_hash(&invalid_task_count.catalog.capabilities).expect("catalog should hash"); invalid_task_count.plan.catalog_hash = invalid_task_count.catalog.catalog_hash; - let error = schedule(invalid_task_count) + let error = materialize_node_page(&invalid_task_count, "lookup", &BTreeMap::new(), 0, 1, None) .expect_err("catalog capability estimates must reserve one task"); assert!(error.to_string().contains("exactly one logical task")); } #[test] -fn scheduler_skips_false_condition_and_resolves_downstream_null() { - // Pins: a false condition is an effective skipped node with JSON null output. - let mut conditional = node( - "conditional", - &[], - ExecutionOperation::Capability { - reference: capability(), - }, - ); - conditional.when = Some(ExecutionCondition::Equals { - reference: ExecutionReference { - path: "$.input.run".to_string(), - }, - value: json!(true), - }); - let mut output = output_node("conditional"); - output.output_schema = json!({}); - let mut plan = canonical(vec![conditional, output]); - plan.definition.output_schema = json!({}); - let mut request = request(Uuid::from_u128(17), plan, BTreeMap::new(), vec![]); - request.run_input = json!({ "run": false }); - - let decision = schedule(request).expect("false condition should be deterministic"); - let ScheduleDecision::Ready(tasks) = decision else { - panic!("expected only the downstream output task, got {decision:?}"); - }; - assert_eq!(tasks.len(), 1); - assert_eq!(tasks[0].node_id, "output"); - assert!(matches!( - &tasks[0].kind, - LogicalTaskKind::Output { value } if value.is_null() - )); -} - -#[test] -fn scheduler_advances_every_hierarchical_reducer_round_to_final_output() { - // Pins: completed reducer batches feed deterministic subsequent rounds until one output remains. - let run_uid = Uuid::from_u128(18); - let mut reduce = node( - "reduce", - &[], - ExecutionOperation::Reduce { - items: json!([1, 2, 3, 4, 5]), - max_items: 5, - reducer: moa_artifacts::execution_plan::ExecutionReducer::Capability { - reference: capability(), - }, - batch_size: 2, - }, - ); - reduce.output_schema = json!({}); - let mut output = output_node("reduce"); - output.output_schema = json!({}); - let mut plan = canonical(vec![reduce, output]); - plan.definition.output_schema = json!({}); - let statuses = BTreeMap::from([ - ("reduce".to_string(), ExecutionNodeStatus::Pending), - ("output".to_string(), ExecutionNodeStatus::Pending), - ]); - let mut completed = vec![ - completed_item_task(run_uid, "reduce", "r1:b0", json!({}), json!(3)), - completed_item_task(run_uid, "reduce", "r1:b1", json!({}), json!(7)), - completed_item_task(run_uid, "reduce", "r1:b2", json!({}), json!(5)), - ]; - - let decision = schedule(request( - run_uid, - plan.clone(), - statuses.clone(), - completed.clone(), - )) - .expect("second reducer round should schedule"); - let ScheduleDecision::Ready(round_two) = decision else { - panic!("expected second reducer round, got {decision:?}"); - }; - assert_eq!(round_two.len(), 2); - assert_eq!(round_two[0].item_key, "r2:b0"); - assert_eq!( - round_two[0].input, - json!({ "round": 2, "batch_index": 0, "items": [3, 7] }) - ); - assert_eq!( - round_two[1].input, - json!({ "round": 2, "batch_index": 1, "items": [5] }) - ); - completed.push(completed_item_task( - run_uid, - "reduce", - "r2:b0", - round_two[0].input.clone(), - json!(10), - )); - completed.push(completed_item_task( - run_uid, - "reduce", - "r2:b1", - round_two[1].input.clone(), - json!(5), - )); - - let decision = schedule(request( - run_uid, - plan.clone(), - statuses.clone(), - completed.clone(), - )) - .expect("final reducer round should schedule"); - let ScheduleDecision::Ready(round_three) = decision else { - panic!("expected final reducer round, got {decision:?}"); - }; - assert_eq!(round_three.len(), 1); - assert_eq!(round_three[0].item_key, "r3:b0"); - assert_eq!( - round_three[0].input, - json!({ "round": 3, "batch_index": 0, "items": [10, 5] }) - ); - completed.push(completed_item_task( - run_uid, - "reduce", - "r3:b0", - round_three[0].input.clone(), - json!(15), - )); - - let decision = schedule(request(run_uid, plan, statuses, completed)) - .expect("completed hierarchy should unlock output"); - let ScheduleDecision::Ready(tasks) = decision else { - panic!("expected terminal output task, got {decision:?}"); - }; - assert_eq!(tasks.len(), 1); - assert!(matches!( - &tasks[0].kind, - LogicalTaskKind::Output { value } if value == &json!(15) - )); -} - -#[test] -fn scheduler_returns_the_effective_projection_used_for_terminal_completion() { - // Pins: callers finalize against the same derived aggregate statuses that selected terminal. - let run_uid = Uuid::from_u128(181); - let mut map = node( - "inspect", - &[], - ExecutionOperation::Map { - items: json!([1]), - item_key: "".to_string(), - max_items: 1, - item_output_schema: json!({ "type": "object" }), - task: MapTask::Capability { - reference: capability(), - }, - }, - ); - map.output_schema = json!({ - "type": "object", - "required": ["items"], - "properties": { - "items": { - "type": "array", - "minItems": 1, - "maxItems": 1 - } - } - }); - let plan = canonical(vec![map, output_node("inspect")]); - let statuses = BTreeMap::from([ - ("inspect".to_string(), ExecutionNodeStatus::Pending), - ("output".to_string(), ExecutionNodeStatus::Completed), - ]); - let tasks = vec![ - completed_item_task( - run_uid, - "inspect", - "number:1", - json!({}), - json!({ "ok": true }), - ), - completed_task(run_uid, "output", json!({ "ok": true })), - ]; - let mut request = request(run_uid, plan, statuses, tasks); - request - .goal - .completion_checks - .retain(|check| matches!(check.kind, CompletionCheckKind::OutputSchema)); - - let outcome = schedule_outcome(request.clone()).expect("schedule completed map run"); - let ScheduleDecision::Terminal(TerminalProjection::Completed { output }) = &outcome.decision - else { - panic!("expected completed terminal, got {:?}", outcome.decision); - }; - assert_eq!( - outcome.effective_projection.node_statuses.get("inspect"), - Some(&ExecutionNodeStatus::Completed) - ); - let evaluation = evaluate_completion(CompletionEvaluationRequest { - goal: request.goal, - plan: request.plan, - run_input: request.run_input, - projection: outcome.effective_projection, - terminal_output: Some(output.clone()), - budget_ledger: request.budget_ledger, - now: request.now, - }) - .expect("evaluate the scheduler's effective terminal projection"); - assert_eq!(evaluation.status, CompletionStatus::Completed); -} - -#[test] -fn scheduler_returns_no_progress_for_unbacked_nonterminal_state() { - // Pins: unfinished state with no runnable or durably waiting task is surfaced as NoProgress. - let plan = canonical(vec![node( - "lookup", - &[], - ExecutionOperation::Capability { - reference: capability(), - }, - )]); - let statuses = BTreeMap::from([("lookup".to_string(), ExecutionNodeStatus::Running)]); - - let decision = schedule(request(Uuid::from_u128(121), plan, statuses, vec![])) - .expect("unbacked running state should remain inspectable"); - assert_eq!( - decision, - ScheduleDecision::NoProgress { - pending_node_ids: vec!["lookup".to_string()] - } - ); -} - -#[test] -fn scheduler_parks_wait_until_then_settles_exactly_at_the_absolute_target() { - // Pins: WaitUntil consumes no executable task slot while early and settles at `at`, not after it. - let run_uid = Uuid::from_u128(122); - let wake = ExecutionTemporalTarget::At { - at: Utc - .with_ymd_and_hms(2026, 7, 13, 1, 0, 0) - .single() - .expect("wake time"), - }; - let plan = canonical(vec![node( - "timer", - &[], - ExecutionOperation::WaitUntil { - wake: wake.clone(), - result: json!({ "ready": true }), - }, - )]); - let statuses = BTreeMap::from([("timer".to_string(), ExecutionNodeStatus::Waiting)]); - let task = ExecutionTaskProjection { - task_id: ExecutionTaskId::derive(run_uid, "timer", "").expect("timer task id"), - node_id: "timer".to_string(), - item_key: String::new(), - status: ExecutionTaskStatus::WaitingTimer, - attempt: 1, - generation: 1, - input: json!({}), - outcome: None, - }; - - let early = schedule(request( - run_uid, - plan.clone(), - statuses.clone(), - vec![task.clone()], - )) - .expect("timer should park before its target"); - assert_eq!( - early, - ScheduleDecision::Waiting(vec![WaitingReason::Timer { - task_id: task.task_id, - wake: wake.clone(), - }]) - ); - - let mut due_request = request(run_uid, plan, statuses, vec![task.clone()]); - due_request.now = Utc - .with_ymd_and_hms(2026, 7, 13, 1, 0, 0) - .single() - .expect("due time"); - let due = schedule(due_request).expect("timer should settle at its target"); - assert_eq!( - due, - ScheduleDecision::SettleWait(WaitSettlement::TimerElapsed { - task_id: task.task_id, - output: json!({ "ready": true }), - }) - ); -} - -#[test] -fn scheduler_selects_input_wait_expiry_and_resolves_relative_targets_at_wait_entry() { - // Pins: storage-only waits deterministically settle, while relative timers anchor on entry. - let run_uid = Uuid::from_u128(123); - let plan = canonical(vec![node( - "lookup", - &[], - ExecutionOperation::Capability { - reference: capability(), - }, - )]); - let statuses = BTreeMap::from([("lookup".to_string(), ExecutionNodeStatus::Waiting)]); - let task = ExecutionTaskProjection { - task_id: ExecutionTaskId::derive(run_uid, "lookup", "").expect("input task id"), - node_id: "lookup".to_string(), - item_key: String::new(), - status: ExecutionTaskStatus::WaitingInput, - attempt: 1, - generation: 1, - input: json!({}), - outcome: Some(ExecutionTaskOutcome { - schema_version: 1, - usage: ExecutionUsage { - cost_microusd: 0, - tokens: 0, - tool_calls: 0, - retrieved_bytes: 0, - }, - result: ExecutionTaskResult::NeedsInput { - question: "Which order?".to_string(), - audience: moa_artifacts::execution_plan::InputAudience::User, - }, - }), - }; - let mut expiry_request = request(run_uid, plan, statuses, vec![task.clone()]); - expiry_request.now = Utc - .with_ymd_and_hms(2026, 7, 13, 12, 0, 0) - .single() - .expect("expiry time"); - assert_eq!( - schedule(expiry_request).expect("input expiry should settle"), - ScheduleDecision::SettleWait(WaitSettlement::WaitExpired { - task_id: task.task_id, - action: ExecutionWaitExpiryAction::FailTask, - }) - ); - +fn relative_temporal_targets_resolve_at_wait_entry_and_fence_on_the_run_deadline() { + // Pins: a wait-entry-relative target is resolved once against the exact entry instant and + // fails closed rather than persisting a due time the run deadline would never reach. let entered_at = Utc .with_ymd_and_hms(2026, 7, 13, 2, 0, 0) .single() @@ -1183,50 +756,6 @@ fn task_transition_helpers_pin_retry_resume_generation_and_replan_supersession() )); } -#[test] -fn scheduler_ignores_cancelled_tasks_from_superseded_plan_revisions() { - // Pins: amendment history keeps cancelled task rows whose nodes are absent from the active - // plan, while the scheduler advances only the replacement branch. - let run_uid = Uuid::from_u128(20); - let replacement = node( - "replacement", - &[], - ExecutionOperation::Capability { - reference: capability(), - }, - ); - let plan = canonical(vec![replacement, output_node("replacement")]); - let superseded = ExecutionTaskProjection { - task_id: ExecutionTaskId::derive(run_uid, "superseded", "").expect("task id"), - node_id: "superseded".to_string(), - item_key: String::new(), - status: ExecutionTaskStatus::Cancelled, - attempt: 1, - generation: 1, - input: json!({}), - outcome: Some(ExecutionTaskOutcome { - schema_version: 1, - usage: ExecutionUsage { - cost_microusd: 0, - tokens: 0, - tool_calls: 0, - retrieved_bytes: 0, - }, - result: ExecutionTaskResult::Cancelled { - reason: "superseded_by_plan_revision".to_string(), - }, - }), - }; - - let decision = schedule(request(run_uid, plan, BTreeMap::new(), vec![superseded])) - .expect("cancelled superseded history must not invalidate the active plan"); - let ScheduleDecision::Ready(tasks) = decision else { - panic!("expected replacement task, got {decision:?}"); - }; - assert_eq!(tasks.len(), 1); - assert_eq!(tasks[0].node_id, "replacement"); -} - fn request( run_uid: Uuid, plan: CanonicalExecutionPlan, @@ -1346,41 +875,6 @@ fn output_node(dependency: &str) -> ExecutionNode { ) } -fn completed_task(run_uid: Uuid, node_id: &str, output: Value) -> ExecutionTaskProjection { - completed_item_task(run_uid, node_id, "", json!({}), output) -} - -fn completed_item_task( - run_uid: Uuid, - node_id: &str, - item_key: &str, - input: Value, - output: Value, -) -> ExecutionTaskProjection { - ExecutionTaskProjection { - task_id: ExecutionTaskId::derive(run_uid, node_id, item_key).expect("task id"), - node_id: node_id.to_string(), - item_key: item_key.to_string(), - status: ExecutionTaskStatus::Completed, - attempt: 1, - generation: 1, - input, - outcome: Some(ExecutionTaskOutcome { - schema_version: 1, - usage: ExecutionUsage { - cost_microusd: 0, - tokens: 0, - tool_calls: 0, - retrieved_bytes: 0, - }, - result: ExecutionTaskResult::Completed { - output, - citations: vec![], - }, - }), - } -} - fn retry() -> RetryPolicy { RetryPolicy { max_attempts: 1, diff --git a/crates/moa-hands/src/adapters/daytona/mod.rs b/crates/moa-hands/src/adapters/daytona/mod.rs index 05a176677..e24139a78 100644 --- a/crates/moa-hands/src/adapters/daytona/mod.rs +++ b/crates/moa-hands/src/adapters/daytona/mod.rs @@ -1211,7 +1211,18 @@ impl HandProvider for DaytonaHandProvider { }) } - async fn pause(&self, handle: &HandHandle) -> Result<()> { + fn supports_suspend(&self) -> bool { + true + } + + /// Stops the sandbox, releasing its CPU and memory while keeping the filesystem. + /// + /// `POST /api/sandbox/{id}/stop` is a real compute release on Daytona's + /// billing model, not a freeze: the sandbox moves to `stopped` (which + /// [`DaytonaHandProvider::status`] maps to [`HandStatus::Stopped`]) and + /// [`DaytonaHandProvider::resume`] brings it back with `/start`. This is the + /// only genuine compute-release-with-filesystem-retention primitive MOA has. + async fn suspend(&self, handle: &HandHandle) -> Result<()> { let workspace_id = handle.daytona_id()?; let (account_id, account_generation) = cloud_account(handle, "Daytona")?; let attempt = self diff --git a/crates/moa-hands/src/adapters/daytona/workspace.rs b/crates/moa-hands/src/adapters/daytona/workspace.rs index d699d3017..a4bb7257c 100644 --- a/crates/moa-hands/src/adapters/daytona/workspace.rs +++ b/crates/moa-hands/src/adapters/daytona/workspace.rs @@ -1091,6 +1091,8 @@ impl SandboxStorageProvider for DaytonaHandProvider { confirmed_disposition: Some(disposition), storage: present.then_some(storage), checkpoint_publication, + // Reconciliation only proves the bytes; it never releases + // compute. See `SandboxStorageProvider::reconcile_workspace_operation`. post_commit_state: Some(WorkspacePostCommitState::AttachmentRetained), }); } diff --git a/crates/moa-hands/src/adapters/e2b/mod.rs b/crates/moa-hands/src/adapters/e2b/mod.rs index 08d8ee80f..ca707785d 100644 --- a/crates/moa-hands/src/adapters/e2b/mod.rs +++ b/crates/moa-hands/src/adapters/e2b/mod.rs @@ -1264,9 +1264,17 @@ impl HandProvider for E2BHandProvider { ) } - async fn pause(&self, _handle: &HandHandle) -> Result<()> { + /// Refuses compute suspension: MOA deliberately opts E2B out of auto-pause. + /// + /// Sandbox creation sets `autoPause: false` and `autoResume.enabled: false` + /// so a sandbox's durable state lives in MOA-owned portable checkpoints + /// rather than in a provider-side paused snapshot MOA cannot fence or + /// account for. Pausing here would reintroduce exactly that hidden state, so + /// the continuation boundary uses the checkpoint path instead. + async fn suspend(&self, _handle: &HandHandle) -> Result<()> { Err(MoaError::Unsupported( - "E2B pause retains process memory and is not a filesystem persistence primitive" + "E2B pause is deliberately disabled (autoPause/autoResume off); MOA carries sandbox \ + state in portable checkpoints instead" .to_string(), )) } diff --git a/crates/moa-hands/src/adapters/e2b/tests.rs b/crates/moa-hands/src/adapters/e2b/tests.rs index 0019d69d6..08cdfa2f3 100644 --- a/crates/moa-hands/src/adapters/e2b/tests.rs +++ b/crates/moa-hands/src/adapters/e2b/tests.rs @@ -7,10 +7,11 @@ use moa_core::{ error::MoaError, traits::{HandProvider, SandboxStorageProvider}, types::{ - hands::{EgressPolicy, HandHandle, SandboxTier}, + hands::{EgressPolicy, HandHandle, HandStatus, SandboxTier}, identifiers::{HandProvisioningOperationId, WorkspaceCheckpointId, WorkspaceOperationId}, sandbox_workspace::{ - WorkspaceCheckpointPublishRequest, WorkspaceOperationKind, WorkspaceRevisionRef, + WorkspaceCheckpointPublishRequest, WorkspaceOperationKind, WorkspaceOperationOutcome, + WorkspacePostCommitState, WorkspaceReconcileRequest, WorkspaceRevisionRef, WorkspaceStorageOperation, }, }, @@ -361,12 +362,8 @@ async fn provisions_executes_and_destroys_sandbox() { .unwrap(); assert_eq!(output.process_stdout(), Some("hello\n")); - // Pins: E2B pause/resume retain process memory and cannot be selected by - // the durable filesystem path. - assert!(matches!( - provider.pause(&handle).await, - Err(MoaError::Unsupported(_)) - )); + // Pins: E2B resume restores process memory and cannot be selected by the + // durable filesystem path. assert!(matches!( provider.resume(&handle).await, Err(MoaError::Unsupported(_)) @@ -578,6 +575,91 @@ async fn exports_reserved_data_root_through_canonical_archive_and_wipes_temp() { .expect("destroy fixture sandbox"); } +// Pins: reconciling an ambiguous non-yield commit proves the published bytes +// without destroying compute the caller is still executing on, and reports the +// same retained disposition the local and Daytona adapters report. +#[tokio::test] +async fn e2b_commit_reconciliation_retains_the_hand_it_reconciles() { + let mut fixture = FixtureE2BApi::start().await; + let store = Arc::new( + crate::core::sandbox_workspace::checkpoint::store::CheckpointObjectStore::new( + Arc::new(object_store::memory::InMemory::new()), + Arc::new(moa_crypto::LocalKmsProvider::new()), + "e2b-reconcile-retention", + crate::core::sandbox_workspace::checkpoint::archive::ArchiveLimits::default(), + crate::core::sandbox_workspace::checkpoint::store::ObservedCheckpointBucketVersioning::Unversioned, + ) + .expect("offline checkpoint store should construct"), + ); + let provider = E2BHandProvider::new(Arc::new( + crate::core::provider_credentials::TestProviderCredentialSource::new( + CloudHandProviderKind::E2b, + fixture.api_url(), + None, + Some("example.e2b.test".to_string()), + Some("base".to_string()), + "test-key", + ), + )) + .with_sandbox_base_url(fixture.api_url()) + .with_checkpoint_store(store); + let mut spec = crate::core::profile::test_support::hand_spec( + SandboxTier::MicroVM, + e2b_test_profile(EgressPolicy::DenyAll), + ); + spec.workspace.current_revision = None; + let binding = spec.workspace.clone(); + let handle = provider.provision(spec).await.expect("provision E2B hand"); + let _ = fixture.next_discovery_request().await; + + let operation = WorkspaceStorageOperation { + operation_id: WorkspaceOperationId::new(), + kind: WorkspaceOperationKind::Commit, + binding, + deadline: chrono::Utc::now() + chrono::Duration::minutes(1), + request_hash: "b".repeat(64), + }; + let published = provider + .publish_workspace_checkpoint(WorkspaceCheckpointPublishRequest { + operation: operation.clone(), + hand: handle.clone(), + parent_revision: None, + release_compute: false, + }) + .await + .expect("publish a non-yield commit checkpoint"); + let storage = published + .storage + .expect("a confirmed commit publishes portable checkpoint storage"); + + let reconciled = provider + .reconcile_workspace_operation( + WorkspaceReconcileRequest::new(operation, Some(handle.clone()), Some(storage)) + .expect("exact-resource reconciliation request validates"), + ) + .await + .expect("reconcile the ambiguous commit against published bytes"); + + assert_eq!(reconciled.outcome, WorkspaceOperationOutcome::Confirmed); + assert_eq!( + reconciled.post_commit_state, + Some(WorkspacePostCommitState::AttachmentRetained), + "reconciliation must never report a compute release it was not asked for" + ); + assert_eq!( + provider + .status(&handle) + .await + .expect("inspect reconciled hand"), + HandStatus::Running, + "reconciling a commit must leave the caller's compute alive" + ); + provider + .destroy(&handle) + .await + .expect("destroy fixture hand"); +} + fn e2b_operation_temp_dirs() -> BTreeSet { std::fs::read_dir(std::env::temp_dir()) .expect("read host temp directory") diff --git a/crates/moa-hands/src/adapters/e2b/workspace.rs b/crates/moa-hands/src/adapters/e2b/workspace.rs index 9ab61b8a6..9adf2620a 100644 --- a/crates/moa-hands/src/adapters/e2b/workspace.rs +++ b/crates/moa-hands/src/adapters/e2b/workspace.rs @@ -506,10 +506,6 @@ impl SandboxStorageProvider for E2BHandProvider { let present = published.is_some(); match operation.kind { WorkspaceOperationKind::Commit | WorkspaceOperationKind::Checkpoint if present => { - if let Some(hand) = request.hand() { - self.kill_exact_workspace_hand(hand, &operation.binding) - .await?; - } let published = published.ok_or_else(|| { MoaError::StorageError( "confirmed E2B checkpoint reconciliation lost its manifest".to_string(), @@ -531,7 +527,11 @@ impl SandboxStorageProvider for E2BHandProvider { confirmed_disposition: Some(WorkspaceConfirmedDisposition::ResourcePresent), storage: Some(storage.clone()), checkpoint_publication: Some(checkpoint_publication), - post_commit_state: Some(WorkspacePostCommitState::ComputeDestroyed), + // Reconciliation only proves the bytes; it never releases + // compute. The caller may still be using this hand, and a + // release-commit finishes its destroy as a separate exact + // step after the durable publication CAS succeeds. + post_commit_state: Some(WorkspacePostCommitState::AttachmentRetained), }); } WorkspaceOperationKind::Delete if !present => { diff --git a/crates/moa-hands/src/adapters/local/mod.rs b/crates/moa-hands/src/adapters/local/mod.rs index d75cd3077..a5f832b9c 100644 --- a/crates/moa-hands/src/adapters/local/mod.rs +++ b/crates/moa-hands/src/adapters/local/mod.rs @@ -1515,26 +1515,19 @@ impl HandProvider for LocalHandProvider { } } - async fn pause(&self, handle: &HandHandle) -> Result<()> { - match handle { - HandHandle::Docker { container_id } => { - let output = Command::new("docker") - .args(["pause", container_id]) - .output() - .await?; - if !output.status.success() { - return Err(MoaError::ProviderError(format!( - "failed to pause docker sandbox: {}", - String::from_utf8_lossy(&output.stderr).trim() - ))); - } - Ok(()) - } - HandHandle::Local { .. } => Ok(()), - _ => Err(MoaError::Unsupported( - "non-local hand handle passed to LocalHandProvider".to_string(), - )), - } + /// Refuses compute suspension: neither local backend can actually release compute. + /// + /// `docker pause` is a cgroup freeze — it stops CPU scheduling but keeps the + /// container's memory and disk allocated, so it releases nothing on any + /// billing model. `docker stop` would release them, but containers are + /// created with `--rm`, which makes a stop a destroy. A local sandbox + /// directory has no compute to release at all. + async fn suspend(&self, _handle: &HandHandle) -> Result<()> { + Err(MoaError::Unsupported( + "local sandboxes cannot release compute: `docker pause` is a cgroup freeze that keeps \ + memory and disk allocated, and `--rm` containers are removed by `docker stop`" + .to_string(), + )) } async fn resume(&self, handle: &HandHandle) -> Result<()> { diff --git a/crates/moa-hands/src/adapters/local/tests.rs b/crates/moa-hands/src/adapters/local/tests.rs index a69a59c1b..3ed7a36e0 100644 --- a/crates/moa-hands/src/adapters/local/tests.rs +++ b/crates/moa-hands/src/adapters/local/tests.rs @@ -3,10 +3,11 @@ use moa_core::{ error::MoaError, traits::{HandProvider, SandboxStorageProvider}, - types::hands::{HandHandle, HandSpec, SandboxProfile, SandboxTier}, + types::hands::{HandHandle, HandSpec, HandStatus, SandboxProfile, SandboxTier}, types::identifiers::{ProviderAccountId, WorkspaceCheckpointId, WorkspaceOperationId}, types::sandbox_workspace::{ ProviderInventoryOwner, WorkspaceCheckpointPublishRequest, WorkspaceOperationKind, + WorkspaceOperationOutcome, WorkspacePostCommitState, WorkspaceReconcileRequest, WorkspaceRevisionRef, WorkspaceStorageOperation, }, }; @@ -235,3 +236,79 @@ async fn local_commit_rejects_a_parent_at_generation_zero_before_storage_work() assert!(matches!(error, MoaError::ValidationError(message) if message.contains("parent"))); } + +// Pins: the local adapter answers an ambiguous non-yield commit exactly as E2B +// and Daytona do — published bytes confirmed, compute untouched — so no future +// adapter can turn reconciliation into a compute-release path. +#[tokio::test] +async fn local_commit_reconciliation_retains_the_hand_it_reconciles() { + let dir = tempdir().expect("create tempdir"); + let store = std::sync::Arc::new( + crate::core::sandbox_workspace::checkpoint::store::CheckpointObjectStore::new( + std::sync::Arc::new(object_store::memory::InMemory::new()), + std::sync::Arc::new(moa_crypto::LocalKmsProvider::new()), + "local-reconcile-retention", + crate::core::sandbox_workspace::checkpoint::archive::ArchiveLimits::default(), + crate::core::sandbox_workspace::checkpoint::store::ObservedCheckpointBucketVersioning::Unversioned, + ) + .expect("offline checkpoint store should construct"), + ); + let provider = LocalHandProvider::new_with_docker_detection(dir.path(), false) + .await + .expect("create local hand provider") + .with_checkpoint_store(store); + let mut spec = hand_spec(SandboxTier::Local); + spec.workspace.current_revision = None; + let binding = spec.workspace.clone(); + let handle = provider + .provision(spec) + .await + .expect("provision local sandbox"); + + let operation = WorkspaceStorageOperation { + operation_id: WorkspaceOperationId::new(), + kind: WorkspaceOperationKind::Commit, + binding, + deadline: chrono::Utc::now() + chrono::Duration::minutes(1), + request_hash: "c".repeat(64), + }; + let published = provider + .publish_workspace_checkpoint(WorkspaceCheckpointPublishRequest { + operation: operation.clone(), + hand: handle.clone(), + parent_revision: None, + release_compute: false, + }) + .await + .expect("publish a non-yield commit checkpoint"); + let storage = published + .storage + .expect("a confirmed commit publishes portable checkpoint storage"); + + let reconciled = provider + .reconcile_workspace_operation( + WorkspaceReconcileRequest::new(operation, Some(handle.clone()), Some(storage)) + .expect("exact-resource reconciliation request validates"), + ) + .await + .expect("reconcile the ambiguous commit against published bytes"); + + assert_eq!(reconciled.outcome, WorkspaceOperationOutcome::Confirmed); + assert_eq!( + reconciled.post_commit_state, + Some(WorkspacePostCommitState::AttachmentRetained), + "reconciliation must never report a compute release it was not asked for" + ); + assert_eq!( + provider + .status(&handle) + .await + .expect("inspect reconciled hand"), + HandStatus::Running, + "reconciling a commit must leave the caller's compute alive" + ); + provider + .destroy(&handle) + .await + .expect("destroy local sandbox"); +} diff --git a/crates/moa-hands/src/adapters/local/workspace.rs b/crates/moa-hands/src/adapters/local/workspace.rs index 25b4112ff..467c99587 100644 --- a/crates/moa-hands/src/adapters/local/workspace.rs +++ b/crates/moa-hands/src/adapters/local/workspace.rs @@ -354,6 +354,8 @@ impl SandboxStorageProvider for LocalHandProvider { manifest_digest: published.manifest_sha256, logical_bytes: published.logical_bytes, }), + // Reconciliation only proves the bytes; it never releases + // compute. See `SandboxStorageProvider::reconcile_workspace_operation`. post_commit_state: Some(WorkspacePostCommitState::AttachmentRetained), }), (WorkspaceOperationKind::Delete, None) => Ok(WorkspaceStorageOperationResult { diff --git a/crates/moa-hands/src/core/dispatch.rs b/crates/moa-hands/src/core/dispatch.rs index 735f4fd27..078f08005 100644 --- a/crates/moa-hands/src/core/dispatch.rs +++ b/crates/moa-hands/src/core/dispatch.rs @@ -13,7 +13,7 @@ use moa_core::{ types::completion::ToolInvocation, types::hands::HandHandle, types::hands::HandStatus, - types::identifiers::{ExecutionRunScopeId, ToolCallId}, + types::identifiers::{ExecutionRunScopeId, ExecutionTaskScopeId, ToolCallId}, types::resource::DeadlineGuard, types::sandbox_workspace::{ExecutionHandReleaseOwner, SandboxWorkspaceScope, WorkspaceEffect}, types::security::ToolCapabilityId, @@ -112,6 +112,34 @@ pub struct JournaledWorkspaceCommit<'a> { pub scope: ToolCallScope<'a>, } +/// One idempotent request to publish an attempt's checkpoint while keeping its compute. +/// +/// This is the continuation-boundary counterpart to +/// [`ExecutionHandReleaseRequest`]. It advances the durable checkpoint head so the +/// next slice can always restore, and deliberately performs no provider teardown: +/// a plain model/tool boundary is not a wait, so destroying and re-provisioning the +/// sandbox between two adjacent slices is pure loss. `retention_deadline_at` is the +/// bound that keeps the retained hand from starving fleet admission. +#[derive(Clone, Copy)] +pub struct ExecutionHandRetentionRequest<'a> { + /// Session whose tenant owns the execution workspace and hand lease. + pub session: &'a SessionMeta, + /// Verified durable execution run. + pub run_id: ExecutionRunScopeId, + /// Verified durable execution task owning the workspace scope. + pub task_id: ExecutionTaskScopeId, + /// Logical task generation the retained compute belongs to. + pub logical_generation: u64, + /// Exact bounded attempt generation publishing this continuation checkpoint. + pub attempt_generation: u64, + /// Absolute instant after which the reaper may destroy the retained sandbox. + /// + /// Never extends the lease's existing idle or hard deadline; it only shortens. + pub retention_deadline_at: chrono::DateTime, + /// Fresh bounded budget for checkpoint publication. + pub scope: ToolCallScope<'a>, +} + /// One idempotent request to release an execution attempt's exact sandbox hand. #[derive(Clone, Copy)] pub struct ExecutionHandReleaseRequest<'a> { diff --git a/crates/moa-hands/src/core/lifecycle.rs b/crates/moa-hands/src/core/lifecycle.rs index f69f4277a..650d642b5 100644 --- a/crates/moa-hands/src/core/lifecycle.rs +++ b/crates/moa-hands/src/core/lifecycle.rs @@ -121,7 +121,8 @@ struct DurableHandProvisionContext<'a> { call_scope: ToolCallScope<'a>, } -fn active_hand_capacity_request( +/// Builds the exact fenced active-compute capacity identity for one lease. +pub(in crate::core) fn active_hand_capacity_request( workspace_binding: &WorkspaceBinding, lease: &HandLease, ) -> Result { @@ -1065,10 +1066,39 @@ impl ToolRouter { if !lease_expired(&lease) && lease_matches_policy(&lease, policy) => { match self - .resume_durable_lease(provider, &lease, &key, call_scope) + .resume_durable_lease( + provider, + &lease, + workspace_binding, + &key, + call_scope, + ) .await { - Ok(handle) => { + // A suspended sandbox has no reserved claim on the slot it + // gave back. Losing that race is terminal for this lease + // rather than something to retry in place: the stopped + // sandbox is handed to the reaper, and admission is refused + // now instead of spinning while the fleet stays full. Safe + // because the continuation boundary published its checkpoint + // before suspending, so a later slice restores the same head + // into fresh compute. + Ok(None) => { + call_scope.admit()?; + let _ = lease_store + .transition_status( + session.tenant_id, + &lease, + HandLeaseStatus::Stale, + ) + .await?; + return Err(MoaError::ValidationError(format!( + "suspended sandbox for session {} provider {provider} lost its \ + active-hands capacity slot to a saturated fleet", + session.id + ))); + } + Ok(Some(handle)) => { call_scope.admit()?; if lease_store .renew_active(HandLeaseRenewRequest { @@ -1265,6 +1295,11 @@ impl ToolRouter { { HandStatus::Running => {} HandStatus::Provisioning => return Ok(None), + // No capacity re-admission here, unlike the active-lease resume path. + // This branch recovers a lease still in `provisioning`, whose + // `ActiveHands` reservation was taken before the provider create and + // is therefore still charged. Only a continuation-boundary suspension + // gives that charge back, and it leaves the lease `active`. HandStatus::Paused | HandStatus::Stopped => { call_scope.admit()?; provider_impl.resume(&handle).await?; @@ -1446,9 +1481,23 @@ impl ToolRouter { && current.attachment == lease.attachment }); if already_active { - if let Some(capacity) = self.hands.workspace_capacity.as_ref() { - let request = active_hand_capacity_request(workspace_binding, lease)?; - let _ = capacity.commit_active_hand(&request).await?; + // The activation replay may already have committed the exact + // charge, but it must never proceed while that charge is still + // pending: the durable release predicates only settle a + // committed reservation, so a lost fence here would strand the + // run with dead compute. + if let Some(capacity) = self.hands.workspace_capacity.as_ref() + && !capacity + .ensure_active_hand_committed(&active_hand_capacity_request( + workspace_binding, + lease, + )?) + .await? + { + return Err(MoaError::StorageError(format!( + "replayed hand activation lost its exact capacity fence for session {} provider {provider}", + session.id + ))); } let active = ActiveHand { handle: handle.clone(), @@ -1649,13 +1698,21 @@ impl ToolRouter { } } + /// Reattaches one live durable lease, resuming it when the sandbox is stopped. + /// + /// Returns `Ok(None)` when a suspended sandbox could not re-win the + /// active-compute slot it gave back at its continuation boundary. That is a + /// distinct outcome from an error: the lease is not retryable in place and + /// the caller must terminalize it, but nothing is lost, because the boundary + /// published a portable checkpoint before suspending. async fn resume_durable_lease( &self, provider: &str, lease: &HandLease, + workspace_binding: &WorkspaceBinding, key: &HandProviderCacheKey, call_scope: ToolCallScope<'_>, - ) -> Result { + ) -> Result> { let lease_handle = lease.handle.as_ref().ok_or_else(|| { MoaError::StorageError(format!( "active hand lease for session {} provider {provider} is missing a handle", @@ -1673,6 +1730,16 @@ impl ToolRouter { match status { HandStatus::Running | HandStatus::Provisioning => {} HandStatus::Paused | HandStatus::Stopped => { + // A continuation boundary that suspends a sandbox releases its + // `ActiveHands` charge so a runnable task can use the slot, which + // makes resuming a fresh admission decision rather than a free + // reattach. Compute must never restart before the slot is re-won. + if !self + .readmit_suspended_hand(lease, workspace_binding) + .await? + { + return Ok(None); + } call_scope.admit()?; provider_impl.resume(&handle).await?; } @@ -1690,7 +1757,26 @@ impl ToolRouter { generation: Some(lease.generation), }, ); - Ok(handle) + Ok(Some(handle)) + } + + /// Re-wins the active-compute admission slot a suspended hand gave back. + /// + /// Reports `false` for a saturated fleet, which is an ordinary outcome and not + /// an error: the caller drops the warm sandbox and a later slice restores the + /// published checkpoint into fresh compute. That is also the honest semantics + /// — a full fleet does not give a warm slot back for free. Deployments with no + /// capacity repository charge nothing and always re-admit. + async fn readmit_suspended_hand( + &self, + lease: &HandLease, + workspace_binding: &WorkspaceBinding, + ) -> Result { + let Some(capacity) = self.hands.workspace_capacity.as_ref() else { + return Ok(true); + }; + let request = active_hand_capacity_request(workspace_binding, lease)?; + capacity.reacquire_suspended_active_hand(&request).await } async fn wait_for_provisioning( diff --git a/crates/moa-hands/src/core/lifecycle/tests.rs b/crates/moa-hands/src/core/lifecycle/tests.rs index 4ec58bf8e..d50d8775a 100644 --- a/crates/moa-hands/src/core/lifecycle/tests.rs +++ b/crates/moa-hands/src/core/lifecycle/tests.rs @@ -509,10 +509,6 @@ impl HandProvider for CountingProvider { Ok(HandStatus::Running) } - async fn pause(&self, _handle: &HandHandle) -> Result<()> { - Ok(()) - } - async fn resume(&self, _handle: &HandHandle) -> Result<()> { Ok(()) } diff --git a/crates/moa-hands/src/core/mod.rs b/crates/moa-hands/src/core/mod.rs index 0c3135ce8..7e45e4902 100644 --- a/crates/moa-hands/src/core/mod.rs +++ b/crates/moa-hands/src/core/mod.rs @@ -49,7 +49,7 @@ use crate::adapters::local::LocalHandProvider; pub use dispatch::{ AuthorizedToolCall, DeferredWorkspaceToolOutput, ExecutionHandReleaseRequest, - JournaledWorkspaceCommit, PendingConnectorToolOutput, + ExecutionHandRetentionRequest, JournaledWorkspaceCommit, PendingConnectorToolOutput, }; use leases::{HAND_LEASE_SESSION_PAGE_SIZE, HandLeaseStore}; pub use maintenance_provider_inventory::SandboxProviderInventory; diff --git a/crates/moa-hands/src/core/profile.rs b/crates/moa-hands/src/core/profile.rs index ad72068e7..3a1e49d0c 100644 --- a/crates/moa-hands/src/core/profile.rs +++ b/crates/moa-hands/src/core/profile.rs @@ -344,10 +344,6 @@ mod tests { Ok(moa_core::types::hands::HandStatus::Running) } - async fn pause(&self, _handle: &moa_core::types::hands::HandHandle) -> Result<()> { - Ok(()) - } - async fn resume(&self, _handle: &moa_core::types::hands::HandHandle) -> Result<()> { Ok(()) } diff --git a/crates/moa-hands/src/core/reaper.rs b/crates/moa-hands/src/core/reaper.rs index 0f08969d8..ddfc1ced8 100644 --- a/crates/moa-hands/src/core/reaper.rs +++ b/crates/moa-hands/src/core/reaper.rs @@ -1229,10 +1229,6 @@ mod tests { Ok(moa_core::types::hands::HandStatus::Running) } - async fn pause(&self, _handle: &HandHandle) -> Result<()> { - Ok(()) - } - async fn resume(&self, _handle: &HandHandle) -> Result<()> { Ok(()) } diff --git a/crates/moa-hands/src/core/recovery/tests.rs b/crates/moa-hands/src/core/recovery/tests.rs index d68e73dee..6b1b6c2ab 100644 --- a/crates/moa-hands/src/core/recovery/tests.rs +++ b/crates/moa-hands/src/core/recovery/tests.rs @@ -169,10 +169,6 @@ impl HandProvider for MockHandProvider { Ok(HandStatus::Running) } - async fn pause(&self, _handle: &HandHandle) -> Result<()> { - Ok(()) - } - async fn resume(&self, _handle: &HandHandle) -> Result<()> { Ok(()) } diff --git a/crates/moa-hands/src/core/sandbox_workspace/capacity.rs b/crates/moa-hands/src/core/sandbox_workspace/capacity.rs index eb0698e23..197d2c32b 100644 --- a/crates/moa-hands/src/core/sandbox_workspace/capacity.rs +++ b/crates/moa-hands/src/core/sandbox_workspace/capacity.rs @@ -20,7 +20,9 @@ use serde_json::Value; use sqlx::{PgPool, Row, types::Json}; use uuid::Uuid; -use crate::core::leases::map_sqlx_error; +use moa_observability::SandboxWorkspaceQuotaDecision; + +use crate::core::{leases::map_sqlx_error, telemetry::record_workspace_quota_decision}; /// One positive capacity quantity requested by an operation. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -457,6 +459,144 @@ impl PostgresWorkspaceCapacityRepository { Ok(changed) } + /// Commits active-hand capacity, treating an earlier identical commit as done. + /// + /// Activation replays legitimately find the reservation already `committed`, + /// which [`Self::commit_active_hand`] reports as "no row changed". Only a + /// lost writer-epoch, instance-generation, or lease fence leaves the charge + /// uncommitted, and the caller must not proceed on that. + pub async fn ensure_active_hand_committed( + &self, + request: &ActiveHandCapacityRequest, + ) -> Result { + validate_active_hand_request(request)?; + let mut conn = self.begin().await?; + let committed = if commit_active_hand_in_transaction(conn.as_mut(), request).await? { + true + } else { + active_hand_reservation_state(conn.as_mut(), request) + .await? + .as_deref() + == Some("committed") + }; + conn.commit().await?; + Ok(committed) + } + + /// Releases active-hand capacity for a hand suspended at a continuation boundary. + /// + /// The lease deliberately stays `active` and keeps its handle: the suspended + /// sandbox still owns its filesystem so the next slice can reattach. Only the + /// compute charge goes back to the fleet, and + /// [`Self::reacquire_suspended_active_hand`] must win it again before the + /// sandbox resumes. Returns `false` when the exact lease or workspace fence + /// no longer holds, in which case the caller must not treat the compute as + /// released. + pub async fn release_suspended_active_hand( + &self, + request: &ActiveHandCapacityRequest, + ) -> Result { + validate_active_hand_request(request)?; + let mut conn = self.begin().await?; + if !active_hand_lease_is_live(conn.as_mut(), request).await? { + conn.rollback().await?; + return Ok(false); + } + let changed = release_active_hand_row(conn.as_mut(), request).await?; + conn.commit().await?; + Ok(changed) + } + + /// Re-admits a suspended hand's active-compute charge before it resumes. + /// + /// Returns `false` when the fleet or tenant has no room. That is an ordinary + /// saturation outcome rather than an error: the caller drops the warm sandbox + /// and lets a later slice restore the published checkpoint into fresh + /// compute. A reservation that is still charged (a replay, or a suspend whose + /// capacity release never committed) is reported as `true` without + /// double-charging. + pub async fn reacquire_suspended_active_hand( + &self, + request: &ActiveHandCapacityRequest, + ) -> Result { + validate_active_hand_request(request)?; + let mut conn = self.begin().await?; + lock_capacity_scope_values( + conn.as_mut(), + request.tenant_id, + request.provider_account_id, + ) + .await?; + if !active_hand_lease_is_live(conn.as_mut(), request).await? { + conn.rollback().await?; + return Err(MoaError::StorageError( + "suspended-hand re-admission lost its exact lease or workspace generation fence" + .to_string(), + )); + } + let state = active_hand_reservation_state(conn.as_mut(), request).await?; + match state.as_deref() { + Some("pending" | "committed" | "reconciling") => { + conn.commit().await?; + return Ok(true); + } + Some("released") => {} + _ => { + conn.rollback().await?; + return Err(MoaError::StorageError( + "suspended hand has no active-hands capacity reservation to re-admit" + .to_string(), + )); + } + } + let quantities = BTreeMap::from([(WorkspaceCapacityDimension::ActiveHands, 1_i64)]); + if let Some(shortfall) = capacity_shortfall( + conn.as_mut(), + request.tenant_id, + request.provider_account_id, + request.provider_account_generation, + &quantities, + ) + .await? + { + conn.rollback().await?; + tracing::info!( + shortfall, + "suspended sandbox lost its active-hands slot to a saturated fleet" + ); + return Ok(false); + } + let readmitted = sqlx::query( + r#" + UPDATE moa.sandbox_capacity_reservations + SET reservation_state = 'committed', expires_at = NULL, updated_at = now() + WHERE tenant_id = $1 AND workspace_id = $2 + AND provider_account_id = $3 AND provider_account_generation = $4 + AND hand_provisioning_operation_id = $5 + AND hand_lease_generation = $6 + AND expected_writer_epoch = $7 + AND expected_instance_generation = $8 + AND resource_dimension = 'active_hands' + AND reservation_state = 'released' + "#, + ) + .bind(request.tenant_id) + .bind(request.workspace_id) + .bind(request.provider_account_id) + .bind(request.provider_account_generation) + .bind(request.provisioning_operation_id) + .bind(request.hand_lease_generation) + .bind(request.expected_writer_epoch) + .bind(request.expected_instance_generation) + .execute(conn.as_mut()) + .await + .map_err(map_sqlx_error)? + .rows_affected() + == 1; + conn.commit().await?; + Ok(readmitted) + } + /// Releases active-hand capacity after exact durable reaper ownership is established. pub async fn release_active_hand_to_reaper( &self, @@ -881,6 +1021,37 @@ async fn load_operation_reservations( .collect() } +/// Reads the persisted state of one exact active-hand charge, when it exists. +async fn active_hand_reservation_state( + conn: &mut sqlx::PgConnection, + request: &ActiveHandCapacityRequest, +) -> Result> { + sqlx::query_scalar( + r#" + SELECT reservation_state + FROM moa.sandbox_capacity_reservations + WHERE tenant_id = $1 AND workspace_id = $2 + AND provider_account_id = $3 AND provider_account_generation = $4 + AND hand_provisioning_operation_id = $5 + AND hand_lease_generation = $6 + AND expected_writer_epoch = $7 + AND expected_instance_generation = $8 + AND resource_dimension = 'active_hands' + "#, + ) + .bind(request.tenant_id) + .bind(request.workspace_id) + .bind(request.provider_account_id) + .bind(request.provider_account_generation) + .bind(request.provisioning_operation_id) + .bind(request.hand_lease_generation) + .bind(request.expected_writer_epoch) + .bind(request.expected_instance_generation) + .fetch_optional(conn) + .await + .map_err(map_sqlx_error) +} + async fn load_active_hand_reservation( conn: &mut sqlx::PgConnection, request: &ActiveHandCapacityRequest, @@ -984,6 +1155,33 @@ async fn enforce_capacity( provider_account_generation: i64, quantities: &BTreeMap, ) -> Result<()> { + match capacity_shortfall( + conn, + tenant_id, + provider_account_id, + provider_account_generation, + quantities, + ) + .await? + { + None => Ok(()), + Some(message) => Err(MoaError::ValidationError(message)), + } +} + +/// Reports the first exceeded limit instead of raising, for callers that treat +/// saturation as an ordinary decision rather than an admission fault. +/// +/// Returns `None` when every requested dimension fits under both the tenant and +/// the provider-account ceiling. Genuine faults — a missing provider-account +/// generation, malformed limits, arithmetic overflow — are still errors. +async fn capacity_shortfall( + conn: &mut sqlx::PgConnection, + tenant_id: TenantId, + provider_account_id: ProviderAccountId, + provider_account_generation: i64, + quantities: &BTreeMap, +) -> Result> { let provider_limits = sqlx::query( r#" SELECT configured_limits @@ -1025,22 +1223,26 @@ async fn enforce_capacity( let tenant_used = reserved_total(conn, Some(tenant_id), None, *dimension).await?; let provider_used = reserved_total(conn, None, Some(provider_account_id), *dimension).await?; - enforce_limit( + if let Some(shortfall) = limit_shortfall( "tenant", *dimension, tenant_used, *quantity, tenant_limits.get(dimension).copied(), - )?; - enforce_limit( + )? { + return Ok(Some(shortfall)); + } + if let Some(shortfall) = limit_shortfall( "provider account", *dimension, provider_used, *quantity, provider_limits.get(dimension).copied(), - )?; + )? { + return Ok(Some(shortfall)); + } } - Ok(()) + Ok(None) } /// Commits one exact active-hand reservation inside an existing transaction. @@ -1090,6 +1292,11 @@ pub async fn commit_active_hand_in_transaction( } /// Releases the active-compute owner held by one exact live durable reaper claim. +/// +/// `released` is accepted as an input state and re-asserted as a no-op update: +/// a hand suspended at a continuation boundary already gave its compute charge +/// back while keeping its lease, so requiring a still-charged reservation here +/// would make every suspended hand's eventual destroy roll back forever. pub(crate) async fn release_active_hand_for_reaper_in_transaction( conn: &mut sqlx::PgConnection, tenant_id: TenantId, @@ -1120,7 +1327,7 @@ pub(crate) async fn release_active_hand_for_reaper_in_transaction( AND reservation.expected_writer_epoch = lease.workspace_writer_epoch AND reservation.expected_instance_generation = lease.workspace_instance_generation AND reservation.resource_dimension = 'active_hands' - AND reservation.reservation_state IN ('pending', 'committed', 'reconciling') + AND reservation.reservation_state IN ('pending', 'committed', 'reconciling', 'released') "#, ) .bind(tenant_id) @@ -1134,6 +1341,48 @@ pub(crate) async fn release_active_hand_for_reaper_in_transaction( == 1) } +/// Locks and verifies that one exact active lease still owns live attached compute. +/// +/// Used by the suspend/reattach pair, which — unlike provisioning and reaping — +/// moves capacity while the lease stays `active` and keeps its handle. +async fn active_hand_lease_is_live( + conn: &mut sqlx::PgConnection, + request: &ActiveHandCapacityRequest, +) -> Result { + Ok(sqlx::query_scalar::<_, bool>( + r#" + SELECT TRUE + FROM moa.hand_leases AS lease + JOIN moa.sandbox_workspaces AS workspace + ON workspace.tenant_id = lease.tenant_id + AND workspace.workspace_id = lease.workspace_id + WHERE lease.tenant_id = $1 + AND lease.provisioning_operation_id = $2 + AND lease.generation = $3 + AND lease.status = 'active' + AND lease.handle IS NOT NULL + AND lease.workspace_id = $4 + AND lease.workspace_writer_epoch = $5 + AND lease.workspace_instance_generation = $6 + AND workspace.provider_account_id = $7 + AND workspace.provider_account_generation = $8 + FOR UPDATE OF lease + "#, + ) + .bind(request.tenant_id) + .bind(request.provisioning_operation_id) + .bind(request.hand_lease_generation) + .bind(request.workspace_id) + .bind(request.expected_writer_epoch) + .bind(request.expected_instance_generation) + .bind(request.provider_account_id) + .bind(request.provider_account_generation) + .fetch_optional(conn) + .await + .map_err(map_sqlx_error)? + .unwrap_or(false)) +} + async fn release_active_hand_row( conn: &mut sqlx::PgConnection, request: &ActiveHandCapacityRequest, @@ -1212,13 +1461,13 @@ async fn reserved_total( .map_err(map_sqlx_error) } -fn enforce_limit( +fn limit_shortfall( scope: &str, dimension: WorkspaceCapacityDimension, used: i64, quantity: i64, limit: Option, -) -> Result<()> { +) -> Result> { let next = used.checked_add(quantity).ok_or_else(|| { MoaError::ValidationError(format!( "{scope} {} capacity arithmetic overflow", @@ -1226,13 +1475,18 @@ fn enforce_limit( )) })?; if limit.is_some_and(|limit| next > limit) { - return Err(MoaError::ValidationError(format!( + // Both outcomes are recorded so the admitted/rejected ratio is meaningful; a + // counter incremented only on rejection cannot distinguish a saturated fleet + // from an idle one. + record_workspace_quota_decision(dimension, SandboxWorkspaceQuotaDecision::Rejected); + return Ok(Some(format!( "{scope} {} capacity exceeded: {used} + {quantity} > {}", dimension.as_str(), limit.unwrap_or_default() ))); } - Ok(()) + record_workspace_quota_decision(dimension, SandboxWorkspaceQuotaDecision::Admitted); + Ok(None) } #[cfg(test)] diff --git a/crates/moa-hands/src/core/sandbox_workspace/lifecycle.rs b/crates/moa-hands/src/core/sandbox_workspace/lifecycle.rs index 5a6450815..1fe121195 100644 --- a/crates/moa-hands/src/core/sandbox_workspace/lifecycle.rs +++ b/crates/moa-hands/src/core/sandbox_workspace/lifecycle.rs @@ -10,12 +10,13 @@ use moa_core::{ WorkspaceOperationId, }, sandbox_workspace::{ - ExecutionHandReleaseOwner, ExecutionHandReleaseReceipt, ProviderStorageKind, - ProviderStorageRef, SandboxWorkspaceScope, SandboxWorkspaceState, - WorkspaceAttachRequest, WorkspaceBinding, WorkspaceCheckpointPublishRequest, - WorkspaceCheckpointState, WorkspaceConfirmedDisposition, WorkspaceOperationKind, - WorkspaceOperationOutcome, WorkspacePostCommitState, WorkspaceReconcileRequest, - WorkspaceRestoreRequest, WorkspaceStorageOperation, WorkspaceStoragePrepareRequest, + ExecutionHandContinuationDisposition, ExecutionHandReleaseOwner, + ExecutionHandReleaseReceipt, ProviderStorageKind, ProviderStorageRef, + SandboxWorkspaceScope, SandboxWorkspaceState, WorkspaceAttachRequest, WorkspaceBinding, + WorkspaceCheckpointPublishRequest, WorkspaceCheckpointState, + WorkspaceConfirmedDisposition, WorkspaceOperationKind, WorkspaceOperationOutcome, + WorkspacePostCommitState, WorkspaceReconcileRequest, WorkspaceRestoreRequest, + WorkspaceStorageOperation, WorkspaceStoragePrepareRequest, }, session::SessionMeta, }, @@ -35,13 +36,23 @@ use super::{ }, operations::WorkspaceOperationIntent, }; +use moa_observability::{ + SandboxWorkspaceCheckpointOperation, SandboxWorkspaceLifecycleOperation, + SandboxWorkspaceMetricResult, +}; + use crate::core::{ - ActiveHand, ExecutionHandReleaseRequest, HandProviderCacheKey, HandRoute, - InstalledManifestMarker, JournaledWorkspaceCommit, ToolCallScope, ToolExecution, ToolRouter, - TrustedSandboxManifest, + ActiveHand, ExecutionHandReleaseRequest, ExecutionHandRetentionRequest, HandProviderCacheKey, + HandRoute, InstalledManifestMarker, JournaledWorkspaceCommit, ToolCallScope, ToolExecution, + ToolRouter, TrustedSandboxManifest, leases::{HandLease, HandLeaseStatus, HandLeaseWorkspaceAttachment}, lifecycle::{ - manifest_scope_key, session_provider_key, workspace_binding_for_hand, workspace_lease_scope, + active_hand_capacity_request, manifest_scope_key, session_provider_key, + workspace_binding_for_hand, workspace_lease_scope, + }, + telemetry::{ + record_workspace_checkpoint, record_workspace_lifecycle, record_workspace_release, + record_workspace_restore, }, }; @@ -783,6 +794,7 @@ impl ToolRouter { workspace: &SandboxWorkspace, call_scope: ToolCallScope<'_>, ) -> Result<()> { + let prepare_started_at = std::time::Instant::now(); let operations = self.hands.workspace_operations.as_ref().ok_or_else(|| { MoaError::StorageError("workspace operation repository missing".to_string()) })?; @@ -846,25 +858,47 @@ impl ToolRouter { }, }) .await?; + // Every arm records an outcome, so the lifecycle counter carries the real + // success/ambiguous ratio rather than only the happy path. match (result.outcome, result.confirmed_disposition) { (WorkspaceOperationOutcome::Confirmed, Some(disposition)) => { operations .confirm_disposition(workspace.tenant_id, operation_id, disposition) .await?; + record_workspace_lifecycle( + &workspace.provider, + SandboxWorkspaceLifecycleOperation::Create, + SandboxWorkspaceMetricResult::Succeeded, + prepare_started_at.elapsed(), + ); Ok(()) } (WorkspaceOperationOutcome::Unknown, None) => { operations .mark_unknown(workspace.tenant_id, operation_id) .await?; + record_workspace_lifecycle( + &workspace.provider, + SandboxWorkspaceLifecycleOperation::Create, + SandboxWorkspaceMetricResult::Ambiguous, + prepare_started_at.elapsed(), + ); Err(MoaError::ExternalEffectUnknownOutcome { operation_id: operation_id.to_string(), }) } - _ => Err(MoaError::ProviderError( - "workspace storage provider returned an inconsistent preparation result" - .to_string(), - )), + _ => { + record_workspace_lifecycle( + &workspace.provider, + SandboxWorkspaceLifecycleOperation::Create, + SandboxWorkspaceMetricResult::Failed, + prepare_started_at.elapsed(), + ); + Err(MoaError::ProviderError( + "workspace storage provider returned an inconsistent preparation result" + .to_string(), + )) + } } } @@ -892,6 +926,7 @@ impl ToolRouter { claim.provider )) })?; + let hydration_started_at = std::time::Instant::now(); let kind = if binding.current_revision.is_some() { WorkspaceOperationKind::Restore } else { @@ -1036,6 +1071,19 @@ impl ToolRouter { "workspace hydration lost its durable operation fence".to_string(), )); } + // Only a confirmed restore counts: an ambiguous or failed provider result + // leaves no verified checkpoint in fresh compute, so counting it here + // would overstate successful restores. + if kind == WorkspaceOperationKind::Restore { + record_workspace_restore(&claim.provider); + record_workspace_checkpoint( + &claim.provider, + SandboxWorkspaceCheckpointOperation::Restore, + SandboxWorkspaceMetricResult::Succeeded, + 0, + hydration_started_at.elapsed(), + ); + } Ok(()) } (WorkspaceOperationOutcome::Unknown, None) => { @@ -1293,6 +1341,263 @@ impl ToolRouter { .await } + /// Publishes one execution-task continuation checkpoint and keeps what it can. + /// + /// A plain model/tool boundary is not a wait: the next slice is enqueued for + /// immediate re-admission, so destroying the sandbox here and restoring it + /// milliseconds later is pure loss — object-store read, decrypt, extract, and a + /// per-file upload round trip on both cloud providers. This publishes the exact + /// same durable checkpoint the release path publishes, so the portable recovery + /// authority advances on every boundary, and then chooses how to keep the + /// sandbox based on what the provider can actually do: + /// + /// * A provider with real compute suspension stops the sandbox **in this + /// call** and hands its `ActiveHands` slot back to the fleet. Release timing + /// is deterministic rather than reaper-lagged, and an idle sandbox stops + /// costing compute and stops competing with runnable work for admission. + /// * A provider without it keeps the hand hot on a deliberately short + /// reaper-owned deadline. That bet only pays off when the next slice arrives + /// fast, so a longer window would only extend the loss. + /// + /// Both paths are safe for the same reason: the checkpoint commits *before* + /// any deadline is armed or any compute is stopped, so losing the warm + /// sandbox is a pure cache miss — the next slice restores from the same head. + pub async fn checkpoint_execution_hand_retaining_compute( + &self, + request: ExecutionHandRetentionRequest<'_>, + ) -> Result { + if request.attempt_generation == 0 || request.logical_generation == 0 { + return Err(MoaError::ValidationError( + "execution task attempt and logical generations must be positive".to_string(), + )); + } + let Some(repository) = self.hands.workspace_repository.as_ref() else { + return Ok(ExecutionHandContinuationDisposition::NoComputeOwned); + }; + let workspace_scope = SandboxWorkspaceScope::ExecutionTask { + run_id: request.run_id, + task_id: request.task_id, + }; + // An attempt that never provisioned a durable workspace or whose lease is no + // longer live has nothing to publish and nothing to keep. Its committed head + // is already the recovery authority, so this is a no-op rather than an error. + let Some(workspace) = repository + .get_by_scope(request.session.tenant_id, &workspace_scope) + .await? + else { + return Ok(ExecutionHandContinuationDisposition::NoComputeOwned); + }; + let lease_scope = workspace_lease_scope(&workspace_scope); + let lease_store = self.hands.hand_leases.as_ref().ok_or_else(|| { + MoaError::StorageError("durable hand lease store missing".to_string()) + })?; + let Some(lease) = lease_store + .get( + request.session.tenant_id, + request.session.id, + &lease_scope, + &workspace.provider, + ) + .await? + else { + return Ok(ExecutionHandContinuationDisposition::NoComputeOwned); + }; + if lease.status != HandLeaseStatus::Active { + return Ok(ExecutionHandContinuationDisposition::NoComputeOwned); + } + let Some(hand) = lease.handle.as_ref().map(|handle| handle.handle.clone()) else { + return Ok(ExecutionHandContinuationDisposition::NoComputeOwned); + }; + + let continuation_key = format!( + "execution-task-continuation-v1:{}:{}:{}", + request.run_id, request.task_id, request.attempt_generation + ); + let tool_call_id = ToolCallId(Uuid::new_v5( + &workspace.workspace_id.0, + continuation_key.as_bytes(), + )); + self.commit_workspace_after_tool(WorkspaceCommitExecution { + session: request.session, + workspace_scope: &workspace_scope, + tool_call_id, + provider_name: &workspace.provider, + hand: &hand, + call_scope: request.scope, + release_compute: false, + }) + .await?; + + let provider_impl = self + .hands + .providers + .get(&workspace.provider) + .ok_or_else(|| { + MoaError::ProviderError(format!( + "hand provider {} is not registered", + workspace.provider + )) + })? + .clone(); + if provider_impl.supports_suspend() { + return self + .suspend_continuation_hand( + &request, + provider_impl.as_ref(), + &workspace, + &lease, + &lease_scope, + &hand, + ) + .await; + } + + // Armed only after the checkpoint commits. Arming it first would let the + // reaper claim and destroy the sandbox in the middle of its own publication. + let started_at = std::time::Instant::now(); + self.bound_retained_hand_lifetime(request, &lease_scope, &workspace.provider) + .await?; + record_workspace_lifecycle( + &workspace.provider, + SandboxWorkspaceLifecycleOperation::Retain, + SandboxWorkspaceMetricResult::Succeeded, + started_at.elapsed(), + ); + Ok(ExecutionHandContinuationDisposition::RetainedHot) + } + + /// Stops a continuation sandbox's compute and returns its admission slot. + /// + /// The provider stop runs before the capacity release on purpose. Releasing + /// first and then failing to stop would under-count a sandbox that is still + /// burning compute; this order can only over-count a sandbox that is already + /// stopped, which the reattach path resolves without double-charging. + async fn suspend_continuation_hand( + &self, + request: &ExecutionHandRetentionRequest<'_>, + provider_impl: &dyn moa_core::traits::HandProvider, + workspace: &SandboxWorkspace, + lease: &HandLease, + lease_scope: &str, + hand: &HandHandle, + ) -> Result { + let started_at = std::time::Instant::now(); + if let Err(error) = self + .run_within_scope(request.scope, provider_impl.suspend(hand)) + .await + { + // Non-fatal by contract: the checkpoint is already published, so the + // caller finishes the ordinary checkpoint-and-destroy path instead of + // leaving a hand hot on a bet that has already lost. + tracing::warn!( + provider = %workspace.provider, + generation = lease.generation, + error = %error, + "continuation sandbox suspension failed; falling back to release" + ); + record_workspace_lifecycle( + &workspace.provider, + SandboxWorkspaceLifecycleOperation::Suspend, + SandboxWorkspaceMetricResult::Failed, + started_at.elapsed(), + ); + return Ok(ExecutionHandContinuationDisposition::SuspendFailed); + } + + // The in-process binding cache hands out an active lease's handle without + // consulting the provider, so a stopped sandbox must be evicted here or the + // next same-process slice would dispatch into compute that is not running. + let cache_key = + session_provider_key(request.session, Some(lease_scope), &workspace.provider); + self.remove_cached_binding_if_matches(&cache_key, hand, Some(lease.generation)) + .await; + + if let Some(capacity) = self.hands.workspace_capacity.as_ref() { + let binding = workspace.binding()?; + let released = capacity + .release_suspended_active_hand(&active_hand_capacity_request(&binding, lease)?) + .await?; + if !released { + // The charge stays held, which is the conservative direction: the + // sandbox really is stopped, so the fleet is only under-admitting. + tracing::warn!( + provider = %workspace.provider, + generation = lease.generation, + "suspended continuation sandbox kept its active-hands charge" + ); + } + } + record_workspace_lifecycle( + &workspace.provider, + SandboxWorkspaceLifecycleOperation::Suspend, + SandboxWorkspaceMetricResult::Succeeded, + started_at.elapsed(), + ); + Ok(ExecutionHandContinuationDisposition::Suspended) + } + + /// Shortens a retained continuation hand's idle deadline to its retention bound. + /// + /// Reuses the ordinary active-lease renewal, which sets the idle deadline under + /// the immutable hard lifetime. The requested bound is additionally clamped to the + /// lease's current idle deadline so retention can only shorten a sandbox's life, + /// never extend it past the policy it was admitted under. + async fn bound_retained_hand_lifetime( + &self, + request: ExecutionHandRetentionRequest<'_>, + lease_scope: &str, + provider: &str, + ) -> Result<()> { + let lease_store = self.hands.hand_leases.as_ref().ok_or_else(|| { + MoaError::StorageError("durable hand lease store missing".to_string()) + })?; + let Some(lease) = lease_store + .get( + request.session.tenant_id, + request.session.id, + lease_scope, + provider, + ) + .await? + else { + return Ok(()); + }; + if lease.status != HandLeaseStatus::Active { + return Ok(()); + } + let Some(attachment) = lease.attachment.clone() else { + return Ok(()); + }; + let retention_deadline_at = lease + .idle_expires_at + .map_or(request.retention_deadline_at, |idle| { + idle.min(request.retention_deadline_at) + }); + if !lease_store + .renew_active(crate::core::leases::HandLeaseRenewRequest { + tenant_id: request.session.tenant_id, + session_id: request.session.id, + worker_id: lease_scope, + provider, + generation: lease.generation, + provisioning_operation_id: lease.provisioning_operation_id, + attachment, + idle_expires_at: retention_deadline_at, + }) + .await? + { + // The lease moved under us, so some other durable owner already governs + // this sandbox's lifetime. The checkpoint is published either way, so the + // worst outcome is that the hand expires on its ordinary idle policy. + tracing::warn!( + provider, + generation = lease.generation, + "retained execution continuation hand kept its ordinary idle deadline" + ); + } + Ok(()) + } + /// Checkpoints one execution-task workspace and releases its exact compute lease. /// /// The returned receipt is the durable proof required before a task may yield to @@ -1542,10 +1847,23 @@ impl ToolRouter { .finalize_task_yield_destroy(&final_workspace.binding()?, &final_lease) .await? { + // The compute is gone but the durable release did not commit, so the + // charge is still held and a reconciler owns it. Recorded as ambiguous + // rather than succeeded so the two are distinguishable on the dashboard. + record_workspace_release( + &initial_workspace.provider, + SandboxWorkspaceMetricResult::Ambiguous, + ); return Err(MoaError::ExternalEffectUnknownOutcome { operation_id: release_key.clone(), }); } + // Counted only after provider destruction is verified AND the release + // receipt commits, which together are what actually free the capacity. + record_workspace_release( + &initial_workspace.provider, + SandboxWorkspaceMetricResult::Succeeded, + ); let key = session_provider_key( request.session, Some(&lease_scope), diff --git a/crates/moa-hands/src/core/sandbox_workspace/maintenance/inventory.rs b/crates/moa-hands/src/core/sandbox_workspace/maintenance/inventory.rs index c88f83201..8336a539f 100644 --- a/crates/moa-hands/src/core/sandbox_workspace/maintenance/inventory.rs +++ b/crates/moa-hands/src/core/sandbox_workspace/maintenance/inventory.rs @@ -114,11 +114,8 @@ impl WorkspaceMaintenanceCoordinator { .or_default() += 1; self.upsert_inventory_finding(&finding).await?; } - self.resolve_unseen_findings( - &observed_keys, - &HashSet::from([(account.id, account.generation)]), - ) - .await?; + self.resolve_unseen_findings(&observed_keys, account.id, account.generation) + .await?; Ok((inventory.resources.len() as u64, counts)) } @@ -486,43 +483,42 @@ impl WorkspaceMaintenanceCoordinator { Ok(()) } + /// Resolves every unresolved finding of one scanned account generation that + /// the completed scan did not observe again. async fn resolve_unseen_findings( &self, observed: &HashSet, - observed_accounts: &HashSet<(ProviderAccountId, u64)>, + account_id: ProviderAccountId, + account_generation: u64, ) -> Result<()> { + let scanned_generation = i64::try_from(account_generation).map_err(|_| { + MoaError::StorageError("provider account generation overflows Postgres".to_string()) + })?; let mut conn = maintenance_conn(&self.pool).await?; let rows = sqlx::query( r#" - SELECT provider_account_id, provider_account_generation, - resource_fingerprint, finding_kind + SELECT resource_fingerprint, finding_kind FROM moa.sandbox_provider_inventory_findings WHERE quarantine_state <> 'resolved' + AND provider_account_id = $1 + AND provider_account_generation = $2 "#, ) + .bind(account_id) + .bind(scanned_generation) .fetch_all(conn.as_mut()) .await .map_err(map_sqlx)?; for row in rows { let key = InventoryFindingKey { - account_id: row.try_get("provider_account_id").map_err(map_sqlx)?, - account_generation: u64::try_from( - row.try_get::("provider_account_generation") - .map_err(map_sqlx)?, - ) - .map_err(|_| { - MoaError::StorageError( - "inventory finding account generation is invalid".to_string(), - ) - })?, + account_id, + account_generation, resource_fingerprint: row.try_get("resource_fingerprint").map_err(map_sqlx)?, kind: InventoryFindingKind::from_label( &row.try_get::("finding_kind").map_err(map_sqlx)?, )?, }; - if !observed_accounts.contains(&(key.account_id, key.account_generation)) - || observed.contains(&key) - { + if observed.contains(&key) { continue; } let digest = format!( @@ -548,10 +544,8 @@ impl WorkspaceMaintenanceCoordinator { AND quarantine_state <> 'resolved' "#, ) - .bind(key.account_id) - .bind(i64::try_from(key.account_generation).map_err(|_| { - MoaError::StorageError("provider account generation overflows Postgres".to_string()) - })?) + .bind(account_id) + .bind(scanned_generation) .bind(&key.resource_fingerprint) .bind(key.kind.as_str()) .bind(digest) diff --git a/crates/moa-hands/src/core/sandbox_workspace/maintenance/mod.rs b/crates/moa-hands/src/core/sandbox_workspace/maintenance/mod.rs index 7d8ec00f4..ec57befbc 100644 --- a/crates/moa-hands/src/core/sandbox_workspace/maintenance/mod.rs +++ b/crates/moa-hands/src/core/sandbox_workspace/maintenance/mod.rs @@ -35,13 +35,13 @@ use moa_core::{ sandbox_workspace::{ ProviderAccountStorageInventory, ProviderInventoryResourceKind, ProviderStorageKind, ProviderStorageRef, SandboxWorkspaceState, TenantStoragePurgeRequest, - WorkspaceConfirmedDisposition, WorkspaceOperationOutcome, WorkspaceReconcileRequest, - WorkspaceStorageOperation, + WorkspaceCapacityDimension, WorkspaceConfirmedDisposition, WorkspaceOperationOutcome, + WorkspaceReconcileRequest, WorkspaceStorageOperation, }, }, }; use moa_db::ScopedConn; -use moa_observability::SandboxWorkspaceInventoryDrift; +use moa_observability::{SandboxStorageResourceMetricState, SandboxWorkspaceInventoryDrift}; use sha2::{Digest, Sha256}; use sqlx::{PgPool, Row, types::Json}; use uuid::Uuid; @@ -57,7 +57,11 @@ use super::{ }; use crate::core::{ leases::{LeaseHandle, PostgresHandLeaseStore}, - telemetry::{record_workspace_inventory_drift, record_workspace_state}, + telemetry::{ + record_workspace_active_hands, record_workspace_inventory_drift, + record_workspace_parked_tasks_with_active_hands, record_workspace_quota_utilization, + record_workspace_state, record_workspace_storage_resource_state, + }, }; /// One bounded checkpoint-retention pass. @@ -319,6 +323,166 @@ impl WorkspaceMaintenanceCoordinator { record_workspace_state(provider, state, count); } } + self.emit_storage_resource_metrics().await?; + self.emit_active_hand_metrics().await?; + self.emit_quota_utilization_metrics().await?; + self.emit_parked_task_compute_violations().await?; + Ok(()) + } + + /// Emits a zero-filled durable storage-resource fleet snapshot. + async fn emit_storage_resource_metrics(&self) -> Result<()> { + let mut conn = maintenance_conn(&self.pool).await?; + let rows = sqlx::query( + "SELECT provider_account.provider AS provider, resource.lifecycle_state, \ + count(*)::BIGINT AS count \ + FROM moa.sandbox_storage_resources AS resource \ + JOIN moa.sandbox_provider_accounts AS provider_account \ + ON provider_account.provider_account_id = resource.provider_account_id \ + GROUP BY provider_account.provider, resource.lifecycle_state", + ) + .fetch_all(conn.as_mut()) + .await + .map_err(map_sqlx)?; + conn.commit().await?; + let mut counts = HashMap::new(); + for row in rows { + let provider = + provider_metric_label(&row.try_get::("provider").map_err(map_sqlx)?) + .to_string(); + let state = row + .try_get::("lifecycle_state") + .map_err(map_sqlx)?; + *counts.entry((provider, state)).or_insert(0u64) += + u64::try_from(row.try_get::("count").map_err(map_sqlx)?).unwrap_or(0); + } + // Zero-fill every provider/state pair: the alerts on these series are + // `absent()`-guarded, so a gauge written only when rows exist would page on a + // healthy fleet that simply has no storage resources in that state. + for provider in ["local", "daytona", "e2b", "other"] { + for state in all_storage_resource_states() { + let count = counts + .get(&(provider.to_string(), state.as_str().to_string())) + .copied() + .unwrap_or(0); + record_workspace_storage_resource_state(provider, state, count); + } + } + Ok(()) + } + + /// Emits the per-provider count of sandbox compute instances holding capacity. + async fn emit_active_hand_metrics(&self) -> Result<()> { + let mut conn = maintenance_conn(&self.pool).await?; + let rows = sqlx::query( + "SELECT provider_account.provider AS provider, count(*)::BIGINT AS count \ + FROM moa.sandbox_capacity_reservations AS reservation \ + JOIN moa.sandbox_provider_accounts AS provider_account \ + ON provider_account.provider_account_id = reservation.provider_account_id \ + WHERE reservation.resource_dimension = 'active_hands' \ + AND reservation.reservation_state <> 'released' \ + GROUP BY provider_account.provider", + ) + .fetch_all(conn.as_mut()) + .await + .map_err(map_sqlx)?; + conn.commit().await?; + let mut counts = HashMap::new(); + for row in rows { + let provider = + provider_metric_label(&row.try_get::("provider").map_err(map_sqlx)?) + .to_string(); + *counts.entry(provider).or_insert(0u64) += + u64::try_from(row.try_get::("count").map_err(map_sqlx)?).unwrap_or(0); + } + for provider in ["local", "daytona", "e2b", "other"] { + record_workspace_active_hands(provider, counts.get(provider).copied().unwrap_or(0)); + } + Ok(()) + } + + /// Emits the fleet-wide utilization ratio for every capacity dimension. + async fn emit_quota_utilization_metrics(&self) -> Result<()> { + let mut conn = maintenance_conn(&self.pool).await?; + let rows = sqlx::query( + "SELECT reservation.resource_dimension, \ + sum(reservation.quantity)::BIGINT AS reserved, \ + max(limits.limit_value)::BIGINT AS limit_value \ + FROM moa.sandbox_capacity_reservations AS reservation \ + LEFT JOIN moa.sandbox_tenant_capacity_limits AS limits \ + ON limits.tenant_id = reservation.tenant_id \ + AND limits.resource_dimension = reservation.resource_dimension \ + WHERE reservation.reservation_state <> 'released' \ + GROUP BY reservation.resource_dimension", + ) + .fetch_all(conn.as_mut()) + .await + .map_err(map_sqlx)?; + conn.commit().await?; + let mut ratios = HashMap::new(); + for row in rows { + let dimension = row + .try_get::("resource_dimension") + .map_err(map_sqlx)?; + let reserved = row.try_get::("reserved").map_err(map_sqlx)?.max(0); + let limit = row + .try_get::, _>("limit_value") + .map_err(map_sqlx)? + .unwrap_or(0); + // An absent or zero limit means the dimension is unbounded for every tenant + // observed, which is 0.0 pressure rather than a division by zero. + let ratio = if limit > 0 { + reserved as f64 / limit as f64 + } else { + 0.0 + }; + ratios.insert(dimension, ratio); + } + for dimension in all_capacity_dimensions() { + let ratio = ratios.get(dimension.as_str()).copied().unwrap_or(0.0); + record_workspace_quota_utilization(dimension, ratio); + } + Ok(()) + } + + /// Emits the count of parked execution tasks that still own sandbox compute. + /// + /// This is the only automated guard on the invariant that a parked run owns no + /// sandbox. A `pending` release receipt is excluded deliberately: that row marks the + /// legitimate in-flight checkpoint-and-release window, so counting it would make every + /// normal yield trip a critical alert. + async fn emit_parked_task_compute_violations(&self) -> Result<()> { + let mut conn = maintenance_conn(&self.pool).await?; + let violations: i64 = sqlx::query_scalar( + "SELECT count(*)::BIGINT \ + FROM moa.hand_leases AS lease \ + JOIN moa.sandbox_workspaces AS workspace \ + ON workspace.workspace_id = lease.workspace_id \ + AND workspace.tenant_id = lease.tenant_id \ + JOIN moa.execution_task AS task \ + ON task.task_id = workspace.scope_task_id \ + AND task.run_uid = workspace.scope_run_id \ + AND task.tenant_id = workspace.tenant_id \ + WHERE lease.status IN ('provisioning', 'active') \ + AND workspace.scope_kind = 'execution_task' \ + AND task.status IN ( \ + 'waiting_input', 'waiting_review', 'waiting_signal', \ + 'waiting_timer', 'waiting_external', 'waiting_replan') \ + AND NOT EXISTS ( \ + SELECT 1 FROM moa.sandbox_execution_hand_release_receipts AS receipt \ + WHERE receipt.tenant_id = task.tenant_id \ + AND receipt.run_uid = task.run_uid \ + AND receipt.task_id = task.task_id \ + AND receipt.owner_kind = 'task' \ + AND receipt.receipt_state = 'pending')", + ) + .fetch_one(conn.as_mut()) + .await + .map_err(map_sqlx)?; + conn.commit().await?; + record_workspace_parked_tasks_with_active_hands( + u64::try_from(violations.max(0)).unwrap_or(0), + ); Ok(()) } @@ -393,6 +557,28 @@ fn all_workspace_states() -> [SandboxWorkspaceState; 10] { ] } +fn all_storage_resource_states() -> [SandboxStorageResourceMetricState; 7] { + [ + SandboxStorageResourceMetricState::Creating, + SandboxStorageResourceMetricState::Ready, + SandboxStorageResourceMetricState::Attached, + SandboxStorageResourceMetricState::Deleting, + SandboxStorageResourceMetricState::Deleted, + SandboxStorageResourceMetricState::Unknown, + SandboxStorageResourceMetricState::Failed, + ] +} + +fn all_capacity_dimensions() -> [WorkspaceCapacityDimension; 5] { + [ + WorkspaceCapacityDimension::Workspaces, + WorkspaceCapacityDimension::ActiveHands, + WorkspaceCapacityDimension::Volumes, + WorkspaceCapacityDimension::Checkpoints, + WorkspaceCapacityDimension::LogicalBytes, + ] +} + fn map_sqlx(error: sqlx::Error) -> MoaError { MoaError::StorageError(error.to_string()) } diff --git a/crates/moa-hands/src/core/sandbox_workspace/repository/checkpoints.rs b/crates/moa-hands/src/core/sandbox_workspace/repository/checkpoints.rs index 4b89249f4..e401d7f5a 100644 --- a/crates/moa-hands/src/core/sandbox_workspace/repository/checkpoints.rs +++ b/crates/moa-hands/src/core/sandbox_workspace/repository/checkpoints.rs @@ -206,7 +206,9 @@ impl PostgresWorkspaceRepository { AND expected_writer_epoch = $7 AND expected_instance_generation = $8 AND resource_dimension = 'active_hands' - AND reservation_state = 'committed' + -- A pending charge is still a charge: releasing compute must + -- settle it rather than roll the whole publication back. + AND reservation_state IN ('pending', 'committed') "#, ) .bind(binding.tenant_id) diff --git a/crates/moa-hands/src/core/sandbox_workspace/repository/lifecycle.rs b/crates/moa-hands/src/core/sandbox_workspace/repository/lifecycle.rs index 608ee5bcd..2052cd9df 100644 --- a/crates/moa-hands/src/core/sandbox_workspace/repository/lifecycle.rs +++ b/crates/moa-hands/src/core/sandbox_workspace/repository/lifecycle.rs @@ -1018,7 +1018,9 @@ impl PostgresWorkspaceRepository { AND hand_lease_generation = $6 AND expected_writer_epoch = $7 AND expected_instance_generation = $8 AND resource_dimension = 'active_hands' - AND reservation_state = 'committed' + -- A pending charge is still a charge: destroying the yielded hand + -- must settle it rather than roll the whole release back. + AND reservation_state IN ('pending', 'committed') "#, ) .bind(binding.tenant_id) diff --git a/crates/moa-hands/src/core/telemetry.rs b/crates/moa-hands/src/core/telemetry.rs index dee8280fa..89f9180b6 100644 --- a/crates/moa-hands/src/core/telemetry.rs +++ b/crates/moa-hands/src/core/telemetry.rs @@ -18,10 +18,12 @@ use moa_observability::{ SandboxWorkspaceInventoryDrift, SandboxWorkspaceLifecycleOperation, SandboxWorkspaceMetricResult, SandboxWorkspaceProviderKind, SandboxWorkspaceQuotaDecision, apply_trace_context_to_span, current_turn_root_span, record_sandbox_storage_resource_state, - record_sandbox_workspace_checkpoint, record_sandbox_workspace_inventory_drift, - record_sandbox_workspace_lifecycle, record_sandbox_workspace_quota_decision, - record_sandbox_workspace_quota_utilization, record_sandbox_workspace_reaper, - record_sandbox_workspace_state, record_tool_call, + record_sandbox_workspace_active_hands, record_sandbox_workspace_checkpoint, + record_sandbox_workspace_inventory_drift, record_sandbox_workspace_lifecycle, + record_sandbox_workspace_parked_tasks_with_active_hands, + record_sandbox_workspace_quota_decision, record_sandbox_workspace_quota_utilization, + record_sandbox_workspace_reaper, record_sandbox_workspace_release, + record_sandbox_workspace_restore, record_sandbox_workspace_state, record_tool_call, }; use opentelemetry::trace::Status; use tracing_opentelemetry::OpenTelemetrySpanExt; @@ -99,6 +101,29 @@ pub fn record_workspace_quota_utilization(dimension: WorkspaceCapacityDimension, record_sandbox_workspace_quota_utilization(dimension, ratio); } +/// Sets one provider's count of sandbox compute instances holding active capacity. +pub fn record_workspace_active_hands(provider: &str, count: u64) { + record_sandbox_workspace_active_hands(sandbox_workspace_provider_kind(provider), count); +} + +/// Sets the count of parked execution tasks that still own sandbox compute. +/// +/// Guards the invariant that a parked run owns no sandbox. Emitted on every +/// maintenance pass including the healthy zero, because the alert is `absent()`-guarded. +pub fn record_workspace_parked_tasks_with_active_hands(count: u64) { + record_sandbox_workspace_parked_tasks_with_active_hands(count); +} + +/// Records one portable-checkpoint restore into fresh sandbox compute. +pub fn record_workspace_restore(provider: &str) { + record_sandbox_workspace_restore(sandbox_workspace_provider_kind(provider)); +} + +/// Records one checkpoint-and-release outcome at an execution yield boundary. +pub fn record_workspace_release(provider: &str, result: SandboxWorkspaceMetricResult) { + record_sandbox_workspace_release(sandbox_workspace_provider_kind(provider), result); +} + /// Sets the complete supervised workspace-reaper health snapshot. pub fn record_workspace_reaper_health( ready: bool, diff --git a/crates/moa-hands/src/lib.rs b/crates/moa-hands/src/lib.rs index 909571dc5..5248246f4 100644 --- a/crates/moa-hands/src/lib.rs +++ b/crates/moa-hands/src/lib.rs @@ -10,10 +10,10 @@ pub use adapters::local::{LOCAL_HAND_CAPABILITIES, LocalHandProvider}; pub use adapters::mcp::{MCPClient, McpDiscoveredTool}; pub use core::{ ActionOrigin, AuthorizedToolCall, CandidateConnector, CatalogDefect, - DeferredWorkspaceToolOutput, ExecutionHandReleaseRequest, FileProviderCredentialSource, - HandLeaseReaper, HandLeaseReaperConfig, HandRoute, JournaledWorkspaceCommit, - MCP_TOOL_REFERENCE_PREFIX, McpCatalogActivation, McpCatalogRefresh, McpConnectorHealth, - PendingConnectorToolOutput, PinnedToolContract, PinnedToolOwner, + DeferredWorkspaceToolOutput, ExecutionHandReleaseRequest, ExecutionHandRetentionRequest, + FileProviderCredentialSource, HandLeaseReaper, HandLeaseReaperConfig, HandRoute, + JournaledWorkspaceCommit, MCP_TOOL_REFERENCE_PREFIX, McpCatalogActivation, McpCatalogRefresh, + McpConnectorHealth, PendingConnectorToolOutput, PinnedToolContract, PinnedToolOwner, PostgresExpiredHandLeaseClaims, PostgresTenantSandboxPolicyStore, PreparedActionInvocation, ProviderCredentialSource, ProviderEndpoint, ProviderHttpAttempt, ProviderSandboxAttempt, SandboxProviderInventory, SessionHandReleasePageOutcome, TenantSandboxPolicyStore, diff --git a/crates/moa-hands/tests/daytona_live.rs b/crates/moa-hands/tests/daytona_live.rs index 00c0116fb..df5a56aa1 100644 --- a/crates/moa-hands/tests/daytona_live.rs +++ b/crates/moa-hands/tests/daytona_live.rs @@ -591,27 +591,6 @@ async fn wait_for_destroyed( } } -async fn wait_for_status( - provider: &DaytonaHandProvider, - handle: &HandHandle, - expected: &[HandStatus], - timeout: Duration, -) -> Result { - let started = Instant::now(); - loop { - if started.elapsed() > timeout { - return Err(MoaError::ProviderError(format!( - "timed out waiting for Daytona status {expected:?}" - ))); - } - let status = provider.status(handle).await?; - if expected.contains(&status) { - return Ok(status); - } - sleep(Duration::from_secs(2)).await; - } -} - async fn destroy_and_wait(provider: &DaytonaHandProvider, handle: &HandHandle) -> Result<()> { provider.destroy(handle).await?; wait_for_destroyed(provider, handle, Duration::from_secs(30)).await @@ -766,23 +745,6 @@ async fn daytona_provider_round_trip() { search.to_text() ); - provider.pause(&handle).await?; - let _ = wait_for_status( - &provider, - &handle, - &[HandStatus::Stopped, HandStatus::Paused], - Duration::from_secs(60), - ) - .await?; - let resumed_read = provider - .execute( - &handle, - "file_read", - &json!({ "path": file_path }).to_string(), - ) - .await?; - assert_eq!(resumed_read.to_text(), marker); - let unsupported_tool = provider .execute( &handle, @@ -1366,35 +1328,6 @@ async fn daytona_router_reuses_and_isolates() { assert_eq!(same_hand_id.as_deref(), Some(handle_one_id.as_str())); assert_eq!(read.to_text(), content_one); - provider.pause(&handle_one).await?; - let _ = wait_for_status( - &provider, - &handle_one, - &[HandStatus::Stopped, HandStatus::Paused], - Duration::from_secs(60), - ) - .await?; - let secured_3 = router - .execute_authorized(moa_hands::AuthorizedToolCall { - session: &session_one, - caller_identity: &identity(), - workspace_scope: Some(&router_workspace_scope(&session_one)), - invocation: &ToolInvocation { - id: None, - name: "file_read".to_string(), - input: json!({ "path": file_one }), - }, - tool_call_id: ToolCallId::new(), - active_canary: None, - catalog: None, - scope: moa_hands::ToolCallScope::unbounded(), - }) - .await?; - let resumed_hand_id = secured_3.hand_id.clone(); - let resumed_read = secured_3.safe_output; - assert_eq!(resumed_hand_id.as_deref(), Some(handle_one_id.as_str())); - assert_eq!(resumed_read.to_text(), content_one); - let secured_4 = router .execute_authorized(moa_hands::AuthorizedToolCall { session: &session_two, diff --git a/crates/moa-hands/tests/docker_hardening_docker.rs b/crates/moa-hands/tests/docker_hardening_docker.rs index 6df6c04f3..bf0e5beec 100644 --- a/crates/moa-hands/tests/docker_hardening_docker.rs +++ b/crates/moa-hands/tests/docker_hardening_docker.rs @@ -58,9 +58,6 @@ async fn docker_container_runs_with_hardening() { .unwrap_or_default(); assert!(mounts.contains("ro")); assert!(rendered.contains("metadata=blocked")); - - provider.pause(&handle).await.unwrap(); - provider.resume(&handle).await.unwrap(); } .await; diff --git a/crates/moa-hands/tests/e2b_live.rs b/crates/moa-hands/tests/e2b_live.rs index b23bdd38e..ffaf35efc 100644 --- a/crates/moa-hands/tests/e2b_live.rs +++ b/crates/moa-hands/tests/e2b_live.rs @@ -388,10 +388,6 @@ async fn e2b_provider_round_trip() { search.to_text() ); - assert!(matches!( - provider.pause(&handle).await, - Err(MoaError::Unsupported(_)) - )); assert!(matches!( provider.resume(&handle).await, Err(MoaError::Unsupported(_)) diff --git a/crates/moa-hands/tests/hands_db/sandbox_workspace/capacity_db.rs b/crates/moa-hands/tests/hands_db/sandbox_workspace/capacity_db.rs index 64b232600..97132b001 100644 --- a/crates/moa-hands/tests/hands_db/sandbox_workspace/capacity_db.rs +++ b/crates/moa-hands/tests/hands_db/sandbox_workspace/capacity_db.rs @@ -4,9 +4,14 @@ use chrono::{Duration as ChronoDuration, Utc}; use moa_core::{ error::MoaError, types::{ + action_policy::CallOrigin, + hands::{ + BuiltinPolicyRevision, CpuLimit, DiskLimit, EgressPolicy, LifetimeLimit, MemoryLimit, + SandboxPolicySnapshot, SandboxProfile, SandboxTier, + }, identifiers::{ ExecutionRunScopeId, ExecutionTaskScopeId, ProviderAccountId, SandboxWorkspaceId, - TenantId, WorkspaceOperationId, + SessionId, TenantId, WorkspaceOperationId, }, sandbox_workspace::{ DurabilityClass, SandboxWorkspaceScope, SandboxWorkspaceState, @@ -14,17 +19,28 @@ use moa_core::{ }, }, }; -use moa_hands::core::sandbox_workspace::{ - capacity::{CapacityQuantity, CapacityReservationRequest, PostgresWorkspaceCapacityRepository}, - model::{CreateWorkspaceRequest, WorkspaceTransition}, - operations::{PostgresWorkspaceOperationRepository, WorkspaceOperationIntent}, - repository::PostgresWorkspaceRepository, - storage_resources::{PostgresWorkspaceStorageResourceRepository, StorageResourceCreateIntent}, +use moa_hands::core::{ + leases::{ + HandLeasePolicy, HandLeaseProvisionRequest, HandLeaseStore, HandLeaseWorkspaceAttachment, + PostgresHandLeaseStore, + }, + sandbox_workspace::{ + capacity::{ + ActiveHandCapacityRequest, CapacityQuantity, CapacityReservationRequest, + PostgresWorkspaceCapacityRepository, + }, + model::{CreateWorkspaceRequest, WorkspaceTransition, WorkspaceWriterClaim}, + operations::{PostgresWorkspaceOperationRepository, WorkspaceOperationIntent}, + repository::PostgresWorkspaceRepository, + storage_resources::{ + PostgresWorkspaceStorageResourceRepository, StorageResourceCreateIntent, + }, + }, }; use sqlx::postgres::PgPoolOptions; use uuid::Uuid; -use super::database_url; +use super::{database_url, seed_session}; #[derive(Debug, Clone)] struct VolumeCandidate { @@ -443,3 +459,222 @@ async fn volume_inventory_overlap_headroom_and_exact_limit_are_atomic_db() { cleanup_volume_account(&pool, account_id).await; pool.close().await; } + +fn active_hand_lease_policy() -> HandLeasePolicy { + let profile = SandboxProfile::new( + CpuLimit::Unbounded, + MemoryLimit::Unbounded, + DiskLimit::Unbounded, + EgressPolicy::DenyAll, + LifetimeLimit::Unbounded, + LifetimeLimit::Unbounded, + ) + .expect("test profile validates"); + HandLeasePolicy::from_effective( + &moa_core::types::hands::resolve_effective_sandbox_profile( + &SandboxPolicySnapshot::new("workspace-capacity-deployment", profile) + .expect("deployment snapshot"), + &SandboxPolicySnapshot::builtin(BuiltinPolicyRevision::TenantUnset), + &SandboxPolicySnapshot::builtin(BuiltinPolicyRevision::AgentUnset), + &SandboxPolicySnapshot::builtin(BuiltinPolicyRevision::RouteUnset), + &SandboxPolicySnapshot::origin(CallOrigin::Production), + "workspace-capacity-capabilities-v1", + ) + .expect("test resolution succeeds"), + ) +} + +/// Drives one workspace to a hydrating writer with a provisioning lease, which +/// is the exact state `reserve_active_hand` admits from. +async fn seed_hydrating_hand( + pool: &sqlx::PgPool, + tenant_id: TenantId, + account_id: ProviderAccountId, + worker_id: &str, +) -> ActiveHandCapacityRequest { + let session_id = SessionId::new(); + seed_session(pool, session_id, tenant_id).await; + let workspaces = PostgresWorkspaceRepository::new(pool.clone()); + let workspace_id = SandboxWorkspaceId::new(); + workspaces + .create(&CreateWorkspaceRequest { + workspace_id, + tenant_id, + scope: SandboxWorkspaceScope::ExecutionTask { + run_id: ExecutionRunScopeId::new(), + task_id: ExecutionTaskScopeId::new(), + }, + provider: "local".to_string(), + provider_account_id: account_id, + provider_account_generation: 1, + durability_class: DurabilityClass::PortableFilesystem, + retention_deadline_at: None, + }) + .await + .expect("persist active-hand test workspace"); + assert!( + workspaces + .transition(WorkspaceTransition { + tenant_id, + workspace_id, + from: SandboxWorkspaceState::Creating, + to: SandboxWorkspaceState::Ready, + writer_epoch: 0, + instance_generation: 0, + }) + .await + .expect("make the workspace claimable") + ); + let restoring = workspaces + .claim_writer(WorkspaceWriterClaim { + tenant_id, + workspace_id, + expected_state: SandboxWorkspaceState::Ready, + expected_writer_epoch: 0, + expected_instance_generation: 0, + }) + .await + .expect("claim the single writer") + .expect("claimed workspace exists"); + let attachment = HandLeaseWorkspaceAttachment::new( + restoring.workspace_id, + restoring.writer_epoch, + restoring.instance_generation, + None, + ) + .expect("claimed workspace attachment validates"); + let provisioning = PostgresHandLeaseStore::new(pool.clone()) + .claim_for_provisioning(HandLeaseProvisionRequest { + session_id, + worker_id, + tenant_id, + provider: "local", + tier: SandboxTier::Local, + attachment, + policy: &active_hand_lease_policy(), + caller_deadline: None, + }) + .await + .expect("claim provisioning lease") + .expect("provisioning lease exists"); + ActiveHandCapacityRequest { + tenant_id, + workspace_id, + provider_account_id: account_id, + provider_account_generation: 1, + provisioning_operation_id: provisioning.provisioning_operation_id, + hand_lease_generation: provisioning.generation, + expected_writer_epoch: restoring.writer_epoch, + expected_instance_generation: restoring.instance_generation, + } +} + +#[tokio::test] +#[ignore = "requires a fresh V60 compose Postgres via MOA_DATABASE_URL"] +async fn concurrent_hand_limit_plus_one_is_rejected_in_both_scopes_db() { + // Pins: `max_active_hands` is charged against the active_hands dimension in + // both the tenant and the provider-account scope, so hand number + // limit-plus-one is refused instead of admitted against an absent limit. + let pool = PgPoolOptions::new() + .max_connections(6) + .connect(&database_url()) + .await + .expect("test Postgres should be reachable"); + let account_id = ProviderAccountId::new(); + let saturating_tenant = TenantId::new(); + let starved_tenant = TenantId::new(); + sqlx::query( + r#" + INSERT INTO moa.sandbox_provider_accounts ( + provider_account_id, generation, provider, isolation_cell, + organization_fingerprint, configured_limits + ) VALUES ($1, 1, 'local', $2, $3, '{"active_hands": 1}'::jsonb) + "#, + ) + .bind(account_id) + .bind(format!("active-hands-{account_id}")) + .bind(format!("org-{account_id}")) + .execute(&pool) + .await + .expect("seed provider-account concurrent-hand ceiling"); + for (tenant_id, limit) in [(saturating_tenant, 1), (starved_tenant, 2)] { + sqlx::query( + "INSERT INTO moa.sandbox_tenant_capacity_limits (tenant_id, configured_limits) VALUES ($1, $2)", + ) + .bind(tenant_id) + .bind(serde_json::json!({ "active_hands": limit })) + .execute(&pool) + .await + .expect("seed tenant concurrent-hand quota"); + } + + let capacity = PostgresWorkspaceCapacityRepository::new(pool.clone()); + let admitted = + seed_hydrating_hand(&pool, saturating_tenant, account_id, "capacity-first").await; + let reservation = capacity + .reserve_active_hand(&admitted) + .await + .expect("the exact concurrent-hand limit is admitted"); + assert_eq!( + reservation.dimension, + WorkspaceCapacityDimension::ActiveHands + ); + assert_eq!(reservation.quantity, 1); + + let over_tenant_limit = + seed_hydrating_hand(&pool, saturating_tenant, account_id, "capacity-second").await; + let tenant_error = capacity + .reserve_active_hand(&over_tenant_limit) + .await + .expect_err("limit plus one must be refused for the saturating tenant"); + assert!( + matches!( + tenant_error, + MoaError::ValidationError(ref detail) + if detail.contains("tenant active_hands capacity exceeded") + ), + "concurrent-hand admission must name the exhausted dimension: {tenant_error}" + ); + + let over_account_limit = + seed_hydrating_hand(&pool, starved_tenant, account_id, "capacity-third").await; + let account_error = capacity + .reserve_active_hand(&over_account_limit) + .await + .expect_err("a second tenant cannot exceed the shared provider-account ceiling"); + assert!( + matches!( + account_error, + MoaError::ValidationError(ref detail) + if detail.contains("provider account active_hands capacity exceeded") + ), + "the provider-account ceiling must be the reported scope: {account_error}" + ); + + let charged = sqlx::query_scalar::<_, i64>( + "SELECT count(*) FROM moa.sandbox_capacity_reservations WHERE provider_account_id = $1 AND resource_dimension = 'active_hands'", + ) + .bind(account_id) + .fetch_one(&pool) + .await + .expect("count durable concurrent-hand charges"); + assert_eq!( + charged, 1, + "a refused admission cannot leave a partial concurrent-hand charge" + ); + + for tenant_id in [saturating_tenant, starved_tenant] { + sqlx::query("DELETE FROM moa.sandbox_tenant_capacity_limits WHERE tenant_id = $1") + .bind(tenant_id) + .execute(&pool) + .await + .expect("clean tenant limits"); + } + sqlx::query("DELETE FROM moa.hand_leases WHERE tenant_id = ANY($1)") + .bind(vec![saturating_tenant, starved_tenant]) + .execute(&pool) + .await + .expect("clean hand leases"); + cleanup_volume_account(&pool, account_id).await; + pool.close().await; +} diff --git a/crates/moa-hands/tests/hands_db/sandbox_workspace/dispatch_db.rs b/crates/moa-hands/tests/hands_db/sandbox_workspace/dispatch_db.rs index bcc5efe0e..9e107dff2 100644 --- a/crates/moa-hands/tests/hands_db/sandbox_workspace/dispatch_db.rs +++ b/crates/moa-hands/tests/hands_db/sandbox_workspace/dispatch_db.rs @@ -1049,10 +1049,6 @@ impl HandProvider for GatedWorkspaceProvider { Ok(HandStatus::Running) } - async fn pause(&self, _handle: &HandHandle) -> Result<()> { - Ok(()) - } - async fn resume(&self, _handle: &HandHandle) -> Result<()> { Ok(()) } diff --git a/crates/moa-hands/tests/hands_db/sandbox_workspace/lifecycle_db.rs b/crates/moa-hands/tests/hands_db/sandbox_workspace/lifecycle_db.rs index 0f1a47283..f35d8078f 100644 --- a/crates/moa-hands/tests/hands_db/sandbox_workspace/lifecycle_db.rs +++ b/crates/moa-hands/tests/hands_db/sandbox_workspace/lifecycle_db.rs @@ -1,26 +1,46 @@ //! Durable workspace lifecycle, fencing, and reconciliation against Postgres. -use std::{path::PathBuf, time::Duration}; +use std::{ + collections::HashMap, + path::PathBuf, + sync::{ + Arc, + atomic::{AtomicBool, AtomicUsize, Ordering}, + }, + time::Duration, +}; +use async_trait::async_trait; use chrono::{Duration as ChronoDuration, Utc}; -use moa_core::error::MoaError; +use moa_core::error::{MoaError, Result as MoaResult}; +use moa_core::traits::{HandProvider, SandboxStorageProvider}; use moa_core::types::{ - action_policy::CallOrigin, + action_policy::{ActionClass, ActionPolicyEffect, CallOrigin, RiskLevel}, hands::{ - BuiltinPolicyRevision, CpuLimit, DiskLimit, EgressPolicy, HandHandle, LifetimeLimit, - MemoryLimit, SandboxPolicySnapshot, SandboxProfile, SandboxTier, + BuiltinPolicyRevision, CpuLimit, DeadlineEnforcement, DiskLimit, EgressMode, EgressPolicy, + HandHandle, HandProviderCapabilities, HandSpec, HandStatus, LifetimeLimit, MemoryLimit, + ResourceSupport, SandboxPolicySnapshot, SandboxProfile, SandboxTier, + SandboxTierCapabilities, }, identifiers::{ ExecutionCompensationScopeId, ExecutionRunScopeId, ExecutionTaskScopeId, - HandProvisioningOperationId, ProviderAccountId, SandboxWorkspaceId, SessionId, TenantId, - WorkspaceCheckpointId, WorkspaceOperationId, + HandProvisioningOperationId, ModelId, ProviderAccountId, SandboxWorkspaceId, SessionId, + TenantId, WorkspaceCheckpointId, WorkspaceOperationId, }, + resource::ResourceBudget, sandbox_workspace::{ - DurabilityClass, ExecutionHandReleaseOwner, ExecutionHandReleaseReceipt, - ProviderStorageKind, ProviderStorageRef, SandboxWorkspaceScope, SandboxWorkspaceState, - WorkspaceBinding, WorkspaceCheckpointPublication, WorkspaceOperationKind, - WorkspacePostCommitState, WorkspaceRevisionRef, WorkspaceStorageOperation, + DurabilityClass, ExecutionHandContinuationDisposition, ExecutionHandReleaseOwner, + ExecutionHandReleaseReceipt, ProviderAccountStorageInventory, ProviderStorageKind, + ProviderStorageRef, SandboxWorkspaceScope, SandboxWorkspaceState, + TenantStoragePurgeRequest, WorkspaceAttachRequest, WorkspaceBinding, + WorkspaceCheckpointPublication, WorkspaceCheckpointPublishRequest, + WorkspaceConfirmedDisposition, WorkspaceOperationKind, WorkspaceOperationOutcome, + WorkspacePostCommitState, WorkspaceReconcileRequest, WorkspaceRestoreRequest, + WorkspaceRevisionRef, WorkspaceStorageDeleteRequest, WorkspaceStorageOperation, + WorkspaceStorageOperationResult, WorkspaceStoragePrepareRequest, }, + session::SessionMeta, + tools::{IdempotencyClass, ToolDiffStrategy, ToolInputShape, ToolOutput, ToolPolicySpec}, }; use moa_hands::core::{ leases::{ @@ -41,7 +61,11 @@ use moa_hands::core::{ repository::PostgresWorkspaceRepository, }, }; -use sqlx::postgres::PgPoolOptions; +use moa_hands::{ + ExecutionHandReleaseRequest, ExecutionHandRetentionRequest, HandRoute, ToolCallScope, + ToolRegistry, ToolRouter, local_development_sandbox_policy, +}; +use sqlx::{Row, postgres::PgPoolOptions}; use super::{database_url, seed_session}; @@ -84,7 +108,7 @@ async fn seed_cancelling_compensation( "output_schema": {}, "input_wait_policy": { "expiry": {"kind": "after", "delay_seconds": 1}, - "on_expiry": {"kind": "fail_run"} + "on_expiry": {"kind": "fail_task"} }, "nodes": [{ "id": "output", "requirement_ids": [], "depends_on": [], "when": null, @@ -359,6 +383,872 @@ fn create_request( } } +const CONTINUATION_PROVIDER: &str = "continuation-retention"; + +/// Provider double that honours `release_compute` and counts real teardown. +/// +/// It mirrors the shipped adapters on the three behaviours these scenarios depend +/// on: the checkpoint capacity charge is reserved before publication, compute is +/// destroyed inside publication only when the caller asked for a release, and +/// suspension is a declared capability rather than something inferred from a call. +struct ContinuationProvider { + capacity: PostgresWorkspaceCapacityRepository, + destroy_calls: AtomicUsize, + reconcile_calls: AtomicUsize, + suspend_calls: AtomicUsize, + resume_calls: AtomicUsize, + /// Mirrors Daytona (`true`) or local/E2B (`false`) compute-release ability. + suspends: bool, + /// Makes a declared suspension fail the way an unreachable provider would. + suspend_fails: bool, + /// Reports a suspended sandbox as stopped so reattach exercises resume. + suspended: AtomicBool, +} + +const CONTINUATION_CHECKPOINT_BYTES: u64 = 11; + +impl ContinuationProvider { + fn new(pool: &sqlx::PgPool) -> Self { + Self::with_suspension(pool, false, false) + } + + fn with_suspension(pool: &sqlx::PgPool, suspends: bool, suspend_fails: bool) -> Self { + Self { + capacity: PostgresWorkspaceCapacityRepository::new(pool.clone()), + destroy_calls: AtomicUsize::new(0), + reconcile_calls: AtomicUsize::new(0), + suspend_calls: AtomicUsize::new(0), + resume_calls: AtomicUsize::new(0), + suspends, + suspend_fails, + suspended: AtomicBool::new(false), + } + } + + fn confirmed(storage: Option) -> WorkspaceStorageOperationResult { + WorkspaceStorageOperationResult { + outcome: WorkspaceOperationOutcome::Confirmed, + confirmed_disposition: Some(WorkspaceConfirmedDisposition::ResourcePresent), + storage, + checkpoint_publication: None, + post_commit_state: None, + } + } + + fn mutable_storage(binding: &WorkspaceBinding) -> ProviderStorageRef { + ProviderStorageRef { + provider_account_id: binding.provider_account_id, + provider_account_generation: binding.provider_account_generation, + kind: ProviderStorageKind::MutableFilesystem, + resource_id: format!("mutable/{}", binding.workspace_id), + workspace_locator: None, + } + } + + fn published( + operation: &WorkspaceStorageOperation, + post_commit_state: WorkspacePostCommitState, + ) -> WorkspaceStorageOperationResult { + let generation = operation + .binding + .current_revision + .as_ref() + .map_or(1, |parent| parent.generation + 1); + let checkpoint_id = WorkspaceCheckpointId(operation.operation_id.0); + let storage = ProviderStorageRef { + provider_account_id: operation.binding.provider_account_id, + provider_account_generation: operation.binding.provider_account_generation, + kind: ProviderStorageKind::PortableCheckpoint, + resource_id: format!("checkpoint/{checkpoint_id}"), + workspace_locator: None, + }; + WorkspaceStorageOperationResult { + outcome: WorkspaceOperationOutcome::Confirmed, + confirmed_disposition: Some(WorkspaceConfirmedDisposition::ResourcePresent), + storage: Some(storage.clone()), + checkpoint_publication: Some(WorkspaceCheckpointPublication { + revision: WorkspaceRevisionRef { + checkpoint_id, + generation, + format_version: 1, + }, + storage, + manifest_digest: format!("sha256:manifest-{checkpoint_id}"), + logical_bytes: CONTINUATION_CHECKPOINT_BYTES, + }), + post_commit_state: Some(post_commit_state), + } + } +} + +#[async_trait] +impl HandProvider for ContinuationProvider { + fn provider_name(&self) -> &str { + CONTINUATION_PROVIDER + } + + fn capabilities(&self) -> HandProviderCapabilities { + HandProviderCapabilities { + revision: "continuation-retention-hands-v1".to_string(), + tiers: vec![SandboxTierCapabilities { + tier: SandboxTier::Container, + cpu: ResourceSupport::unbounded_only(), + memory: ResourceSupport::unbounded_only(), + ephemeral_disk: ResourceSupport::unbounded_only(), + egress_modes: vec![ + EgressMode::DenyAll, + EgressMode::AllowList, + EgressMode::Unrestricted, + ], + idle_enforcement: DeadlineEnforcement::DurableReaper, + max_lifetime_enforcement: DeadlineEnforcement::DurableReaper, + }], + } + } + + async fn provision(&self, spec: HandSpec) -> MoaResult { + Ok(HandHandle::docker(format!( + "continuation-retention-{}", + spec.provisioning_operation_id + ))) + } + + async fn provisioned_hands( + &self, + _provider_account_id: ProviderAccountId, + _provider_account_generation: u64, + _operation_id: HandProvisioningOperationId, + ) -> MoaResult> { + Ok(Vec::new()) + } + + async fn execute( + &self, + _handle: &HandHandle, + _tool: &str, + _input: &str, + ) -> MoaResult { + Err(MoaError::Unsupported( + "tool execution is outside the continuation-retention scenario".to_string(), + )) + } + + async fn status(&self, _handle: &HandHandle) -> MoaResult { + Ok(if self.suspended.load(Ordering::SeqCst) { + HandStatus::Stopped + } else { + HandStatus::Running + }) + } + + fn supports_suspend(&self) -> bool { + self.suspends + } + + async fn suspend(&self, _handle: &HandHandle) -> MoaResult<()> { + self.suspend_calls.fetch_add(1, Ordering::SeqCst); + if !self.suspends { + return Err(MoaError::Unsupported( + "this continuation provider cannot release compute".to_string(), + )); + } + if self.suspend_fails { + return Err(MoaError::ProviderError( + "continuation provider suspend is unreachable".to_string(), + )); + } + self.suspended.store(true, Ordering::SeqCst); + Ok(()) + } + + async fn resume(&self, _handle: &HandHandle) -> MoaResult<()> { + self.resume_calls.fetch_add(1, Ordering::SeqCst); + self.suspended.store(false, Ordering::SeqCst); + Ok(()) + } + + async fn destroy(&self, _handle: &HandHandle) -> MoaResult<()> { + self.destroy_calls.fetch_add(1, Ordering::SeqCst); + self.suspended.store(false, Ordering::SeqCst); + Ok(()) + } +} + +#[async_trait] +impl SandboxStorageProvider for ContinuationProvider { + fn storage_provider_name(&self) -> &str { + CONTINUATION_PROVIDER + } + + async fn enumerate_account_storage( + &self, + provider_account_id: ProviderAccountId, + provider_account_generation: u64, + ) -> MoaResult { + Ok(ProviderAccountStorageInventory { + provider_account_id, + provider_account_generation, + observed_at: Utc::now(), + resources: Vec::new(), + }) + } + + async fn prepare_workspace_storage( + &self, + request: WorkspaceStoragePrepareRequest, + ) -> MoaResult { + Ok(Self::confirmed(Some(Self::mutable_storage( + &request.operation.binding, + )))) + } + + async fn attach_workspace( + &self, + request: WorkspaceAttachRequest, + ) -> MoaResult { + Ok(Self::confirmed(Some(Self::mutable_storage( + &request.operation.binding, + )))) + } + + async fn publish_workspace_checkpoint( + &self, + request: WorkspaceCheckpointPublishRequest, + ) -> MoaResult { + self.capacity + .reserve_checkpoint_publication(&request.operation, CONTINUATION_CHECKPOINT_BYTES) + .await?; + let post_commit_state = if request.release_compute { + ::destroy(self, &request.hand).await?; + WorkspacePostCommitState::ComputeDestroyed + } else { + WorkspacePostCommitState::AttachmentRetained + }; + Ok(Self::published(&request.operation, post_commit_state)) + } + + async fn restore_workspace( + &self, + _request: WorkspaceRestoreRequest, + ) -> MoaResult { + Ok(Self::confirmed(None)) + } + + async fn delete_workspace_storage( + &self, + _request: WorkspaceStorageDeleteRequest, + ) -> MoaResult { + Err(MoaError::Unsupported( + "delete is outside the continuation-retention scenario".to_string(), + )) + } + + async fn delete_tenant_storage_resource( + &self, + _request: TenantStoragePurgeRequest, + ) -> MoaResult { + Err(MoaError::Unsupported( + "tenant purge is outside the continuation-retention scenario".to_string(), + )) + } + + async fn reconcile_workspace_operation( + &self, + request: WorkspaceReconcileRequest, + ) -> MoaResult { + self.reconcile_calls.fetch_add(1, Ordering::SeqCst); + Ok(Self::published( + request.operation(), + WorkspacePostCommitState::AttachmentRetained, + )) + } + + async fn verify_workspace_storage(&self, _storage: &ProviderStorageRef) -> MoaResult { + Ok(true) + } +} + +fn continuation_router(pool: &sqlx::PgPool, provider: Arc) -> ToolRouter { + let mut registry = ToolRegistry::new(); + registry.register_hand( + "continuation_route_anchor", + "exposes the configured continuation provider route", + serde_json::json!({ "type": "object", "additionalProperties": false }), + ToolPolicySpec { + risk_level: RiskLevel::Low, + default_effect: ActionPolicyEffect::Allow, + action_class: ActionClass::Read, + input_shape: ToolInputShape::Json, + diff_strategy: ToolDiffStrategy::None, + }, + IdempotencyClass::Idempotent, + ); + registry.retarget_hand_tools(vec![HandRoute { + provider: CONTINUATION_PROVIDER.to_string(), + tier: SandboxTier::Container, + policy: SandboxPolicySnapshot::builtin(BuiltinPolicyRevision::RouteUnset), + }]); + let mut hand_providers: HashMap> = HashMap::new(); + hand_providers.insert( + CONTINUATION_PROVIDER.to_string(), + Arc::clone(&provider) as Arc, + ); + ToolRouter::new(registry, hand_providers, local_development_sandbox_policy()) + .with_sandbox_storage_provider(Arc::clone(&provider) as Arc) + .expect("register continuation storage provider") + .with_workspace_repositories(pool.clone()) + .with_hand_lease_store(Arc::new(PostgresHandLeaseStore::new(pool.clone()))) +} + +async fn committed_active_hand_reservations( + pool: &sqlx::PgPool, + tenant_id: TenantId, + workspace_id: SandboxWorkspaceId, +) -> i64 { + sqlx::query( + "SELECT count(*)::BIGINT AS live FROM moa.sandbox_capacity_reservations \ + WHERE tenant_id = $1 AND workspace_id = $2 \ + AND resource_dimension = 'active_hands' \ + AND reservation_state IN ('pending', 'committed')", + ) + .bind(tenant_id) + .bind(workspace_id) + .fetch_one(pool) + .await + .expect("count live active-hand reservations") + .try_get::("live") + .expect("decode live active-hand reservations") +} + +/// One attached execution-task sandbox ready for a continuation-boundary scenario. +struct ContinuationFixture { + _test_db: moa_test_support::postgres::TestDb, + pool: sqlx::PgPool, + tenant_id: TenantId, + session_id: SessionId, + account_id: ProviderAccountId, + workspace_id: SandboxWorkspaceId, + workspace_scope: SandboxWorkspaceScope, + session: SessionMeta, + lease_scope: String, + provider: Arc, + router: ToolRouter, + workspaces: PostgresWorkspaceRepository, + leases: PostgresHandLeaseStore, +} + +/// Attaches one execution-task workspace onto a provider with the given suspend ability. +async fn continuation_fixture(suspends: bool, suspend_fails: bool) -> ContinuationFixture { + let test_db = moa_test_support::postgres::bootstrap_test_db() + .await + .expect("bootstrap isolated current-schema Postgres"); + let pool = test_db.store().pool().clone(); + let tenant_id = TenantId::new(); + let session_id = SessionId::new(); + let account_id = ProviderAccountId::new(); + let workspace_id = SandboxWorkspaceId::new(); + seed_session(&pool, session_id, tenant_id).await; + sqlx::query( + r#" + INSERT INTO moa.sandbox_provider_accounts ( + provider_account_id, generation, provider, isolation_cell, + organization_fingerprint, configured_limits + ) VALUES ($1, 1, $2, $3, $4, '{}'::jsonb) + "#, + ) + .bind(account_id) + .bind(CONTINUATION_PROVIDER) + .bind(format!("continuation-{account_id}")) + .bind(format!("continuation-org-{account_id}")) + .execute(&pool) + .await + .expect("seed continuation provider account"); + let (run_id, _, _) = seed_cancelling_compensation(&pool, tenant_id, session_id).await; + let task_id = seed_cancelling_task(&pool, tenant_id, run_id, "continuation-suspend").await; + let workspace_scope = SandboxWorkspaceScope::ExecutionTask { run_id, task_id }; + let workspaces = PostgresWorkspaceRepository::new(pool.clone()); + workspaces + .create(&CreateWorkspaceRequest { + workspace_id, + tenant_id, + scope: workspace_scope.clone(), + provider: CONTINUATION_PROVIDER.to_string(), + provider_account_id: account_id, + provider_account_generation: 1, + durability_class: DurabilityClass::PortableFilesystem, + retention_deadline_at: None, + }) + .await + .expect("create continuation task workspace"); + + let provider = Arc::new(ContinuationProvider::with_suspension( + &pool, + suspends, + suspend_fails, + )); + let router = continuation_router(&pool, Arc::clone(&provider)); + let session = SessionMeta { + id: session_id, + tenant_id, + model: ModelId::new("continuation-suspend-model"), + ..SessionMeta::default() + }; + router + .attach_managed_workspace(&session, &workspace_scope, workspace_id) + .await + .expect("attach must materialize provider compute and storage"); + let leases = PostgresHandLeaseStore::new(pool.clone()); + ContinuationFixture { + _test_db: test_db, + pool, + tenant_id, + session_id, + account_id, + workspace_id, + workspace_scope, + session, + lease_scope: format!("execution:{run_id}:{task_id}"), + provider, + router, + workspaces, + leases, + } +} + +fn continuation_retention_request<'a>( + fixture: &'a ContinuationFixture, + retention_deadline_at: chrono::DateTime, +) -> ExecutionHandRetentionRequest<'a> { + let SandboxWorkspaceScope::ExecutionTask { run_id, task_id } = fixture.workspace_scope else { + panic!("continuation fixture always owns an execution-task workspace"); + }; + ExecutionHandRetentionRequest { + session: &fixture.session, + run_id, + task_id, + logical_generation: 1, + attempt_generation: 1, + retention_deadline_at, + scope: ToolCallScope::unbounded().with_budget(ResourceBudget::until( + Utc::now() + ChronoDuration::minutes(5), + )), + } +} + +#[tokio::test] +#[ignore = "requires Postgres for an isolated current-schema test database"] +async fn continuation_suspends_compute_and_returns_its_admission_slot_db() { + // Pins: on a provider that can genuinely release compute, a continuation boundary + // stops the sandbox inside the yield rather than leaving it hot for the reaper, + // keeps the lease and handle so the next slice reattaches, and hands the + // `ActiveHands` slot back so a runnable task can be admitted into it. + let fixture = continuation_fixture(true, false).await; + assert_eq!( + committed_active_hand_reservations(&fixture.pool, fixture.tenant_id, fixture.workspace_id) + .await, + 1 + ); + + let disposition = fixture + .router + .checkpoint_execution_hand_retaining_compute(continuation_retention_request( + &fixture, + Utc::now() + ChronoDuration::minutes(2), + )) + .await + .expect("a continuation boundary must publish its checkpoint"); + + assert_eq!( + disposition, + ExecutionHandContinuationDisposition::Suspended, + "a suspend-capable provider must take the suspend path" + ); + assert_eq!(fixture.provider.suspend_calls.load(Ordering::SeqCst), 1); + assert_eq!( + fixture.provider.destroy_calls.load(Ordering::SeqCst), + 0, + "suspension must not destroy the sandbox the next slice will reattach to" + ); + let retained = fixture + .workspaces + .get(fixture.tenant_id, fixture.workspace_id) + .await + .expect("load suspended workspace") + .expect("suspended workspace exists"); + assert_eq!( + retained.checkpoint_generation, 1, + "suspension must still advance the portable recovery head" + ); + let lease = fixture + .leases + .get( + fixture.tenant_id, + fixture.session_id, + &fixture.lease_scope, + CONTINUATION_PROVIDER, + ) + .await + .expect("load suspended continuation lease") + .expect("suspended continuation lease exists"); + assert_eq!( + lease.status, + HandLeaseStatus::Active, + "the lease must survive so the next slice can reattach the same filesystem" + ); + assert!(lease.handle.is_some()); + assert_eq!( + committed_active_hand_reservations(&fixture.pool, fixture.tenant_id, fixture.workspace_id) + .await, + 0, + "a suspended hand must not keep holding fleet admission capacity" + ); +} + +#[tokio::test] +#[ignore = "requires Postgres for an isolated current-schema test database"] +async fn suspended_continuation_hand_reattaches_by_rewinning_capacity_db() { + // Pins: reattaching a suspended sandbox is a fresh admission decision — the charge + // released at the boundary is re-won and the provider is actually resumed, rather + // than the hand being handed back warm while the fleet gauge under-counts it. + let fixture = continuation_fixture(true, false).await; + fixture + .router + .checkpoint_execution_hand_retaining_compute(continuation_retention_request( + &fixture, + Utc::now() + ChronoDuration::minutes(2), + )) + .await + .expect("suspend the continuation sandbox"); + assert_eq!( + committed_active_hand_reservations(&fixture.pool, fixture.tenant_id, fixture.workspace_id) + .await, + 0 + ); + + fixture + .router + .attach_managed_workspace( + &fixture.session, + &fixture.workspace_scope, + fixture.workspace_id, + ) + .await + .expect("the next slice must reattach the suspended sandbox"); + + assert_eq!( + fixture.provider.resume_calls.load(Ordering::SeqCst), + 1, + "a stopped sandbox must be resumed, not dispatched into while stopped" + ); + assert_eq!( + fixture.provider.destroy_calls.load(Ordering::SeqCst), + 0, + "an admitted reattach must reuse the warm sandbox" + ); + assert_eq!( + committed_active_hand_reservations(&fixture.pool, fixture.tenant_id, fixture.workspace_id) + .await, + 1, + "resuming compute must charge the fleet for it again" + ); +} + +#[tokio::test] +#[ignore = "requires Postgres for an isolated current-schema test database"] +async fn saturated_reattach_drops_the_suspended_hand_instead_of_resuming_db() { + // Pins: a suspended sandbox has no reserved claim on its old slot. When the fleet + // filled up while it was stopped, reattach refuses to resume it — which is safe + // only because the boundary published its checkpoint before suspending, making the + // eviction a cache miss rather than lost work. + let fixture = continuation_fixture(true, false).await; + fixture + .router + .checkpoint_execution_hand_retaining_compute(continuation_retention_request( + &fixture, + Utc::now() + ChronoDuration::minutes(2), + )) + .await + .expect("suspend the continuation sandbox"); + + // Every active-hands slot in the provider account is now spoken for. + sqlx::query( + "UPDATE moa.sandbox_provider_accounts \ + SET configured_limits = '{\"active_hands\": 0}'::jsonb \ + WHERE provider_account_id = $1 AND generation = 1", + ) + .bind(fixture.account_id) + .execute(&fixture.pool) + .await + .expect("saturate the provider account"); + + let error = fixture + .router + .attach_managed_workspace( + &fixture.session, + &fixture.workspace_scope, + fixture.workspace_id, + ) + .await + .expect_err("a saturated fleet must not hand back the warm slot for free"); + + assert!( + error.to_string().contains("capacity"), + "reattach must fail on admission, not on some unrelated fault: {error}" + ); + assert_eq!( + fixture.provider.resume_calls.load(Ordering::SeqCst), + 0, + "compute must never restart before its admission slot is re-won" + ); + assert_eq!( + committed_active_hand_reservations(&fixture.pool, fixture.tenant_id, fixture.workspace_id) + .await, + 0, + "a refused reattach must leave the fleet charge released" + ); +} + +#[tokio::test] +#[ignore = "requires Postgres for an isolated current-schema test database"] +async fn failed_suspension_falls_back_to_release_instead_of_staying_hot_db() { + // Pins: a declared-but-failing suspension never leaves a hand hot on a bet that + // already lost. It reports the fallback so the caller finishes the ordinary + // checkpoint-and-destroy path, which returns the capacity. + let fixture = continuation_fixture(true, true).await; + + let disposition = fixture + .router + .checkpoint_execution_hand_retaining_compute(continuation_retention_request( + &fixture, + Utc::now() + ChronoDuration::minutes(2), + )) + .await + .expect("a failed suspension is non-fatal; the checkpoint still publishes"); + + assert_eq!( + disposition, + ExecutionHandContinuationDisposition::SuspendFailed + ); + assert_eq!(fixture.provider.suspend_calls.load(Ordering::SeqCst), 1); + + let SandboxWorkspaceScope::ExecutionTask { run_id, task_id } = fixture.workspace_scope else { + panic!("continuation fixture always owns an execution-task workspace"); + }; + fixture + .router + .checkpoint_and_release_execution_hand(ExecutionHandReleaseRequest { + session: &fixture.session, + run_id, + owner: ExecutionHandReleaseOwner::Task { + task_id, + logical_generation: 1, + }, + attempt_generation: 1, + scope: ToolCallScope::unbounded().with_budget(ResourceBudget::until( + Utc::now() + ChronoDuration::minutes(5), + )), + }) + .await + .expect("the fallback release must complete"); + + assert_eq!(fixture.provider.destroy_calls.load(Ordering::SeqCst), 1); + assert_eq!( + committed_active_hand_reservations(&fixture.pool, fixture.tenant_id, fixture.workspace_id) + .await, + 0, + "the fallback release must return the active-hands slot" + ); +} + +#[tokio::test] +#[ignore = "requires Postgres for an isolated current-schema test database"] +async fn continuation_retains_the_hand_while_a_park_releases_it_db() { + // Pins: on a provider that cannot release compute, a plain model/tool boundary + // publishes its checkpoint and keeps the exact sandbox, its lease, and its admitted + // `ActiveHands` slot — bounded by a shortened idle deadline the reaper owns — while + // the very next genuine park on the same sandbox destroys compute and gives the + // capacity back. + let test_db = moa_test_support::postgres::bootstrap_test_db() + .await + .expect("bootstrap isolated current-schema Postgres"); + let pool = test_db.store().pool().clone(); + let tenant_id = TenantId::new(); + let session_id = SessionId::new(); + let account_id = ProviderAccountId::new(); + let workspace_id = SandboxWorkspaceId::new(); + seed_session(&pool, session_id, tenant_id).await; + sqlx::query( + r#" + INSERT INTO moa.sandbox_provider_accounts ( + provider_account_id, generation, provider, isolation_cell, + organization_fingerprint, configured_limits + ) VALUES ($1, 1, $2, $3, $4, '{}'::jsonb) + "#, + ) + .bind(account_id) + .bind(CONTINUATION_PROVIDER) + .bind(format!("continuation-{account_id}")) + .bind(format!("continuation-org-{account_id}")) + .execute(&pool) + .await + .expect("seed continuation provider account"); + let (run_id, _, _) = seed_cancelling_compensation(&pool, tenant_id, session_id).await; + let task_id = seed_cancelling_task(&pool, tenant_id, run_id, "continuation-retention").await; + let workspace_scope = SandboxWorkspaceScope::ExecutionTask { run_id, task_id }; + let workspaces = PostgresWorkspaceRepository::new(pool.clone()); + workspaces + .create(&CreateWorkspaceRequest { + workspace_id, + tenant_id, + scope: workspace_scope.clone(), + provider: CONTINUATION_PROVIDER.to_string(), + provider_account_id: account_id, + provider_account_generation: 1, + durability_class: DurabilityClass::PortableFilesystem, + retention_deadline_at: None, + }) + .await + .expect("create continuation task workspace"); + + let provider = Arc::new(ContinuationProvider::new(&pool)); + let router = continuation_router(&pool, Arc::clone(&provider)); + let session = SessionMeta { + id: session_id, + tenant_id, + model: ModelId::new("continuation-retention-model"), + ..SessionMeta::default() + }; + router + .attach_managed_workspace(&session, &workspace_scope, workspace_id) + .await + .expect("attach must materialize provider compute and storage"); + let leases = PostgresHandLeaseStore::new(pool.clone()); + let lease_scope = format!("execution:{run_id}:{task_id}"); + let provisioned = leases + .get(tenant_id, session_id, &lease_scope, CONTINUATION_PROVIDER) + .await + .expect("load provisioned continuation lease") + .expect("continuation lease exists"); + assert_eq!(provisioned.status, HandLeaseStatus::Active); + assert_eq!( + committed_active_hand_reservations(&pool, tenant_id, workspace_id).await, + 1 + ); + + let retention_deadline_at = Utc::now() + ChronoDuration::minutes(2); + let disposition = router + .checkpoint_execution_hand_retaining_compute(ExecutionHandRetentionRequest { + session: &session, + run_id, + task_id, + logical_generation: 1, + attempt_generation: 1, + retention_deadline_at, + scope: ToolCallScope::unbounded().with_budget(ResourceBudget::until( + Utc::now() + ChronoDuration::minutes(5), + )), + }) + .await + .expect("a continuation boundary must publish its checkpoint"); + + assert_eq!( + disposition, + ExecutionHandContinuationDisposition::RetainedHot, + "a provider that cannot release compute must fall back to bounded hot retention" + ); + assert_eq!( + provider.suspend_calls.load(Ordering::SeqCst), + 0, + "a provider that declares no suspension must never be asked to suspend" + ); + assert_eq!( + provider.destroy_calls.load(Ordering::SeqCst), + 0, + "a continuation boundary must not destroy the sandbox it is about to resume in" + ); + let retained_workspace = workspaces + .get(tenant_id, workspace_id) + .await + .expect("load retained workspace") + .expect("retained workspace exists"); + assert_eq!(retained_workspace.state, SandboxWorkspaceState::Active); + assert_eq!( + retained_workspace.checkpoint_generation, 1, + "retention must still advance the portable recovery head" + ); + let retained_lease = leases + .get(tenant_id, session_id, &lease_scope, CONTINUATION_PROVIDER) + .await + .expect("load retained continuation lease") + .expect("retained continuation lease exists"); + assert_eq!(retained_lease.status, HandLeaseStatus::Active); + assert!(retained_lease.handle.is_some()); + assert_eq!(retained_lease.generation, provisioned.generation); + let bounded_idle = retained_lease + .idle_expires_at + .expect("a retained hand must carry a reaper-owned retention deadline"); + assert!( + bounded_idle <= retention_deadline_at, + "retention must bound the retained hand at or before its requested deadline" + ); + assert!( + bounded_idle > Utc::now(), + "retention must leave the next slice a window to reattach" + ); + assert_eq!( + committed_active_hand_reservations(&pool, tenant_id, workspace_id).await, + 1, + "a retained hand keeps its admitted active-hands slot" + ); + + // The park path now runs against the exact sandbox the continuation kept alive. + let receipt = router + .checkpoint_and_release_execution_hand(ExecutionHandReleaseRequest { + session: &session, + run_id, + owner: ExecutionHandReleaseOwner::Task { + task_id, + logical_generation: 1, + }, + attempt_generation: 1, + scope: ToolCallScope::unbounded().with_budget(ResourceBudget::until( + Utc::now() + ChronoDuration::minutes(5), + )), + }) + .await + .expect("a genuine park must release the retained sandbox"); + + assert_eq!( + provider.destroy_calls.load(Ordering::SeqCst), + 1, + "a park must destroy exactly the hand the continuation retained" + ); + assert_eq!( + receipt.checkpoint_generation, + Some(2), + "the park publishes the next head on top of the retained continuation checkpoint" + ); + let released_workspace = workspaces + .get(tenant_id, workspace_id) + .await + .expect("load released workspace") + .expect("released workspace exists"); + assert_eq!(released_workspace.state, SandboxWorkspaceState::Ready); + let released_lease = leases + .get(tenant_id, session_id, &lease_scope, CONTINUATION_PROVIDER) + .await + .expect("load released continuation lease") + .expect("released continuation lease exists"); + assert_eq!(released_lease.status, HandLeaseStatus::Destroyed); + assert!(released_lease.handle.is_none()); + assert_eq!( + committed_active_hand_reservations(&pool, tenant_id, workspace_id).await, + 0, + "a park must return the active-hands slot to the fleet" + ); + assert_eq!(provider.reconcile_calls.load(Ordering::SeqCst), 0); +} + #[tokio::test] #[ignore = "requires a fresh V60 compose Postgres via MOA_DATABASE_URL"] async fn cancelling_task_without_owned_compute_gets_exact_absence_receipt_db() { diff --git a/crates/moa-hands/tests/hands_offline/sandbox_workspace_recovery_offline.rs b/crates/moa-hands/tests/hands_offline/sandbox_workspace_recovery_offline.rs index 081c52d54..492aa6eca 100644 --- a/crates/moa-hands/tests/hands_offline/sandbox_workspace_recovery_offline.rs +++ b/crates/moa-hands/tests/hands_offline/sandbox_workspace_recovery_offline.rs @@ -45,7 +45,6 @@ struct ProviderIoSnapshot { execute: u32, health: u32, status: u32, - pause: u32, resume: u32, destroy: u32, } @@ -172,15 +171,6 @@ impl HandProvider for ObservableProvider { Ok(HandStatus::Running) } - async fn pause(&self, _handle: &HandHandle) -> Result<()> { - self.state - .lock() - .expect("observable provider state should not be poisoned") - .io - .pause += 1; - Ok(()) - } - async fn resume(&self, _handle: &HandHandle) -> Result<()> { self.state .lock() diff --git a/crates/moa-memory/ingest/prompts/judge.txt b/crates/moa-memory/ingest/prompts/judge.md similarity index 100% rename from crates/moa-memory/ingest/prompts/judge.txt rename to crates/moa-memory/ingest/prompts/judge.md diff --git a/crates/moa-memory/ingest/src/contradiction.rs b/crates/moa-memory/ingest/src/contradiction.rs index 47768ed64..13ff09288 100644 --- a/crates/moa-memory/ingest/src/contradiction.rs +++ b/crates/moa-memory/ingest/src/contradiction.rs @@ -34,7 +34,7 @@ const DEFAULT_FAST_BUDGET: Duration = Duration::from_millis(250); const DEFAULT_SLOW_BUDGET: Duration = Duration::from_secs(5); const DEFAULT_JUDGE_BUDGET: Duration = Duration::from_millis(200); const CACHE_CAPACITY: u64 = 10_000; -const JUDGE_PROMPT: &str = include_str!("../prompts/judge.txt"); +const JUDGE_PROMPT: &str = include_str!("../prompts/judge.md"); /// Conflict routing decision returned by contradiction detection. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] diff --git a/crates/moa-migrations/migrations/postgres/V000059__long_horizon_execution.sql b/crates/moa-migrations/migrations/postgres/V000059__long_horizon_execution.sql index 5bb25760e..a6df84670 100644 --- a/crates/moa-migrations/migrations/postgres/V000059__long_horizon_execution.sql +++ b/crates/moa-migrations/migrations/postgres/V000059__long_horizon_execution.sql @@ -165,8 +165,6 @@ AS $$ SELECT CASE candidate ->> 'kind' WHEN 'fail_task' THEN candidate = '{"kind":"fail_task"}'::JSONB - WHEN 'fail_run' THEN - candidate = '{"kind":"fail_run"}'::JSONB WHEN 'continue_with' THEN moa.execution_json_object_has_exact_keys(candidate, ARRAY['kind', 'output']) ELSE FALSE @@ -310,6 +308,13 @@ ALTER TABLE moa.execution_run ADD COLUMN last_progress_at TIMESTAMPTZ NOT NULL DEFAULT now(), ADD COLUMN pause_requested_at TIMESTAMPTZ, ADD COLUMN paused_at TIMESTAMPTZ, + -- Consecutive crashed activations since the last acknowledged wake. This is + -- deliberately not generation-scoped: the reset rides the existing healthy + -- wake acknowledgement, so failures can carry across a generation bump, but + -- one healthy activation clears it and a run that cannot complete even one + -- is genuinely not progressing. + ADD COLUMN activation_failure_count BIGINT NOT NULL DEFAULT 0 + CHECK (activation_failure_count >= 0), ADD COLUMN ready_task_count BIGINT NOT NULL DEFAULT 0 CHECK (ready_task_count >= 0), ADD COLUMN active_task_count BIGINT NOT NULL DEFAULT 0 @@ -430,6 +435,67 @@ ALTER TABLE moa.execution_run 'completed', 'partial', 'blocked', 'unsupported', 'failed', 'cancelled' )); +-- Retire the two terminal-vocabulary values this architecture supersedes. +-- +-- `scheduler_no_progress` was produced only by the deleted whole-plan scheduler's +-- `ScheduleDecision::NoProgress`. The incremental controller cannot reach the state it +-- named: `cancel_unmaterialized_dependents_in_tx` terminalizes the whole transitive +-- dependent closure in the same transaction as the node failure and raises on a partial +-- cascade, and any residual stall is settled by the always-armed run deadline as +-- `deadline_exceeded`, which is the more actionable terminal. +-- +-- `dependency_failed` is a task-level failure class, but a dependency failure is now a +-- node-level cancellation: the cascade asserts every dependent it cancels carries zero +-- tasks, so there is no task to classify. Producing it would require synthesizing failed +-- rows for work that never ran, which would displace the real root-cause class that +-- `load_earliest_typed_task_failure` surfaces on the run terminal. +-- +-- This patches the live function rather than editing V27 in place: V55 already rewrites +-- this body through `pg_get_functiondef` + `replace`, so an in-place V27 edit would reach +-- a fresh database and never reach one that already ran V27. Each removal is guarded on +-- its exact predecessor text so a drifted baseline fails loudly instead of silently +-- leaving an accepted value behind. `execution_run_terminal_evidence` needs no companion +-- change: V55 already replaced that constraint with one that delegates entirely to this +-- function and never repeats the class list. +DO $execution_long_horizon_terminal_reason$ +DECLARE + definition TEXT; + -- Each anchor starts at the branch indent and ends with its own newline, so the + -- removal takes the whole branch and leaves the surrounding lines intact. + old_no_progress_validation TEXT := $old$ WHEN 'scheduler_no_progress' THEN + terminal_cause = '{"kind":"scheduler_no_progress"}'::JSONB +$old$; + old_no_progress_reason TEXT := $old$ WHEN 'scheduler_no_progress' THEN + RETURN CASE status_value + WHEN 'unsupported' THEN 'unsupported_plan' + WHEN 'partial' THEN 'no_progress' + WHEN 'blocked' THEN 'no_progress' + WHEN 'failed' THEN 'no_progress' + ELSE NULL + END; +$old$; + old_failure_classes TEXT := + $old$ 'retryable','dependency_failed','invalid_input','invalid_output',$old$; + new_failure_classes TEXT := + $new$ 'retryable','invalid_input','invalid_output',$new$; +BEGIN + SELECT pg_get_functiondef( + 'moa.execution_terminal_reason_for(text,jsonb,text)'::REGPROCEDURE + ) INTO definition; + IF position(old_no_progress_validation IN definition) = 0 + OR position(old_no_progress_reason IN definition) = 0 + OR position(old_failure_classes IN definition) = 0 THEN + RAISE EXCEPTION + 'execution terminal reason function drifted before V59' + USING ERRCODE = '55000'; + END IF; + definition := replace(definition, old_no_progress_validation, ''); + definition := replace(definition, old_no_progress_reason, ''); + definition := replace(definition, old_failure_classes, new_failure_classes); + EXECUTE definition; +END +$execution_long_horizon_terminal_reason$; + DROP INDEX moa.execution_run_nonterminal_idx; CREATE INDEX execution_run_nonterminal_idx ON moa.execution_run (status, updated_at, run_uid) @@ -439,6 +505,18 @@ CREATE INDEX execution_run_nonterminal_idx 'waiting_replan', 'pause_requested', 'pausing', 'paused', 'compensating' ); +-- The exact-deadline invariant guard scans for overdue nonterminal runs each +-- reconcile. It shares this predicate with execution_run_nonterminal_idx, but +-- that index leads on status and so cannot answer a budget_deadline_at range; +-- leading on the deadline keeps the guard an index-only scan. +CREATE INDEX execution_run_overdue_deadline_idx + ON moa.execution_run (budget_deadline_at, run_uid) + WHERE budget_deadline_at IS NOT NULL AND status IN ( + 'awaiting_confirmation', 'queued', 'running', 'waiting_input', + 'waiting_review', 'waiting_signal', 'waiting_timer', 'waiting_external', + 'waiting_replan', 'pause_requested', 'pausing', 'paused', 'compensating' + ); + CREATE INDEX execution_run_activation_idx ON moa.execution_run (activation_state, next_wake_at, updated_at, run_uid) WHERE activation_state IN ('queued', 'advancing'); @@ -644,13 +722,22 @@ CREATE INDEX execution_task_ready_idx ON moa.execution_task (tenant_id, run_uid, ready_at, node_id, item_key, task_id) WHERE status = 'ready'; -CREATE INDEX execution_task_active_attempt_watchdog_idx - ON moa.execution_task (attempt_deadline_at, tenant_id, run_uid, task_id) - WHERE status = 'running' AND attempt_state = 'running'; - -CREATE INDEX execution_task_cancelling_reconciliation_idx - ON moa.execution_task (last_progress_at, tenant_id, run_uid, task_id) - WHERE attempt_state = 'cancelling'; +-- Oldest live attempt, for the stuck-attempt guard. A deadline-keyed index over +-- the same rows was deliberately dropped as unused: nothing orders or filters on +-- attempt_deadline_at, and a btree leading on it could not answer this ordering +-- anyway. This one is backed by a measured index-only scan. +CREATE INDEX execution_task_active_attempt_started_idx + ON moa.execution_task (attempt_started_at, task_id) + WHERE status = 'running' AND attempt_state = 'running' + AND attempt_started_at IS NOT NULL; + +-- Fleet admission scans the ready queue per tenant, not per run: the fair-tenant +-- probe tests one tenant's due ready work and the per-item pick orders that +-- tenant's queue by ready_at. Leading on run_uid forces those to sort the whole +-- tenant queue once per admitted item, so give them their own ordered window. +CREATE INDEX execution_task_tenant_ready_order_idx + ON moa.execution_task (tenant_id, ready_at, task_id) + WHERE status = 'ready' AND ready_at IS NOT NULL; CREATE INDEX execution_task_terminal_retention_idx ON moa.execution_task (tenant_id, completed_at, run_uid, task_id) @@ -711,17 +798,13 @@ SET attempt_generation = generation, END, last_progress_at = updated_at; -CREATE INDEX execution_compensation_active_watchdog_idx - ON moa.execution_compensation ( - attempt_deadline_at, tenant_id, run_uid, compensation_id - ) - WHERE status = 'running' AND attempt_state = 'running'; - -CREATE INDEX execution_compensation_cancelling_reconciliation_idx - ON moa.execution_compensation ( - last_progress_at, tenant_id, run_uid, compensation_id - ) - WHERE attempt_state = 'cancelling'; +-- Rollback twin of execution_task_active_attempt_started_idx: a compensation +-- attempt holds the same active-compute reservation, so the stuck-attempt guard +-- takes an ordered minimum from each table and reduces the two rows. +CREATE INDEX execution_compensation_active_attempt_started_idx + ON moa.execution_compensation (attempt_started_at, compensation_id) + WHERE status = 'running' AND attempt_state = 'running' + AND attempt_started_at IS NOT NULL; CREATE INDEX execution_compensation_terminal_retention_idx ON moa.execution_compensation (tenant_id, completed_at, run_uid, compensation_id) @@ -1376,8 +1459,11 @@ CREATE TABLE moa.execution_trigger ( 'external_reconcile', 'external_start_recovery', 'schedule_occurrence', 'compensation_watchdog' )), + -- Triggers are never claimed and never dead-letter. Claiming, retry accounting, and + -- dead-lettering live entirely on moa.execution_dispatch_outbox, and a trigger + -- delivery always requires durable retry, so a trigger row only ever settles. state TEXT NOT NULL DEFAULT 'pending' CHECK (state IN ( - 'pending', 'dispatching', 'delivered', 'superseded', 'cancelled', 'dead_letter' + 'pending', 'delivered', 'superseded', 'cancelled' )), controller_generation BIGINT CHECK (controller_generation >= 1), attempt_generation BIGINT CHECK (attempt_generation >= 1), @@ -1387,11 +1473,9 @@ CREATE TABLE moa.execution_trigger ( occurrence_sequence BIGINT CHECK (occurrence_sequence >= 1), due_at TIMESTAMPTZ NOT NULL, payload JSONB NOT NULL DEFAULT '{}'::JSONB CHECK (jsonb_typeof(payload) = 'object'), - claim_owner TEXT, - claimed_at TIMESTAMPTZ, - claim_expires_at TIMESTAMPTZ, - delivery_attempts INTEGER NOT NULL DEFAULT 0 CHECK (delivery_attempts >= 0), delivered_at TIMESTAMPTZ, + -- Operator-facing only: rearm_external_start_recovery records why a start-recovery + -- trigger keeps rearming. Nothing in Rust reads it back. last_error TEXT CHECK (last_error IS NULL OR octet_length(last_error) <= 4096), created_at TIMESTAMPTZ NOT NULL DEFAULT now(), updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), @@ -1487,14 +1571,6 @@ CREATE TABLE moa.execution_trigger ( AND occurrence_sequence IS NULL ) ), - CONSTRAINT execution_trigger_claim_pair_check CHECK ( - (claim_owner IS NULL AND claimed_at IS NULL AND claim_expires_at IS NULL) - OR ( - claim_owner IS NOT NULL AND btrim(claim_owner) <> '' - AND claimed_at IS NOT NULL AND claim_expires_at IS NOT NULL - AND claim_expires_at > claimed_at - ) - ), CONSTRAINT execution_trigger_delivery_pair_check CHECK ( (state = 'delivered') = (delivered_at IS NOT NULL) ), @@ -1524,7 +1600,7 @@ CREATE UNIQUE INDEX execution_trigger_current_run_generation_uidx COALESCE(compensation_generation, 0), COALESCE(compensation_attempt_generation, 0) ) - WHERE state IN ('pending', 'dispatching') AND run_uid IS NOT NULL; + WHERE state = 'pending' AND run_uid IS NOT NULL; CREATE UNIQUE INDEX execution_trigger_schedule_occurrence_uidx ON moa.execution_trigger ( @@ -1536,17 +1612,9 @@ CREATE INDEX execution_trigger_due_idx ON moa.execution_trigger (due_at, tenant_id, trigger_uid) WHERE state = 'pending'; -CREATE INDEX execution_trigger_claim_expiry_idx - ON moa.execution_trigger (claim_expires_at, due_at, trigger_uid) - WHERE state = 'dispatching'; - -CREATE INDEX execution_trigger_dead_letter_idx - ON moa.execution_trigger (created_at, tenant_id, trigger_uid) - WHERE state = 'dead_letter'; - CREATE INDEX execution_trigger_run_wake_idx ON moa.execution_trigger (run_uid, due_at, trigger_uid) - WHERE run_uid IS NOT NULL AND state IN ('pending', 'dispatching'); + WHERE run_uid IS NOT NULL AND state = 'pending'; CREATE OR REPLACE FUNCTION moa.execution_attempt_cancel_payload_is_valid( candidate JSONB, @@ -1634,6 +1702,13 @@ CREATE TABLE moa.execution_dispatch_outbox ( CHECK (compensation_attempt_generation >= 1), not_before_at TIMESTAMPTZ NOT NULL DEFAULT now(), payload JSONB NOT NULL DEFAULT '{}'::JSONB CHECK (jsonb_typeof(payload) = 'object'), + -- Restate persists a completed invocation's response and replays it for any + -- later request carrying the same idempotency key. A reconciler repair that + -- returned a row to `pending` under its original `dispatch_uid` would attach + -- to that memoized completion instead of re-executing the target. This + -- counter gives each repair a distinct delivery identity to fold into the + -- key; it advances only on repair, so ordinary redelivery stays idempotent. + repair_epoch INTEGER NOT NULL DEFAULT 0 CHECK (repair_epoch >= 0), claim_owner TEXT, claimed_at TIMESTAMPTZ, claim_expires_at TIMESTAMPTZ, @@ -1972,7 +2047,6 @@ CREATE TABLE moa.execution_tenant_dispatch_state ( tenant_id UUID PRIMARY KEY, weight NUMERIC(20, 6) NOT NULL DEFAULT 1 CHECK (weight > 0), virtual_finish NUMERIC(30, 6) NOT NULL DEFAULT 0 CHECK (virtual_finish >= 0), - deficit NUMERIC(30, 6) NOT NULL DEFAULT 0, last_dispatched_at TIMESTAMPTZ, version BIGINT NOT NULL DEFAULT 1 CHECK (version >= 1), created_at TIMESTAMPTZ NOT NULL DEFAULT now(), @@ -2347,9 +2421,12 @@ BEGIN END IF; transition_allowed := CASE OLD.status + -- 'failed' without an attempt: a relative wait can resolve at wait entry + -- to a due time at or past the run deadline, which fails the task on its + -- own node rather than entering a wait that could never settle. WHEN 'pending' THEN NEW.status IN ( 'ready', 'reserved', 'waiting_review', 'waiting_signal', - 'waiting_timer', 'skipped', 'cancelled' + 'waiting_timer', 'failed', 'skipped', 'cancelled' ) WHEN 'ready' THEN NEW.status IN ('dispatching', 'reserved', 'cancelled') WHEN 'reserved' THEN NEW.status IN ('dispatching', 'running', 'cancelled') @@ -2421,7 +2498,15 @@ BEGIN IF NEW.ready_task_count < 0 OR NEW.active_task_count < 0 THEN RAISE EXCEPTION 'execution run task counters cannot be negative'; END IF; - IF OLD.status = 'pausing' AND NEW.active_task_count = 0 THEN + -- Draining the last active attempt completes a pause. Attempt settlement only + -- adjusts counters, so the promotion has to happen here. It must never rewrite + -- a status the writer chose: a terminal write that also zeroes the counter + -- (deadline, budget, terminal fence) would otherwise be swallowed into + -- 'paused', and the run could then never leave it, because 'paused' admits + -- only 'queued' and 'cancelled'. + IF OLD.status = 'pausing' + AND NEW.status = OLD.status + AND NEW.active_task_count = 0 THEN NEW.status := 'paused'; NEW.activation_state := 'paused'; NEW.paused_at := COALESCE(NEW.paused_at, now()); diff --git a/crates/moa-migrations/migrations/postgres/V000060__sandbox_active_compute_capacity.sql b/crates/moa-migrations/migrations/postgres/V000060__sandbox_active_compute_capacity.sql index 0d61a7f05..64d17458b 100644 --- a/crates/moa-migrations/migrations/postgres/V000060__sandbox_active_compute_capacity.sql +++ b/crates/moa-migrations/migrations/postgres/V000060__sandbox_active_compute_capacity.sql @@ -283,6 +283,74 @@ CREATE TRIGGER sandbox_execution_hand_release_receipt_delete_guard BEFORE DELETE ON moa.sandbox_execution_hand_release_receipts FOR EACH ROW EXECUTE FUNCTION moa.reject_execution_immutable_payload(); +-- Release receipts carry tenant_id, so tenant offboarding owns them and the +-- purge catalog must list them; the drift check in run_tenant_purge_batch scans +-- every tenant-scoped table in moa/public/analytics/pii_vault and aborts the +-- whole offboarding for an unregistered one. `sandbox_provider_inventory_claims` +-- has no tenant_id -- it is global maintenance authority keyed by provider +-- account -- so its absence from the catalog is correct. +CREATE TRIGGER moa_tenant_purge_fence_insert +AFTER INSERT ON moa.sandbox_execution_hand_release_receipts +REFERENCING NEW TABLE AS tenant_purge_new_rows +FOR EACH STATEMENT EXECUTE FUNCTION moa.guard_tenant_write_statement('tenant_id'); +CREATE TRIGGER moa_tenant_purge_fence_update +AFTER UPDATE ON moa.sandbox_execution_hand_release_receipts +REFERENCING OLD TABLE AS tenant_purge_old_rows NEW TABLE AS tenant_purge_new_rows +FOR EACH STATEMENT EXECUTE FUNCTION moa.guard_tenant_write_statement('tenant_id'); + +-- Every receipt FK is ON DELETE RESTRICT, so receipts must drain before +-- execution_task, execution_compensation, sandbox_workspaces, and +-- sandbox_workspace_checkpoints. The checkpoint stage is the earliest of the +-- four; shift through a remote range and take the slot immediately before it. +UPDATE moa.tenant_purge_catalog +SET stage_order = stage_order + 1000 +WHERE stage_order >= ( + SELECT stage_order FROM moa.tenant_purge_catalog + WHERE stage_name = 'moa.sandbox_workspace_checkpoints' +); + +UPDATE moa.tenant_purge_catalog +SET stage_order = stage_order - 999 +WHERE stage_order >= 1000; + +INSERT INTO moa.tenant_purge_catalog ( + stage_order, stage_name, table_schema, table_name, scope_mode, action_mode +) +SELECT checkpoint_stage.stage_order - 1, + 'moa.sandbox_execution_hand_release_receipts', + 'moa', + 'sandbox_execution_hand_release_receipts', + 'tenant_id', + 'delete' +FROM moa.tenant_purge_catalog AS checkpoint_stage +WHERE checkpoint_stage.stage_name = 'moa.sandbox_workspace_checkpoints'; + +COMMENT ON TABLE moa.tenant_purge_catalog IS + 'Closed 158-table tenant-offboarding residue surface. Fleet capacity-bucket rows, sandbox provider accounts, sandbox provider inventory claims, and inventory findings are global maintenance authority; the two nullable-scope simulator certification authority tables are also intentionally global and absent.'; + +DO $sandbox_hand_release_receipt_purge_function$ +DECLARE + predecessor TEXT; + replacement TEXT; +BEGIN + SELECT pg_get_functiondef('moa.run_tenant_purge_batch(uuid,text)'::REGPROCEDURE) + INTO predecessor; + IF predecessor NOT LIKE '%catalog_count <> 157%' + OR predecessor NOT LIKE '%exactly 157 tables%' THEN + RAISE EXCEPTION 'unexpected V59 tenant purge function definition' + USING ERRCODE = '55000'; + END IF; + replacement := replace(predecessor, 'catalog_count <> 157', 'catalog_count <> 158'); + replacement := replace(replacement, 'exactly 157 tables', 'exactly 158 tables'); + EXECUTE replacement; +END +$sandbox_hand_release_receipt_purge_function$; + +ALTER FUNCTION moa.run_tenant_purge_batch(UUID, TEXT) OWNER TO moa_owner; +REVOKE ALL ON FUNCTION moa.run_tenant_purge_batch(UUID, TEXT) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION moa.run_tenant_purge_batch(UUID, TEXT) + TO moa_app, moa_promoter, moa_workspace_maintenance; + CREATE FUNCTION moa.guard_pending_task_hand_release_attempt() RETURNS TRIGGER LANGUAGE plpgsql diff --git a/crates/moa-migrations/tests/run_idempotency_db/execution_and_security_catalog.rs b/crates/moa-migrations/tests/run_idempotency_db/execution_and_security_catalog.rs index a01f4d93a..8b9e0cda6 100644 --- a/crates/moa-migrations/tests/run_idempotency_db/execution_and_security_catalog.rs +++ b/crates/moa-migrations/tests/run_idempotency_db/execution_and_security_catalog.rs @@ -564,7 +564,7 @@ async fn execution_analytics_fresh_cutover_and_exact_contract_db() { ('failed','deadline_exceeded','deadline_exceeded') ), failure_class(value) AS ( VALUES - ('retryable'),('dependency_failed'),('invalid_input'), + ('retryable'),('invalid_input'), ('invalid_output'),('authorization_denied'),('budget_exceeded'), ('deadline_exceeded'),('cancelled'),('unsupported'),('terminal') ), task_failure AS ( @@ -607,17 +607,6 @@ async fn execution_analytics_fresh_cutover_and_exact_contract_db() { UNION ALL SELECT status,cause,'generated_plan',expected FROM task_failure UNION ALL - SELECT - status,'{"kind":"scheduler_no_progress"}'::JSONB, - 'generated_plan', - CASE status - WHEN 'unsupported' THEN 'unsupported_plan' - ELSE 'no_progress' - END - FROM ( - VALUES ('partial'),('blocked'),('unsupported'),('failed') - ) projection(status) - UNION ALL SELECT status, jsonb_build_object( @@ -673,6 +662,15 @@ async fn execution_analytics_fresh_cutover_and_exact_contract_db() { 'generated_plan'), ('failed', '{"kind":"task_failure","class":"not_a_class"}', + 'generated_plan'), + -- The two terminal-vocabulary values V59 retires. Both were accepted by + -- the V27 baseline, so these cells fail if the V59 patch to + -- moa.execution_terminal_reason_for stops applying. + ('blocked', + '{"kind":"scheduler_no_progress"}', + 'generated_plan'), + ('failed', + '{"kind":"task_failure","class":"dependency_failed"}', 'generated_plan') ) cell(status,cause,source_kind) WHERE moa.execution_terminal_reason_for( @@ -844,7 +842,7 @@ async fn execution_analytics_fresh_cutover_and_exact_contract_db() { 'expiry',jsonb_build_object( 'kind','after','delay_seconds',1 ), - 'on_expiry',jsonb_build_object('kind','fail_run') + 'on_expiry',jsonb_build_object('kind','fail_task') ), 'nodes','[]'::JSONB ), @@ -859,7 +857,7 @@ async fn execution_analytics_fresh_cutover_and_exact_contract_db() { 'expiry',jsonb_build_object( 'kind','after','delay_seconds',1 ), - 'on_expiry',jsonb_build_object('kind','fail_run') + 'on_expiry',jsonb_build_object('kind','fail_task') ), 'nodes','[]'::JSONB ), @@ -899,13 +897,13 @@ async fn execution_analytics_fresh_cutover_and_exact_contract_db() { 0,0,1,0,0 ); INSERT INTO moa.execution_action_review_outbox ( - review_uid,tenant_id,contact_id,run_uid,task_id,generation, + review_uid,tenant_id,contact_id,run_uid,operation_id,owner_kind,generation, resolution,traceparent,tracestate,task_traceparent,task_tracestate ) VALUES ( '00000000-0000-0000-0000-000000337043', '00000000-0000-0000-0000-000000337020',NULL, '00000000-0000-0000-0000-000000337041', - '00000000-0000-0000-0000-000000337042',1,'{}', + '00000000-0000-0000-0000-000000337042','task',1,'{}', '00-11111111111111111111111111111111-2222222222222222-01', 'a=one', '00-33333333333333333333333333333333-4444444444444444-00', @@ -988,7 +986,7 @@ async fn execution_analytics_fresh_cutover_and_exact_contract_db() { 'expiry',jsonb_build_object(\ 'kind','after','delay_seconds',1\ ),\ - 'on_expiry',jsonb_build_object('kind','fail_run')\ + 'on_expiry',jsonb_build_object('kind','fail_task')\ ),\ 'nodes','[]'::JSONB\ ),\ @@ -1003,7 +1001,7 @@ async fn execution_analytics_fresh_cutover_and_exact_contract_db() { 'expiry',jsonb_build_object(\ 'kind','after','delay_seconds',1\ ),\ - 'on_expiry',jsonb_build_object('kind','fail_run')\ + 'on_expiry',jsonb_build_object('kind','fail_task')\ ),\ 'nodes','[]'::JSONB\ ),\ @@ -1051,20 +1049,20 @@ async fn execution_analytics_fresh_cutover_and_exact_contract_db() { ) .await .is_err(); - let outbox_scope_rejected = target + let outbox_scope_rejected = postgres_error_fact(target .execute( "INSERT INTO moa.execution_action_review_outbox (\ - review_uid,tenant_id,contact_id,run_uid,task_id,generation,resolution\ + review_uid,tenant_id,contact_id,run_uid,operation_id,owner_kind,generation,resolution\ ) VALUES (\ '00000000-0000-0000-0000-000000337054',\ '00000000-0000-0000-0000-000000337020',\ '00000000-0000-0000-0000-000000337051',\ '00000000-0000-0000-0000-000000337041',\ - '00000000-0000-0000-0000-000000337042',1,'{}'\ + '00000000-0000-0000-0000-000000337042','task',1,'{}'\ )", ) .await - .is_err(); + .expect_err("a cross-scope action review row must be rejected")); target .execute( @@ -1294,8 +1292,8 @@ async fn execution_analytics_fresh_cutover_and_exact_contract_db() { assert_eq!(removed_mode_sql_state.as_deref(), Some("42703")); assert_eq!(invalid_insert_residue, 0); assert!(!old_route_envelope_valid); - assert_eq!(valid_terminal_cells, 71); - assert_eq!(invalid_terminal_cells, 7); + assert_eq!(valid_terminal_cells, 63); + assert_eq!(invalid_terminal_cells, 9); assert_eq!(provenance_matrix, (true, true, true, false, false, false)); assert_eq!(json_vectors, (true, false, false, false, true)); assert_eq!( @@ -1305,7 +1303,18 @@ async fn execution_analytics_fresh_cutover_and_exact_contract_db() { assert!(second_run_seq > first_run_seq); assert!(planning_context_scope_rejected); assert!(task_scope_rejected); - assert!(outbox_scope_rejected); + // Pinned by constraint, not just `is_err`: while the column was still named + // `task_id` this probe passed on an undefined-column error, so it proved + // nothing about scoping. Assert the composite + // (run_uid, tenant_id, contact_scope_id) fence is what rejects the row. + assert_eq!( + outbox_scope_rejected, + ( + Some("23503".to_string()), + Some("execution_action_review_outbox_run_normalized_scope_fkey".to_string()) + ), + "a cross-scope action review row must be rejected by the scope fence" + ); assert!(outbox_trace_mutation_rejected); assert!(review_trace_mutation_rejected); assert_eq!( @@ -1697,14 +1706,15 @@ async fn long_horizon_execution_cutover_rejects_live_runs_and_installs_fenced_ca let catalog_shape: (bool, bool, bool, bool, bool, bool) = sqlx::query_as( r#" SELECT - (SELECT count(*) = 142 + (SELECT count(*) = 143 FROM information_schema.columns WHERE table_schema = 'moa' AND ( (table_name = 'execution_run' AND column_name IN ( 'admitted_identity', 'controller_generation', 'activation_state', 'next_wake_at', 'waiting_since', 'last_progress_at', - 'pause_requested_at', 'paused_at', 'ready_task_count', + 'pause_requested_at', 'paused_at', + 'activation_failure_count', 'ready_task_count', 'active_task_count', 'waiting_task_count', 'waiting_input_task_count', 'waiting_input_user_task_count', 'waiting_input_tenant_admin_task_count', @@ -1835,13 +1845,13 @@ async fn long_horizon_execution_cutover_rejects_live_runs_and_installs_fenced_ca 'execution_task_checkpoint', 'execution_terminal_archive', 'execution_terminal_archive_segment' )), - (SELECT count(*) = 46 + (SELECT count(*) = 44 FROM pg_indexes WHERE schemaname = 'moa' AND indexname IN ( 'execution_run_terminal_retention_idx', 'execution_task_ready_idx', - 'execution_task_active_attempt_watchdog_idx', + 'execution_task_tenant_ready_order_idx', 'execution_trigger_due_idx', 'execution_dispatch_outbox_pending_idx', 'execution_dispatch_outbox_task_attempt_uidx', @@ -1849,14 +1859,11 @@ async fn long_horizon_execution_cutover_rejects_live_runs_and_installs_fenced_ca 'execution_schedule_due_idx', 'execution_task_terminal_retention_idx', 'execution_dispatch_outbox_compensation_attempt_uidx', - 'execution_compensation_active_watchdog_idx', 'execution_capacity_bucket_lock_order_idx', 'execution_tenant_dispatch_fairness_idx', - 'execution_dispatch_outbox_claim_expiry_idx', - 'execution_trigger_claim_expiry_idx' + 'execution_dispatch_outbox_claim_expiry_idx' ,'execution_run_schedule_occurrence_uidx' ,'execution_external_job_callback_receipt_retention_idx' - ,'execution_trigger_dead_letter_idx' ,'execution_dispatch_outbox_dead_letter_idx' ,'execution_dispatch_outbox_task_attempt_cancel_uidx' ,'execution_dispatch_outbox_compensation_attempt_cancel_uidx' @@ -1873,8 +1880,6 @@ async fn long_horizon_execution_cutover_rejects_live_runs_and_installs_fenced_ca ,'execution_capacity_reservation_parked_run_owner_uidx' ,'execution_capacity_reservation_trigger_owner_uidx' ,'execution_capacity_reservation_external_job_owner_uidx' - ,'execution_task_cancelling_reconciliation_idx' - ,'execution_compensation_cancelling_reconciliation_idx' ,'execution_completion_scan_actionable_idx' ,'execution_task_failure_fingerprint_idx' ,'execution_amendment_receipt_retention_idx' @@ -1885,6 +1890,9 @@ async fn long_horizon_execution_cutover_rejects_live_runs_and_installs_fenced_ca ,'execution_task_waiting_projection_idx' ,'execution_trigger_run_wake_idx' ,'execution_dispatch_outbox_external_cancel_uidx' + ,'execution_run_overdue_deadline_idx' + ,'execution_task_active_attempt_started_idx' + ,'execution_compensation_active_attempt_started_idx' )), moa.execution_admitted_identity_is_valid(admitted_identity, tenant_id) AND activation_state = 'terminal' @@ -2043,7 +2051,8 @@ async fn long_horizon_execution_cutover_rejects_live_runs_and_installs_fenced_ca FROM pg_trigger AS trigger JOIN pg_proc AS proc ON proc.oid = trigger.tgfoid WHERE trigger.tgrelid = 'moa.execution_task'::REGCLASS - AND NOT trigger.tgisinternal) + AND NOT trigger.tgisinternal + AND proc.proname LIKE 'enforce_execution_task%') AND to_regprocedure('moa.enforce_execution_task_long_horizon_update()') IS NULL AND @@ -2073,6 +2082,67 @@ async fn long_horizon_execution_cutover_rejects_live_runs_and_installs_fenced_ca .fetch_one(&target) .await?; + // A tenant-scoped table that is not registered in moa.tenant_purge_catalog + // makes run_tenant_purge_batch raise 55000 in its last stage, for every + // tenant, after rows are already deleted -- so right-to-erasure can never + // discharge. Reproduce both halves of that gate here: the count constant + // compiled into the function, and the drift scan it runs. + let purge_catalog: (i64, Option, Option>, bool) = sqlx::query_as( + r#" + SELECT + (SELECT count(*) FROM moa.tenant_purge_catalog), + (substring( + pg_get_functiondef( + 'moa.run_tenant_purge_batch(uuid,text)'::REGPROCEDURE + ) + FROM 'catalog_count <> ([0-9]+)' + ))::BIGINT, + (SELECT array_agg( + format('%I.%I', namespace.nspname, table_row.relname) + ORDER BY 1 + ) + FROM pg_class AS table_row + JOIN pg_namespace AS namespace + ON namespace.oid = table_row.relnamespace + JOIN pg_attribute AS column_row ON column_row.attrelid = table_row.oid + WHERE table_row.relkind IN ('r', 'p') + AND NOT table_row.relispartition + AND namespace.nspname IN ('public', 'moa', 'analytics', 'pii_vault') + AND column_row.attnum > 0 + AND NOT column_row.attisdropped + AND column_row.attname IN ('tenant_id', 'storage_partition_id') + AND NOT ( + namespace.nspname = 'moa' + AND table_row.relname IN ( + 'simulator_certification_mandate', + 'simulator_certification_evidence_import' + ) + ) + AND NOT EXISTS ( + SELECT 1 FROM moa.tenant_purge_catalog AS catalog + WHERE catalog.table_schema = namespace.nspname + AND catalog.table_name = table_row.relname + )), + COALESCE( + (SELECT count(*) = 4 AND bool_and( + parent.stage_order > ( + SELECT stage_order FROM moa.tenant_purge_catalog + WHERE stage_name + = 'moa.sandbox_execution_hand_release_receipts' + ) + ) + FROM moa.tenant_purge_catalog AS parent + WHERE parent.stage_name IN ( + 'moa.execution_task', 'moa.execution_compensation', + 'moa.sandbox_workspaces', 'moa.sandbox_workspace_checkpoints' + )), + FALSE + ) + "#, + ) + .fetch_one(&target) + .await?; + sqlx::query( "INSERT INTO moa.execution_dispatch_outbox ( \ dispatch_uid, tenant_id, run_uid, dispatch_kind, \ @@ -2107,6 +2177,7 @@ async fn long_horizon_execution_cutover_rejects_live_runs_and_installs_fenced_ca input_resume_counters, invalid_attempt_generation_rejected, catalog_shape, + purge_catalog, duplicate_activation_rejected, )) } @@ -2124,6 +2195,7 @@ async fn long_horizon_execution_cutover_rejects_live_runs_and_installs_fenced_ca input_resume_counters, invalid_attempt_generation_rejected, catalog_shape, + purge_catalog, duplicate_activation_rejected, ) = outcome.expect("long-horizon migration assertions should complete"); assert!( @@ -2146,8 +2218,234 @@ async fn long_horizon_execution_cutover_rejects_live_runs_and_installs_fenced_ca "attempt generation may only advance one fence at a time" ); assert_eq!(catalog_shape, (true, true, true, true, true, true)); + let (purge_catalog_count, purge_batch_constant, purge_catalog_drift, receipts_drain_first) = + purge_catalog; + assert_eq!( + purge_batch_constant, + Some(purge_catalog_count), + "run_tenant_purge_batch's catalog-count constant must equal the catalog it guards" + ); + assert_eq!( + purge_catalog_drift, None, + "every tenant-scoped table must be registered in moa.tenant_purge_catalog" + ); + assert!( + receipts_drain_first, + "sandbox hand-release receipts must purge before every ON DELETE RESTRICT parent" + ); assert!( duplicate_activation_rejected, "one run generation/wake epoch must have exactly one dispatch" ); } + +/// Seeds one confirmed-shape queued execution run and returns its `run_uid`. +async fn seed_queued_execution_run( + target: &sqlx::PgPool, + tenant_id: uuid::Uuid, + session_id: uuid::Uuid, + planning_context_uid: uuid::Uuid, +) -> Result> { + let run_uid = uuid::Uuid::new_v4(); + let plan_hash = "1".repeat(64); + let plan = serde_json::json!({ + "definition": { + "cancel_policy": "retain_effects", + "input_schema": {}, + "output_schema": {}, + "input_wait_policy": { + "expiry": {"kind": "after", "delay_seconds": 3600}, + "on_expiry": {"kind": "fail_task"} + }, + "nodes": [{ + "id": "output", + "requirement_ids": [], + "depends_on": [], + "when": null, + "input": {}, + "output_schema": {}, + "operation": {"kind": "output", "value": {}}, + "compensation": null, + "retry": { + "max_attempts": 1, + "initial_backoff_ms": 1, + "max_backoff_ms": 1 + }, + "budget": null + }] + }, + "plan_hash": plan_hash, + "catalog_hash": "0".repeat(64), + "estimate": { + "cost_microusd": 0, + "tokens": 0, + "tool_calls": 0, + "retrieved_bytes": 0, + "tasks": 1 + }, + "report": {"issues": []} + }); + sqlx::query( + "INSERT INTO moa.execution_run ( \ + run_uid, tenant_id, session_id, originating_user_sequence_num, \ + planning_context_uid, planning_context_hash, owner_user_id, goal_contract, \ + initial_plan, active_plan, initial_plan_hash, active_plan_hash, \ + capability_catalog, authorization_envelope, source_provenance, source_kind, \ + input, status, admitted_identity \ + ) VALUES ( \ + $1, $2, $3, 0, $4, $5, 'migration-test', $6, $7, $7, $8, $8, \ + $9, $10, $11, 'generated_plan', '{}'::JSONB, 'queued', $12 \ + )", + ) + .bind(run_uid) + .bind(tenant_id) + .bind(session_id) + .bind(planning_context_uid) + .bind("2".repeat(64)) + .bind(serde_json::json!({ + "objective": "migration", + "requirements": [], + "deliverables": [], + "coverage": [], + "constraints": [], + "completion_checks": [] + })) + .bind(&plan) + .bind(&plan_hash) + .bind(serde_json::json!({ + "capabilities": [], + "catalog_hash": "0".repeat(64) + })) + .bind(serde_json::json!({"capability_refs": [], "skill_refs": []})) + .bind(serde_json::json!({ + "kind": "generated_plan", + "planner": { + "model": "migration-test", + "prompt_version": "planner", + "candidate_hash": "3".repeat(64), + "compiler_report_hash": "4".repeat(64), + "final_plan_hash": plan_hash, + "repair_attempts": 0 + } + })) + .bind(serde_json::json!({ + "identity_type": "operator", + "id": uuid::Uuid::new_v4(), + "tenant_id": tenant_id, + "api_key_id": null, + "acting_on_behalf_of": null + })) + .execute(target) + .await?; + sqlx::query( + "UPDATE moa.execution_run SET status = 'pause_requested', \ + activation_state = 'paused', active_task_count = 1, \ + pause_requested_at = now(), updated_at = now() WHERE run_uid = $1", + ) + .bind(run_uid) + .execute(target) + .await?; + sqlx::query( + "UPDATE moa.execution_run SET status = 'pausing', updated_at = now() \ + WHERE run_uid = $1", + ) + .bind(run_uid) + .execute(target) + .await?; + Ok(run_uid) +} + +#[tokio::test] +#[ignore = "requires a superuser-capable local Postgres via MOA_DATABASE_URL"] +async fn draining_pausing_run_promotes_only_an_unchosen_status_db() { + // Pins: attempt settlement only adjusts counters, so draining the last active + // attempt must still complete the pause; but a writer that chooses its own + // status out of `pausing` keeps it. Swallowing a terminal write into `paused` + // wedges the run, because `paused` admits only `queued` and `cancelled`. + let admin_url = test_database_url(); + let db_name = unique_db_name(); + let admin = PgPoolOptions::new() + .max_connections(1) + .connect(&admin_url) + .await + .expect("connect pause-promotion maintenance database"); + admin + .execute(format!("CREATE DATABASE \"{db_name}\"").as_str()) + .await + .expect("create pause-promotion database"); + let target_url = with_database(&admin_url, &db_name); + + let outcome = async { + install_required_extensions(&target_url).await?; + run_reporting_applied_serialized(&target_url).await?; + let target = PgPoolOptions::new() + .max_connections(1) + .connect(&target_url) + .await?; + + let tenant_id = uuid::Uuid::new_v4(); + let session_id = uuid::Uuid::new_v4(); + let planning_context_uid = uuid::Uuid::new_v4(); + sqlx::query( + "INSERT INTO moa.execution_planning_context ( \ + planning_context_uid, tenant_id, session_id, \ + originating_user_sequence_num, originating_user_event_hash, \ + owner_user_id, planning_context_hash, snapshot \ + ) VALUES ($1, $2, $3, 0, $4, 'migration-test', $4, '{}'::JSONB)", + ) + .bind(planning_context_uid) + .bind(tenant_id) + .bind(session_id) + .bind("2".repeat(64)) + .execute(&target) + .await?; + + let drained = + seed_queued_execution_run(&target, tenant_id, session_id, planning_context_uid).await?; + let terminalized = + seed_queued_execution_run(&target, tenant_id, session_id, planning_context_uid).await?; + + // Attempt settlement writes counters only; the run keeps `pausing`. + let promoted: (String, String, bool) = sqlx::query_as( + "UPDATE moa.execution_run SET active_task_count = 0, updated_at = now() \ + WHERE run_uid = $1 \ + RETURNING status, activation_state, paused_at IS NOT NULL", + ) + .bind(drained) + .fetch_one(&target) + .await?; + + // A pending terminal commits its status alongside the same zeroed counter. + let terminal: (String, i64) = sqlx::query_as( + "UPDATE moa.execution_run SET status = 'failed', active_task_count = 0, \ + terminal_reason = 'internal_failure', \ + terminal_cause = '{\"kind\":\"internal_failure\"}'::JSONB, \ + terminal_satisfied_requirement_count = 0, \ + terminal_requirement_count = 0, activation_state = 'terminal', \ + completed_at = now(), updated_at = now() \ + WHERE run_uid = $1 RETURNING status, active_task_count", + ) + .bind(terminalized) + .fetch_one(&target) + .await?; + + target.close().await; + Ok::<_, Box>((promoted, terminal)) + } + .await; + + drop_database_with_zero_connections(&admin, &db_name).await; + admin.close().await; + + let (promoted, terminal) = outcome.expect("pause promotion assertions should complete"); + assert_eq!( + promoted, + ("paused".to_string(), "paused".to_string(), true), + "draining the last active attempt must complete the pause" + ); + assert_eq!( + terminal, + ("failed".to_string(), 0), + "a chosen terminal status must survive the pausing promotion" + ); +} diff --git a/crates/moa-migrations/tests/run_idempotency_db/execution_compensation.rs b/crates/moa-migrations/tests/run_idempotency_db/execution_compensation.rs index 2552be64b..8a1504e9c 100644 --- a/crates/moa-migrations/tests/run_idempotency_db/execution_compensation.rs +++ b/crates/moa-migrations/tests/run_idempotency_db/execution_compensation.rs @@ -147,7 +147,7 @@ async fn seed_execution_run(target: &PgPool) -> TestResult { "output_schema": {}, "input_wait_policy": { "expiry": {"kind": "after", "delay_seconds": 1}, - "on_expiry": {"kind": "fail_run"} + "on_expiry": {"kind": "fail_task"} }, "nodes": [{ "id": "output", diff --git a/crates/moa-migrations/tests/run_idempotency_db/tenant_purge.rs b/crates/moa-migrations/tests/run_idempotency_db/tenant_purge.rs index fda4165b7..e72dc5f43 100644 --- a/crates/moa-migrations/tests/run_idempotency_db/tenant_purge.rs +++ b/crates/moa-migrations/tests/run_idempotency_db/tenant_purge.rs @@ -709,7 +709,7 @@ async fn seed_tenant_purge_activated_release_chain( #[tokio::test] #[ignore = "requires a superuser-capable local Postgres via MOA_DATABASE_URL"] async fn bounded_tenant_purge_final_schema_executes_bounded_batches_db() { - // Pins: a pristine final schema persists exactly 142 purge stages, installs + // Pins: a pristine final schema persists exactly 158 purge stages, installs // statement fences, and advances a real purge in fixed-size batches. let admin_url = test_database_url(); let db_name = unique_db_name(); @@ -1308,7 +1308,7 @@ async fn bounded_tenant_purge_final_schema_executes_bounded_batches_db() { true, ) ); - assert_eq!(catalog_count, 142); + assert_eq!(catalog_count, 158); assert_eq!( trigger_kinds, vec![ @@ -1419,9 +1419,9 @@ async fn bounded_tenant_purge_final_schema_executes_bounded_batches_db() { #[tokio::test] #[ignore = "requires a superuser-capable local Postgres via MOA_DATABASE_URL"] async fn sandbox_workspace_purge_catalog_db() { - // Pins: V58 extends the current 134-stage catalog to exactly 142 with all - // workspace rows fenced and checkpoint head/parent ordering encoded in the - // bounded owner-only purge function. + // Pins: a full clean apply lands a 158-stage catalog (V58 took it to 142, V59 + // to 157, V60 to 158) with all workspace rows fenced and checkpoint + // head/parent ordering encoded in the bounded owner-only purge function. let admin_url = test_database_url(); let db_name = unique_db_name(); let tenant_id = uuid::Uuid::new_v4(); @@ -1559,7 +1559,7 @@ async fn sandbox_workspace_purge_catalog_db() { ) = outcome.expect("sandbox workspace purge schema assertions should complete"); assert_eq!(first, expected_migration_labels()); assert!(second.is_empty(), "V58 must not reapply: {second:?}"); - assert_eq!(catalog_count, 142); + assert_eq!(catalog_count, 158); assert_eq!( workspace_stages, vec![ @@ -1596,8 +1596,8 @@ async fn sandbox_workspace_purge_catalog_db() { ); assert_eq!(fence_count, 14); assert_eq!(global_catalog_count, 0); - assert!(purge_definition.contains("catalog_count <> 142")); - assert!(purge_definition.contains("exactly 142 tables")); + assert!(purge_definition.contains("catalog_count <> 158")); + assert!(purge_definition.contains("exactly 158 tables")); assert!(purge_definition.contains("SET current_checkpoint_id = NULL")); assert!(purge_definition.contains("ORDER BY target.generation DESC")); assert_eq!(checkpoint_columns.len(), 7); diff --git a/crates/moa-observability/src/lib.rs b/crates/moa-observability/src/lib.rs index c2c85ad76..dc2b2e8b5 100644 --- a/crates/moa-observability/src/lib.rs +++ b/crates/moa-observability/src/lib.rs @@ -30,14 +30,17 @@ pub use runtime_metrics::{ record_genai_client_time_to_first_chunk, record_genai_client_token_usage, record_knowledge_sync_run, record_llm_cost_cents, record_memory_operation, record_sandbox_provision_duration, record_sandbox_storage_resource_state, - record_sandbox_workspace_checkpoint, record_sandbox_workspace_inventory_drift, - record_sandbox_workspace_lifecycle, record_sandbox_workspace_quota_decision, - record_sandbox_workspace_quota_utilization, record_sandbox_workspace_reaper, - record_sandbox_workspace_state, record_session_error, record_session_event_append, - record_session_event_append_phase_duration, record_sessions_active, - record_simulation_cost_cents, record_simulation_tokens, record_simulation_turn, - record_tool_call, record_tool_failure, record_tool_reprovision, record_turn_completed, - record_turn_latency, record_turn_step_duration, record_turn_workflow_outcome, + record_sandbox_workspace_active_hands, record_sandbox_workspace_checkpoint, + record_sandbox_workspace_inventory_drift, record_sandbox_workspace_lifecycle, + record_sandbox_workspace_parked_tasks_with_active_hands, + record_sandbox_workspace_quota_decision, record_sandbox_workspace_quota_utilization, + record_sandbox_workspace_reaper, record_sandbox_workspace_release, + record_sandbox_workspace_restore, record_sandbox_workspace_state, record_session_error, + record_session_event_append, record_session_event_append_phase_duration, + record_sessions_active, record_simulation_cost_cents, record_simulation_tokens, + record_simulation_turn, record_tool_call, record_tool_failure, record_tool_reprovision, + record_turn_completed, record_turn_latency, record_turn_step_duration, + record_turn_workflow_outcome, }; pub use telemetry::{TelemetryConfig, TelemetryGuard, init_observability}; pub use trace_context::apply_trace_context_to_span; diff --git a/crates/moa-observability/src/runtime_metrics.rs b/crates/moa-observability/src/runtime_metrics.rs index 921a8ad7b..35e54b567 100644 --- a/crates/moa-observability/src/runtime_metrics.rs +++ b/crates/moa-observability/src/runtime_metrics.rs @@ -95,8 +95,16 @@ impl WorkerFanInSettledKind { } /// Bounded nonterminal execution phases exported for fleet run counts. +/// +/// The variants are exactly the thirteen nonterminal `ExecutionRunStatus` +/// values carried by the durable `execution_run_nonterminal_idx` predicate, in +/// that order. The mapping is total on purpose: a status with no phase would be +/// dropped from the census, and `sum(moa_execution_runs)` would quietly stop +/// equalling the live nonterminal fleet. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ExecutionRunMetricPhase { + /// The displayed plan and estimate await owning-user confirmation. + AwaitingConfirmation, /// Accepted work waiting for controller activation. Queued, /// Work currently advancing or running an attempt. @@ -111,6 +119,8 @@ pub enum ExecutionRunMetricPhase { WaitingTimer, /// Storage-only wait for an asynchronous external job. WaitingExternal, + /// Storage-only wait for a compiler-validated plan amendment. + WaitingReplan, /// A pause has been requested but active work is still settling. PauseRequested, /// The run is checkpointing and releasing resources before pausing. @@ -126,6 +136,7 @@ impl ExecutionRunMetricPhase { #[must_use] pub const fn as_str(self) -> &'static str { match self { + Self::AwaitingConfirmation => "awaiting_confirmation", Self::Queued => "queued", Self::Running => "running", Self::WaitingInput => "waiting_input", @@ -133,6 +144,7 @@ impl ExecutionRunMetricPhase { Self::WaitingSignal => "waiting_signal", Self::WaitingTimer => "waiting_timer", Self::WaitingExternal => "waiting_external", + Self::WaitingReplan => "waiting_replan", Self::PauseRequested => "pause_requested", Self::Pausing => "pausing", Self::Paused => "paused", @@ -146,8 +158,8 @@ impl ExecutionRunMetricPhase { pub enum ExecutionAdmissionResource { /// Nonterminal runs that are not fully parked. ActiveRuns, - /// Task attempts currently holding active-compute reservations. - ActiveAttempts, + /// Forward and compensation attempts holding active-compute reservations. + ActiveTasks, /// Runs retained in storage-only waiting or paused states. ParkedRuns, /// Pending durable trigger rows. @@ -158,11 +170,16 @@ pub enum ExecutionAdmissionResource { impl ExecutionAdmissionResource { /// Returns the stable low-cardinality resource label. + /// + /// The labels are exactly the durable + /// `moa.execution_capacity_bucket.resource_dimension` discriminators, so an + /// operator reading an alert can query the originating bucket row without a + /// translation table. #[must_use] pub const fn as_str(self) -> &'static str { match self { Self::ActiveRuns => "active_runs", - Self::ActiveAttempts => "active_attempts", + Self::ActiveTasks => "active_tasks", Self::ParkedRuns => "parked_runs", Self::ScheduledTriggers => "scheduled_triggers", Self::ExternalJobs => "external_jobs", @@ -237,6 +254,10 @@ pub enum SandboxWorkspaceLifecycleOperation { Purge, /// Apply checkpoint retention and garbage collection. Retention, + /// Release compute at a continuation boundary while keeping the filesystem. + Suspend, + /// Keep compute hot at a continuation boundary because suspend is unavailable. + Retain, } impl SandboxWorkspaceLifecycleOperation { @@ -253,6 +274,8 @@ impl SandboxWorkspaceLifecycleOperation { Self::Reconcile => "reconcile", Self::Purge => "purge", Self::Retention => "retention", + Self::Suspend => "suspend", + Self::Retain => "retain", } } } @@ -765,18 +788,10 @@ const HISTOGRAM_BOUNDARIES: &[(&str, &[f64])] = &[ "moa_worker_terminal_parent_ack_seconds", COORDINATION_ACK_DURATION_SECONDS_BUCKETS, ), - ( - "moa_execution_mutation_run_wake_ack_seconds", - COORDINATION_ACK_DURATION_SECONDS_BUCKETS, - ), ( "moa_execution_dispatch_batch_size", EXECUTION_TASK_COUNT_BUCKETS, ), - ( - "moa_execution_owned_in_flight_tasks", - EXECUTION_TASK_COUNT_BUCKETS, - ), ( "moa_sandbox_provision_seconds", SERVICE_DURATION_SECONDS_BUCKETS, @@ -1041,25 +1056,17 @@ pub fn record_worker_terminal_parent_ack(duration: Duration) { histogram!("moa_worker_terminal_parent_ack_seconds").record(duration.as_secs_f64()); } -/// Records persisted-execution-mutation-to-run-wake acknowledgement latency. -/// -/// The duration begins after the user-facing mutation commits and ends when -/// the exact `ExecutionRun` wake is acknowledged by Restate. -pub fn record_execution_mutation_run_wake_ack(duration: Duration) { - histogram!("moa_execution_mutation_run_wake_ack_seconds").record(duration.as_secs_f64()); -} - -/// Records how many ready execution tasks were dispatched in one bounded refill. +/// Records how many ready execution tasks the dispatcher admitted in one +/// bounded refill. pub fn record_execution_dispatch_batch_size(size: usize) { histogram!("moa_execution_dispatch_batch_size").record(size as f64); } -/// Records the number of task calls currently owned by one `ExecutionRun`. -pub fn record_execution_owned_in_flight_tasks(count: usize) { - histogram!("moa_execution_owned_in_flight_tasks").record(count as f64); -} - /// Sets the current fleet count for one bounded nonterminal execution phase. +/// +/// Callers must record every phase on each fleet snapshot including the healthy +/// zero, so a phase that drains reports zero rather than keeping its last value +/// forever. pub fn record_execution_run_phase(phase: ExecutionRunMetricPhase, count: u64) { gauge!("moa_execution_runs", "phase" => phase.as_str()).set(count as f64); } @@ -1070,37 +1077,30 @@ pub fn record_execution_oldest_ready_age(age: Duration) { } /// Sets the number of nonterminal runs whose absolute deadline has elapsed. +/// +/// This is the exact-deadline invariant guard, so callers must record it on +/// every fleet snapshot including the healthy zero. pub fn record_execution_overdue_deadlines(count: u64) { gauge!("moa_execution_overdue_deadlines").set(count as f64); } /// Sets trigger delivery lag, capped depth, and sample-completeness from one fleet snapshot. +/// +/// Triggers carry no dead-letter state: claiming and retry exhaustion live entirely on +/// the dispatch outbox, so only the due sample is observed here. pub fn record_execution_trigger_queue( lag: Duration, due_triggers: u64, due_sample_saturated: bool, - dead_letters: u64, - dead_letter_sample_saturated: bool, ) { gauge!("moa_execution_trigger_lag_seconds").set(lag.as_secs_f64()); gauge!("moa_execution_trigger_due").set(due_triggers as f64); - gauge!("moa_execution_trigger_dead_letters").set(dead_letters as f64); gauge!( "moa_execution_queue_sample_saturated", "queue" => "trigger", "sample" => "due" ) .set(if due_sample_saturated { 1.0 } else { 0.0 }); - gauge!( - "moa_execution_queue_sample_saturated", - "queue" => "trigger", - "sample" => "dead_letter" - ) - .set(if dead_letter_sample_saturated { - 1.0 - } else { - 0.0 - }); } /// Sets outbox delivery lag, capped depth, and sample-completeness from one fleet snapshot. @@ -1133,11 +1133,20 @@ pub fn record_execution_outbox_queue( } /// Sets the age of the oldest active task-attempt lease. +/// +/// Callers must record this on every fleet snapshot, using `Duration::ZERO` +/// when no attempt is active. A gauge only written while work exists keeps its +/// last value forever once the fleet drains, so its alert would page on a queue +/// that emptied hours earlier. pub fn record_execution_active_attempt_oldest_age(age: Duration) { gauge!("moa_execution_active_attempt_oldest_age_seconds").set(age.as_secs_f64()); } /// Sets the age of the oldest nonterminal asynchronous external job. +/// +/// Callers must record this on every fleet snapshot, using `Duration::ZERO` +/// when no external job is outstanding, so the alert can tell a quiet fleet +/// from a stopped producer. pub fn record_execution_external_job_oldest_age(age: Duration) { gauge!("moa_execution_external_job_oldest_age_seconds").set(age.as_secs_f64()); } @@ -1188,21 +1197,33 @@ pub fn record_execution_retention(ready: bool, last_success_age: Option "commit").record(0.002); histogram!("moa_turn_latency_seconds").record(0.025); record_worker_terminal_parent_ack(Duration::from_millis(4)); - record_execution_mutation_run_wake_ack(Duration::from_millis(5)); record_execution_dispatch_batch_size(32); - record_execution_owned_in_flight_tasks(64); provider .force_flush() .expect("in-memory metric exporter should flush"); @@ -2174,9 +2178,7 @@ mod tests { for metric in [ "moa_turn_latency_seconds", "moa_worker_terminal_parent_ack_seconds", - "moa_execution_mutation_run_wake_ack_seconds", "moa_execution_dispatch_batch_size", - "moa_execution_owned_in_flight_tasks", ] { assert!( names.contains(metric), @@ -2199,9 +2201,7 @@ mod tests { metrics::with_local_recorder(&recorder, || { register_metric_descriptions(); record_worker_terminal_parent_ack(Duration::from_millis(4)); - record_execution_mutation_run_wake_ack(Duration::from_millis(5)); record_execution_dispatch_batch_size(32); - record_execution_owned_in_flight_tasks(64); record_worker_terminal_delivery(WorkerTerminalDeliveryResult::Accepted); record_worker_terminal_delivery(WorkerTerminalDeliveryResult::Duplicate); record_worker_fan_in_settled(WorkerFanInSettledKind::Completed); @@ -2211,9 +2211,7 @@ mod tests { let coordination_metrics = [ "moa_worker_terminal_parent_ack_seconds", - "moa_execution_mutation_run_wake_ack_seconds", "moa_execution_dispatch_batch_size", - "moa_execution_owned_in_flight_tasks", "moa_worker_terminal_deliveries_total", "moa_worker_fan_in_settled_total", ]; @@ -2261,13 +2259,17 @@ mod tests { #[test] fn long_horizon_metrics_export_descriptions_and_only_bounded_labels() { - // Pins: execution, drain, and sandbox-yield health reaches production - // exporters without tenant, run, task, deployment-version, or provider-account IDs. + // Pins: execution fleet health, drain cost, and the sandbox-yield and + // parked-task hand invariants reach production exporters without tenant, + // run, task, deployment-version, or provider-account IDs. That each + // recorder also has a caller outside this crate is pinned separately by + // `validate-observability.sh`, not by this test. let recorder = PrometheusBuilder::new().build_recorder(); let handle = recorder.handle(); metrics::with_local_recorder(&recorder, || { register_metric_descriptions(); for phase in [ + ExecutionRunMetricPhase::AwaitingConfirmation, ExecutionRunMetricPhase::Queued, ExecutionRunMetricPhase::Running, ExecutionRunMetricPhase::WaitingInput, @@ -2275,6 +2277,7 @@ mod tests { ExecutionRunMetricPhase::WaitingSignal, ExecutionRunMetricPhase::WaitingTimer, ExecutionRunMetricPhase::WaitingExternal, + ExecutionRunMetricPhase::WaitingReplan, ExecutionRunMetricPhase::PauseRequested, ExecutionRunMetricPhase::Pausing, ExecutionRunMetricPhase::Paused, @@ -2284,12 +2287,12 @@ mod tests { } record_execution_oldest_ready_age(Duration::from_secs(31)); record_execution_overdue_deadlines(2); - record_execution_trigger_queue(Duration::from_secs(7), 11, true, 1, false); + record_execution_trigger_queue(Duration::from_secs(7), 11, true); record_execution_outbox_queue(Duration::from_secs(8), 12, false, 1, true); record_execution_active_attempt_oldest_age(Duration::from_secs(61)); record_execution_external_job_oldest_age(Duration::from_secs(62)); record_execution_admission_utilization( - ExecutionAdmissionResource::ActiveAttempts, + ExecutionAdmissionResource::ActiveTasks, ExecutionAdmissionScope::Fleet, 0.75, ); @@ -2303,7 +2306,7 @@ mod tests { record_execution_maintenance(false, None); record_execution_retention(true, Some(Duration::from_secs(3_600))); record_execution_retention(false, None); - record_restate_draining_deployments(2, 3, Duration::from_secs(3_600), 4.5); + record_restate_draining_deployments(2, 17, Duration::from_secs(3_600)); record_sandbox_workspace_active_hands(SandboxWorkspaceProviderKind::E2b, 2); record_sandbox_workspace_parked_tasks_with_active_hands(0); record_sandbox_workspace_restore(SandboxWorkspaceProviderKind::E2b); @@ -2320,7 +2323,6 @@ mod tests { "moa_execution_overdue_deadlines", "moa_execution_trigger_lag_seconds", "moa_execution_trigger_due", - "moa_execution_trigger_dead_letters", "moa_execution_outbox_lag_seconds", "moa_execution_outbox_claimable", "moa_execution_outbox_dead_letters", @@ -2334,9 +2336,8 @@ mod tests { "moa_execution_retention_ready", "moa_execution_retention_last_success_age_seconds", "moa_restate_draining_deployments", - "moa_restate_draining_deployment_replicas", + "moa_restate_draining_deployment_blocking_invocations", "moa_restate_draining_deployment_oldest_age_seconds", - "moa_restate_draining_deployment_replica_hours", "moa_sandbox_workspace_active_hands", "moa_sandbox_workspace_parked_tasks_with_active_hands", "moa_sandbox_workspace_restores_total", @@ -2367,8 +2368,13 @@ mod tests { ); for label in [ + // The census is only total if the two statuses that carry no active + // compute still get a phase; dropping either desyncs + // `sum(moa_execution_runs)` from the durable nonterminal predicate. + "phase=\"awaiting_confirmation\"", + "phase=\"waiting_replan\"", "phase=\"waiting_timer\"", - "resource=\"active_attempts\"", + "resource=\"active_tasks\"", "scope=\"fleet\"", "scope=\"tenant_peak\"", "provider_kind=\"e2b\"", diff --git a/crates/moa-orchestrator/src/external_job_ingress.rs b/crates/moa-orchestrator/src/external_job_ingress.rs index e90b31923..ac037bd36 100644 --- a/crates/moa-orchestrator/src/external_job_ingress.rs +++ b/crates/moa-orchestrator/src/external_job_ingress.rs @@ -1035,6 +1035,7 @@ mod tests { not_before_at: Utc::now(), payload: serde_json::json!({}), delivery_attempts: 0, + repair_epoch: 0, claim_owner: None, claimed_at: None, claim_expires_at: None, diff --git a/crates/moa-orchestrator/src/main.rs b/crates/moa-orchestrator/src/main.rs index 4964b9044..ef8fe9d70 100644 --- a/crates/moa-orchestrator/src/main.rs +++ b/crates/moa-orchestrator/src/main.rs @@ -38,6 +38,7 @@ use moa_orchestrator::{ start_hand_lease_reaper, start_mcp_catalog_refresh, start_workspace_reaper, }, kms::KmsRuntime, + restate_drain::{resolve_admin_url, spawn_restate_drain_observer}, sandbox_workspace_rollout::validate_startup_state as validate_sandbox_workspace_rollout, }, }; @@ -654,6 +655,21 @@ async fn run_maintenance( }, ); let shutdown = CancellationToken::new(); + // Pure fleet observation, so an unresolvable or unreachable Restate admin + // API is logged rather than made fatal, and the observer is deliberately + // left out of the supervised `select!` below: losing drain telemetry must + // not take down the single replica that owns workspace reaping, + // authorization outbox delivery, and action-review timeouts. + let restate_drain_observer = match resolve_admin_url(&restate_ingress_url) { + Ok(admin_url) => Some(spawn_restate_drain_observer(admin_url, shutdown.clone())), + Err(error) => { + tracing::warn!( + %error, + "could not resolve the Restate admin API; deployment drain telemetry is disabled" + ); + None + } + }; let mut execution_cron_reconciler = spawn_execution_cron_reconciler( restate_ingress_url, config.execution.trigger_reconciliation_cadence_seconds, @@ -784,6 +800,11 @@ async fn run_maintenance( let _ = join_task_bounded("execution CronJob reconciler", execution_cron_reconciler).await; }, + async move { + if let Some(handle) = restate_drain_observer { + let _ = join_task_bounded("Restate deployment drain observer", handle).await; + } + }, ); Ok(()) diff --git a/crates/moa-orchestrator/src/objects/execution_run_controller.rs b/crates/moa-orchestrator/src/objects/execution_run_controller.rs index 3a6d2385b..362237db2 100644 --- a/crates/moa-orchestrator/src/objects/execution_run_controller.rs +++ b/crates/moa-orchestrator/src/objects/execution_run_controller.rs @@ -126,6 +126,20 @@ impl ExecutionRunController for ExecutionRunControllerImpl { .into_inner(); progress::deliver(&ctx, &self.repository, &request, &committed).await?; + // A run that parked without enqueueing more work may be waiting for a replacement plan + // that nothing else in the product will propose. Selecting that bounded slice is one + // indexed read and one durable send; the paid planner call happens in its own service. + if committed.terminal_delivery.is_none() && !committed.response.continuation_enqueued { + crate::services::execution_amendment_planner::dispatch_parked_replan_planning( + &ctx, + &self.repository, + request.tenant_id, + request.run_uid, + committed.response.controller_generation, + committed.response.wake_epoch, + ) + .await?; + } Ok(Json::from(committed.response)) } } diff --git a/crates/moa-orchestrator/src/objects/execution_run_controller/advance.rs b/crates/moa-orchestrator/src/objects/execution_run_controller/advance.rs index 635f24923..e1b378ee2 100644 --- a/crates/moa-orchestrator/src/objects/execution_run_controller/advance.rs +++ b/crates/moa-orchestrator/src/objects/execution_run_controller/advance.rs @@ -11,7 +11,7 @@ use moa_execution::{ budget::BudgetLedger, materialize_node_page, repository::{ - ExecutionRepository, ExecutionScope, RunControllerClaimOutcome, + ExecutionRepository, ExecutionRunRecord, ExecutionScope, RunControllerClaimOutcome, RunControllerCompletionOutcome, RunControllerCompletionRequest, RunDeadlineArmOutcome, completion::{CompletionAdvanceOutcome, CompletionAdvanceRequest}, outbox::{ExecutionDispatchKind, ExecutionDispatchRecord}, @@ -19,16 +19,73 @@ use moa_execution::{ ExecutionReduceMaterializationCursor, MapAggregatePageOutcome, MapAggregatePageRequest, ReadyMaterializationOutcome, ReadyMaterializationRequest, ReduceRoundInputPageRequest, }, + run::{ResumedControllerRecoveryOutcome, ResumedControllerRecoveryRequest}, terminal::{ FinalizationOutcome, PendingTerminalAdvanceOutcome, PendingTerminalAdvanceStage, RunTriggerDrainOutcome, RunTriggerDrainRequest, }, }, - state::{ExecutionProjection, ExecutionTaskStatus}, + state::{ + ExecutionProjection, ExecutionRunStatus, ExecutionTaskStatus, ExecutionTerminalCause, + ExecutionTerminalEvidence, ExecutionTerminalReason, PendingExecutionTerminal, + }, }; use serde::{Deserialize, Serialize}; use serde_json::json; +/// Records that a declared node condition evaluated false and its branch was skipped. +/// +/// A skipped branch produces no logical task, no attempt, and no output, so the only +/// other evidence that the plan branched at all is the absence of rows. "Why did this +/// branch not run" has to stay answerable after the fact — that is the whole reason a +/// declared condition is preferable to an agent turn making the same choice silently. +/// The node id is carried only on the span event; the counter keeps plan-defined ids +/// out of metric labels. +fn record_condition_skip( + run: &moa_execution::repository::ExecutionRunRecord, + node_id: &str, + plan_node: &moa_artifacts::execution_plan::ExecutionNode, +) { + use moa_artifacts::execution_plan::{ExecutionCondition, ExecutionOperation}; + + let operation = match plan_node.operation { + ExecutionOperation::Capability { .. } => "capability", + ExecutionOperation::Agent { .. } => "agent", + ExecutionOperation::Map { .. } => "map", + ExecutionOperation::Reduce { .. } => "reduce", + ExecutionOperation::Review { .. } => "review", + ExecutionOperation::WaitSignal { .. } => "wait_signal", + ExecutionOperation::WaitUntil { .. } => "wait_until", + ExecutionOperation::Output { .. } => "output", + }; + let condition = match plan_node.when { + Some(ExecutionCondition::Exists { .. }) => "exists", + Some(ExecutionCondition::Equals { .. }) => "equals", + None => "none", + }; + tracing::info!( + run_uid = %run.run_uid, + node_id, + operation, + condition, + outcome = "false", + "execution node condition evaluated false; branch skipped" + ); + metrics::counter!( + "moa_execution_node_condition_skipped_total", + "operation" => operation, + "condition" => condition, + ) + .increment(1); +} + +/// Consecutive crashed controller activations tolerated before a run is failed for repair. +/// +/// A resumed claim always means the prior activation of the exact same wake never acknowledged it. +/// Transient crashes clear on the next attempt, so a run that consumes this whole budget is failing +/// deterministically and can only be repaired by hand. +const MAXIMUM_RESUMED_ACTIVATION_RECOVERIES: u64 = 5; + /// Journaled database commit and the bounded side effects selected by it. #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] #[serde(deny_unknown_fields)] @@ -39,19 +96,22 @@ pub(super) struct ControllerAdvanceCommit { } #[derive(Clone, Copy, Debug, Eq, PartialEq)] -struct ActivationLimits { - remaining_steps: usize, - remaining_tasks: usize, +pub(super) struct ActivationLimits { + pub(super) remaining_steps: usize, + pub(super) remaining_tasks: usize, } #[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum ActivationPreflight { +pub(super) enum ActivationPreflight { PendingTerminal, DueDeadline, Ordinary, } -fn completion_scan_steps(scanned_tasks: u32, scanned_nodes: u32) -> moa_execution::Result { +pub(super) fn completion_scan_steps( + scanned_tasks: u32, + scanned_nodes: u32, +) -> moa_execution::Result { usize::try_from(scanned_tasks) .ok() .and_then(|tasks| { @@ -64,7 +124,9 @@ fn completion_scan_steps(scanned_tasks: u32, scanned_nodes: u32) -> moa_executio }) } -fn terminal_trigger_page_limit(remaining_steps: usize) -> moa_execution::Result> { +pub(super) fn terminal_trigger_page_limit( + remaining_steps: usize, +) -> moa_execution::Result> { if remaining_steps == 0 { return Ok(None); } @@ -75,7 +137,7 @@ fn terminal_trigger_page_limit(remaining_steps: usize) -> moa_execution::Result< }) } -fn pending_terminal_step_count( +pub(super) fn pending_terminal_step_count( settled_task_count: u64, drained_trigger_count: u32, cancellation_dispatch_count: usize, @@ -95,7 +157,7 @@ fn pending_terminal_step_count( }) } -fn map_aggregate_requires_continuation( +pub(super) fn map_aggregate_requires_continuation( outcome: &MapAggregatePageOutcome, ) -> moa_execution::Result { match outcome { @@ -112,7 +174,7 @@ fn map_aggregate_requires_continuation( } } -fn validate_resumed_recovery_commit( +pub(super) fn validate_resumed_recovery_commit( prior_wake_epoch: u64, current_wake_epoch: u64, continuation_enqueued: bool, @@ -130,7 +192,7 @@ fn validate_resumed_recovery_commit( Ok(()) } -fn validate_trigger_drain_continuation( +pub(super) fn validate_trigger_drain_continuation( prior_wake_epoch: u64, current_wake_epoch: u64, drained_trigger_count: u32, @@ -151,7 +213,7 @@ fn validate_trigger_drain_continuation( } fn validate_replan_stop_continuation( - run: &moa_execution::repository::ExecutionRunRecord, + run: &ExecutionRunRecord, continuation: &ExecutionDispatchRecord, continuation_wake_epoch: u64, ) -> moa_execution::Result<()> { @@ -163,7 +225,7 @@ fn validate_replan_stop_continuation( validate_replan_stop_continuation_fields(run.wake_epoch, continuation_wake_epoch, exact_owner) } -fn validate_replan_stop_continuation_fields( +pub(super) fn validate_replan_stop_continuation_fields( prior_wake_epoch: u64, continuation_wake_epoch: u64, exact_owner: bool, @@ -177,7 +239,7 @@ fn validate_replan_stop_continuation_fields( Ok(()) } -fn activation_preflight( +pub(super) fn activation_preflight( has_pending_terminal: bool, deadline_at: Option>, now: DateTime, @@ -192,7 +254,10 @@ fn activation_preflight( } impl ActivationLimits { - fn new(maximum_steps: usize, dispatch_batch_size: usize) -> moa_execution::Result { + pub(super) fn new( + maximum_steps: usize, + dispatch_batch_size: usize, + ) -> moa_execution::Result { if maximum_steps == 0 || dispatch_batch_size == 0 { return Err(moa_execution::Error::InvalidRepositoryInput { message: "controller activation bounds must both be greater than zero".to_string(), @@ -204,7 +269,7 @@ impl ActivationLimits { }) } - fn inspect_nodes(&mut self, count: usize) -> usize { + pub(super) fn inspect_nodes(&mut self, count: usize) -> usize { let inspected = self.remaining_steps.min(count); self.remaining_steps -= inspected; inspected @@ -219,7 +284,7 @@ impl ActivationLimits { Ok(()) } - fn task_page_limit(&self) -> moa_execution::Result { + pub(super) fn task_page_limit(&self) -> moa_execution::Result { u32::try_from(self.remaining_tasks.min(1_000)).map_err(|_| { moa_execution::Error::ArithmeticOverflow { context: "controller task page limit".to_string(), @@ -227,7 +292,7 @@ impl ActivationLimits { }) } - fn record_tasks(&mut self, count: usize) -> moa_execution::Result<()> { + pub(super) fn record_tasks(&mut self, count: usize) -> moa_execution::Result<()> { self.remaining_tasks = self.remaining_tasks.checked_sub(count).ok_or_else(|| { moa_execution::Error::InvalidRepositoryData { message: "controller materialized beyond its dispatch bound".to_string(), @@ -657,6 +722,7 @@ pub(super) async fn advance( source_exhausted, reduce_cursor, terminal_output, + condition_skipped, .. } = materialize_node_page( &schedule, @@ -685,6 +751,7 @@ pub(super) async fn advance( }), source_exhausted, terminal_output, + condition_skipped, tasks, }, ) @@ -692,6 +759,9 @@ pub(super) async fn advance( { ReadyMaterializationOutcome::Applied { tasks, .. } | ReadyMaterializationOutcome::Replayed { tasks, .. } => { + if condition_skipped { + record_condition_skip(&run, &node.node_id, plan_node); + } if tasks.len() != task_count { return Err(moa_execution::Error::InvalidRepositoryData { message: "ready materialization returned a different task page" @@ -1066,75 +1136,173 @@ async fn resume_with_bounded_continuation( scope: ExecutionScope, config: &moa_config::ExecutionConfig, request: &ExecutionRunAdvanceRequest, - run: &moa_execution::repository::ExecutionRunRecord, + run: &ExecutionRunRecord, ) -> moa_execution::Result { - let completion = repository - .complete_controller_wake( + let recovery = repository + .recover_resumed_controller_wake( scope, config, run.run_uid, - RunControllerCompletionRequest { + ResumedControllerRecoveryRequest { controller_generation: request.controller_generation, wake_epoch: request.wake_epoch, checkpoint: settlement::continuation_checkpoint(run), - continuation_payload: Some(json!({ + continuation_payload: json!({ "cause": "resumed_activation_recovery", "prior_dispatch_uid": request.dispatch_uid, - })), + }), continuation_not_before_at: Utc::now(), + maximum_consecutive_failures: MAXIMUM_RESUMED_ACTIVATION_RECOVERIES, }, ) .await?; - match completion { - RunControllerCompletionOutcome::Applied { run, continuation } => { - let continuation_enqueued = continuation.is_some(); - validate_resumed_recovery_commit( - request.wake_epoch, - run.wake_epoch, - continuation_enqueued, - )?; + match recovery { + ResumedControllerRecoveryOutcome::Recovered { + run: recovered, + consecutive_failures, + .. + } => { + validate_resumed_recovery_commit(request.wake_epoch, recovered.wake_epoch, true)?; + tracing::warn!( + run_uid = %run.run_uid, + controller_generation = recovered.controller_generation, + wake_epoch = recovered.wake_epoch, + consecutive_failures, + "recovered a crashed controller activation with one replacement wake" + ); Ok(ControllerAdvanceCommit { response: ExecutionRunAdvanceResponse { outcome: ExecutionRunAdvanceOutcome::Advanced, - controller_generation: run.controller_generation, - wake_epoch: run.wake_epoch, + controller_generation: recovered.controller_generation, + wake_epoch: recovered.wake_epoch, activation_steps: 0, materialized_tasks: 0, - continuation_enqueued, + continuation_enqueued: true, }, publish_progress: true, terminal_delivery: None, }) } - RunControllerCompletionOutcome::Replayed(run) => { - Ok(noop_commit(ExecutionRunAdvanceOutcome::Replayed, &run)) + ResumedControllerRecoveryOutcome::BudgetExhausted { + consecutive_failures, + } => { + fail_unrecoverable_activation(repository, scope, config, run, consecutive_failures) + .await } - RunControllerCompletionOutcome::CapacitySaturated { dimension } => { - Err(moa_execution::Error::CapacitySaturated { - dimension: dimension.as_str(), - }) + ResumedControllerRecoveryOutcome::Replayed(replayed) => { + Ok(noop_commit(ExecutionRunAdvanceOutcome::Replayed, &replayed)) } - RunControllerCompletionOutcome::StaleGeneration { current_generation } => { + ResumedControllerRecoveryOutcome::StaleGeneration { current_generation } => { Ok(stale_commit(current_generation, run.wake_epoch)) } - RunControllerCompletionOutcome::StaleWake { + ResumedControllerRecoveryOutcome::StaleWake { current_wake_epoch, .. } => Ok(stale_commit(run.controller_generation, current_wake_epoch)), - RunControllerCompletionOutcome::NotFound => { + ResumedControllerRecoveryOutcome::NotFound => { Err(moa_execution::Error::InvalidRepositoryData { message: "execution run disappeared during resumed activation recovery".to_string(), }) } - RunControllerCompletionOutcome::InvalidState => { + ResumedControllerRecoveryOutcome::InvalidState => { Ok(stale_commit(run.controller_generation, run.wake_epoch)) } } } +/// Fails a run whose activation has crashed past its bounded recovery budget. +/// +/// The claimed wake is still unacknowledged here, so the terminal intent and its first bounded +/// settlement page commit against it in one Postgres transaction. Only after that transition +/// succeeds does the caller observe the product failure, which the pending-terminal drain then +/// carries to a real terminal status and Session delivery. +async fn fail_unrecoverable_activation( + repository: &ExecutionRepository, + scope: ExecutionScope, + config: &moa_config::ExecutionConfig, + run: &ExecutionRunRecord, + consecutive_failures: u64, +) -> moa_execution::Result { + let now = Utc::now(); + let limits = + ActivationLimits::new(config.maximum_activation_steps, config.dispatch_batch_size)?; + let page_limit = u32::try_from( + limits + .remaining_steps + .min(limits.remaining_tasks) + .min(1_000), + ) + .map_err(|_| moa_execution::Error::ArithmeticOverflow { + context: "controller unrecoverable-activation page limit".to_string(), + })?; + tracing::error!( + run_uid = %run.run_uid, + controller_generation = run.controller_generation, + wake_epoch = run.wake_epoch, + consecutive_failures, + "controller activation exhausted its recovery budget; failing the run for manual repair" + ); + let outcome = if run.pending_terminal.is_some() { + // A terminal intent already owns this run; drive its bounded drain on the claimed wake + // rather than fencing a second, conflicting intent that could never be installed. + repository + .advance_pending_terminal_settlement( + config, + scope, + run.run_uid, + run.controller_generation, + run.wake_epoch, + now, + page_limit, + ) + .await? + } else { + repository + .fence_completion_terminal_and_enqueue_settlement( + config, + scope, + run.run_uid, + run.controller_generation, + run.wake_epoch, + unrecoverable_activation_terminal(run, consecutive_failures)?, + now, + page_limit, + ) + .await? + }; + pending_terminal_commit(repository, scope, run, outcome).await +} + +/// Builds the terminal intent for a run whose controller can no longer advance it. +fn unrecoverable_activation_terminal( + run: &ExecutionRunRecord, + consecutive_failures: u64, +) -> moa_execution::Result { + let requirement_count = u64::try_from(run.goal.requirements.len()).map_err(|_| { + moa_execution::Error::ArithmeticOverflow { + context: "controller unrecoverable-activation requirement count".to_string(), + } + })?; + Ok(PendingExecutionTerminal { + status: ExecutionRunStatus::Failed, + reason: ExecutionTerminalReason::InternalFailure, + terminal_evidence: ExecutionTerminalEvidence { + cause: ExecutionTerminalCause::InternalFailure, + satisfied_requirement_count: 0, + requirement_count, + }, + completion_check_results: Vec::new(), + terminal_gaps: vec![format!( + "controller activation failed {consecutive_failures} consecutive times and requires manual repair" + )], + output: run.output.clone(), + cancellation_reason: None, + }) +} + async fn pending_terminal_commit( repository: &ExecutionRepository, scope: ExecutionScope, - claimed_run: &moa_execution::repository::ExecutionRunRecord, + claimed_run: &ExecutionRunRecord, outcome: PendingTerminalAdvanceOutcome, ) -> moa_execution::Result { let (commit, response_outcome) = match outcome { @@ -1193,7 +1361,7 @@ async fn pending_terminal_commit( async fn terminal_commit( repository: &ExecutionRepository, scope: ExecutionScope, - run: &moa_execution::repository::ExecutionRunRecord, + run: &ExecutionRunRecord, ) -> moa_execution::Result { let terminal_delivery = repository .load_bounded_terminal_delivery(scope, run.run_uid) @@ -1217,7 +1385,7 @@ async fn terminal_commit( fn noop_commit( outcome: ExecutionRunAdvanceOutcome, - run: &moa_execution::repository::ExecutionRunRecord, + run: &ExecutionRunRecord, ) -> ControllerAdvanceCommit { ControllerAdvanceCommit { response: ExecutionRunAdvanceResponse { @@ -1233,7 +1401,7 @@ fn noop_commit( } } -fn stale_commit(controller_generation: u64, wake_epoch: u64) -> ControllerAdvanceCommit { +pub(super) fn stale_commit(controller_generation: u64, wake_epoch: u64) -> ControllerAdvanceCommit { ControllerAdvanceCommit { response: ExecutionRunAdvanceResponse { outcome: ExecutionRunAdvanceOutcome::Stale, @@ -1247,114 +1415,3 @@ fn stale_commit(controller_generation: u64, wake_epoch: u64) -> ControllerAdvanc terminal_delivery: None, } } - -#[cfg(test)] -pub(super) fn stale_commit_for_test( - controller_generation: u64, - wake_epoch: u64, -) -> ControllerAdvanceCommit { - stale_commit(controller_generation, wake_epoch) -} - -#[cfg(test)] -pub(super) fn consume_limits_for_test( - maximum_steps: usize, - dispatch_batch_size: usize, - node_counts: &[usize], - task_counts: &[usize], -) -> moa_execution::Result<(usize, usize, usize, usize)> { - let mut limits = ActivationLimits::new(maximum_steps, dispatch_batch_size)?; - let mut inspected = 0usize; - let mut tasks = 0usize; - for count in node_counts { - inspected += limits.inspect_nodes(*count); - } - for count in task_counts { - let accepted = limits.remaining_tasks.min(*count); - limits.record_tasks(accepted)?; - tasks += accepted; - } - Ok(( - inspected, - tasks, - limits.remaining_steps, - limits.remaining_tasks, - )) -} - -#[cfg(test)] -pub(super) fn completion_scan_steps_for_test( - scanned_tasks: u32, - scanned_nodes: u32, -) -> moa_execution::Result { - completion_scan_steps(scanned_tasks, scanned_nodes) -} - -#[cfg(test)] -pub(super) fn validate_resumed_recovery_for_test( - prior_wake_epoch: u64, - current_wake_epoch: u64, - continuation_enqueued: bool, -) -> moa_execution::Result<()> { - validate_resumed_recovery_commit(prior_wake_epoch, current_wake_epoch, continuation_enqueued) -} - -#[cfg(test)] -pub(super) fn validate_trigger_drain_for_test( - prior_wake_epoch: u64, - current_wake_epoch: u64, - drained_trigger_count: u32, -) -> moa_execution::Result<()> { - validate_trigger_drain_continuation(prior_wake_epoch, current_wake_epoch, drained_trigger_count) -} - -#[cfg(test)] -pub(super) fn terminal_trigger_page_limit_for_test( - remaining_steps: usize, -) -> moa_execution::Result> { - terminal_trigger_page_limit(remaining_steps) -} - -#[cfg(test)] -pub(super) fn pending_terminal_step_count_for_test( - settled_task_count: u64, - drained_trigger_count: u32, - cancellation_dispatch_count: usize, - compensation_admitted: bool, -) -> moa_execution::Result { - pending_terminal_step_count( - settled_task_count, - drained_trigger_count, - cancellation_dispatch_count, - compensation_admitted, - ) -} - -#[cfg(test)] -pub(super) fn map_aggregate_requires_continuation_for_test( - outcome: &MapAggregatePageOutcome, -) -> moa_execution::Result { - map_aggregate_requires_continuation(outcome) -} - -#[cfg(test)] -pub(super) fn validate_replan_stop_continuation_for_test( - prior_wake_epoch: u64, - continuation_wake_epoch: u64, - exact_owner: bool, -) -> moa_execution::Result<()> { - validate_replan_stop_continuation_fields(prior_wake_epoch, continuation_wake_epoch, exact_owner) -} - -#[cfg(test)] -pub(super) fn activation_preflight_for_test( - has_pending_terminal: bool, - deadline_at: Option>, - now: DateTime, -) -> &'static str { - match activation_preflight(has_pending_terminal, deadline_at, now) { - ActivationPreflight::PendingTerminal => "pending_terminal", - ActivationPreflight::DueDeadline => "due_deadline", - ActivationPreflight::Ordinary => "ordinary", - } -} diff --git a/crates/moa-orchestrator/src/objects/execution_run_controller/settlement.rs b/crates/moa-orchestrator/src/objects/execution_run_controller/settlement.rs index fe8f897a9..0e0ca39b4 100644 --- a/crates/moa-orchestrator/src/objects/execution_run_controller/settlement.rs +++ b/crates/moa-orchestrator/src/objects/execution_run_controller/settlement.rs @@ -43,7 +43,7 @@ pub(super) fn continuation_checkpoint( run: &ExecutionRunRecord, ) -> ExecutionRunActivationCheckpoint { ExecutionRunActivationCheckpoint { - status: ExecutionRunStatus::Running, + status: continuation_status(run.pending_terminal.is_some()), activation_state: ExecutionActivationState::Queued, next_wake_at: run.next_wake_at, waiting_since: run.waiting_since, @@ -52,6 +52,21 @@ pub(super) fn continuation_checkpoint( } } +/// Chooses the run phase a bounded continuation must preserve rather than overwrite. +/// +/// A fenced terminal intent means the only remaining work is its compensation drain, and +/// `trigger_is_current` requires `compensating` for that run's compensation watchdog to stay +/// current; overwriting the phase silently disarms the watchdog for the in-flight attempt. +/// Without a terminal intent a continuation means forward scheduler work remains, which is +/// `running` for every non-terminal phase. +pub(super) fn continuation_status(has_pending_terminal: bool) -> ExecutionRunStatus { + if has_pending_terminal { + ExecutionRunStatus::Compensating + } else { + ExecutionRunStatus::Running + } +} + pub(super) fn waiting_status(waiting: &[WaitingReason]) -> ExecutionRunStatus { if waiting .iter() diff --git a/crates/moa-orchestrator/src/objects/execution_run_controller/tests.rs b/crates/moa-orchestrator/src/objects/execution_run_controller/tests.rs index f48af3556..895d2ce5f 100644 --- a/crates/moa-orchestrator/src/objects/execution_run_controller/tests.rs +++ b/crates/moa-orchestrator/src/objects/execution_run_controller/tests.rs @@ -12,6 +12,7 @@ use moa_execution::{ use super::{ ExecutionRunAdvanceOutcome, ExecutionRunAdvanceRequest, ExecutionRunAdvanceResponse, advance, + advance::{ActivationLimits, ActivationPreflight}, settlement, }; @@ -25,25 +26,49 @@ fn request(run_uid: Uuid) -> ExecutionRunAdvanceRequest { } } -#[test] -fn activation_limits_are_independent_hard_ceilings() { - // Pins: a large node page cannot consume more scheduler transitions than the activation - // bound, and a large ready page cannot borrow that unused budget to exceed dispatch_batch_size. - let observed = advance::consume_limits_for_test(3, 2, &[2, 4], &[1, 9]) - .expect("positive limits are valid"); - - assert_eq!(observed, (3, 2, 0, 0)); -} - #[test] fn activation_limits_reject_zero_instead_of_creating_a_busy_loop() { // Pins: invalid zero bounds fail before the controller can enqueue an endless continuation. - let error = advance::consume_limits_for_test(0, 2, &[1], &[1]) - .expect_err("zero activation steps must fail closed"); + // The controller pump re-enqueues itself, so a zero bound would advance no work and schedule + // another activation forever. + for (steps, batch) in [(0, 2), (2, 0), (0, 0)] { + let error = ActivationLimits::new(steps, batch) + .expect_err("zero activation bounds must fail closed"); + assert_eq!( + error.to_string(), + "invalid execution repository request: controller activation bounds must both be greater than zero" + ); + } + ActivationLimits::new(1, 1).expect("the smallest positive bounds are valid"); +} +#[test] +fn activation_limits_are_independent_hard_ceilings() { + // Pins: a large node page cannot consume more scheduler transitions than the activation + // bound, and a large ready page cannot borrow that unused budget to exceed dispatch_batch_size. + let mut limits = ActivationLimits::new(3, 2).expect("positive limits are valid"); + assert_eq!(limits.inspect_nodes(2), 2); + assert_eq!( + limits.inspect_nodes(4), + 1, + "node inspection is clamped by the remaining activation steps" + ); + assert_eq!(limits.remaining_steps, 0); assert_eq!( - error.to_string(), - "invalid execution repository request: controller activation bounds must both be greater than zero" + limits.task_page_limit().expect("task page limit fits u32"), + 2, + "an exhausted step budget must not shrink the independent dispatch bound" + ); + limits + .record_tasks(2) + .expect("the whole dispatch batch fits its own ceiling"); + assert_eq!(limits.remaining_tasks, 0); + assert_eq!( + limits + .record_tasks(1) + .expect_err("a ready page cannot borrow activation steps for dispatch") + .to_string(), + "invalid execution repository data: controller materialized beyond its dispatch bound" ); } @@ -52,17 +77,53 @@ fn completion_projection_counts_every_bounded_row_against_activation_work() { // Pins: task-evidence and node-evidence scans share the activation-step ceiling; neither // page can be omitted from accounting and turn terminal evaluation into unbounded work. assert_eq!( - advance::completion_scan_steps_for_test(7, 11) - .expect("bounded completion counts fit in usize"), + advance::completion_scan_steps(7, 11).expect("bounded completion counts fit in usize"), 18 ); } +#[test] +fn exhausted_activation_budget_defers_terminal_trigger_drain() { + // Pins: if completion evaluation consumes the last activation step, the controller must not + // call the repository's nonzero-page drain API. It checkpoints one continuation so the fresh + // wake starts with a real drain budget instead of failing or spinning on page_limit=0. + assert_eq!( + advance::terminal_trigger_page_limit(0).expect("zero remaining work is a valid deferral"), + None + ); + assert_eq!( + advance::terminal_trigger_page_limit(1).expect("one remaining step permits one trigger"), + Some(1) + ); + assert_eq!( + advance::terminal_trigger_page_limit(2_000).expect("repository page size is bounded"), + Some(1_000) + ); +} + +#[test] +fn pending_terminal_pages_charge_every_bounded_transition() { + // Pins: forward storage settlement, trigger cleanup, cancellation dispatch, and the single + // reverse-order compensation admission all share maximum_activation_steps. A compensation + // success/retry wake may admit only one slice, while review/external waits charge no phantom + // work and remain parked until their persisted resolution enqueues a fresh wake. + assert_eq!( + advance::pending_terminal_step_count(2, 3, 4, true) + .expect("bounded terminal page accounting fits"), + 10 + ); + assert_eq!( + advance::pending_terminal_step_count(0, 0, 0, false) + .expect("a parked review or external wait performs no controller work"), + 0 + ); +} + #[test] fn stale_or_paused_activation_has_no_controller_side_effects() { // Pins: a stale delivery—including a defensive activation delivered while the run is // paused—must acknowledge successfully without polling, trigger, or progress work. - let commit = advance::stale_commit_for_test(9, 14); + let commit = advance::stale_commit(9, 14); assert_eq!( commit.response, @@ -83,16 +144,16 @@ fn stale_or_paused_activation_has_no_controller_side_effects() { fn resumed_activation_recovery_enqueues_exactly_one_fresh_wake() { // Pins: after a crash following any committed page, the resumed wake performs no second page; // it can only ACK wake 5 and create wake 6 as the bounded continuation. - advance::validate_resumed_recovery_for_test(5, 6, true) + advance::validate_resumed_recovery_commit(5, 6, true) .expect("one exact fresh continuation is valid"); - let skipped_wake = advance::validate_resumed_recovery_for_test(5, 7, true) + let skipped_wake = advance::validate_resumed_recovery_commit(5, 7, true) .expect_err("recovery cannot skip to a second continuation wake"); assert_eq!( skipped_wake.to_string(), "invalid execution repository data: resumed activation recovery must enqueue exactly one fresh wake" ); - let missing = advance::validate_resumed_recovery_for_test(5, 6, false) + let missing = advance::validate_resumed_recovery_commit(5, 6, false) .expect_err("recovery cannot ACK without a continuation"); assert_eq!( missing.to_string(), @@ -105,17 +166,17 @@ fn replan_stop_page_transfers_exactly_one_fresh_wake_to_its_run_activation() { // Pins: a bounded replan-stop scan commits its cursor, ACKs the source wake, rebinds the // durable intent, and creates exactly one new RunActivation in the same transaction. The // controller must reject both a skipped epoch and a dispatch owned by another run boundary. - advance::validate_replan_stop_continuation_for_test(12, 13, true) + advance::validate_replan_stop_continuation_fields(12, 13, true) .expect("one exact replan-stop continuation is valid"); - let skipped = advance::validate_replan_stop_continuation_for_test(12, 14, true) + let skipped = advance::validate_replan_stop_continuation_fields(12, 14, true) .expect_err("a replan-stop page cannot skip a fresh wake"); assert_eq!( skipped.to_string(), "invalid execution repository data: resumed activation recovery must enqueue exactly one fresh wake" ); - let wrong_owner = advance::validate_replan_stop_continuation_for_test(12, 13, false) + let wrong_owner = advance::validate_replan_stop_continuation_fields(12, 13, false) .expect_err("a replan-stop continuation must own the exact run wake"); assert_eq!( wrong_owner.to_string(), @@ -128,16 +189,16 @@ fn successful_terminal_trigger_drain_is_nonempty_and_owns_one_fresh_wake() { // Pins: successful finalization drains active deadline/wait triggers in bounded pages. A // committed page must settle real work and transfer ownership to exactly the next wake; a // zero-row page or skipped epoch could otherwise hot-loop or strand terminal finalization. - advance::validate_trigger_drain_for_test(8, 9, 2) + advance::validate_trigger_drain_continuation(8, 9, 2) .expect("one nonempty drain page and one exact continuation are valid"); - let empty = advance::validate_trigger_drain_for_test(8, 9, 0) + let empty = advance::validate_trigger_drain_continuation(8, 9, 0) .expect_err("a page continuation cannot be committed without trigger progress"); assert_eq!( empty.to_string(), "invalid execution repository data: terminal trigger drain must settle a nonempty page and enqueue one fresh wake" ); - let skipped = advance::validate_trigger_drain_for_test(8, 10, 2) + let skipped = advance::validate_trigger_drain_continuation(8, 10, 2) .expect_err("a drain page cannot skip a controller wake"); assert_eq!( skipped.to_string(), @@ -145,53 +206,13 @@ fn successful_terminal_trigger_drain_is_nonempty_and_owns_one_fresh_wake() { ); } -#[test] -fn exhausted_activation_budget_defers_terminal_trigger_drain() { - // Pins: if completion evaluation consumes the last activation step, the controller must not - // call the repository's nonzero-page drain API. It checkpoints one continuation so the fresh - // wake starts with a real drain budget instead of failing or spinning on page_limit=0. - assert_eq!( - advance::terminal_trigger_page_limit_for_test(0) - .expect("zero remaining work is a valid deferral"), - None - ); - assert_eq!( - advance::terminal_trigger_page_limit_for_test(1) - .expect("one remaining step permits one trigger"), - Some(1) - ); - assert_eq!( - advance::terminal_trigger_page_limit_for_test(2_000) - .expect("repository page size is bounded"), - Some(1_000) - ); -} - -#[test] -fn pending_terminal_pages_charge_every_bounded_transition() { - // Pins: forward storage settlement, trigger cleanup, cancellation dispatch, and the single - // reverse-order compensation admission all share maximum_activation_steps. A compensation - // success/retry wake may admit only one slice, while review/external waits charge no phantom - // work and remain parked until their persisted resolution enqueues a fresh wake. - assert_eq!( - advance::pending_terminal_step_count_for_test(2, 3, 4, true) - .expect("bounded terminal page accounting fits"), - 10 - ); - assert_eq!( - advance::pending_terminal_step_count_for_test(0, 0, 0, false) - .expect("a parked review or external wait performs no controller work"), - 0 - ); -} - #[test] fn bounded_map_aggregate_pages_continue_only_after_a_completed_page() { // Pins: partial/replayed partial pages, overflow, and a cursor conflict end this activation and // enqueue one continuation. Only a completed page may spend remaining steps on another node; // a missing run is corruption rather than a retry loop. assert!( - advance::map_aggregate_requires_continuation_for_test(&MapAggregatePageOutcome::Applied { + advance::map_aggregate_requires_continuation(&MapAggregatePageOutcome::Applied { next_cursor_item_key: Some("item-16".to_string()), aggregated_tasks: 16, aggregate_complete: false, @@ -199,25 +220,22 @@ fn bounded_map_aggregate_pages_continue_only_after_a_completed_page() { .expect("partial aggregate page is valid") ); assert!( - !advance::map_aggregate_requires_continuation_for_test( - &MapAggregatePageOutcome::Replayed { - next_cursor_item_key: Some("item-32".to_string()), - aggregate_complete: true, - }, - ) + !advance::map_aggregate_requires_continuation(&MapAggregatePageOutcome::Replayed { + next_cursor_item_key: Some("item-32".to_string()), + aggregate_complete: true, + },) .expect("completed replay is valid") ); assert!( - advance::map_aggregate_requires_continuation_for_test(&MapAggregatePageOutcome::Overflow,) + advance::map_aggregate_requires_continuation(&MapAggregatePageOutcome::Overflow,) .expect("overflow persists a failed node for the next wake") ); assert!( - advance::map_aggregate_requires_continuation_for_test(&MapAggregatePageOutcome::Conflict,) + advance::map_aggregate_requires_continuation(&MapAggregatePageOutcome::Conflict,) .expect("cursor conflict yields to a fresh wake") ); - let missing = - advance::map_aggregate_requires_continuation_for_test(&MapAggregatePageOutcome::NotFound) - .expect_err("a claimed run cannot disappear"); + let missing = advance::map_aggregate_requires_continuation(&MapAggregatePageOutcome::NotFound) + .expect_err("a claimed run cannot disappear"); assert_eq!( missing.to_string(), "invalid execution repository data: execution run disappeared during bounded map aggregation" @@ -240,7 +258,7 @@ fn parked_wait_phase_prioritizes_human_review_over_a_timer() { expiry: ExecutionTemporalTarget::After { delay_seconds: 3_600, }, - on_expiry: ExecutionWaitExpiryAction::FailRun, + on_expiry: ExecutionWaitExpiryAction::FailTask, }, }, ]; @@ -267,6 +285,22 @@ fn parked_replan_phase_survives_a_bounded_empty_reason_sample() { ); } +#[test] +fn bounded_continuation_keeps_a_fenced_terminal_run_compensating() { + // Pins: a controller crash while draining a compensating run must not rewrite the run phase. + // `trigger_is_current` requires status='compensating' for a CompensationWatchdog, so a + // continuation that checkpoints 'running' lets prepare_watchdog_trigger supersede the + // in-flight watchdog and permanently disarms ambiguity resolution for that attempt. + assert_eq!( + settlement::continuation_status(true), + ExecutionRunStatus::Compensating + ); + assert_eq!( + settlement::continuation_status(false), + ExecutionRunStatus::Running + ); +} + #[test] fn checkpoint_preserves_the_persisted_exact_wait_wake() { // Pins: controller replay never re-resolves an After target from a new wall clock; the exact @@ -298,16 +332,16 @@ fn terminal_fences_run_before_any_ordinary_scheduler_work() { .expect("test timestamp is valid"); assert_eq!( - advance::activation_preflight_for_test(true, Some(now), now), - "pending_terminal" + advance::activation_preflight(true, Some(now), now), + ActivationPreflight::PendingTerminal ); assert_eq!( - advance::activation_preflight_for_test(false, Some(now), now), - "due_deadline" + advance::activation_preflight(false, Some(now), now), + ActivationPreflight::DueDeadline ); assert_eq!( - advance::activation_preflight_for_test(false, Some(now + TimeDelta::seconds(1)), now,), - "ordinary" + advance::activation_preflight(false, Some(now + TimeDelta::seconds(1)), now), + ActivationPreflight::Ordinary ); } diff --git a/crates/moa-orchestrator/src/runtime/endpoint.rs b/crates/moa-orchestrator/src/runtime/endpoint.rs index 3dc03a009..272e667b9 100644 --- a/crates/moa-orchestrator/src/runtime/endpoint.rs +++ b/crates/moa-orchestrator/src/runtime/endpoint.rs @@ -52,6 +52,7 @@ use crate::{ contacts::{Contacts, ContactsImpl}, durable_timeout::{DurableTimeout, DurableTimeoutImpl}, execution::{Execution, ExecutionImpl}, + execution_amendment_planner::{ExecutionAmendmentPlanner, ExecutionAmendmentPlannerImpl}, execution_dispatcher::{ ExecutionDispatchDrain, ExecutionDispatchDrainImpl, ExecutionDispatchReconciler, ExecutionDispatchReconcilerImpl, ExecutionDispatcher, ExecutionDispatcherImpl, @@ -109,6 +110,7 @@ const CORE_BODY_SERVICE_NAMES: &[&str] = &[ "ExecutionDispatcher", "ExecutionDispatchDrain", "ExecutionDispatchReconciler", + "ExecutionAmendmentPlanner", "DurableTimeout", "GraphMemoryMaint", "Knowledge", @@ -147,6 +149,7 @@ const INGRESS_PRIVATE_SERVICE_NAMES: &[&str] = &[ "ExecutionDispatcher", "ExecutionDispatchDrain", "ExecutionDispatchReconciler", + "ExecutionAmendmentPlanner", "DurableTimeout", ]; #[cfg(test)] @@ -557,6 +560,21 @@ pub fn build_endpoint(runtime_deps: &RuntimeDeps) -> Endpoint { ExecutionDispatchReconcilerImpl::new(pool.clone(), &config.execution).serve(), high_cost_internal_service_options(), ) + .bind_with_options( + ExecutionAmendmentPlannerImpl::new( + pool.clone(), + config.execution.clone(), + moa_core::types::identifiers::ModelId::new( + config + .models + .auxiliary + .clone() + .unwrap_or_else(|| config.models.main.clone()), + ), + ) + .serve(), + high_cost_internal_service_options(), + ) .bind_with_options( DurableTimeoutImpl::new(pool.clone()).serve(), high_cost_internal_service_options(), @@ -1041,6 +1059,7 @@ mod tests { "ExecutionDispatcher", "ExecutionDispatchDrain", "ExecutionDispatchReconciler", + "ExecutionAmendmentPlanner", "DurableTimeout", ] ); diff --git a/crates/moa-orchestrator/src/runtime/mod.rs b/crates/moa-orchestrator/src/runtime/mod.rs index 418e355dd..2e850bd05 100644 --- a/crates/moa-orchestrator/src/runtime/mod.rs +++ b/crates/moa-orchestrator/src/runtime/mod.rs @@ -8,4 +8,5 @@ pub mod endpoint; pub mod execution_dispatch; pub mod jobs; pub mod kms; +pub mod restate_drain; pub mod sandbox_workspace_rollout; diff --git a/crates/moa-orchestrator/src/runtime/restate_drain.rs b/crates/moa-orchestrator/src/runtime/restate_drain.rs new file mode 100644 index 000000000..73593f88a --- /dev/null +++ b/crates/moa-orchestrator/src/runtime/restate_drain.rs @@ -0,0 +1,384 @@ +//! Restate deployment-drain observation owned by the maintenance role. +//! +//! Bounded activations exist so that a handler deployment stops accepting new +//! work the moment a newer revision registers, then retires once the +//! invocations already pinned to it finish. Nothing else in the runtime +//! observes whether that retirement actually completes: a wedged old revision +//! holding pinned invocations keeps serving forever and looks identical to a +//! healthy fleet. This lane is the only observer of that state. +//! +//! It reads Restate's admin introspection tables, never Kubernetes. Everything +//! it exports is a fleet-level aggregate with no `deployment_id` label, because +//! deployment identity is unbounded over the lifetime of a cluster. + +use std::time::Duration; + +use anyhow::{Context as AnyhowContext, Result}; +use reqwest::Client; +use serde::Deserialize; +use tokio::task::JoinHandle; +use tokio_util::sync::CancellationToken; + +/// Restate's default admin port, used when only the ingress URL is configured. +const RESTATE_ADMIN_PORT: u16 = 9070; +/// Environment variable naming the Restate admin API, when it is not derivable. +const RESTATE_ADMIN_URL_ENV: &str = "MOA_RESTATE_ADMIN_URL"; +/// Per-request ceiling for one admin introspection call. +const ADMIN_HTTP_TIMEOUT: Duration = Duration::from_secs(10); +/// Steady cadence between drain observations. +/// +/// Deployment registration is operator-paced, so this is deliberately far +/// slower than the correctness lanes. The drain alert compares against an hour; +/// five minutes resolves it with room to spare while keeping this lane's cost +/// on the Restate query engine negligible. +const OBSERVE_INTERVAL: Duration = Duration::from_secs(5 * 60); +/// Upper bound on the exponential backoff after failed observations. +const OBSERVE_MAX_BACKOFF: Duration = Duration::from_secs(60 * 60); + +/// One row of per-deployment drain state from Restate's introspection tables. +/// +/// `age_seconds` is the age of the deployment registration itself. Supersession +/// age — the number that answers "how long has this been draining" — is derived +/// from it in [`aggregate_drain_state`], because a deployment only begins +/// draining when a newer one registers. +/// +/// `blocking_invocations` counts non-terminal invocations that still name this +/// deployment, using the same predicate +/// `crate::runtime::bootstrap::active_invocations_query` and +/// `scripts/cutover-long-horizon-execution.sh` use to refuse deregistration. +/// The gauge therefore reads as work remaining before the revision may retire. +const DRAIN_QUERY: &str = "SELECT d.id AS deployment_id, \ + date_part('epoch', now() - d.created_at) AS age_seconds, \ + COUNT(i.id) AS blocking_invocations \ + FROM sys_deployment d \ + LEFT JOIN sys_invocation i \ + ON (i.pinned_deployment_id = d.id OR i.last_attempt_deployment_id = d.id) \ + AND i.status NOT IN ('completed', 'killed') \ + GROUP BY d.id, d.created_at"; + +#[derive(Debug, Deserialize)] +struct DrainQueryResponse { + rows: Vec, +} + +#[derive(Debug, Deserialize)] +struct DeploymentDrainRow { + deployment_id: String, + age_seconds: f64, + blocking_invocations: i64, +} + +/// Fleet-level drain aggregate exported as unlabeled gauges. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DrainObservation { + /// Superseded deployments that still hold non-terminal invocations. + pub deployments: u64, + /// Non-terminal invocations blocking retirement of those deployments. + pub blocking_invocations: u64, + /// Time since the longest-draining of those deployments was superseded. + pub oldest_drain_age: Duration, + /// Identities of the draining deployments, for logs only — never a label. + pub draining_deployment_ids: Vec, +} + +impl DrainObservation { + /// The healthy fleet: one current revision and nothing left to retire. + fn drained() -> Self { + Self { + deployments: 0, + blocking_invocations: 0, + oldest_drain_age: Duration::ZERO, + draining_deployment_ids: Vec::new(), + } + } +} + +/// Resolves the Restate admin API from the environment or the ingress URL. +/// +/// The maintenance pod is given an ingress URL, not an admin URL, so the +/// derived form is the one that works unconfigured: the admin API is the same +/// host on Restate's admin port. `MOA_RESTATE_ADMIN_URL` overrides it for +/// deployments whose two ports are not colocated, such as a local stack that +/// remaps both. +pub fn resolve_admin_url(ingress_url: &str) -> Result { + if let Ok(configured) = std::env::var(RESTATE_ADMIN_URL_ENV) { + let configured = configured.trim().trim_end_matches('/'); + if !configured.is_empty() { + return Ok(configured.to_string()); + } + } + derive_admin_url_from_ingress(ingress_url) +} + +/// Rewrites an ingress URL onto Restate's admin port. +fn derive_admin_url_from_ingress(ingress_url: &str) -> Result { + let mut derived = reqwest::Url::parse(ingress_url) + .with_context(|| format!("parse Restate ingress URL `{ingress_url}`"))?; + derived + .set_port(Some(RESTATE_ADMIN_PORT)) + .map_err(|()| anyhow::anyhow!("Restate ingress URL `{ingress_url}` accepts no port"))?; + Ok(derived.as_str().trim_end_matches('/').to_string()) +} + +/// Starts the maintenance-owned Restate drain observer. +/// +/// The returned task never fails. An unreachable or erroring admin API is +/// logged and retried with bounded backoff, because the maintenance role runs +/// as a single non-redundant replica under a `Recreate` strategy and must not +/// be taken down by a transient dependency it only observes. The task returns +/// only when `shutdown` is cancelled. +pub fn spawn_restate_drain_observer( + admin_url: String, + shutdown: CancellationToken, +) -> JoinHandle<()> { + tokio::spawn(async move { + let client = match Client::builder().timeout(ADMIN_HTTP_TIMEOUT).build() { + Ok(client) => client, + Err(error) => { + tracing::warn!( + %error, + "Restate drain observer could not build its HTTP client; drain telemetry is disabled" + ); + return; + } + }; + tracing::info!( + admin_url = %admin_url, + interval_secs = OBSERVE_INTERVAL.as_secs(), + "Restate deployment drain observer started" + ); + + let mut consecutive_failures = 0_u32; + loop { + match observe_drain_state(&client, &admin_url).await { + Ok(observation) => { + consecutive_failures = 0; + export(&observation); + } + Err(error) => { + consecutive_failures = consecutive_failures.saturating_add(1); + // The gauges keep their last observed values rather than + // reporting a fabricated zero: an unreachable admin API is + // not evidence that nothing is draining. + tracing::warn!( + %error, + consecutive_failures, + retry_delay_secs = observe_delay(consecutive_failures).as_secs(), + "Restate deployment drain observation failed; retrying with bounded backoff" + ); + } + } + + let delay = observe_delay(consecutive_failures); + tokio::select! { + () = shutdown.cancelled() => return, + () = tokio::time::sleep(delay) => {} + } + } + }) +} + +/// Writes the aggregate to the drain gauges, including the healthy zero. +/// +/// The zero must be written on every successful pass. The drain alert is +/// `absent()`-guarded, so a fleet with nothing draining that never reports +/// would page exactly like a fleet that stopped reporting. +fn export(observation: &DrainObservation) { + moa_observability::runtime_metrics::record_restate_draining_deployments( + observation.deployments, + observation.blocking_invocations, + observation.oldest_drain_age, + ); + if observation.deployments > 0 { + tracing::info!( + draining_deployments = observation.deployments, + blocking_invocations = observation.blocking_invocations, + oldest_drain_age_secs = observation.oldest_drain_age.as_secs(), + deployment_ids = %observation.draining_deployment_ids.join(","), + "Restate deployment revisions are still draining" + ); + } +} + +/// Runs one admin introspection pass and aggregates the result. +async fn observe_drain_state(client: &Client, admin_url: &str) -> Result { + let response = client + .post(format!("{admin_url}/query")) + .header("content-type", "application/json") + .header("accept", "application/json") + .json(&serde_json::json!({ "query": DRAIN_QUERY })) + .send() + .await + .context("query Restate deployment drain state")? + .error_for_status() + .context("Restate deployment drain query failed")?; + let payload = response + .json::() + .await + .context("decode Restate deployment drain query")?; + Ok(aggregate_drain_state(payload.rows)) +} + +/// Reduces per-deployment rows to the unlabeled fleet aggregate. +/// +/// A deployment is draining when a newer deployment exists — Restate routes new +/// invocations to the newest revision registering a service — and non-terminal +/// invocations still name it. Its drain age is measured from the moment it was +/// superseded, which is the registration age of the next-newer deployment, not +/// from its own registration. Measuring from its own registration would report +/// the deployment's entire lifetime as drain time and put every rollout +/// instantly over the alert threshold. +fn aggregate_drain_state(mut rows: Vec) -> DrainObservation { + if rows.len() < 2 { + // A single registered deployment is the current one, and no deployment + // at all is a fleet mid-bootstrap. Neither can be draining. + return DrainObservation::drained(); + } + // Oldest first. The last row is the current revision, which by definition + // still accepts new work and is therefore never counted as draining. + rows.sort_by(|left, right| { + right + .age_seconds + .total_cmp(&left.age_seconds) + .then_with(|| left.deployment_id.cmp(&right.deployment_id)) + }); + + let mut observation = DrainObservation::drained(); + for index in 0..rows.len() - 1 { + let deployment = &rows[index]; + if deployment.blocking_invocations <= 0 { + // Superseded and fully drained. It may still be registered awaiting + // deregistration, but it is holding nothing. + continue; + } + // The next-newer deployment's own age is the elapsed time since it + // registered, which is exactly when this one stopped taking new work. + let drain_age = Duration::try_from_secs_f64(rows[index + 1].age_seconds.max(0.0)) + .unwrap_or(Duration::ZERO); + observation.deployments = observation.deployments.saturating_add(1); + observation.blocking_invocations = observation + .blocking_invocations + .saturating_add(deployment.blocking_invocations.unsigned_abs()); + observation.oldest_drain_age = observation.oldest_drain_age.max(drain_age); + observation + .draining_deployment_ids + .push(deployment.deployment_id.clone()); + } + observation +} + +/// Exponential backoff from the steady cadence, capped at [`OBSERVE_MAX_BACKOFF`]. +fn observe_delay(consecutive_failures: u32) -> Duration { + let multiplier = 1_u32 + .checked_shl(consecutive_failures.min(31)) + .unwrap_or(u32::MAX); + OBSERVE_INTERVAL + .saturating_mul(multiplier) + .min(OBSERVE_MAX_BACKOFF) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn row(id: &str, age_seconds: f64, blocking_invocations: i64) -> DeploymentDrainRow { + DeploymentDrainRow { + deployment_id: id.to_string(), + age_seconds, + blocking_invocations, + } + } + + // Pins: drain age is measured from supersession, not from registration. A + // long-lived deployment superseded one minute ago has been draining for one + // minute; reporting its registration age would put every rollout instantly + // past the one-hour drain alert. + #[test] + fn drain_age_measures_time_since_supersession_offline() { + let observation = aggregate_drain_state(vec![ + row("dp_old", 30.0 * 24.0 * 60.0 * 60.0, 4), + row("dp_current", 60.0, 0), + ]); + + assert_eq!(observation.deployments, 1); + assert_eq!(observation.blocking_invocations, 4); + assert_eq!(observation.oldest_drain_age, Duration::from_secs(60)); + assert_eq!(observation.draining_deployment_ids, vec!["dp_old"]); + } + + // Pins: the healthy fleet reports an explicit zero rather than no sample, + // because the drain alert is absent()-guarded and silence pages. + #[test] + fn fully_drained_fleet_reports_explicit_zero_offline() { + let observation = aggregate_drain_state(vec![ + row("dp_retired", 7_200.0, 0), + row("dp_current", 300.0, 11), + ]); + + assert_eq!(observation, DrainObservation::drained()); + } + + // Pins: the newest deployment is never draining however much work it holds, + // and a fleet that has only ever registered one deployment reports zero. + #[test] + fn current_deployment_is_never_counted_as_draining_offline() { + assert_eq!( + aggregate_drain_state(vec![row("dp_current", 42.0, 99)]), + DrainObservation::drained() + ); + assert_eq!( + aggregate_drain_state(Vec::new()), + DrainObservation::drained() + ); + } + + // Pins: with three live revisions, each draining deployment is aged from the + // revision that superseded it, so the oldest drain age is the supersession + // age of the earliest still-blocking revision, and counts sum across both. + #[test] + fn multiple_draining_revisions_aggregate_without_deployment_labels_offline() { + let observation = aggregate_drain_state(vec![ + row("dp_current", 100.0, 3), + row("dp_v1", 9_000.0, 2), + row("dp_v2", 5_000.0, 5), + ]); + + assert_eq!(observation.deployments, 2); + assert_eq!(observation.blocking_invocations, 7); + // dp_v1 was superseded when dp_v2 registered 5000s ago; dp_v2 was + // superseded when dp_current registered 100s ago. + assert_eq!(observation.oldest_drain_age, Duration::from_secs(5_000)); + assert_eq!(observation.draining_deployment_ids, vec!["dp_v1", "dp_v2"]); + } + + // Pins: clock skew between Restate and this process cannot produce a + // negative duration or panic the maintenance singleton. + #[test] + fn negative_registration_age_from_clock_skew_clamps_to_zero_offline() { + let observation = + aggregate_drain_state(vec![row("dp_old", 600.0, 1), row("dp_current", -5.0, 0)]); + + assert_eq!(observation.deployments, 1); + assert_eq!(observation.oldest_drain_age, Duration::ZERO); + } + + // Pins: the admin URL is derivable from the ingress URL the maintenance pod + // already receives, so drain telemetry needs no new deployment setting. + #[test] + fn admin_url_derives_restate_admin_port_from_ingress_offline() { + let derived = + derive_admin_url_from_ingress("http://restate.moa-restate.svc.cluster.local:8080") + .expect("derive admin URL"); + + assert_eq!(derived, "http://restate.moa-restate.svc.cluster.local:9070"); + } + + // Pins: repeated admin failures back off toward the hour cap instead of + // hammering an unhealthy Restate, and one success returns to the cadence. + #[test] + fn failed_observations_back_off_to_the_bounded_ceiling_offline() { + assert_eq!(observe_delay(0), OBSERVE_INTERVAL); + assert_eq!(observe_delay(1), OBSERVE_INTERVAL * 2); + assert_eq!(observe_delay(31), OBSERVE_MAX_BACKOFF); + } +} diff --git a/crates/moa-orchestrator/src/runtime/sandbox_workspace_rollout.rs b/crates/moa-orchestrator/src/runtime/sandbox_workspace_rollout.rs index 4c1bad2ac..195830d65 100644 --- a/crates/moa-orchestrator/src/runtime/sandbox_workspace_rollout.rs +++ b/crates/moa-orchestrator/src/runtime/sandbox_workspace_rollout.rs @@ -160,18 +160,21 @@ async fn bootstrap_provider_account( || format!("local:{}", account.isolation_cell), ToOwned::to_owned, ); - let headroom = if account.provider == "daytona" { - config - .cloud - .daytona_storage - .account(account.provider_account_id) - .map_or_else( - || json!({}), - |storage| json!({ "volumes": storage.admission_headroom }), - ) - } else { - json!({}) - }; + // Volumes are a Daytona-only lifetime resource with an operator-authored + // organization ceiling. Non-Daytona accounts have no volume inventory, so + // they carry neither a volume limit nor volume headroom. + let daytona_storage = (account.provider == "daytona") + .then(|| { + config + .cloud + .daytona_storage + .account(account.provider_account_id) + }) + .flatten(); + let headroom = daytona_storage.map_or_else( + || json!({}), + |storage| json!({ "volumes": storage.admission_headroom }), + ); let generation = i64::try_from(account.generation) .context("sandbox provider-account generation exceeds Postgres bigint")?; sqlx::query("SELECT moa.bootstrap_sandbox_provider_account($1, $2, $3, $4, $5, $6, $7, $8)") @@ -181,7 +184,7 @@ async fn bootstrap_provider_account( .bind(account.isolation_cell) .bind(organization_fingerprint) .bind(account.project_fingerprint) - .bind(limits.as_json()) + .bind(limits.as_json(daytona_storage.map(|storage| storage.volume_ceiling))) .bind(headroom) .execute(connection) .await @@ -197,7 +200,7 @@ async fn bootstrap_provider_account( #[derive(Debug, Default)] struct ProviderCapacityLimits { workspaces: u64, - volumes: u64, + active_hands: u64, checkpoints: u64, logical_bytes: u64, } @@ -205,27 +208,35 @@ struct ProviderCapacityLimits { impl ProviderCapacityLimits { fn add_route(&mut self, route: &moa_config::SandboxWorkspaceQuotaRouteConfig) -> Result<()> { self.workspaces = checked_sum(self.workspaces, route.max_workspaces, "workspaces")?; - self.volumes = checked_sum(self.volumes, route.max_active_hands, "volumes")?; + self.active_hands = checked_sum(self.active_hands, route.max_active_hands, "active_hands")?; self.checkpoints = checked_sum(self.checkpoints, route.max_checkpoints, "checkpoints")?; self.logical_bytes = checked_sum(self.logical_bytes, route.max_logical_bytes, "logical_bytes")?; Ok(()) } - fn as_json(&self) -> Value { - json!({ + /// Renders the account-wide ceiling, including the Daytona volume ceiling + /// when the account owns a configured volume isolation cell. + fn as_json(&self, volume_ceiling: Option) -> Value { + let mut limits = json!({ "workspaces": self.workspaces, - "volumes": self.volumes, + "active_hands": self.active_hands, "checkpoints": self.checkpoints, "logical_bytes": self.logical_bytes, - }) + }); + if let Some(volume_ceiling) = volume_ceiling + && let Some(object) = limits.as_object_mut() + { + object.insert("volumes".to_string(), json!(volume_ceiling)); + } + limits } } fn tenant_limits(route: &moa_config::SandboxWorkspaceQuotaRouteConfig) -> Value { json!({ "workspaces": route.max_workspaces, - "volumes": route.max_active_hands, + "active_hands": route.max_active_hands, "checkpoints": route.max_checkpoints, "logical_bytes": route.max_logical_bytes, }) @@ -265,13 +276,25 @@ mod tests { .expect("bounded route should aggregate"); } assert_eq!( - limits.as_json(), + limits.as_json(None), json!({ "workspaces": 8, - "volumes": 4, + "active_hands": 4, "checkpoints": 20, "logical_bytes": 2_048, - }) + }), + "concurrent-hand quotas must charge active_hands, never the Daytona volume ceiling" + ); + assert_eq!( + limits.as_json(Some(12)), + json!({ + "workspaces": 8, + "active_hands": 4, + "checkpoints": 20, + "logical_bytes": 2_048, + "volumes": 12, + }), + "a Daytona account's volume ceiling is the operator-authored organization cap" ); } } diff --git a/crates/moa-orchestrator/src/services/execution/handlers.rs b/crates/moa-orchestrator/src/services/execution/handlers.rs index ae1b8eb43..99e06dd1e 100644 --- a/crates/moa-orchestrator/src/services/execution/handlers.rs +++ b/crates/moa-orchestrator/src/services/execution/handlers.rs @@ -928,7 +928,7 @@ async fn kick_control_dispatcher( kick_execution_dispatcher(ctx, run_uid, generation, action).await } -async fn kick_execution_dispatcher( +pub(crate) async fn kick_execution_dispatcher( ctx: &Context<'_>, run_uid: uuid::Uuid, durable_fence: u64, @@ -1144,7 +1144,7 @@ pub(super) async fn external_wait_mutation( Ok(mutation_from_task_write(write)) } -pub(super) async fn apply_amendment_inner( +pub(crate) async fn apply_amendment_inner( pool: sqlx::PgPool, config: ExecutionConfig, request: ExecutionAmendmentRequest, diff --git a/crates/moa-orchestrator/src/services/execution/support.rs b/crates/moa-orchestrator/src/services/execution/support.rs index 5a6b2613a..3cdf76255 100644 --- a/crates/moa-orchestrator/src/services/execution/support.rs +++ b/crates/moa-orchestrator/src/services/execution/support.rs @@ -13,13 +13,13 @@ pub(super) fn scoped_catalog_error( } #[derive(Clone, Debug, Deserialize, Serialize)] -pub(super) struct ExecutionMutationHandoff { +pub(crate) struct ExecutionMutationHandoff { wake_epoch: u64, task_ids_to_release: Vec, } #[derive(Clone, Debug, Deserialize, Serialize)] -pub(super) enum ExecutionMutationAccepted { +pub(crate) enum ExecutionMutationAccepted { Accepted { response: ExecutionMutationResponse, handoff: ExecutionMutationHandoff, @@ -30,14 +30,14 @@ pub(super) enum ExecutionMutationAccepted { } impl ExecutionMutationAccepted { - pub(super) fn wake_epoch(&self) -> Option { + pub(crate) fn wake_epoch(&self) -> Option { match self { Self::Accepted { handoff, .. } => Some(handoff.wake_epoch), Self::Rejected { .. } => None, } } - pub(super) fn with_task_ids_to_release( + pub(crate) fn with_task_ids_to_release( mut self, task_ids_to_release: Vec, ) -> Self { @@ -47,7 +47,7 @@ impl ExecutionMutationAccepted { self } - pub(super) fn into_response(self) -> ExecutionMutationResponse { + pub(crate) fn into_response(self) -> ExecutionMutationResponse { match self { Self::Accepted { response, .. } | Self::Rejected { response } => response, } diff --git a/crates/moa-orchestrator/src/services/execution/tests.rs b/crates/moa-orchestrator/src/services/execution/tests.rs index e224aa085..cabdf6846 100644 --- a/crates/moa-orchestrator/src/services/execution/tests.rs +++ b/crates/moa-orchestrator/src/services/execution/tests.rs @@ -265,7 +265,7 @@ fn skill_revision(name: &str, revision_uid: u128) -> StoredArtifactRevision { delay_seconds: 86_400, }, on_expiry: - moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, }, input_schema: json!({"type": "object"}), output_schema: json!({"type": "object"}), @@ -638,7 +638,7 @@ fn accepted_turn_requires_skill_template_provenance_from_planning_snapshot() { expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { delay_seconds: 86_400, }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, }, input_schema: json!({"type": "object"}), output_schema: json!({"type": "object"}), @@ -719,7 +719,7 @@ fn pinned_execution_template( expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { delay_seconds: 86_400, }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, }, input_schema: json!({"type": "object"}), output_schema: json!({"type": "object"}), diff --git a/crates/moa-orchestrator/src/services/execution_amendment_planner.rs b/crates/moa-orchestrator/src/services/execution_amendment_planner.rs new file mode 100644 index 000000000..67c58f4f8 --- /dev/null +++ b/crates/moa-orchestrator/src/services/execution_amendment_planner.rs @@ -0,0 +1,556 @@ +//! Bounded self-healing amendment planning for runs parked in `WaitingReplan`. +//! +//! A task that returns `NeedsReplan` parks its run. Nothing else in the product +//! proposes a replacement plan, so without this slice the run waits until its +//! absolute deadline kills it. The controller activation that observes the park +//! is required to stay short and must never issue a model call, so it only +//! *selects* the work: it sends one durable invocation here, and this service +//! performs the paid planner call and submits the resulting candidate through +//! the ordinary `Execution/apply_amendment` boundary, which owns validation, +//! the three replan-stop conditions, and the revision fence. +//! +//! Exactly one planning invocation exists per `(run_uid, plan_revision)`. Every +//! accepted amendment advances the revision, so the number of paid planner calls +//! is bounded by the number of revisions, which the replan-stop evaluation and +//! the pre-call exhaustion check both bound. + +mod preparation; + +#[cfg(test)] +mod tests; + +use std::sync::atomic::{AtomicUsize, Ordering}; + +use async_trait::async_trait; +use moa_artifacts::execution_plan::PlanAmendment; +use moa_brain::execution_planning::{ + ExecutionAmendmentPlanningRequest, ExecutionAmendmentPlanningResultKind, plan_amendment, +}; +use moa_config::ExecutionConfig; +use moa_core::{ + traits::{Identity, LLMProvider}, + types::{ + completion::{CompletionRequest, CompletionStream, SharedCompletionRequest}, + execution_planning::ExecutionPlanningAuditEnvelope, + identifiers::ModelId, + model::ModelCapabilities, + }, +}; +use moa_execution::{ + ReplanStopReason, + capability::amendment_hash, + repository::{ + ExecutionRepository, ExecutionRunRecord, ExecutionScope, + replan_stop::{NewExecutionReplanStopIntent, ReplanStopIntentWriteOutcome}, + }, + state::ExecutionRunStatus, + wire::{ExecutionAmendmentRequest, ExecutionMutationResponse, ExecutionRunRequest}, +}; +use moa_observability::restate_observability::annotate_restate_handler_span; +use restate_sdk::prelude::*; +use serde::{Deserialize, Serialize}; +use serde_json::json; + +use crate::services::{ + execution::ExecutionClient, + llm_gateway::{LLMCompletionAction, LLMGatewayClient, completion_idempotency_key}, +}; + +pub use preparation::{ + AmendmentPlanningInputs, AmendmentPlanningOrigin, AmendmentPlanningTarget, + PreparedAmendmentPlanning, prepare_amendment_planning, +}; + +/// Closed disposition of one bounded amendment-planning slice. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[serde(tag = "outcome", rename_all = "snake_case", deny_unknown_fields)] +pub enum ExecutionAmendmentPlanningResponse { + /// A planner-authored amendment reached the public amendment boundary. + Submitted { + /// Exact durable disposition returned by `Execution/apply_amendment`. + response: Box, + }, + /// Replanning stopped without a candidate and the run was fenced to terminalize. + Stopped { + /// Typed deterministic stop reason. + reason: ReplanStopReason, + }, + /// The revision was no longer the live `WaitingReplan` revision. + Skipped, +} + +/// Restate service that owns the paid amendment-planning slice for one run revision. +#[restate_sdk::service] +#[name = "ExecutionAmendmentPlanner"] +pub trait ExecutionAmendmentPlanner { + /// Plans and submits at most one amendment for the exact parked run revision. + async fn plan( + request: Json, + ) -> Result, HandlerError>; +} + +/// PostgreSQL-backed bounded amendment planner. +#[derive(Clone)] +pub struct ExecutionAmendmentPlannerImpl { + repository: ExecutionRepository, + config: ExecutionConfig, + planner_model: ModelId, +} + +impl ExecutionAmendmentPlannerImpl { + /// Creates the planner over the shared execution repository and auxiliary model. + #[must_use] + pub fn new(pool: sqlx::PgPool, config: ExecutionConfig, planner_model: ModelId) -> Self { + Self { + repository: ExecutionRepository::new(pool), + config, + planner_model, + } + } +} + +impl ExecutionAmendmentPlanner for ExecutionAmendmentPlannerImpl { + #[tracing::instrument(skip(self, ctx, request), fields(run_uid = %request.0.run_uid))] + // SAFETY: ingress-private slice selected only by the trusted controller activation; every + // authority it uses is reloaded from the locked run, and the amendment it proposes is + // submitted under the run's own admitted identity through the authorized public boundary. + async fn plan( + &self, + ctx: Context<'_>, + request: Json, + ) -> Result, HandlerError> { + crate::ctx::adopt_incoming_trace_parent(&ctx); + annotate_restate_handler_span("ExecutionAmendmentPlanner", "plan"); + let target = request.into_inner(); + let scope = target.scope(); + + let repository = self.repository.clone(); + let config = self.config.clone(); + let inputs = ctx + .run(|| async move { + prepare_amendment_planning(&repository, &config, target, chrono::Utc::now()) + .await + .map(Json::from) + }) + .name(format!( + "execution_amendment_inputs_{}_{}", + target.run_uid, target.base_plan_revision + )) + .await? + .into_inner(); + + let prepared = match inputs { + AmendmentPlanningInputs::Skip => { + return Ok(Json::from(ExecutionAmendmentPlanningResponse::Skipped)); + } + AmendmentPlanningInputs::Exhausted { origin, exhaustion } => { + return self + .record_planner_stop( + &ctx, + scope, + origin, + exhaustion.reason, + exhaustion.description, + ) + .await + .map(Json::from); + } + AmendmentPlanningInputs::Ready(prepared) => *prepared, + }; + + let provider = RestateAmendmentPlannerProvider { + ctx: &ctx, + run_uid: target.run_uid, + plan_revision: target.base_plan_revision, + next_attempt: AtomicUsize::new(0), + }; + let planned = plan_amendment( + &provider, + ExecutionAmendmentPlanningRequest { + run_uid: target.run_uid, + base_plan_revision: target.base_plan_revision, + context: prepared.context, + evidence: prepared.evidence, + remaining_budget: prepared.remaining_budget, + planner_model: self.planner_model.clone(), + config: self.config.clone(), + now: prepared.now, + }, + ) + .await + .map_err(crate::workflows::errors::moa_error_to_handler_error)?; + for (ordinal, audit) in planned.audits.into_iter().enumerate() { + self.persist_audit(&ctx, scope, &target, ordinal, audit) + .await?; + } + + let amendment = match planned.kind { + ExecutionAmendmentPlanningResultKind::Ready { amendment, .. } => amendment, + ExecutionAmendmentPlanningResultKind::NeedsInput { message } + | ExecutionAmendmentPlanningResultKind::Unsupported { message } => { + // Planner-authored verdict text is safe to carry into the replan-stop reason. + return self + .record_planner_stop( + &ctx, + scope, + prepared.origin, + ReplanStopReason::NoProgress, + format!("amendment planner stopped: {message}"), + ) + .await + .map(Json::from); + } + ExecutionAmendmentPlanningResultKind::ProviderFailure { message } => { + // Infrastructure failure, not a semantic verdict: the raw provider string must not + // reach the user-surfaced replan-stop gaps. Record the detail for operators (the + // persisted planner audit already carries the ProviderError outcome) and stop the + // replan with a bounded, user-safe description. + tracing::error!( + run_uid = %target.run_uid, + plan_revision = target.base_plan_revision, + detail = %message, + "amendment planner provider failure" + ); + return self + .record_planner_stop( + &ctx, + scope, + prepared.origin, + ReplanStopReason::NoProgress, + "an internal error interrupted amendment planning".to_string(), + ) + .await + .map(Json::from); + } + }; + + let response = submit_amendment(&ctx, &target, &prepared.admitted_identity, amendment) + .await? + .into_inner(); + Ok(Json::from(ExecutionAmendmentPlanningResponse::Submitted { + response: Box::new(response), + })) + } +} + +impl ExecutionAmendmentPlannerImpl { + /// Persists one planner or compile audit produced by the amendment operation. + async fn persist_audit( + &self, + ctx: &Context<'_>, + scope: ExecutionScope, + target: &AmendmentPlanningTarget, + ordinal: usize, + audit: ExecutionPlanningAuditEnvelope, + ) -> Result<(), HandlerError> { + let repository = self.repository.clone(); + ctx.run(|| { + let repository = repository.clone(); + let audit = audit.clone(); + async move { + preparation::persist_amendment_audit(&repository, scope, &audit) + .await + .map(Json::from) + } + }) + .name(format!( + "execution_amendment_audit_{}_{}_{ordinal}", + target.run_uid, target.base_plan_revision + )) + .await?; + Ok(()) + } + + /// Fences the run to terminalize when no further amendment may be attempted. + async fn record_planner_stop( + &self, + ctx: &Context<'_>, + scope: ExecutionScope, + origin: AmendmentPlanningOrigin, + reason: ReplanStopReason, + description: String, + ) -> Result { + let amendment_digest = + amendment_hash(&planner_stop_amendment(origin, reason, &description)) + .map_err(crate::workflows::errors::execution_error_to_handler_error)?; + let repository = self.repository.clone(); + let config = self.config.clone(); + let detail = description.clone(); + let write = ctx + .run(|| { + let repository = repository.clone(); + let config = config.clone(); + let detail = detail.clone(); + async move { + repository + .request_replan_stop( + scope, + &config, + NewExecutionReplanStopIntent { + run_uid: origin.run_uid, + session_id: origin.session_id, + base_plan_revision: origin.base_plan_revision, + origin_task_id: origin.task_id, + task_generation: origin.task_generation, + amendment_hash: amendment_digest, + stop_reason: reason, + detail: Some(detail), + }, + ) + .await + .map(|outcome| Json::from(JournaledReplanStopWrite::from(&outcome))) + .map_err(crate::workflows::errors::execution_error_to_handler_error) + } + }) + .name(format!( + "execution_amendment_stop_{}_{}", + origin.run_uid, origin.base_plan_revision + )) + .await? + .into_inner(); + match write { + JournaledReplanStopWrite::Applied | JournaledReplanStopWrite::Replayed => { + // The intent committed its own exact controller wake; this only shortens the + // latency before the fleet drain delivers it. + crate::services::execution::handlers::kick_execution_dispatcher( + ctx, + origin.run_uid, + origin.base_plan_revision, + "amendment-planner-stop", + ) + .await?; + Ok(ExecutionAmendmentPlanningResponse::Stopped { reason }) + } + JournaledReplanStopWrite::NotFound | JournaledReplanStopWrite::Conflict => { + Ok(ExecutionAmendmentPlanningResponse::Skipped) + } + } + } +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +enum JournaledReplanStopWrite { + Applied, + Replayed, + NotFound, + Conflict, +} + +impl From<&ReplanStopIntentWriteOutcome> for JournaledReplanStopWrite { + fn from(value: &ReplanStopIntentWriteOutcome) -> Self { + match value { + ReplanStopIntentWriteOutcome::Applied(_) => Self::Applied, + ReplanStopIntentWriteOutcome::Replayed(_) => Self::Replayed, + ReplanStopIntentWriteOutcome::NotFound => Self::NotFound, + ReplanStopIntentWriteOutcome::Conflict => Self::Conflict, + } + } +} + +/// Submits one planner-authored amendment through the authorized public boundary. +/// +/// The run's admitted identity travels with the call so `Execution/apply_amendment` +/// performs the same participant check an external amendment would, and the planner +/// never gains authority the run itself did not already hold. +async fn submit_amendment( + ctx: &Context<'_>, + target: &AmendmentPlanningTarget, + identity: &Identity, + amendment: PlanAmendment, +) -> Result, HandlerError> { + let call = ctx + .service_client::() + .apply_amendment(Json::from(ExecutionAmendmentRequest { + run: ExecutionRunRequest { + tenant_id: target.tenant_id, + contact_id: target.contact_id, + session_id: target.session_id, + run_uid: target.run_uid, + }, + expected_plan_revision: target.base_plan_revision, + amendment, + })); + crate::restate_identity::with_identity_headers(call, identity) + .call() + .await + .map_err(HandlerError::from) +} + +/// Builds the deterministic empty amendment identifying one planner-authored stop. +/// +/// `request_replan_stop` keys replay on the amendment hash, so a stop that carries no +/// candidate still needs a stable identity derived only from its own frozen evidence. +fn planner_stop_amendment( + origin: AmendmentPlanningOrigin, + reason: ReplanStopReason, + description: &str, +) -> PlanAmendment { + PlanAmendment { + base_plan_revision: origin.base_plan_revision, + reason: description.to_string(), + evidence: json!({ + "source": "amendment_planner", + "stop_reason": reason.as_str(), + "origin_task_id": origin.task_id, + }), + operations: Vec::new(), + } +} + +/// Returns the exact planning target for a run parked in `WaitingReplan`. +/// +/// Nothing else may reach the planner: a run holding a terminal intent or awaiting manual +/// repair is on its way out, and re-planning it would race the settlement that owns it. +fn waiting_replan_target(run: &ExecutionRunRecord) -> Option { + parked_run_needs_amendment( + run.status, + run.pending_terminal.is_some(), + run.manual_repair_required, + run.waiting_replan_task_count, + ) + .then_some(AmendmentPlanningTarget { + tenant_id: run.tenant_id, + contact_id: run.contact_id, + session_id: run.session_id, + run_uid: run.run_uid, + base_plan_revision: run.plan_revision, + }) +} + +/// Decides whether one parked run is a legitimate amendment-planning candidate. +const fn parked_run_needs_amendment( + status: ExecutionRunStatus, + has_pending_terminal: bool, + manual_repair_required: bool, + waiting_replan_task_count: u64, +) -> bool { + matches!(status, ExecutionRunStatus::WaitingReplan) + && !has_pending_terminal + && !manual_repair_required + // The compiler admits an amendment that supersedes exactly one WaitingReplan node, so a + // run holding any other count cannot be repaired by this path. + && waiting_replan_task_count == 1 +} + +/// Returns the stable Restate identity of the sole planning slice for one run revision. +fn amendment_planning_identity(run_uid: uuid::Uuid, base_plan_revision: u64) -> String { + format!("moa:execution-amendment-plan:v1:{run_uid}:{base_plan_revision}") +} + +/// Selects the bounded planning slice for one controller activation that parked a run. +/// +/// This is the controller's only participation in self-healing: it performs one indexed +/// read and one durable send, and never waits for the planner. If the run is not parked in +/// `WaitingReplan` the read is the whole cost. +pub(crate) async fn dispatch_parked_replan_planning( + ctx: &ObjectContext<'_>, + repository: &ExecutionRepository, + tenant_id: moa_core::types::identifiers::TenantId, + run_uid: uuid::Uuid, + controller_generation: u64, + wake_epoch: u64, +) -> Result<(), HandlerError> { + if automatic_amendment_planner_paused() { + return Ok(()); + } + let repository = repository.clone(); + let target = ctx + .run(|| { + let repository = repository.clone(); + async move { + let run = repository + .load_run(ExecutionScope::ControlPlane, run_uid) + .await + .map_err(crate::workflows::errors::execution_error_to_handler_error)?; + let Some(run) = run else { + return Ok(Json::from(None)); + }; + if run.tenant_id != tenant_id { + return Err(TerminalError::new_with_code( + 409, + "amendment planning owner does not match activation", + ) + .into()); + } + Ok::<_, HandlerError>(Json::from(waiting_replan_target(&run))) + } + }) + .name(format!( + "execution_amendment_planning_select_{controller_generation}_{wake_epoch}" + )) + .await? + .into_inner(); + let Some(target) = target else { + return Ok(()); + }; + let handle = crate::restate_identity::replay_safe_request( + ctx.service_client::() + .plan(Json::from(target)) + .idempotency_key(amendment_planning_identity( + target.run_uid, + target.base_plan_revision, + )), + ) + .send(); + let _planning_invocation_id = handle.invocation_id().await?; + Ok(()) +} + +/// Restate-journaled planner provider that routes every model call through the gateway. +struct RestateAmendmentPlannerProvider<'a, 'ctx> { + ctx: &'a Context<'ctx>, + run_uid: uuid::Uuid, + plan_revision: u64, + next_attempt: AtomicUsize, +} + +#[async_trait] +impl LLMProvider for RestateAmendmentPlannerProvider<'_, '_> { + fn name(&self) -> &'static str { + "restate-llm-gateway" + } + + fn capabilities(&self) -> ModelCapabilities { + ModelCapabilities::default() + } + + async fn complete( + &self, + request: SharedCompletionRequest, + ) -> moa_core::error::Result { + // Restate's JSON transport requires the owned durable DTO. This is the + // explicit serialization boundary after in-process shared routing. + let request = CompletionRequest::from_view(&request); + // Amendment generation and its bounded repair are sequential planner calls. + let attempt = self.next_attempt.fetch_add(1, Ordering::Relaxed); + let response = crate::restate_identity::replay_safe_request( + self.ctx + .service_client::() + .complete(Json::from(request)) + .idempotency_key(completion_idempotency_key( + self.ctx.invocation_id(), + LLMCompletionAction::ExecutionAmendment { + run_uid: self.run_uid, + plan_revision: self.plan_revision, + attempt, + }, + )), + ) + .call() + .await + .map_err(|error| moa_core::error::MoaError::ProviderError(error.to_string()))? + .into_inner(); + Ok(CompletionStream::from_response(response)) + } +} + +#[cfg(feature = "integration")] +fn automatic_amendment_planner_paused() -> bool { + std::env::var("MOA_EXECUTION_TEST_PAUSE_AMENDMENT_PLANNER").as_deref() == Ok("true") +} + +#[cfg(not(feature = "integration"))] +const fn automatic_amendment_planner_paused() -> bool { + false +} diff --git a/crates/moa-orchestrator/src/services/execution_amendment_planner/preparation.rs b/crates/moa-orchestrator/src/services/execution_amendment_planner/preparation.rs new file mode 100644 index 000000000..54029130c --- /dev/null +++ b/crates/moa-orchestrator/src/services/execution_amendment_planner/preparation.rs @@ -0,0 +1,374 @@ +//! Bounded planner inputs frozen from one persisted `WaitingReplan` revision. +//! +//! Every value the amendment planner may observe is derived here, inside one +//! journaled database step, so the model call itself owns no authority: the +//! capability catalog, the authorization envelope, the remaining budget, and the +//! failure evidence all come from the locked run rather than from a caller. + +use std::collections::BTreeSet; + +use chrono::{DateTime, Utc}; +use moa_artifacts::execution_plan::{ExecutionBudgetLimit, ExecutionTaskResult}; +use moa_brain::execution_planning::AmendmentPlanningEvidence; +use moa_config::ExecutionConfig; +use moa_core::{ + traits::Identity, + types::{ + contact::ContactId, + execution_planning::{ + EXECUTION_REPORT_MAX_BYTES, ExecutionPlanningAuditEnvelope, + ExecutionPlanningAuditPayload, + }, + identifiers::{SessionId, TenantId}, + }, +}; +use moa_execution::{ + replan::{ReplanExhaustion, replan_exhaustion_reason}, + repository::{ + ExecutionRepository, ExecutionScope, + amendment::{AmendmentProjectionOutcome, AmendmentProjectionRequest}, + audit::{CompileAuditWriteOutcome, PlannerCallAuditWriteOutcome}, + }, + state::ExecutionTaskId, + wire::ExecutionPlanningContextSnapshot, +}; +use restate_sdk::prelude::*; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use uuid::Uuid; + +/// Exact persisted run revision one bounded planning slice may act on. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct AmendmentPlanningTarget { + /// Owning tenant. + pub tenant_id: TenantId, + /// Optional owning contact. + pub contact_id: Option, + /// Parent session that owns the amendment mutation boundary. + pub session_id: SessionId, + /// Run parked in `WaitingReplan`. + pub run_uid: Uuid, + /// Active plan revision the amendment must fence. + pub base_plan_revision: u64, +} + +impl AmendmentPlanningTarget { + /// Returns the durable repository scope that owns this run. + #[must_use] + pub fn scope(&self) -> ExecutionScope { + self.contact_id.map_or( + ExecutionScope::Tenant { + tenant_id: self.tenant_id, + }, + |contact_id| ExecutionScope::Contact { + tenant_id: self.tenant_id, + contact_id, + }, + ) + } +} + +/// Exact `WaitingReplan` origin every replan-stop intent must fence. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct AmendmentPlanningOrigin { + /// Owning run. + pub run_uid: Uuid, + /// Parent session that owns the mutation boundary. + pub session_id: SessionId, + /// Plan revision that stopped. + pub base_plan_revision: u64, + /// Originating `WaitingReplan` task. + pub task_id: ExecutionTaskId, + /// Exact logical task generation. + pub task_generation: u64, +} + +/// Frozen planner input for one bounded amendment call. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct PreparedAmendmentPlanning { + /// Persisted planning authority narrowed to currently available capabilities. + pub context: ExecutionPlanningContextSnapshot, + /// Immutable goal, plan, projection, and bounded failure evidence. + pub evidence: AmendmentPlanningEvidence, + /// Resources still available for replacement work. + pub remaining_budget: ExecutionBudgetLimit, + /// Exact replan origin used by a later stop intent. + pub origin: AmendmentPlanningOrigin, + /// Principal admitted when the run was created. + pub admitted_identity: Identity, + /// Journaled planner time. + pub now: DateTime, +} + +/// Closed disposition of one bounded planning-input load. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(tag = "inputs", rename_all = "snake_case")] +pub enum AmendmentPlanningInputs { + /// The revision is ready for exactly one bounded planner call. + Ready(Box), + /// Another replan attempt is already knowably exhausted. + Exhausted { + /// Exact replan origin the stop intent fences. + origin: AmendmentPlanningOrigin, + /// Immediately knowable typed exhaustion. + exhaustion: ReplanExhaustion, + }, + /// The revision is no longer the live `WaitingReplan` revision. + Skip, +} + +/// Freezes every planner input for one exact `WaitingReplan` revision. +/// +/// Returns [`AmendmentPlanningInputs::Skip`] whenever the run moved on, already +/// carries a replan-stop intent, or is being terminalized: a planning slice is +/// always allowed to arrive late and must never resurrect a settled revision. +pub async fn prepare_amendment_planning( + repository: &ExecutionRepository, + config: &ExecutionConfig, + target: AmendmentPlanningTarget, + now: DateTime, +) -> Result { + let scope = target.scope(); + let snapshot = match repository + .load_amendment_projection_for_session( + scope, + config, + AmendmentProjectionRequest { + run_uid: target.run_uid, + session_id: target.session_id, + expected_plan_revision: target.base_plan_revision, + }, + ) + .await + .map_err(execution_error)? + { + AmendmentProjectionOutcome::Ready(snapshot) => *snapshot, + AmendmentProjectionOutcome::NotFound | AmendmentProjectionOutcome::Conflict => { + return Ok(AmendmentPlanningInputs::Skip); + } + }; + if snapshot.run.tenant_id != target.tenant_id || snapshot.run.contact_id != target.contact_id { + return Err(TerminalError::new_with_code(409, "execution scope mismatch").into()); + } + if snapshot.run.pending_terminal.is_some() || snapshot.run.manual_repair_required { + return Ok(AmendmentPlanningInputs::Skip); + } + // A recorded stop intent owns the exact fresh wake that terminalizes this run. Proposing an + // amendment against the same revision would apply work the controller is about to discard. + if repository + .load_replan_stop_intent( + scope, + snapshot.run.run_uid, + snapshot.run.controller_generation, + snapshot.run.wake_epoch, + ) + .await + .map_err(execution_error)? + .is_some() + { + return Ok(AmendmentPlanningInputs::Skip); + } + let [waiting_task] = snapshot.projection.replan_tasks.as_slice() else { + return Ok(AmendmentPlanningInputs::Skip); + }; + let origin = AmendmentPlanningOrigin { + run_uid: snapshot.run.run_uid, + session_id: snapshot.run.session_id, + base_plan_revision: snapshot.run.plan_revision, + task_id: waiting_task.task_id, + task_generation: waiting_task.generation, + }; + // The cheapest possible verdict: an elapsed deadline or a fully consumed dimension makes + // another paid planner call pointless, so exhaustion is decided before any provider work. + if let Some(exhaustion) = replan_exhaustion_reason(&snapshot.budget_ledger, now) { + return Ok(AmendmentPlanningInputs::Exhausted { origin, exhaustion }); + } + let Some(outcome) = waiting_task.outcome.as_ref() else { + return Err(TerminalError::new("WaitingReplan task has no persisted outcome").into()); + }; + let ExecutionTaskResult::NeedsReplan { reason, evidence } = &outcome.result else { + return Err(TerminalError::new("WaitingReplan task has no NeedsReplan evidence").into()); + }; + let failure_evidence = bounded_failure_evidence(reason, evidence)?; + let planning_context = repository + .load_planning_context(scope, snapshot.run.planning_context_uid) + .await + .map_err(execution_error)? + .ok_or_else(|| TerminalError::new("execution planning context does not exist"))?; + if planning_context.planning_context_hash != snapshot.run.planning_context_hash + || planning_context.snapshot.tenant_id != snapshot.run.tenant_id + || planning_context.snapshot.contact_id != snapshot.run.contact_id + || planning_context.snapshot.session_id != snapshot.run.session_id + || planning_context.snapshot.originating_user_sequence_num + != snapshot.run.originating_user_sequence_num + || planning_context.snapshot.owner_user_id != snapshot.run.owner_user_id + || planning_context.snapshot.catalog != snapshot.run.catalog + || planning_context.snapshot.authorization != snapshot.run.authorization + || planning_context.snapshot.pinned_instruction_skills + != snapshot.run.pinned_instruction_skills + { + return Err(TerminalError::new_with_code( + 409, + "persisted amendment planning authority does not match the active run", + ) + .into()); + } + let mut effective_context = planning_context.snapshot; + // Confirmation may replace the budget frozen at planning time, so the planner is shown the + // approved run budget rather than the immutable admission budget. + effective_context.budget = snapshot.run.approved_budget.clone(); + // Amendment planning is governed by the exact capability catalog persisted with + // the run. Consulting the deployment router here would make the same wake + // compile differently after a catalog refresh and would drop installed + // connector provenance before dispatch can generation-fence it. + let admitted_tool_names = effective_context + .catalog + .capabilities + .iter() + .filter_map(|capability| capability.source.model_visible_tool_name()) + .map(ToString::to_string) + .collect(); + let context = narrow_amendment_context(effective_context, &admitted_tool_names) + .map_err(execution_error)?; + let remaining_budget = snapshot + .budget_ledger + .remaining_limit() + .map_err(execution_error)?; + Ok(AmendmentPlanningInputs::Ready(Box::new( + PreparedAmendmentPlanning { + context, + evidence: AmendmentPlanningEvidence { + goal: snapshot.run.goal, + active_plan: snapshot.run.active_plan, + projection: snapshot.projection, + failure_evidence, + waiting_task: origin.task_id, + }, + remaining_budget, + origin, + admitted_identity: snapshot.run.admitted_identity, + now, + }, + ))) +} + +/// Wraps `NeedsReplan` evidence for planner use, rejecting an over-cap payload first. +/// +/// The planner envelope is bounded, so an oversized projection must fail here rather +/// than after a paid provider call has already been issued with a truncated prompt. +pub fn bounded_failure_evidence(reason: &str, evidence: &Value) -> Result { + let failure_evidence = json!({"reason": reason, "evidence": evidence}); + let encoded = moa_core::canonical_json::canonical_json_bytes(&failure_evidence) + .map_err(|error| TerminalError::new(error.to_string()))?; + if encoded.len() > EXECUTION_REPORT_MAX_BYTES { + return Err(TerminalError::new_with_code( + 422, + "WaitingReplan failure evidence exceeds the bounded planner envelope", + ) + .into()); + } + Ok(failure_evidence) +} + +/// Retains only persisted capabilities whose governed tool is currently available. +pub fn narrow_amendment_context( + mut context: ExecutionPlanningContextSnapshot, + available_tool_names: &BTreeSet, +) -> moa_execution::Result { + use moa_execution::capability::CapabilitySource; + + let retained_refs = context + .catalog + .capabilities + .iter() + .filter(|capability| match &capability.source { + CapabilitySource::BuiltInTool { name } | CapabilitySource::HandTool { name } => { + available_tool_names.contains(name) + } + // `tool_name`, not `remote_name`: availability is membership in the + // router's registered names, and a connector tool is registered + // under its server-qualified reference. + CapabilitySource::McpTool { tool_name, .. } + | CapabilitySource::ActionArtifact { tool_name, .. } + | CapabilitySource::ConnectorAction { tool_name, .. } + | CapabilitySource::InstalledConnectorAction { tool_name, .. } + | CapabilitySource::SkillAction { tool_name, .. } + | CapabilitySource::Memory { tool_name, .. } => { + available_tool_names.contains(tool_name) + } + CapabilitySource::SkillCode { .. } + | CapabilitySource::Knowledge { .. } + | CapabilitySource::Model => true, + }) + .map(|capability| capability.reference.clone()) + .collect::>(); + narrow_authorized_capability_refs(&mut context.authorization.capability_refs, &retained_refs); + context + .validate() + .map_err(|error| moa_execution::Error::InvalidRepositoryInput { + message: error.to_string(), + })?; + Ok(context) +} + +/// Intersects persisted authorization with a live availability set. +/// +/// This is deliberately a retain and never an extend: an availability observation +/// may only remove authority the planning context already froze. +pub fn narrow_authorized_capability_refs( + authorized: &mut Vec, + live: &[moa_artifacts::execution_plan::CapabilityReference], +) { + authorized.retain(|reference| live.contains(reference)); +} + +/// Persists one planner or compiler audit produced by amendment planning. +pub async fn persist_amendment_audit( + repository: &ExecutionRepository, + scope: ExecutionScope, + envelope: &ExecutionPlanningAuditEnvelope, +) -> Result<(), HandlerError> { + match &envelope.payload { + ExecutionPlanningAuditPayload::PlannerCall { .. } => { + let result = repository + .write_planner_call_audit(scope, envelope) + .await + .map_err(execution_error)?; + if matches!(result, PlannerCallAuditWriteOutcome::Conflict { .. }) { + return Err(TerminalError::new_with_code( + 409, + "execution amendment planner audit conflicts with first persisted evidence", + ) + .into()); + } + } + ExecutionPlanningAuditPayload::Compile { .. } => { + let result = repository + .write_compile_audit(scope, envelope) + .await + .map_err(execution_error)?; + if matches!(result, CompileAuditWriteOutcome::Conflict { .. }) { + return Err(TerminalError::new_with_code( + 409, + "execution amendment compile audit conflicts with first persisted evidence", + ) + .into()); + } + } + ExecutionPlanningAuditPayload::Route { .. } => { + return Err(TerminalError::new_with_code( + 422, + "execution amendment planning produced a route audit", + ) + .into()); + } + } + Ok(()) +} + +fn execution_error(error: moa_execution::Error) -> HandlerError { + crate::workflows::errors::execution_error_to_handler_error(error) +} diff --git a/crates/moa-orchestrator/src/services/execution_amendment_planner/tests.rs b/crates/moa-orchestrator/src/services/execution_amendment_planner/tests.rs new file mode 100644 index 000000000..99755e2ab --- /dev/null +++ b/crates/moa-orchestrator/src/services/execution_amendment_planner/tests.rs @@ -0,0 +1,822 @@ +//! Inline regressions for the bounded amendment-planning slice. + +use super::preparation::{ + bounded_failure_evidence, narrow_amendment_context, narrow_authorized_capability_refs, +}; +use super::{ + AmendmentPlanningOrigin, ReplanStopReason, amendment_planning_identity, + parked_run_needs_amendment, planner_stop_amendment, +}; + +use std::collections::BTreeSet; + +use moa_artifacts::execution_plan::{CapabilityReference, ExecutionBudgetLimit}; +use moa_core::types::action_policy::{ActionClass, ActionPolicyEffect, RiskLevel}; +use moa_core::types::tools::{IdempotencyClass, ToolAsyncMode}; +use moa_execution::capability::{ + CapabilityPolicyContext, CapabilitySource, ExecutionCapability, ExecutionCapabilityCatalog, + ExecutionClass, ExecutionEstimate, amendment_hash, +}; +use moa_execution::state::{ExecutionRunStatus, ExecutionTaskId}; +use serde_json::json; +use uuid::Uuid; + +fn unbounded_budget() -> ExecutionBudgetLimit { + ExecutionBudgetLimit { + max_cost_microusd: None, + max_tokens: None, + max_tasks: None, + max_tool_calls: None, + max_retrieved_bytes: None, + deadline_at: None, + } +} + +fn reference(name: &str) -> CapabilityReference { + CapabilityReference { + name: name.to_string(), + version: "1".to_string(), + } +} + +fn tool_capability(reference_name: &str, tool_name: &str) -> ExecutionCapability { + let source = CapabilitySource::BuiltInTool { + name: tool_name.to_string(), + }; + ExecutionCapability { + reference: reference(reference_name), + contract_revision: "contract-v1".to_string(), + description: format!("Capability {reference_name}"), + input_schema: json!({"type": "object"}), + output_schema: json!({"type": "object"}), + action_class: ActionClass::Read, + risk_level: RiskLevel::Low, + default_effect: ActionPolicyEffect::Allow, + idempotency_class: IdempotencyClass::Idempotent, + async_mode: ToolAsyncMode::SynchronousOnly, + execution_class: ExecutionClass::Data, + requires_sandbox: false, + policy_context: CapabilityPolicyContext::registered(source.clone()), + source, + estimate: ExecutionEstimate { + tool_calls: 1, + tasks: 1, + ..ExecutionEstimate::default() + }, + rollback: None, + } +} + +fn planning_context( + capabilities: Vec, + capability_refs: Vec, +) -> moa_execution::wire::ExecutionPlanningContextSnapshot { + moa_execution::wire::ExecutionPlanningContextSnapshot { + schema_version: 1, + tenant_id: moa_core::types::identifiers::TenantId::new(), + contact_id: None, + session_id: moa_core::types::identifiers::SessionId::new(), + originating_user_sequence_num: 1, + originating_user_event_hash: moa_execution::capability::ExecutionHash::from_bytes([7; 32]) + .to_string(), + owner_user_id: moa_core::types::identifiers::UserId::new("planner-owner"), + catalog: ExecutionCapabilityCatalog::build(capabilities) + .expect("fixture capability catalog should build"), + authorization: moa_execution::capability::ExecutionAuthorizationEnvelope { + capability_refs, + skill_refs: Vec::new(), + }, + pinned_instruction_skills: Vec::new(), + execution_templates: Vec::new(), + budget: unbounded_budget(), + } +} + +#[test] +fn amendment_live_authority_check_only_removes_persisted_capabilities_offline() { + // Pins: a live availability set is an intersection with persisted planning authority; it + // cannot introduce a caller- or model-selected reference the planning context never froze. + let persisted_a = reference("persisted-a"); + let persisted_b = reference("persisted-b"); + let live_only = reference("live-only"); + let mut authorized = vec![persisted_a, persisted_b.clone()]; + + narrow_authorized_capability_refs(&mut authorized, &[persisted_b.clone(), live_only]); + + assert_eq!(authorized, vec![persisted_b]); +} + +#[test] +fn amendment_context_drops_capabilities_whose_tool_is_no_longer_registered_offline() { + // Pins: the narrowed snapshot the planner is shown may not advertise a capability whose + // governed tool disappeared, so the model cannot propose work dispatch would reject. + let context = planning_context( + vec![ + tool_capability("kept", "registered_tool"), + tool_capability("dropped", "unregistered_tool"), + ], + vec![reference("kept"), reference("dropped")], + ); + let available = BTreeSet::from(["registered_tool".to_string()]); + + let narrowed = narrow_amendment_context(context, &available) + .expect("narrowed authority should remain a valid planning context"); + + assert_eq!( + narrowed.authorization.capability_refs, + vec![reference("kept")], + "only the capability whose tool is still registered may stay authorized" + ); +} + +#[test] +fn amendment_failure_evidence_is_preserved_and_bounded_before_provider_use_offline() { + // Pins: runtime planning keeps exact structured NeedsReplan evidence, while rejecting an + // over-cap value before any paid model call can be issued for it. + let evidence = json!({"shape": ["a", "b"]}); + assert_eq!( + bounded_failure_evidence("shape changed", &evidence) + .expect("small evidence should remain available"), + json!({"reason": "shape changed", "evidence": evidence}) + ); + let oversized = json!({ + "body": "x".repeat(moa_core::types::execution_planning::EXECUTION_REPORT_MAX_BYTES) + }); + let error = bounded_failure_evidence("shape changed", &oversized) + .expect_err("oversized evidence should fail before planning"); + let message = >::as_ref(&error) + .to_string(); + assert!( + message.contains("exceeds the bounded planner envelope"), + "unexpected rejection: {message}" + ); +} + +#[test] +fn one_planning_slice_exists_per_run_revision_offline() { + // Pins: the planner loop is bounded by plan revision. Repeated controller activations of the + // same parked revision must coalesce onto one paid invocation, while an accepted amendment + // that advances the revision must be allowed exactly one more. + let run_uid = Uuid::from_u128(3); + let first = amendment_planning_identity(run_uid, 1); + assert_eq!(first, amendment_planning_identity(run_uid, 1)); + assert_ne!(first, amendment_planning_identity(run_uid, 2)); + assert_ne!(first, amendment_planning_identity(Uuid::from_u128(4), 1)); +} + +#[test] +fn planner_stop_identity_is_stable_per_reason_and_description_offline() { + // Pins: `request_replan_stop` keys replay on the amendment hash, so a candidate-free planner + // stop needs an identity derived only from its own frozen evidence. Two different verdicts + // must not collide onto one persisted intent. + let origin = AmendmentPlanningOrigin { + run_uid: Uuid::from_u128(9), + session_id: moa_core::types::identifiers::SessionId(Uuid::from_u128(10)), + base_plan_revision: 2, + task_id: ExecutionTaskId::from_uuid(Uuid::from_u128(11)), + task_generation: 1, + }; + let stop = |reason, detail: &str| { + amendment_hash(&planner_stop_amendment(origin, reason, detail)) + .expect("planner stop amendment should hash") + }; + + assert_eq!( + stop(ReplanStopReason::NoProgress, "planner stopped"), + stop(ReplanStopReason::NoProgress, "planner stopped") + ); + assert_ne!( + stop(ReplanStopReason::NoProgress, "planner stopped"), + stop(ReplanStopReason::BudgetExhausted, "planner stopped") + ); + assert_ne!( + stop(ReplanStopReason::NoProgress, "planner stopped"), + stop(ReplanStopReason::NoProgress, "planner stopped differently") + ); + assert!( + planner_stop_amendment(origin, ReplanStopReason::NoProgress, "detail") + .operations + .is_empty(), + "a planner stop must never carry plan operations" + ); +} + +#[test] +fn only_a_clean_waiting_replan_park_selects_the_planner_offline() { + // Pins: a run holding a terminal intent or awaiting manual repair is already being settled, + // and selecting it for planning would race the settlement that owns it. A run parked for any + // other reason must not trigger a paid planner call at all. + assert!(parked_run_needs_amendment( + ExecutionRunStatus::WaitingReplan, + false, + false, + 1 + )); + for parked in [ + ExecutionRunStatus::Running, + ExecutionRunStatus::WaitingInput, + ExecutionRunStatus::WaitingTimer, + ExecutionRunStatus::WaitingExternal, + ExecutionRunStatus::Paused, + ExecutionRunStatus::Completed, + ] { + assert!( + !parked_run_needs_amendment(parked, false, false, 1), + "{parked:?} must not select amendment planning" + ); + } + assert!( + !parked_run_needs_amendment(ExecutionRunStatus::WaitingReplan, true, false, 1), + "a run already carrying a terminal intent must not be replanned" + ); + assert!( + !parked_run_needs_amendment(ExecutionRunStatus::WaitingReplan, false, true, 1), + "a run awaiting manual repair must not be replanned" + ); + for count in [0, 2] { + assert!( + !parked_run_needs_amendment(ExecutionRunStatus::WaitingReplan, false, false, count), + "an amendment may supersede exactly one WaitingReplan task, not {count}" + ); + } +} + +#[tokio::test] +async fn waiting_replan_uses_confirmed_budget_for_planning_apply_and_replay_db() { + // Pins: confirmation may replace the budget frozen at planning time. Amendment planning must + // then show the planner only the persisted run ledger, apply the resulting candidate through + // the production amendment boundary, and replay the exact same submission idempotently. + use moa_artifacts::execution_plan::{ + CompletionCheck, CompletionCheckKind, ExecutionBudgetLimit, ExecutionCancelPolicy, + ExecutionGoalContract, ExecutionNode, ExecutionOperation, ExecutionPlanDefinition, + ExecutionRequirement, ExecutionTaskOutcome, ExecutionTaskResult, ExecutionTemporalTarget, + ExecutionUsage, ExecutionWaitExpiryAction, ExecutionWaitPolicy, + GeneratedAmendmentCandidate, RetryPolicy, + }; + use moa_core::types::execution_planning::{ + ExecutionSourceProvenance, GeneratedPlanPlannerProvenance, + }; + use moa_core::types::identifiers::{ModelId, SessionId, TenantId, UserId}; + use moa_execution::compiler::{CompileExecutionRequest, compile}; + use moa_execution::repository::{ + ConfirmationOutcome, ExecutionRepository, ExecutionScope, NewExecutionRun, + audit::{NewExecutionPlanningContext, PlanningContextWriteOutcome}, + ready::{ReadyMaterializationOutcome, ReadyMaterializationRequest}, + run::RunAdmissionOutcome, + task::{ + TaskAttemptFence, TaskAttemptReleaseClaimOutcome, TaskAttemptSettlementOutcome, + TaskAttemptStartOutcome, + }, + }; + use moa_execution::state::{LogicalTask, LogicalTaskKind}; + use moa_execution::wire::{ + ExecutionAmendmentRequest, ExecutionMutationResponse, ExecutionRunRequest, + planning_context_hash, + }; + use moa_providers::ScriptedProvider; + + fn replan_budget(max_resource: u64, max_tasks: u64) -> ExecutionBudgetLimit { + // Truncated to microseconds so equality against a Postgres round-trip is exact: + // nanosecond-granular CI clocks otherwise fail assertions local clocks let pass. + let deadline = chrono::Utc::now() + chrono::TimeDelta::hours(1); + let deadline = + chrono::DateTime::::from_timestamp_micros(deadline.timestamp_micros()) + .expect("hour-offset deadline is representable at microsecond precision"); + ExecutionBudgetLimit { + max_cost_microusd: Some(max_resource), + max_tokens: Some(max_resource / 10), + max_tasks: Some(max_tasks), + max_tool_calls: Some(max_resource / 10_000), + max_retrieved_bytes: Some(max_resource.saturating_mul(20)), + deadline_at: Some(deadline), + } + } + + fn replan_goal() -> ExecutionGoalContract { + ExecutionGoalContract { + objective: "repair with the confirmed budget".to_string(), + requirements: vec![ + ExecutionRequirement { + id: "req_inputs".to_string(), + description: "prepare report inputs".to_string(), + }, + ExecutionRequirement { + id: "req_report".to_string(), + description: "produce the repaired report".to_string(), + }, + ], + deliverables: Vec::new(), + coverage: Vec::new(), + constraints: Vec::new(), + // Both requirements are covered by the terminal output check so the linkage + // survives amendments that rename the prepare/output nodes. + completion_checks: vec![CompletionCheck { + id: "check_output".to_string(), + description: "validate the repaired output".to_string(), + requirement_ids: vec!["req_inputs".to_string(), "req_report".to_string()], + constraint_ids: Vec::new(), + kind: CompletionCheckKind::OutputSchema, + }], + } + } + + fn replan_node(id: &str, depends_on: Vec, value: serde_json::Value) -> ExecutionNode { + ExecutionNode { + id: id.to_string(), + requirement_ids: vec!["req_report".to_string()], + depends_on, + when: None, + input: json!({}), + output_schema: json!({"type": "object"}), + operation: ExecutionOperation::Output { value }, + compensation: None, + retry: RetryPolicy { + max_attempts: 1, + initial_backoff_ms: 0, + max_backoff_ms: 0, + }, + budget: None, + } + } + + fn replan_plan() -> ExecutionPlanDefinition { + ExecutionPlanDefinition { + cancel_policy: ExecutionCancelPolicy::RetainEffects, + input_wait_policy: ExecutionWaitPolicy { + expiry: ExecutionTemporalTarget::At { + at: chrono::Utc::now() + chrono::TimeDelta::minutes(30), + }, + on_expiry: ExecutionWaitExpiryAction::FailTask, + }, + input_schema: json!({"type": "object"}), + output_schema: json!({"type": "object"}), + nodes: vec![ + ExecutionNode { + id: "prepare".to_string(), + requirement_ids: vec!["req_inputs".to_string()], + depends_on: Vec::new(), + when: None, + input: json!({}), + output_schema: json!({"type": "object"}), + operation: ExecutionOperation::Agent { + instructions: "prepare report inputs".to_string(), + skill_refs: Vec::new(), + capability_refs: Vec::new(), + max_turns: 1, + }, + compensation: None, + retry: RetryPolicy { + max_attempts: 1, + initial_backoff_ms: 0, + max_backoff_ms: 0, + }, + budget: None, + }, + replan_node( + "output", + vec!["prepare".to_string()], + json!({"value": "stale"}), + ), + ], + } + } + + fn replan_task(run_uid: Uuid, node_id: &str, value: serde_json::Value) -> LogicalTask { + LogicalTask { + task_id: ExecutionTaskId::derive(run_uid, node_id, "") + .expect("fixture task id should derive"), + node_id: node_id.to_string(), + item_key: String::new(), + requirement_ids: if node_id == "prepare" { + vec!["req_inputs".to_string()] + } else { + vec!["req_report".to_string()] + }, + plan_revision: 1, + generation: 1, + input: json!({}), + kind: if node_id == "prepare" { + LogicalTaskKind::Agent { + instructions: "prepare report inputs".to_string(), + skill_refs: Vec::new(), + capability_refs: Vec::new(), + max_turns: 1, + } + } else { + LogicalTaskKind::Output { value } + }, + compensation: None, + retry: RetryPolicy { + max_attempts: 1, + initial_backoff_ms: 0, + max_backoff_ms: 0, + }, + reservation: ExecutionEstimate { + cost_microusd: 2, + tokens: 2, + tasks: 1, + tool_calls: 2, + retrieved_bytes: 2, + }, + } + } + + fn replan_outcome(result: ExecutionTaskResult) -> ExecutionTaskOutcome { + ExecutionTaskOutcome { + schema_version: 1, + usage: ExecutionUsage { + cost_microusd: 1, + tokens: 1, + tool_calls: 1, + retrieved_bytes: 1, + }, + result, + } + } + + fn replan_amendment_value() -> serde_json::Value { + json!({ + "base_plan_revision": 1, + "reason": "replace stale output", + "evidence": {"shape": "changed"}, + "operations": [ + {"kind": "remove_pending_node", "node_id": "output"}, + { + "kind": "add_node", + "node": { + "id": "replacement_output", + "requirement_ids": ["req_report"], + "depends_on": ["prepare"], + "when": null, + "input": {}, + "output_schema": {"type": "object"}, + "operation": {"kind": "output", "value": {"value": "repaired"}}, + "compensation": null, + "retry": { + "max_attempts": 1, + "initial_backoff_ms": 0, + "max_backoff_ms": 0 + }, + "budget": null + } + } + ] + }) + } + + /// Drives one node through the exact production materialize/admit/settle path. + async fn run_node_to_outcome( + repository: &ExecutionRepository, + scope: ExecutionScope, + config: &moa_config::ExecutionConfig, + run_uid: Uuid, + task: LogicalTask, + outcome: ExecutionTaskOutcome, + ) { + let node_id = task.node_id.clone(); + assert!(matches!( + repository + .materialize_ready_page( + scope, + config, + ReadyMaterializationRequest { + run_uid, + plan_revision: 1, + node_id: node_id.clone(), + expected_cursor: 0, + reduce_cursor: None, + source_exhausted: true, + terminal_output: None, + condition_skipped: false, + tasks: vec![task], + }, + ) + .await + .expect("ready page should materialize"), + ReadyMaterializationOutcome::Applied { .. } + )); + let admission = repository + .admit_ready_attempts(config, 1, chrono::Utc::now()) + .await + .expect("ready admission should succeed") + .admitted + .into_iter() + .next() + .unwrap_or_else(|| panic!("node `{node_id}` must admit exactly one attempt")); + let fence = TaskAttemptFence { + tenant_id: admission.tenant_id, + run_uid: admission.run_uid, + task_id: admission.task_id, + controller_generation: admission.controller_generation, + attempt_generation: admission.attempt_generation, + dispatch_uid: admission.dispatch_uid, + capacity_reservation_uid: admission.capacity_reservation_uid, + watchdog_trigger_uid: admission.watchdog_trigger_uid, + attempt_deadline_at: admission.attempt_deadline_at, + }; + let TaskAttemptStartOutcome::Started(started) = repository + .start_task_attempt(fence) + .await + .expect("attempt start should succeed") + else { + panic!("node `{node_id}` attempt must start"); + }; + let settled_at = chrono::Utc::now(); + assert!(matches!( + repository + .begin_task_attempt_release( + fence, + started.task.generation, + "fixture_settlement", + settled_at, + ) + .await + .expect("attempt release should claim"), + TaskAttemptReleaseClaimOutcome::Applied(_) + )); + assert!(matches!( + repository + .settle_released_task_attempt(config, fence, outcome, None, settled_at, None) + .await + .expect("attempt settlement should succeed"), + TaskAttemptSettlementOutcome::Applied { .. } + )); + } + + let test_db = moa_test_support::postgres::bootstrap_test_db() + .await + .expect("execution test database should bootstrap"); + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let config = moa_config::ExecutionConfig::default(); + let tenant_id = TenantId::new(); + let session_id = SessionId::new(); + let owner_user_id = UserId::new("confirmed-replan-owner"); + let scope = ExecutionScope::Tenant { tenant_id }; + let catalog = + ExecutionCapabilityCatalog::build(Vec::new()).expect("empty catalog should be valid"); + let authorization = moa_execution::capability::ExecutionAuthorizationEnvelope { + capability_refs: Vec::new(), + skill_refs: Vec::new(), + }; + let planning_budget = replan_budget(1_000_000, 10); + let confirmed_budget = replan_budget(2_000_000, 3); + let goal = replan_goal(); + let compile_outcome = compile(CompileExecutionRequest { + goal: goal.clone(), + plan: replan_plan(), + run_input: json!({}), + catalog: catalog.clone(), + authorization: authorization.clone(), + approved_budget: planning_budget.clone(), + config: config.clone(), + now: chrono::Utc::now(), + }); + let compiled = compile_outcome.compiled.unwrap_or_else(|| { + panic!( + "replan fixture should compile within the initial planning budget: {:?}", + compile_outcome.report.issues + ) + }); + let compiled_plan = compiled.plan; + let planning_snapshot = moa_execution::wire::ExecutionPlanningContextSnapshot { + schema_version: 1, + tenant_id, + contact_id: None, + session_id, + originating_user_sequence_num: 17, + originating_user_event_hash: moa_execution::capability::ExecutionHash::from_bytes([17; 32]) + .to_string(), + owner_user_id: owner_user_id.clone(), + catalog: catalog.clone(), + authorization: authorization.clone(), + pinned_instruction_skills: Vec::new(), + execution_templates: Vec::new(), + budget: planning_budget.clone(), + }; + let planning_hash = planning_context_hash(&planning_snapshot) + .expect("planning snapshot should have a canonical hash"); + let PlanningContextWriteOutcome::Created(planning_context) = repository + .create_planning_context( + scope, + NewExecutionPlanningContext { + snapshot: planning_snapshot, + planning_context_hash: planning_hash, + }, + ) + .await + .expect("planning context should persist") + else { + panic!("fresh planning context should be created"); + }; + let admitted_identity = moa_core::traits::Identity { + identity_type: moa_core::traits::IdentityType::Operator, + id: Uuid::from_u128(1), + tenant_id, + api_key_id: None, + acting_on_behalf_of: None, + }; + let RunAdmissionOutcome::Admitted(run) = repository + .create_run( + scope, + &config, + NewExecutionRun { + tenant_id, + contact_id: None, + session_id, + originating_user_sequence_num: 17, + planning_context_uid: planning_context.planning_context_uid, + planning_context_hash: planning_context.planning_context_hash, + owner_user_id, + admitted_identity: admitted_identity.clone(), + goal, + plan: compiled_plan.clone(), + catalog, + authorization, + pinned_instruction_skills: Vec::new(), + source_provenance: ExecutionSourceProvenance::GeneratedPlan { + planner: GeneratedPlanPlannerProvenance { + model: "scripted-confirmed-replan".to_string(), + prompt_version: "confirmed-replan".to_string(), + candidate_hash: "a".repeat(64), + compiler_report_hash: "b".repeat(64), + final_plan_hash: compiled_plan.plan_hash.to_string(), + repair_attempts: 0, + }, + }, + input: json!({}), + status: moa_execution::state::ExecutionRunStatus::AwaitingConfirmation, + approved_budget: planning_budget.clone(), + idempotency_key: Some("confirmed-replan-budget".to_string()), + }, + ) + .await + .expect("awaiting-confirmation run should persist") + else { + panic!("a fresh idempotency key should admit a new run"); + }; + let ConfirmationOutcome::Confirmed(confirmed) = repository + .confirm_run( + scope, + run.run_uid, + &run.active_plan_hash, + confirmed_budget.clone(), + ) + .await + .expect("confirmation write should succeed") + else { + panic!("confirmation should replace the approved budget"); + }; + assert_eq!(confirmed.approved_budget, confirmed_budget); + + let prepare_task = replan_task(run.run_uid, "prepare", json!({"value": "prepared"})); + let output_task = replan_task(run.run_uid, "output", json!({"value": "stale"})); + let waiting_task_id = output_task.task_id; + run_node_to_outcome( + &repository, + scope, + &config, + run.run_uid, + prepare_task, + replan_outcome(ExecutionTaskResult::Completed { + output: json!({"value": "prepared"}), + citations: Vec::new(), + }), + ) + .await; + run_node_to_outcome( + &repository, + scope, + &config, + run.run_uid, + output_task, + replan_outcome(ExecutionTaskResult::NeedsReplan { + reason: "shape changed".to_string(), + evidence: json!({"kind": "confirmed-budget"}), + }), + ) + .await; + + let target = super::AmendmentPlanningTarget { + tenant_id, + contact_id: None, + session_id, + run_uid: run.run_uid, + base_plan_revision: 1, + }; + let super::AmendmentPlanningInputs::Ready(prepared) = + super::prepare_amendment_planning(&repository, &config, target, chrono::Utc::now()) + .await + .expect("confirmed WaitingReplan should prepare amendment planning") + else { + panic!("an active WaitingReplan revision should produce planner input"); + }; + assert_eq!(prepared.context.budget, confirmed_budget); + assert_eq!( + repository + .load_planning_context(scope, planning_context.planning_context_uid) + .await + .expect("immutable planning context should reload") + .expect("immutable planning context should remain present") + .snapshot + .budget, + planning_budget, + "the admission planning context must stay immutable" + ); + assert_eq!(prepared.remaining_budget.max_cost_microusd, Some(1_999_997)); + assert_eq!(prepared.remaining_budget.max_tokens, Some(199_997)); + assert_eq!(prepared.remaining_budget.max_tasks, Some(1)); + assert_eq!(prepared.admitted_identity, admitted_identity); + assert_eq!(prepared.origin.task_id, waiting_task_id); + assert_eq!( + prepared.evidence.failure_evidence, + json!({"reason": "shape changed", "evidence": {"kind": "confirmed-budget"}}), + "exact NeedsReplan evidence must reach the planner" + ); + + let provider = ScriptedProvider::new(moa_core::types::model::ModelCapabilities::default()) + .push_text(json!({"amendment": replan_amendment_value()}).to_string()); + let planned = moa_brain::execution_planning::plan_amendment( + &provider, + moa_brain::execution_planning::ExecutionAmendmentPlanningRequest { + run_uid: run.run_uid, + base_plan_revision: 1, + context: prepared.context, + evidence: prepared.evidence, + remaining_budget: prepared.remaining_budget, + planner_model: ModelId::new("scripted-confirmed-replan"), + config: config.clone(), + now: prepared.now, + }, + ) + .await + .expect("persisted confirmed budget should permit amendment planning"); + assert_eq!(provider.recorded_requests().len(), 1); + let planner_prompt = serde_json::to_string(&provider.recorded_requests()[0].messages) + .expect("recorded amendment request should serialize"); + assert!( + planner_prompt.contains("1999997"), + "amendment planner prompt must carry the reconciled confirmed budget" + ); + let moa_brain::execution_planning::ExecutionAmendmentPlanningResultKind::Ready { + amendment, + .. + } = planned.kind + else { + panic!("a valid confirmed-budget amendment should be ready"); + }; + + let amendment_request = ExecutionAmendmentRequest { + run: ExecutionRunRequest { + tenant_id, + contact_id: None, + session_id, + run_uid: run.run_uid, + }, + expected_plan_revision: 1, + amendment, + }; + let applied = crate::services::execution::handlers::apply_amendment_inner( + pool.clone(), + config.clone(), + amendment_request.clone(), + ) + .await + .expect("planned amendment should apply through the production service boundary") + .into_response(); + assert!( + matches!(applied, ExecutionMutationResponse::Applied { ref run } if run.plan_revision == 2), + "planned amendment should apply revision two: {applied:?}" + ); + let replayed = crate::services::execution::handlers::apply_amendment_inner( + pool, + config, + amendment_request.clone(), + ) + .await + .expect("exact amendment replay should remain idempotent") + .into_response(); + assert!( + matches!(replayed, ExecutionMutationResponse::Replayed { ref run } if run.plan_revision == 2), + "exact replay must not create a third revision: {replayed:?}" + ); + + let mut injected = + serde_json::to_value(amendment_request).expect("amendment request should serialize"); + injected + .as_object_mut() + .expect("amendment request wire shape should be an object") + .insert( + "approved_budget".to_string(), + json!({"max_tasks": 1_000_000}), + ); + serde_json::from_value::(injected) + .expect_err("caller-supplied amendment budget authority must be rejected"); + serde_json::from_value::(json!({ + "amendment": replan_amendment_value(), + "approved_budget": {"max_tasks": 1_000_000} + })) + .expect_err("model-supplied amendment budget authority must be rejected"); +} diff --git a/crates/moa-orchestrator/src/services/execution_dispatcher.rs b/crates/moa-orchestrator/src/services/execution_dispatcher.rs index 6c56dfe7f..0db2428b7 100644 --- a/crates/moa-orchestrator/src/services/execution_dispatcher.rs +++ b/crates/moa-orchestrator/src/services/execution_dispatcher.rs @@ -6,9 +6,10 @@ use chrono::{DateTime, Utc}; use moa_execution::repository::{ ExecutionRepository, ExecutionScope, outbox::{ + ExecutionAdmissionResourceDimension, ExecutionAdmissionUtilizationSample, ExecutionDispatchFailureOutcome, ExecutionDispatchRetryPolicy, ExecutionMaintenanceJobKind, ExecutionMaintenanceSettlementOutcome, ExecutionQueueBacklogSample, - ExecutionQueueHealthSnapshot, + ExecutionQueueHealthSnapshot, ExecutionRunPhaseDimension, ExecutionRunPhaseSample, }, }; use restate_sdk::prelude::*; @@ -62,6 +63,8 @@ pub struct DrainExecutionDispatchesResponse { pub dead_lettered: usize, /// Claims changed ownership before settlement. pub stale_claims: usize, + /// Claims whose durable settlement failed and was deferred to claim expiry. + pub settlement_failures: usize, } /// Operational request for one bounded due-trigger repair pass. @@ -99,14 +102,20 @@ pub struct ExecutionQueueHealthReport { pub claimable_dispatches_saturated: bool, /// Age of the oldest observed claimable dispatch. pub outbox_lag_seconds: f64, - /// Trigger dead letters observed up to the sample cap. - pub dead_letter_triggers: u32, - /// Whether trigger dead letters exceeded the sample cap. - pub dead_letter_triggers_saturated: bool, /// Dispatch dead letters observed up to the sample cap. pub dead_letter_dispatches: u32, /// Whether dispatch dead letters exceeded the sample cap. pub dead_letter_dispatches_saturated: bool, + /// Nonterminal runs whose absolute deadline has elapsed, capped at the sample limit. + pub overdue_deadlines: u32, + /// Age of the oldest active forward or compensation attempt, zero when none is active. + pub active_attempt_oldest_age_seconds: f64, + /// Live run count for every bounded nonterminal phase, including idle zeroes. + pub run_phases: Vec, + /// Age of the oldest nonterminal external job, zero when none is live. + pub external_job_oldest_age_seconds: f64, + /// Ceiling utilization for every bounded admission resource, including idle zeroes. + pub admission_utilization: Vec, } #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] @@ -114,11 +123,23 @@ struct JournaledDispatchAckBatch { delivered_dispatch_uids: Vec, } +/// One journaled claimed row paired with the repair generation of its persisted identity. +/// +/// The repair epoch cannot be recovered from the row after the claim, so it is journaled +/// alongside it: replay must address the same Restate invocation this episode did. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +struct ClaimedExecutionDispatch { + dispatch: JournaledExecutionDispatch, + repair_epoch: u32, +} + #[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] #[serde(rename_all = "snake_case")] enum JournaledDispatchFailure { RetryScheduled, DeadLettered, + DeadLetteredWithoutOwnerRepair, StaleClaim, } @@ -282,7 +303,10 @@ impl ExecutionDispatchDrain for ExecutionDispatchDrainImpl { Json::from( records .into_iter() - .map(JournaledExecutionDispatch::from) + .map(|record| ClaimedExecutionDispatch { + repair_epoch: record.repair_epoch, + dispatch: JournaledExecutionDispatch::from(record), + }) .collect::>(), ) }) @@ -299,14 +323,15 @@ impl ExecutionDispatchDrain for ExecutionDispatchDrainImpl { retry_scheduled: 0, dead_lettered: 0, stale_claims: 0, + settlement_failures: 0, }; let delivery_results = accept_batch(&ctx, &journaled).await?; let mut delivered_dispatch_uids = Vec::with_capacity(journaled.len()); let mut failed_dispatches = Vec::new(); - for (dispatch, accepted) in journaled.into_iter().zip(delivery_results) { + for (claimed, accepted) in journaled.into_iter().zip(delivery_results) { match accepted { - Ok(()) => delivered_dispatch_uids.push(dispatch.dispatch_uid), - Err(error) => failed_dispatches.push((dispatch, error)), + Ok(()) => delivered_dispatch_uids.push(claimed.dispatch.dispatch_uid), + Err(error) => failed_dispatches.push((claimed.dispatch, error)), } } settle_delivered_batch( @@ -317,6 +342,8 @@ impl ExecutionDispatchDrain for ExecutionDispatchDrainImpl { &mut response, ) .await?; + // One unsettleable row must never abort the pass: the successor scheduled below is the + // fleet's only self-perpetuating pump, and an unsettled claim is recovered at expiry. for (dispatch, error) in failed_dispatches { settle_failure( &ctx, @@ -326,7 +353,7 @@ impl ExecutionDispatchDrain for ExecutionDispatchDrainImpl { error, &mut response, ) - .await?; + .await; } // Synchronous trigger/controller deliveries can materialize Ready tasks without another // outbox continuation. Admit once more inside this bounded drain episode so the resulting @@ -360,11 +387,14 @@ impl ExecutionDispatchDrain for ExecutionDispatchDrainImpl { moa_observability::runtime_metrics::record_execution_dispatch_batch_size( admission.admitted_count, ); - if let Some(age) = admission.oldest_ready_age_millis { - moa_observability::runtime_metrics::record_execution_oldest_ready_age( - Duration::from_millis(age), - ); - } + // Written on every drain, including the empty one. A gauge set only while work exists + // holds its last value forever once the queue drains, so its SLO alert would page on a + // queue that emptied hours earlier. + moa_observability::runtime_metrics::record_execution_oldest_ready_age( + admission + .oldest_ready_age_millis + .map_or(Duration::ZERO, Duration::from_millis), + ); let repository = self.repository.clone(); let wake = ctx .run(|| async move { @@ -413,6 +443,20 @@ fn dispatch_head_idempotency_key( ) } +/// Returns the repair-scoped Restate delivery identity for one claimed outbox row. +/// +/// Restate retains a completed invocation's response under its idempotency key for the +/// endpoint's retention window, so a recovery requeue that reused the bare `dispatch_uid` +/// would attach to that memoized response and never execute its target. Each requeue +/// advances the row's repair epoch; the steady-state identity is unchanged at epoch zero. +fn delivery_identity(dispatch_uid: uuid::Uuid, repair_epoch: u32) -> String { + if repair_epoch == 0 { + dispatch_uid.to_string() + } else { + format!("{dispatch_uid}:{repair_epoch}") + } +} + fn reconciliation_drain_idempotency_key(generation: u64) -> String { format!("execution-reconcile-drain:{generation}") } @@ -712,16 +756,34 @@ fn queue_health_report(snapshot: ExecutionQueueHealthSnapshot) -> ExecutionQueue claimable_dispatches_saturated: snapshot.claimable_dispatches.saturated, outbox_lag_seconds: backlog_age(snapshot.observed_at, &snapshot.claimable_dispatches) .as_secs_f64(), - dead_letter_triggers: snapshot.dead_letter_triggers.observed_count, - dead_letter_triggers_saturated: snapshot.dead_letter_triggers.saturated, dead_letter_dispatches: snapshot.dead_letter_dispatches.observed_count, dead_letter_dispatches_saturated: snapshot.dead_letter_dispatches.saturated, + overdue_deadlines: snapshot.overdue_deadlines, + active_attempt_oldest_age_seconds: age_since( + snapshot.observed_at, + snapshot.oldest_active_attempt_at, + ) + .as_secs_f64(), + run_phases: snapshot.run_phases, + external_job_oldest_age_seconds: age_since( + snapshot.observed_at, + snapshot.oldest_external_job_at, + ) + .as_secs_f64(), + admission_utilization: snapshot.admission_utilization, } } fn backlog_age(observed_at: DateTime, sample: &ExecutionQueueBacklogSample) -> Duration { - sample - .oldest_at + age_since(observed_at, sample.oldest_at) +} + +/// Returns the age of one observed timestamp, reporting zero when nothing was observed. +/// +/// An absent observation is a healthy zero rather than a gap: these ages back `absent()` +/// guarded alerts, so a quiet fleet must still publish a value. +fn age_since(observed_at: DateTime, oldest_at: Option>) -> Duration { + oldest_at .map(|oldest_at| observed_at.signed_duration_since(oldest_at)) .and_then(|age| age.to_std().ok()) .unwrap_or(Duration::ZERO) @@ -732,8 +794,6 @@ fn record_queue_health(health: &ExecutionQueueHealthReport) { Duration::from_secs_f64(health.trigger_lag_seconds), u64::from(health.due_triggers), health.due_triggers_saturated, - u64::from(health.dead_letter_triggers), - health.dead_letter_triggers_saturated, ); moa_observability::runtime_metrics::record_execution_outbox_queue( Duration::from_secs_f64(health.outbox_lag_seconds), @@ -742,6 +802,83 @@ fn record_queue_health(health: &ExecutionQueueHealthReport) { u64::from(health.dead_letter_dispatches), health.dead_letter_dispatches_saturated, ); + // Every gauge below is written on each snapshot, including its healthy zero: the alerts + // carry `absent()`, so a gauge written only while work exists would page on a quiet fleet. + moa_observability::runtime_metrics::record_execution_overdue_deadlines(u64::from( + health.overdue_deadlines, + )); + moa_observability::runtime_metrics::record_execution_active_attempt_oldest_age( + Duration::from_secs_f64(health.active_attempt_oldest_age_seconds), + ); + moa_observability::runtime_metrics::record_execution_external_job_oldest_age( + Duration::from_secs_f64(health.external_job_oldest_age_seconds), + ); + for sample in &health.run_phases { + moa_observability::runtime_metrics::record_execution_run_phase( + run_phase_metric(sample.phase), + sample.run_count, + ); + } + for sample in &health.admission_utilization { + let resource = admission_resource_metric(sample.resource); + moa_observability::runtime_metrics::record_execution_admission_utilization( + resource, + moa_observability::runtime_metrics::ExecutionAdmissionScope::Fleet, + sample.fleet_ratio, + ); + moa_observability::runtime_metrics::record_execution_admission_utilization( + resource, + moa_observability::runtime_metrics::ExecutionAdmissionScope::TenantPeak, + sample.tenant_peak_ratio, + ); + moa_observability::runtime_metrics::record_execution_tenant_max_share( + resource, + sample.tenant_max_share_ratio, + ); + } +} + +/// Maps one durable nonterminal run status to its metric phase label. +/// +/// Both sides are the exact nonterminal `moa.execution_run.status` labels; the match exists +/// only because the repository cannot depend on the observability crate. It is total, so a +/// new durable phase cannot reach the census without being given a label. +fn run_phase_metric( + phase: ExecutionRunPhaseDimension, +) -> moa_observability::runtime_metrics::ExecutionRunMetricPhase { + use moa_observability::runtime_metrics::ExecutionRunMetricPhase as Metric; + match phase { + ExecutionRunPhaseDimension::AwaitingConfirmation => Metric::AwaitingConfirmation, + ExecutionRunPhaseDimension::Queued => Metric::Queued, + ExecutionRunPhaseDimension::Running => Metric::Running, + ExecutionRunPhaseDimension::WaitingInput => Metric::WaitingInput, + ExecutionRunPhaseDimension::WaitingReview => Metric::WaitingReview, + ExecutionRunPhaseDimension::WaitingSignal => Metric::WaitingSignal, + ExecutionRunPhaseDimension::WaitingTimer => Metric::WaitingTimer, + ExecutionRunPhaseDimension::WaitingExternal => Metric::WaitingExternal, + ExecutionRunPhaseDimension::WaitingReplan => Metric::WaitingReplan, + ExecutionRunPhaseDimension::PauseRequested => Metric::PauseRequested, + ExecutionRunPhaseDimension::Pausing => Metric::Pausing, + ExecutionRunPhaseDimension::Paused => Metric::Paused, + ExecutionRunPhaseDimension::Compensating => Metric::Compensating, + } +} + +/// Maps one durable capacity dimension to its metric label. +/// +/// Both sides are the exact `moa.execution_capacity_bucket.resource_dimension` labels; the +/// match exists only because the repository cannot depend on the observability crate. +fn admission_resource_metric( + resource: ExecutionAdmissionResourceDimension, +) -> moa_observability::runtime_metrics::ExecutionAdmissionResource { + use moa_observability::runtime_metrics::ExecutionAdmissionResource as Metric; + match resource { + ExecutionAdmissionResourceDimension::ActiveRuns => Metric::ActiveRuns, + ExecutionAdmissionResourceDimension::ActiveTasks => Metric::ActiveTasks, + ExecutionAdmissionResourceDimension::ParkedRuns => Metric::ParkedRuns, + ExecutionAdmissionResourceDimension::ScheduledTriggers => Metric::ScheduledTriggers, + ExecutionAdmissionResourceDimension::ExternalJobs => Metric::ExternalJobs, + } } async fn settle_delivered_batch( @@ -788,6 +925,10 @@ async fn settle_delivered_batch( Ok(()) } +/// Records one delivery failure, tolerating a durable settlement that cannot be applied. +/// +/// A row whose settlement fails keeps its claim until expiry and is reclaimed by a later +/// drain, so the failure is counted and the pass continues rather than aborting the fleet. async fn settle_failure( ctx: &ObjectContext<'_>, repository: &ExecutionRepository, @@ -795,11 +936,11 @@ async fn settle_failure( dispatch: JournaledExecutionDispatch, error: String, response: &mut DrainExecutionDispatchesResponse, -) -> Result<(), HandlerError> { +) { let repository = repository.clone(); let claim_owner = claim_owner.to_string(); let dispatch_uid = dispatch.dispatch_uid; - let outcome = ctx + let settlement = ctx .run(|| async move { repository .record_dispatch_failure( @@ -818,6 +959,9 @@ async fn settle_failure( ExecutionDispatchFailureOutcome::DeadLettered => { JournaledDispatchFailure::DeadLettered } + ExecutionDispatchFailureOutcome::DeadLetteredWithoutOwnerRepair => { + JournaledDispatchFailure::DeadLetteredWithoutOwnerRepair + } ExecutionDispatchFailureOutcome::StaleClaim => { JournaledDispatchFailure::StaleClaim } @@ -826,23 +970,47 @@ async fn settle_failure( .map_err(execution_error_to_handler_error) }) .name(format!("execution_dispatch_fail_{dispatch_uid}")) - .await? - .into_inner(); + .await; + let outcome = match settlement { + Ok(outcome) => outcome.into_inner(), + Err(settlement_error) => { + tracing::warn!( + dispatch_uid = %dispatch_uid, + error = %settlement_error, + "execution dispatch delivery failure could not be settled; deferring to claim expiry" + ); + response.settlement_failures += 1; + return; + } + }; match outcome { JournaledDispatchFailure::RetryScheduled => response.retry_scheduled += 1, JournaledDispatchFailure::DeadLettered => response.dead_lettered += 1, + JournaledDispatchFailure::DeadLetteredWithoutOwnerRepair => { + response.dead_lettered += 1; + tracing::warn!( + dispatch_uid = %dispatch_uid, + "execution dispatch dead-lettered while its attempt owner was already settled" + ); + } JournaledDispatchFailure::StaleClaim => response.stale_claims += 1, } - Ok(()) } async fn accept_batch( ctx: &ObjectContext<'_>, - dispatches: &[JournaledExecutionDispatch], + dispatches: &[ClaimedExecutionDispatch], ) -> Result>, HandlerError> { let targets = dispatches .iter() - .map(JournaledExecutionDispatch::target) + .map(|claimed| { + claimed.dispatch.target().map(|target| { + ( + target, + delivery_identity(claimed.dispatch.dispatch_uid, claimed.repair_epoch), + ) + }) + }) .collect::>(); let mut results = (0..targets.len()).map(|_| None).collect::>(); let mut run_slots = Vec::new(); @@ -855,8 +1023,7 @@ async fn accept_batch( // are retained in homogeneous durable fan-ins and reassembled by stable slot below. for (slot, target) in targets.into_iter().enumerate() { match target { - Ok(ExecutionDispatchTarget::RunActivation(request)) => { - let dispatch_uid = request.dispatch_uid; + Ok((ExecutionDispatchTarget::RunActivation(request), identity)) => { run_slots.push(slot); run_calls.push( crate::restate_identity::replay_safe_request( @@ -864,24 +1031,25 @@ async fn accept_batch( request.run_uid.to_string(), ) .advance(Json::from(request)) - .idempotency_key(dispatch_uid.to_string()), + .idempotency_key(identity), ) .call(), ); } - Ok(ExecutionDispatchTarget::TriggerDelivery(request)) => { - let dispatch_uid = request.dispatch_uid; + Ok((ExecutionDispatchTarget::TriggerDelivery(request), identity)) => { trigger_slots.push(slot); trigger_calls.push( crate::restate_identity::replay_safe_request( ctx.service_client::() .fire(Json::from(request)) - .idempotency_key(dispatch_uid.to_string()), + .idempotency_key(identity), ) .call(), ); } - Ok(target) => results[slot] = Some(accept_target(ctx, target).await.map(|_| ())), + Ok((target, identity)) => { + results[slot] = Some(accept_target(ctx, target, identity).await.map(|_| ())); + } Err(error) => results[slot] = Some(Err(error.to_string())), } } @@ -909,17 +1077,24 @@ async fn accept_batch( async fn accept_target( ctx: &ObjectContext<'_>, target: ExecutionDispatchTarget, + identity: String, ) -> Result { match target { ExecutionDispatchTarget::RunActivation(_) | ExecutionDispatchTarget::TriggerDelivery(_) => { Err("synchronous dispatch target bypassed bounded fan-out".to_string()) } ExecutionDispatchTarget::TaskAttempt(request) => { - let dispatch_uid = request.dispatch_uid; + // The workflow key stays the bare dispatch UID: `ExecutionTaskAttempt::run` asserts + // `ctx.key() == request.dispatch_uid`, and cancellation addresses the same workflow + // through the task row's `active_dispatch_uid`. A repair therefore only restarts + // this attempt while Restate holds no completed `run` for that key — see + // `requeue_current_accepted_dispatches_in_conn` for the two cases and the watchdog + // backstop that covers the other one. + let workflow_key = request.dispatch_uid.to_string(); let handle = crate::restate_identity::replay_safe_request( - ctx.workflow_client::(dispatch_uid.to_string()) + ctx.workflow_client::(workflow_key) .run(Json::from(request)) - .idempotency_key(dispatch_uid.to_string()), + .idempotency_key(identity), ) .send(); handle @@ -928,12 +1103,11 @@ async fn accept_target( .map_err(|error| error.to_string()) } ExecutionDispatchTarget::TaskAttemptCancel(request) => { - let dispatch_uid = request.cancellation_dispatch_uid; let workflow_key = request.active_dispatch_uid.to_string(); let handle = crate::restate_identity::replay_safe_request( ctx.workflow_client::(workflow_key) .cancel(Json::from(request)) - .idempotency_key(dispatch_uid.to_string()), + .idempotency_key(identity), ) .send(); handle @@ -942,11 +1116,13 @@ async fn accept_target( .map_err(|error| error.to_string()) } ExecutionDispatchTarget::CompensationAttempt(request) => { - let dispatch_uid = request.dispatch_uid; + // See the task-attempt arm: the compensation workflow asserts the same key identity + // and carries the same repair split. + let workflow_key = request.dispatch_uid.to_string(); let handle = crate::restate_identity::replay_safe_request( - ctx.workflow_client::(dispatch_uid.to_string()) + ctx.workflow_client::(workflow_key) .run(Json::from(request)) - .idempotency_key(dispatch_uid.to_string()), + .idempotency_key(identity), ) .send(); handle @@ -955,12 +1131,11 @@ async fn accept_target( .map_err(|error| error.to_string()) } ExecutionDispatchTarget::CompensationAttemptCancel(request) => { - let dispatch_uid = request.cancellation_dispatch_uid; let workflow_key = request.active_dispatch_uid.to_string(); let handle = crate::restate_identity::replay_safe_request( ctx.workflow_client::(workflow_key) .cancel(Json::from(request)) - .idempotency_key(dispatch_uid.to_string()), + .idempotency_key(identity), ) .send(); handle @@ -968,14 +1143,11 @@ async fn accept_target( .await .map_err(|error| error.to_string()) } - ExecutionDispatchTarget::ExternalCancel { - dispatch_uid, - request, - } => { + ExecutionDispatchTarget::ExternalCancel { request, .. } => { let handle = crate::restate_identity::replay_safe_request( ctx.service_client::() .cancel_external_job(Json::from(request)) - .idempotency_key(dispatch_uid.to_string()), + .idempotency_key(identity), ) .send(); handle @@ -989,8 +1161,9 @@ async fn accept_target( #[cfg(test)] mod tests { use super::{ - dispatch_head_idempotency_key, next_dispatch_delay, next_dispatch_successor, - reconciliation_drain_idempotency_key, + ExecutionRunPhaseDimension, delivery_identity, dispatch_head_idempotency_key, + next_dispatch_delay, next_dispatch_successor, queue_health_report, + reconciliation_drain_idempotency_key, run_phase_metric, }; use chrono::{TimeDelta, Utc}; use std::time::Duration; @@ -1062,6 +1235,20 @@ mod tests { assert_eq!(normal_delay, Duration::from_micros(500)); } + #[test] + fn repaired_delivery_identity_leaves_steady_state_keys_untouched() { + // Pins: Restate memoizes a completed invocation's response under its idempotency key, + // so every recovery requeue must address a distinct identity while an unrepaired row + // keeps the bare dispatch UID that producers and replays already coalesce on. + let dispatch_uid = uuid::Uuid::from_u128(1); + assert_eq!(delivery_identity(dispatch_uid, 0), dispatch_uid.to_string()); + let first_repair = delivery_identity(dispatch_uid, 1); + assert_ne!(first_repair, dispatch_uid.to_string()); + assert_eq!(first_repair, delivery_identity(dispatch_uid, 1)); + assert_ne!(first_repair, delivery_identity(dispatch_uid, 2)); + assert_ne!(first_repair, delivery_identity(uuid::Uuid::from_u128(2), 1)); + } + #[test] fn reconciliation_redrive_does_not_reuse_a_completed_head_identity() { // Pins: repair can requeue the same dispatch UID and due time after downstream loss; each @@ -1073,4 +1260,117 @@ mod tests { assert_eq!(first_repair, reconciliation_drain_idempotency_key(7)); assert_ne!(first_repair, reconciliation_drain_idempotency_key(8)); } + + /// Builds the snapshot a quiet fleet produces, with the two live-work fields injectable. + fn quiet_snapshot( + observed_at: chrono::DateTime, + running_runs: u64, + oldest_external_job_at: Option>, + ) -> moa_execution::repository::outbox::ExecutionQueueHealthSnapshot { + use moa_execution::repository::outbox::{ + ExecutionAdmissionResourceDimension, ExecutionAdmissionUtilizationSample, + ExecutionQueueBacklogSample, ExecutionQueueHealthSnapshot, ExecutionRunPhaseSample, + }; + + let idle_backlog = ExecutionQueueBacklogSample { + oldest_at: None, + observed_count: 0, + saturated: false, + }; + // Exactly what the repository fold produces: every bounded phase present, and every + // phase holding no runs carrying its explicit zero. + let run_phases = ExecutionRunPhaseDimension::ALL + .into_iter() + .map(|phase| ExecutionRunPhaseSample { + phase, + run_count: if phase == ExecutionRunPhaseDimension::Running { + running_runs + } else { + 0 + }, + }) + .collect(); + ExecutionQueueHealthSnapshot { + observed_at, + due_triggers: idle_backlog.clone(), + claimable_dispatches: idle_backlog.clone(), + dead_letter_dispatches: idle_backlog, + overdue_deadlines: 0, + oldest_active_attempt_at: None, + run_phases, + oldest_external_job_at, + admission_utilization: vec![ExecutionAdmissionUtilizationSample { + resource: ExecutionAdmissionResourceDimension::ActiveRuns, + fleet_ratio: 0.1, + tenant_peak_ratio: 1.0, + tenant_max_share_ratio: 0.75, + }], + } + } + + #[test] + fn quiet_fleet_still_reports_every_run_phase_and_external_job_age() { + // Pins: the reconciliation pass publishes these gauges, and their alerts carry + // `absent()`. A phase holding no runs and a fleet with no live external job must + // still produce a written zero, so the report may neither drop an idle phase nor + // leave the external-job age unset when the repository observed nothing. + let observed_at = Utc::now(); + + let report = queue_health_report(quiet_snapshot(observed_at, 3, None)); + assert_eq!( + report.run_phases.len(), + ExecutionRunPhaseDimension::ALL.len(), + "an idle phase must survive the report boundary and publish its zero" + ); + assert_eq!( + report + .run_phases + .iter() + .filter(|sample| sample.run_count == 0) + .count(), + ExecutionRunPhaseDimension::ALL.len() - 1 + ); + assert_eq!( + report.run_phases.iter().map(|s| s.run_count).sum::(), + 3, + "the census must still sum to the live fleet" + ); + assert_eq!(report.external_job_oldest_age_seconds, 0.0); + assert_eq!( + report.admission_utilization[0].tenant_max_share_ratio, 0.75, + "tenant concentration must reach the metric layer unmodified" + ); + + // A live external job is reported as its real age, not as the same healthy zero. + let live = queue_health_report(quiet_snapshot( + observed_at, + 0, + Some(observed_at - TimeDelta::seconds(90)), + )); + assert_eq!(live.external_job_oldest_age_seconds, 90.0); + assert_eq!( + live.run_phases.len(), + ExecutionRunPhaseDimension::ALL.len(), + "a fleet with no runs at all still reports every phase" + ); + } + + #[test] + fn every_bounded_run_phase_maps_to_a_distinct_metric_label() { + // Pins: the census is one gauge series per phase label. If two durable phases mapped + // onto one label they would overwrite each other's series, and the census would + // report a number smaller than the live fleet while every gauge still looked healthy. + let labels = ExecutionRunPhaseDimension::ALL + .into_iter() + .map(|phase| run_phase_metric(phase).as_str()) + .collect::>(); + assert_eq!(labels.len(), ExecutionRunPhaseDimension::ALL.len()); + for phase in ExecutionRunPhaseDimension::ALL { + assert_eq!( + run_phase_metric(phase).as_str(), + phase.as_str(), + "the metric label must be the durable status label" + ); + } + } } diff --git a/crates/moa-orchestrator/src/services/llm_gateway.rs b/crates/moa-orchestrator/src/services/llm_gateway.rs index 17752bc06..2053bdb25 100644 --- a/crates/moa-orchestrator/src/services/llm_gateway.rs +++ b/crates/moa-orchestrator/src/services/llm_gateway.rs @@ -329,6 +329,15 @@ pub(crate) enum LLMCompletionAction { ExecutionRouting { attempt: usize }, /// One initial durable-plan generation or repair attempt. InitialPlanning { attempt: usize }, + /// One plan-amendment generation or repair attempt for a parked run revision. + ExecutionAmendment { + /// Run whose plan is being amended. + run_uid: Uuid, + /// Active plan revision the amendment fences. + plan_revision: u64, + /// Sequential attempt within the same planning slice. + attempt: usize, + }, /// The single root-turn input guardrail evaluation. RootInputGuardrail, /// One root model-loop turn. @@ -348,6 +357,11 @@ impl LLMCompletionAction { match self { Self::ExecutionRouting { attempt } => format!("execution-routing:{attempt}"), Self::InitialPlanning { attempt } => format!("initial-planning:{attempt}"), + Self::ExecutionAmendment { + run_uid, + plan_revision, + attempt, + } => format!("execution-amendment:{run_uid}:{plan_revision}:{attempt}"), Self::RootInputGuardrail => "root-input-guardrail".to_string(), Self::RootModel { turn } => format!("root-model:{turn}"), Self::RootOutputGuardrail { turn } => format!("root-output-guardrail:{turn}"), diff --git a/crates/moa-orchestrator/src/services/mod.rs b/crates/moa-orchestrator/src/services/mod.rs index db4d754e5..4244c4375 100644 --- a/crates/moa-orchestrator/src/services/mod.rs +++ b/crates/moa-orchestrator/src/services/mod.rs @@ -18,6 +18,7 @@ pub mod contacts; pub mod dual_control; pub mod durable_timeout; pub mod execution; +pub mod execution_amendment_planner; pub mod execution_dispatcher; pub mod execution_retention; pub mod execution_schedule; diff --git a/crates/moa-orchestrator/src/services/tool_executor.rs b/crates/moa-orchestrator/src/services/tool_executor.rs index 58ae59875..511fa4b86 100644 --- a/crates/moa-orchestrator/src/services/tool_executor.rs +++ b/crates/moa-orchestrator/src/services/tool_executor.rs @@ -25,6 +25,7 @@ use moa_core::{ ExecutionCompensationScopeId, ExecutionRunScopeId, ExecutionTaskScopeId, SessionId, TenantId, ToolCallId, }, + types::sandbox_workspace::ExecutionHandContinuationDisposition, types::sandbox_workspace::ExecutionHandReleaseOwner, types::sandbox_workspace::ExecutionHandReleaseReceipt, types::sandbox_workspace::SandboxWorkspaceScope, @@ -67,9 +68,9 @@ use moa_execution::wire::{ ExecutionToolDispatchRejection, }; use moa_hands::{ - DeferredWorkspaceToolOutput, ExecutionHandReleaseRequest, JournaledWorkspaceCommit, - PendingConnectorToolOutput, SessionHandReleasePageOutcome, ToolCallScope, ToolCatalogPin, - ToolCatalogSnapshot, ToolExecution, ToolRouter, + DeferredWorkspaceToolOutput, ExecutionHandReleaseRequest, ExecutionHandRetentionRequest, + JournaledWorkspaceCommit, PendingConnectorToolOutput, SessionHandReleasePageOutcome, + ToolCallScope, ToolCatalogPin, ToolCatalogSnapshot, ToolExecution, ToolRouter, }; use moa_security::{ OutputClassification, ToolInputCanaryScreening, classify_tool_output, @@ -80,6 +81,7 @@ use moa_wire::tools::{ToolDescriptor, tool_descriptor}; use restate_sdk::prelude::*; use serde_json::Value; use sha2::{Digest, Sha256}; +use uuid::Uuid; use crate::services::execution_dispatcher::{DispatchExecutionsRequest, ExecutionDispatcherClient}; use crate::services::sandbox_workspaces::SandboxWorkspaceManagement; @@ -549,6 +551,11 @@ pub trait ToolExecutor { request: Json, ) -> Result, HandlerError>; + /// Publishes one continuation checkpoint and keeps its sandbox for the next slice. + async fn checkpoint_execution_hands_retaining_compute( + request: Json, + ) -> Result, HandlerError>; + /// Releases the generation-independent hand scope owned by one compensation. async fn release_execution_compensation_hands( request: Json, @@ -680,6 +687,28 @@ pub struct CheckpointAndReleaseExecutionHandsRequest { pub release_deadline_at: chrono::DateTime, } +/// Exact bounded execution sandbox to checkpoint and keep across a continuation. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CheckpointExecutionHandsRetainingComputeRequest { + /// Verified tenant that owns the execution run and lease. + pub tenant_id: TenantId, + /// Authoritative parent session loaded from the run. + pub session_id: SessionId, + /// Owning execution run. + pub run_uid: uuid::Uuid, + /// Task whose durable workspace scope owns the retained sandbox. + pub task_id: ExecutionTaskScopeId, + /// Logical task generation the retained compute belongs to. + pub logical_generation: u64, + /// Exact active-attempt generation publishing this continuation checkpoint. + pub attempt_generation: u64, + /// Fresh absolute bound for checkpoint publication. + pub publish_deadline_at: chrono::DateTime, + /// Absolute instant after which the reaper may destroy the retained sandbox. + pub retention_deadline_at: chrono::DateTime, +} + /// Request to release one settled compensation's scoped hands. #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] #[serde(deny_unknown_fields)] @@ -2506,6 +2535,71 @@ impl ToolExecutor for ToolExecutorImpl { .await?) } + #[tracing::instrument(skip(self, ctx, request))] + // SAFETY: internal bounded-attempt continuation; the authoritative Session is loaded and its + // exact tenant/run/task generation is fenced again by the durable workspace repository. + async fn checkpoint_execution_hands_retaining_compute( + &self, + ctx: Context<'_>, + request: Json, + ) -> Result, HandlerError> { + crate::ctx::adopt_incoming_trace_parent(&ctx); + annotate_restate_handler_span( + "ToolExecutor", + "checkpoint_execution_hands_retaining_compute", + ); + let request = request.into_inner(); + let session_store = self.session_access.sessions.clone(); + let session_id = request.session_id; + let session = ctx + .run(|| async move { + session_store + .get_session(session_id) + .await + .map(Json::from) + .map_err(moa_error_to_handler_error) + }) + .name(format!("load_task_continuation_session:{session_id}")) + .await? + .into_inner(); + if session.tenant_id != request.tenant_id { + return Err( + TerminalError::new("execution continuation session tenant mismatch").into(), + ); + } + let router = self.router.clone(); + let run_id = ExecutionRunScopeId(request.run_uid); + let task_id = request.task_id; + let logical_generation = request.logical_generation; + let attempt_generation = request.attempt_generation; + let publish_deadline_at = request.publish_deadline_at; + let retention_deadline_at = request.retention_deadline_at; + Ok(ctx + .run(|| async move { + router + .checkpoint_execution_hand_retaining_compute(ExecutionHandRetentionRequest { + session: &session, + run_id, + task_id, + logical_generation, + attempt_generation, + retention_deadline_at, + scope: ToolCallScope::unbounded().with_budget( + moa_core::types::resource::ResourceBudget::until(publish_deadline_at), + ), + }) + .await + .map(Json::from) + .map_err(moa_error_to_handler_error) + }) + .name(format!( + "checkpoint_retaining_execution_hand:{}:{}:{}", + request.run_uid, request.task_id, request.attempt_generation + )) + .retry_policy(RunRetryPolicy::new().max_attempts(1)) + .await?) + } + #[tracing::instrument(skip(self, ctx, request))] // SAFETY: internal terminal-task teardown reclaims only the typed run/task hand scope and returns no caller-owned data. async fn release_execution_task_hands( @@ -2516,12 +2610,7 @@ impl ToolExecutor for ToolExecutorImpl { crate::ctx::adopt_incoming_trace_parent(&ctx); annotate_restate_handler_span("ToolExecutor", "release_execution_task_hands"); let request = request.into_inner(); - let scope = execution_task_hand_scope(ExecutionTaskOrigin { - run_uid: request.run_uid, - task_uid: request.task_id.as_uuid(), - generation: 1, - attempt_generation: 1, - }); + let scope = execution_task_hand_scope(request.run_uid, request.task_id.as_uuid()); if !self .router .reclaim_hands(request.tenant_id, &request.session_id, Some(scope.as_str())) @@ -2542,12 +2631,8 @@ impl ToolExecutor for ToolExecutorImpl { crate::ctx::adopt_incoming_trace_parent(&ctx); annotate_restate_handler_span("ToolExecutor", "release_execution_compensation_hands"); let request = request.into_inner(); - let scope = execution_compensation_hand_scope(ExecutionCompensationOrigin { - run_uid: request.run_uid, - compensation_id: request.compensation_id.as_uuid(), - generation: 1, - attempt_generation: 1, - }); + let scope = + execution_compensation_hand_scope(request.run_uid, request.compensation_id.as_uuid()); if !self .router .reclaim_hands(request.tenant_id, &request.session_id, Some(scope.as_str())) @@ -2656,23 +2741,30 @@ pub fn tool_run_name( } /// Builds the isolated hand scope shared by generations of one execution task. -pub fn execution_task_hand_scope(origin: ExecutionTaskOrigin) -> String { - format!("execution:{}:{}", origin.run_uid, origin.task_uid) +/// +/// Takes the two identifiers it actually uses rather than a full origin, so +/// callers that only know the run and task cannot invent generation values. +pub fn execution_task_hand_scope(run_uid: Uuid, task_uid: Uuid) -> String { + format!("execution:{run_uid}:{task_uid}") } /// Builds the isolated hand scope shared by generations of one compensation. -pub fn execution_compensation_hand_scope(origin: ExecutionCompensationOrigin) -> String { - format!( - "execution_compensation:{}:{}", - origin.run_uid, origin.compensation_id - ) +/// +/// Takes the two identifiers it actually uses rather than a full origin, so +/// callers that only know the run and compensation cannot invent generations. +pub fn execution_compensation_hand_scope(run_uid: Uuid, compensation_id: Uuid) -> String { + format!("execution_compensation:{run_uid}:{compensation_id}") } /// Builds the isolated hand scope for one typed execution operation. pub fn execution_hand_scope(origin: ExecutionToolCallOrigin) -> String { match origin { - ExecutionToolCallOrigin::Task(origin) => execution_task_hand_scope(origin), - ExecutionToolCallOrigin::Compensation(origin) => execution_compensation_hand_scope(origin), + ExecutionToolCallOrigin::Task(origin) => { + execution_task_hand_scope(origin.run_uid, origin.task_uid) + } + ExecutionToolCallOrigin::Compensation(origin) => { + execution_compensation_hand_scope(origin.run_uid, origin.compensation_id) + } } } @@ -3976,10 +4068,6 @@ mod tests { Ok(HandStatus::Running) } - async fn pause(&self, _handle: &HandHandle) -> moa_core::error::Result<()> { - Ok(()) - } - async fn resume(&self, _handle: &HandHandle) -> moa_core::error::Result<()> { Ok(()) } @@ -4222,13 +4310,16 @@ mod tests { ..first }; - assert_eq!( - execution_task_hand_scope(first), - execution_task_hand_scope(next_generation) - ); + // Generations of one task share a scope by construction now that + // `execution_task_hand_scope` takes only the run and task identifiers, + // so the surviving assertion is the one with content: siblings differ. assert_ne!( - execution_task_hand_scope(first), - execution_task_hand_scope(sibling) + execution_task_hand_scope(first.run_uid, first.task_uid), + execution_task_hand_scope(sibling.run_uid, sibling.task_uid) + ); + assert_eq!( + execution_hand_scope(ExecutionToolCallOrigin::Task(first)), + execution_hand_scope(ExecutionToolCallOrigin::Task(next_generation)) ); assert_eq!( execution_workspace_scope(ExecutionToolCallOrigin::Task(first)), @@ -4328,9 +4419,9 @@ mod tests { execution_hand_scope(first_origin), execution_hand_scope(next_origin) ); - assert_eq!( - execution_compensation_hand_scope(first), - execution_compensation_hand_scope(next) + assert_ne!( + execution_compensation_hand_scope(first.run_uid, first.compensation_id), + execution_compensation_hand_scope(first.run_uid, Uuid::from_u128(31)) ); let first_name = execution_tool_run_name(&definition, &request, first_origin); let next_name = execution_tool_run_name(&definition, &request, next_origin); diff --git a/crates/moa-orchestrator/src/workflows/attempt_slice.rs b/crates/moa-orchestrator/src/workflows/attempt_slice.rs new file mode 100644 index 000000000..1ee17bf2a --- /dev/null +++ b/crates/moa-orchestrator/src/workflows/attempt_slice.rs @@ -0,0 +1,78 @@ +//! Helpers shared by the bounded task- and compensation-attempt slices. +//! +//! Restate models the exclusive and shared halves of a workflow as two distinct +//! context types. Both satisfy the SDK's blanket-implemented [`ContextSideEffects`] +//! and [`ContextClient`] traits, but bounding a helper on `ContextSideEffects<'ctx>` +//! makes the journal lifetime early-bound while `ContextSideEffects::run` requires +//! the journaled closure to outlive that same `'ctx`, so the generated handlers fail +//! rustc's higher-ranked check (`rust-lang/rust#100013`). Helpers that journal are +//! therefore written per context type; helpers that do not stay generic. + +use chrono::{DateTime, Utc}; +use restate_sdk::prelude::*; +use uuid::Uuid; + +use crate::services::execution_dispatcher::{DispatchExecutionsRequest, ExecutionDispatcherClient}; + +/// Idempotency-key family for task-attempt dispatcher wakes. +pub(crate) const TASK_ATTEMPT_DISPATCH_KICK: &str = "task-attempt-dispatch"; + +/// Idempotency-key family for compensation-attempt dispatcher wakes. +pub(crate) const COMPENSATION_ATTEMPT_DISPATCH_KICK: &str = "compensation-attempt-dispatch"; + +/// Journals the current wall clock from a shared watchdog or cancellation handler. +/// +/// Shared twin of [`crate::workflows::durable_utc_now`]; `step_name` becomes the +/// Restate journal entry name and must stay stable, because renaming a durable step +/// changes the replay journal key. +pub(crate) async fn durable_utc_now_shared( + ctx: &SharedWorkflowContext<'_>, + step_name: &'static str, +) -> Result, HandlerError> { + Ok(ctx + .run(|| async { Ok::<_, HandlerError>(Json::from(Utc::now())) }) + .name(step_name) + .await? + .into_inner()) +} + +/// Wakes the fleet dispatcher immediately after an attempt boundary commits. +/// +/// Attempt workflows run asynchronously from the dispatcher, which indexes the +/// persisted future head and owns the only delayed wake. `prefix` keeps the task +/// and compensation families on disjoint idempotency keys so one family's kick can +/// never attach to the other's completed invocation. +pub(crate) async fn kick_dispatcher<'ctx, C>( + ctx: &C, + prefix: &'static str, + dispatch_uid: Uuid, + boundary: &'static str, +) -> Result<(), HandlerError> +where + C: ContextClient<'ctx>, +{ + let handle = crate::restate_identity::replay_safe_request( + ctx.service_client::() + .dispatch(Json::from(DispatchExecutionsRequest::default())) + .idempotency_key(format!("{prefix}:{dispatch_uid}:{boundary}")), + ) + .send(); + let _invocation_id = handle.invocation_id().await?; + Ok(()) +} + +/// Rejects any delivery whose workflow key is not the immutable dispatch identity. +/// +/// `mismatch_message` names the owning attempt family so an operator can tell which +/// durable surface rejected the delivery. +pub(crate) fn require_dispatch_key( + key: &str, + dispatch_uid: Uuid, + mismatch_message: &'static str, +) -> Result<(), HandlerError> { + if key == dispatch_uid.to_string() { + Ok(()) + } else { + Err(TerminalError::new_with_code(404, mismatch_message).into()) + } +} diff --git a/crates/moa-orchestrator/src/workflows/execution_compensation_attempt.rs b/crates/moa-orchestrator/src/workflows/execution_compensation_attempt.rs index cb69c108c..f23d02047 100644 --- a/crates/moa-orchestrator/src/workflows/execution_compensation_attempt.rs +++ b/crates/moa-orchestrator/src/workflows/execution_compensation_attempt.rs @@ -6,7 +6,6 @@ mod yielding; use std::{collections::HashMap, sync::Arc}; -use chrono::Utc; use moa_config::SessionLimitsConfig; use moa_core::{traits::ChannelAdapter, types::channel::Channel}; use moa_execution::repository::{ @@ -25,11 +24,15 @@ use moa_session::PostgresSessionStore; use restate_sdk::prelude::*; use uuid::Uuid; -use crate::workflows::errors::execution_error_to_handler_error; -use crate::{ - restate_identity::replay_safe_request, - services::execution_dispatcher::{DispatchExecutionsRequest, ExecutionDispatcherClient}, +use crate::workflows::attempt_slice::{ + COMPENSATION_ATTEMPT_DISPATCH_KICK, durable_utc_now_shared, kick_dispatcher, + require_dispatch_key, }; +use crate::workflows::durable_utc_now; +use crate::workflows::errors::execution_error_to_handler_error; + +/// Operator-visible rejection for a delivery aimed at another dispatch identity. +const DISPATCH_KEY_MISMATCH: &str = "compensation attempt dispatch mismatch"; /// Durable surface for one strict reverse-order compensation slice. #[restate_sdk::workflow] @@ -89,8 +92,8 @@ impl ExecutionCompensationAttempt for ExecutionCompensationAttemptImpl { crate::ctx::adopt_incoming_trace_parent(&ctx); annotate_restate_handler_span("ExecutionCompensationAttempt", "run"); let request = request.into_inner(); - require_dispatch_key(ctx.key(), request.dispatch_uid)?; - let now = journal_now(&ctx, "compensation_attempt_started_at").await?; + require_dispatch_key(ctx.key(), request.dispatch_uid, DISPATCH_KEY_MISMATCH)?; + let now = durable_utc_now(&ctx, "compensation_attempt_started_at").await?; let repository = self.repository.clone(); let fence = compensation_attempt_fence(&request); let started = ctx @@ -110,7 +113,7 @@ impl ExecutionCompensationAttempt for ExecutionCompensationAttemptImpl { }; validate_authoritative_attempt(&request, &started)?; let exit = active::execute_compensation_attempt(self, &ctx, &request, &started).await?; - let progress_at = journal_now(&ctx, "compensation_attempt_progress_at").await?; + let progress_at = durable_utc_now(&ctx, "compensation_attempt_progress_at").await?; let repository = self.repository.clone(); let fence = compensation_attempt_fence(&request); ctx.run(|| async move { @@ -127,7 +130,13 @@ impl ExecutionCompensationAttempt for ExecutionCompensationAttemptImpl { .name("record_compensation_attempt_progress") .await?; settle_active_exit(self, &ctx, &request, &started, exit).await?; - kick_dispatcher(&ctx, request.dispatch_uid, "run").await + kick_dispatcher( + &ctx, + COMPENSATION_ATTEMPT_DISPATCH_KICK, + request.dispatch_uid, + "run", + ) + .await } #[tracing::instrument(skip(self, ctx, request))] @@ -141,7 +150,12 @@ impl ExecutionCompensationAttempt for ExecutionCompensationAttemptImpl { crate::ctx::adopt_incoming_trace_parent(&ctx); annotate_restate_handler_span("ExecutionCompensationAttempt", "watchdog"); let request = request.into_inner(); - require_dispatch_key(ctx.key(), request.dispatch_uid)?; + require_dispatch_key(ctx.key(), request.dispatch_uid, DISPATCH_KEY_MISMATCH)?; + if !watchdog_is_due(self, &ctx, &request).await? { + return Ok(Json::from(ExecutionAttemptWatchdogResponse { + outcome: ExecutionAttemptWatchdogResponseOutcome::RetryDelivery, + })); + } let release_request = ExecutionCompensationAttemptCancelRequest { cancellation_dispatch_uid: Uuid::new_v5( &request.dispatch_uid, @@ -182,7 +196,11 @@ impl ExecutionCompensationAttempt for ExecutionCompensationAttemptImpl { crate::ctx::adopt_incoming_trace_parent(&ctx); annotate_restate_handler_span("ExecutionCompensationAttempt", "cancel"); let request = request.into_inner(); - require_dispatch_key(ctx.key(), request.active_dispatch_uid)?; + require_dispatch_key( + ctx.key(), + request.active_dispatch_uid, + DISPATCH_KEY_MISMATCH, + )?; let outcome = yielding::cancel_compensation_attempt(self, &ctx, request.clone()).await?; if outcome == ExecutionAttemptWatchdogResponseOutcome::RetryDelivery { return Err(anyhow::anyhow!( @@ -190,7 +208,13 @@ impl ExecutionCompensationAttempt for ExecutionCompensationAttemptImpl { ) .into()); } - kick_dispatcher_shared(&ctx, request.cancellation_dispatch_uid, "cancel").await + kick_dispatcher( + &ctx, + COMPENSATION_ATTEMPT_DISPATCH_KICK, + request.cancellation_dispatch_uid, + "cancel", + ) + .await } } @@ -220,7 +244,7 @@ async fn settle_active_exit( else { return Ok(()); }; - let now = journal_now(ctx, "compensation_attempt_settled_at").await?; + let now = durable_utc_now(ctx, "compensation_attempt_settled_at").await?; let repository = workflow.repository.clone(); ctx.run(|| async move { repository @@ -247,6 +271,40 @@ async fn settle_active_exit( } } +/// Reports whether this watchdog delivery has actually reached its attempt deadline. +/// +/// `prepare_watchdog_trigger` already gates delivery on the Postgres clock one layer +/// up, so this is defense in depth: it keeps the compensation path symmetric with the +/// task watchdog, which re-checks its own due time before releasing anything, instead +/// of depending solely on a guard in another module. An absent trigger means delivery +/// already settled it, so the caller proceeds exactly as before. The compensation +/// watchdog trigger is created with `due_at` equal to the attempt deadline, so this is +/// the same comparison the task path makes against `attempt_deadline_at`. +async fn watchdog_is_due( + workflow: &ExecutionCompensationAttemptImpl, + ctx: &SharedWorkflowContext<'_>, + request: &ExecutionCompensationAttemptWatchdogRequest, +) -> Result { + let repository = workflow.repository.clone(); + let scope = ExecutionScope::Tenant { + tenant_id: request.tenant_id, + }; + let watchdog_trigger_uid = request.watchdog_trigger_uid; + let due_at = ctx + .run(|| async move { + repository + .load_trigger(scope, watchdog_trigger_uid) + .await + .map(|trigger| Json::from(trigger.map(|trigger| trigger.due_at))) + .map_err(execution_error_to_handler_error) + }) + .name("load_compensation_watchdog_trigger") + .await? + .into_inner(); + let observed_at = durable_utc_now_shared(ctx, "compensation_watchdog_observed_at").await?; + Ok(!due_at.is_some_and(|due_at| due_at > observed_at)) +} + fn compensation_attempt_fence( request: &ExecutionCompensationAttemptRequest, ) -> CompensationAttemptFence { @@ -288,67 +346,3 @@ fn validate_authoritative_attempt( } Ok(()) } - -async fn journal_now( - ctx: &WorkflowContext<'_>, - name: &'static str, -) -> Result, HandlerError> { - Ok(ctx - .run(|| async { Ok::<_, HandlerError>(Json::from(Utc::now())) }) - .name(name) - .await? - .into_inner()) -} - -async fn journal_now_shared( - ctx: &SharedWorkflowContext<'_>, - name: &'static str, -) -> Result, HandlerError> { - Ok(ctx - .run(|| async { Ok::<_, HandlerError>(Json::from(Utc::now())) }) - .name(name) - .await? - .into_inner()) -} - -fn require_dispatch_key(key: &str, dispatch_uid: Uuid) -> Result<(), HandlerError> { - if key == dispatch_uid.to_string() { - Ok(()) - } else { - Err(TerminalError::new_with_code(404, "compensation attempt dispatch mismatch").into()) - } -} - -async fn kick_dispatcher( - ctx: &WorkflowContext<'_>, - dispatch_uid: Uuid, - boundary: &'static str, -) -> Result<(), HandlerError> { - let handle = replay_safe_request( - ctx.service_client::() - .dispatch(Json::from(DispatchExecutionsRequest::default())) - .idempotency_key(format!( - "compensation-attempt-dispatch:{dispatch_uid}:{boundary}" - )), - ) - .send(); - let _invocation_id = handle.invocation_id().await?; - Ok(()) -} - -async fn kick_dispatcher_shared( - ctx: &SharedWorkflowContext<'_>, - dispatch_uid: Uuid, - boundary: &'static str, -) -> Result<(), HandlerError> { - let handle = replay_safe_request( - ctx.service_client::() - .dispatch(Json::from(DispatchExecutionsRequest::default())) - .idempotency_key(format!( - "compensation-attempt-dispatch:{dispatch_uid}:{boundary}" - )), - ) - .send(); - let _invocation_id = handle.invocation_id().await?; - Ok(()) -} diff --git a/crates/moa-orchestrator/src/workflows/execution_compensation_attempt/external.rs b/crates/moa-orchestrator/src/workflows/execution_compensation_attempt/external.rs index 647fbf50c..5652cac56 100644 --- a/crates/moa-orchestrator/src/workflows/execution_compensation_attempt/external.rs +++ b/crates/moa-orchestrator/src/workflows/execution_compensation_attempt/external.rs @@ -9,9 +9,10 @@ use uuid::Uuid; use crate::services::tool_executor::ToolExecutorClient; use crate::workflows::{ + durable_utc_now, errors::execution_error_to_handler_error, execution_compensation_attempt::{ - ExecutionCompensationAttemptImpl, journal_now, + ExecutionCompensationAttemptImpl, yielding::{release_hands_request, release_request}, }, }; @@ -24,7 +25,7 @@ pub(super) async fn yield_external_job( external_job_uid: Uuid, ) -> Result<(), HandlerError> { let release_request = release_request(request, ExecutionCompensationReleaseIntent::ExternalJob); - let claimed_at = journal_now(ctx, "compensation_external_release_claimed_at").await?; + let claimed_at = durable_utc_now(ctx, "compensation_external_release_claimed_at").await?; let repository = workflow.repository.clone(); let request_for_claim = release_request.clone(); let started = ctx @@ -55,7 +56,7 @@ pub(super) async fn yield_external_job( .call() .await? .into_inner(); - let yielded_at = journal_now(ctx, "compensation_external_job_yielded_at").await?; + let yielded_at = durable_utc_now(ctx, "compensation_external_job_yielded_at").await?; let repository = workflow.repository.clone(); ctx.run(|| async move { repository diff --git a/crates/moa-orchestrator/src/workflows/execution_compensation_attempt/yielding.rs b/crates/moa-orchestrator/src/workflows/execution_compensation_attempt/yielding.rs index 7648413bb..3ca33cfe2 100644 --- a/crates/moa-orchestrator/src/workflows/execution_compensation_attempt/yielding.rs +++ b/crates/moa-orchestrator/src/workflows/execution_compensation_attempt/yielding.rs @@ -23,8 +23,9 @@ use crate::{ services::tool_executor::{CheckpointAndReleaseExecutionHandsRequest, ToolExecutorClient}, tool_invocation::governed::GovernedReviewPending, workflows::{ + attempt_slice::durable_utc_now_shared, durable_utc_now, errors::execution_error_to_handler_error, - execution_compensation_attempt::{ExecutionCompensationAttemptImpl, journal_now}, + execution_compensation_attempt::ExecutionCompensationAttemptImpl, }, }; @@ -46,7 +47,7 @@ pub(super) async fn park_compensation_review( else { return Ok(()); }; - let now = journal_now(ctx, "compensation_review_parked_at").await?; + let now = durable_utc_now(ctx, "compensation_review_parked_at").await?; let repository = workflow.repository.clone(); let parked = ctx .run(|| async move { @@ -106,7 +107,7 @@ pub(super) async fn release_compensation_hands_workflow( )>, HandlerError, > { - let claimed_at = journal_now(ctx, "compensation_release_claimed_at").await?; + let claimed_at = durable_utc_now(ctx, "compensation_release_claimed_at").await?; let release_request = release_request(request, intent); let repository = workflow.repository.clone(); let claim_request = release_request.clone(); @@ -205,7 +206,7 @@ pub(super) async fn release_and_settle_compensation_shared( request: ExecutionCompensationAttemptCancelRequest, settlement: SharedReleaseSettlement, ) -> Result { - let claimed_at = super::journal_now_shared(ctx, "compensation_release_claimed_at").await?; + let claimed_at = durable_utc_now_shared(ctx, "compensation_release_claimed_at").await?; let repository = workflow.repository.clone(); let claim_request = request.clone(); let claim = ctx @@ -238,7 +239,7 @@ pub(super) async fn release_and_settle_compensation_shared( .call() .await? .into_inner(); - let settled_at = super::journal_now_shared(ctx, "compensation_cancel_settled_at").await?; + let settled_at = durable_utc_now_shared(ctx, "compensation_cancel_settled_at").await?; let outcome = match settlement { SharedReleaseSettlement::Cancelled => ExecutionCompensationOutcome::Failed { message: format!( diff --git a/crates/moa-orchestrator/src/workflows/execution_task_attempt.rs b/crates/moa-orchestrator/src/workflows/execution_task_attempt.rs index 8f32de8f9..28925264f 100644 --- a/crates/moa-orchestrator/src/workflows/execution_task_attempt.rs +++ b/crates/moa-orchestrator/src/workflows/execution_task_attempt.rs @@ -7,7 +7,7 @@ mod yielding; use std::{collections::HashMap, sync::Arc}; -use chrono::{Duration, Utc}; +use chrono::Duration; use moa_artifacts::execution_plan::{ExecutionFailureClass, ExecutionTaskResult}; use moa_config::{ExecutionConfig, SessionLimitsConfig}; use moa_core::{traits::ChannelAdapter, types::channel::Channel}; @@ -31,11 +31,15 @@ use moa_observability::restate_observability::annotate_restate_handler_span; use moa_session::PostgresSessionStore; use restate_sdk::prelude::*; -use crate::{ - services::execution_dispatcher::{DispatchExecutionsRequest, ExecutionDispatcherClient}, - workflows::errors::execution_error_to_handler_error, +use crate::workflows::{ + attempt_slice::{TASK_ATTEMPT_DISPATCH_KICK, kick_dispatcher, require_dispatch_key}, + durable_utc_now, + errors::execution_error_to_handler_error, }; +/// Operator-visible rejection for a delivery aimed at another dispatch identity. +const DISPATCH_KEY_MISMATCH: &str = "execution attempt dispatch mismatch"; + /// Returns the catalog-owned model-visible name for one governed capability. pub(crate) fn capability_tool_name( capability: &ExecutionCapability, @@ -112,7 +116,7 @@ impl ExecutionTaskAttempt for ExecutionTaskAttemptImpl { crate::ctx::adopt_incoming_trace_parent(&ctx); annotate_restate_handler_span("ExecutionTaskAttempt", "run"); let request = request.into_inner(); - require_dispatch_key(ctx.key(), request.dispatch_uid)?; + require_dispatch_key(ctx.key(), request.dispatch_uid, DISPATCH_KEY_MISMATCH)?; let fence = task_attempt_fence(&request); let repository = self.repository.clone(); let started = ctx @@ -172,7 +176,7 @@ impl ExecutionTaskAttempt for ExecutionTaskAttemptImpl { crate::ctx::adopt_incoming_trace_parent(&ctx); annotate_restate_handler_span("ExecutionTaskAttempt", "watchdog"); let request = request.into_inner(); - require_dispatch_key(ctx.key(), request.dispatch_uid)?; + require_dispatch_key(ctx.key(), request.dispatch_uid, DISPATCH_KEY_MISMATCH)?; let watchdog = watchdog::handle_task_attempt_watchdog(self, &ctx, request).await?; // TriggerDelivery awaits this handler, so its owning dispatcher observes every outbox row // committed by watchdog settlement before selecting the next durable timing head. @@ -192,10 +196,14 @@ impl ExecutionTaskAttempt for ExecutionTaskAttemptImpl { crate::ctx::adopt_incoming_trace_parent(&ctx); annotate_restate_handler_span("ExecutionTaskAttempt", "cancel"); let request = request.into_inner(); - require_dispatch_key(ctx.key(), request.active_dispatch_uid)?; + require_dispatch_key( + ctx.key(), + request.active_dispatch_uid, + DISPATCH_KEY_MISMATCH, + )?; let dispatch_uid = request.active_dispatch_uid; yielding::cancel_task_attempt(self, &ctx, request).await?; - kick_dispatcher_shared(&ctx, dispatch_uid, "cancel").await + kick_dispatcher(&ctx, TASK_ATTEMPT_DISPATCH_KICK, dispatch_uid, "cancel").await } } @@ -229,7 +237,7 @@ async fn settle_active_exit( outcome, ); let outcome = exhaust_retry_outcome(started.task.attempt, &started.task.retry, outcome); - let settled_at = journal_now(ctx, "task_attempt_settled_at").await?; + let settled_at = durable_utc_now(ctx, "task_attempt_settled_at").await?; let retry_at = matches!( outcome.result, ExecutionTaskResult::Failed { @@ -259,8 +267,7 @@ async fn settle_active_exit( return Ok(()); }; let release_receipt = - yielding::checkpoint_task_hands_workflow(workflow, ctx, request, &releasing) - .await?; + yielding::checkpoint_task_hands_workflow(ctx, request, &releasing).await?; let Some(release_receipt) = release_receipt else { return Err(TerminalError::new( "normal task outcome omitted its durable hand-release receipt", @@ -364,56 +371,11 @@ async fn settle_active_exit( } active::ActiveTaskAttemptExit::OwnershipLost => return Ok(()), }; - // Task workflows run asynchronously from the dispatcher. Wake it immediately after the - // transaction commits; it indexes the persisted future head and owns the only delayed wake. - kick_dispatcher(ctx, request.dispatch_uid, boundary).await -} - -async fn kick_dispatcher( - ctx: &WorkflowContext<'_>, - dispatch_uid: uuid::Uuid, - boundary: &'static str, -) -> Result<(), HandlerError> { - let handle = crate::restate_identity::replay_safe_request( - ctx.service_client::() - .dispatch(Json::from(DispatchExecutionsRequest::default())) - .idempotency_key(format!("task-attempt-dispatch:{dispatch_uid}:{boundary}")), + kick_dispatcher( + ctx, + TASK_ATTEMPT_DISPATCH_KICK, + request.dispatch_uid, + boundary, ) - .send(); - let _invocation_id = handle.invocation_id().await?; - Ok(()) -} - -async fn kick_dispatcher_shared( - ctx: &SharedWorkflowContext<'_>, - dispatch_uid: uuid::Uuid, - boundary: &'static str, -) -> Result<(), HandlerError> { - let handle = crate::restate_identity::replay_safe_request( - ctx.service_client::() - .dispatch(Json::from(DispatchExecutionsRequest::default())) - .idempotency_key(format!("task-attempt-dispatch:{dispatch_uid}:{boundary}")), - ) - .send(); - let _invocation_id = handle.invocation_id().await?; - Ok(()) -} - -async fn journal_now( - ctx: &WorkflowContext<'_>, - name: &'static str, -) -> Result, HandlerError> { - Ok(ctx - .run(|| async { Ok::<_, HandlerError>(Json::from(Utc::now())) }) - .name(name) - .await? - .into_inner()) -} - -fn require_dispatch_key(key: &str, dispatch_uid: uuid::Uuid) -> Result<(), HandlerError> { - if key == dispatch_uid.to_string() { - Ok(()) - } else { - Err(TerminalError::new_with_code(404, "execution attempt dispatch mismatch").into()) - } + .await } diff --git a/crates/moa-orchestrator/src/workflows/execution_task_attempt/active.rs b/crates/moa-orchestrator/src/workflows/execution_task_attempt/active.rs index 884f122cc..3e63e3f76 100644 --- a/crates/moa-orchestrator/src/workflows/execution_task_attempt/active.rs +++ b/crates/moa-orchestrator/src/workflows/execution_task_attempt/active.rs @@ -45,9 +45,10 @@ use crate::{ GovernedInvocationRequest, invoke_governed_tool, }, workflows::{ + durable_utc_now, errors::moa_error_to_handler_error, execution_task_attempt::{ - ExecutionTaskAttemptImpl, capability_tool_name, journal_now, task_attempt_fence, + ExecutionTaskAttemptImpl, capability_tool_name, task_attempt_fence, }, }, }; @@ -355,7 +356,7 @@ async fn persist_external_start_checkpoint( schema_version: continuation.schema_version, payload: continuation.to_bounded_json().map_err(TerminalError::new)?, workspace_release_receipt: None, - created_at: journal_now(ctx, "task_external_start_checkpointed_at").await?, + created_at: durable_utc_now(ctx, "task_external_start_checkpointed_at").await?, }; let repository = workflow.repository.clone(); Ok(ctx diff --git a/crates/moa-orchestrator/src/workflows/execution_task_attempt/external.rs b/crates/moa-orchestrator/src/workflows/execution_task_attempt/external.rs index 739f3b583..cb5e90c56 100644 --- a/crates/moa-orchestrator/src/workflows/execution_task_attempt/external.rs +++ b/crates/moa-orchestrator/src/workflows/execution_task_attempt/external.rs @@ -11,9 +11,10 @@ use restate_sdk::prelude::*; use uuid::Uuid; use crate::workflows::{ + durable_utc_now, errors::execution_error_to_handler_error, execution_task_attempt::{ - ExecutionTaskAttemptImpl, active::TaskAttemptContinuation, journal_now, task_attempt_fence, + ExecutionTaskAttemptImpl, active::TaskAttemptContinuation, task_attempt_fence, yielding::checkpoint_task_hands_workflow, }, }; @@ -32,7 +33,7 @@ pub(super) async fn yield_external_job( .bind_external_job(external_job_uid) .map_err(TerminalError::new)?; } - let claimed_at = journal_now(ctx, "task_external_job_release_claimed_at").await?; + let claimed_at = durable_utc_now(ctx, "task_external_job_release_claimed_at").await?; let repository = workflow.repository.clone(); let fence = task_attempt_fence(request); let task_generation = started.task.generation; @@ -66,8 +67,8 @@ pub(super) async fn yield_external_job( let Some(started) = started else { return Ok(()); }; - let release_receipt = checkpoint_task_hands_workflow(workflow, ctx, request, &started).await?; - let yielded_at = journal_now(ctx, "task_external_job_yielded_at").await?; + let release_receipt = checkpoint_task_hands_workflow(ctx, request, &started).await?; + let yielded_at = durable_utc_now(ctx, "task_external_job_yielded_at").await?; let continuation_checkpoint = continuation .map(|mut continuation| { continuation.workspace_release_receipt_id = diff --git a/crates/moa-orchestrator/src/workflows/execution_task_attempt/watchdog.rs b/crates/moa-orchestrator/src/workflows/execution_task_attempt/watchdog.rs index aeca00171..5609682b7 100644 --- a/crates/moa-orchestrator/src/workflows/execution_task_attempt/watchdog.rs +++ b/crates/moa-orchestrator/src/workflows/execution_task_attempt/watchdog.rs @@ -26,10 +26,11 @@ use uuid::Uuid; use crate::{ services::llm_gateway::{LLMCompletionOwner, cancel_completion_owner}, workflows::{ + attempt_slice::durable_utc_now_shared, errors::execution_error_to_handler_error, execution_task_attempt::{ ExecutionTaskAttemptImpl, task_attempt_fence, - yielding::{begin_release_shared, checkpoint_task_hands_shared, journal_now_shared}, + yielding::{begin_release_shared, checkpoint_task_hands_shared}, }, }, }; @@ -106,7 +107,7 @@ pub(super) async fn handle_task_attempt_watchdog( ExecutionAttemptWatchdogResponseOutcome::ReplayedOrStale, )); } - let now = journal_now_shared(ctx, "task_watchdog_observed_at").await?; + let now = durable_utc_now_shared(ctx, "task_watchdog_observed_at").await?; if deadline > now { return Ok(watchdog_result( ExecutionAttemptWatchdogResponseOutcome::RetryDelivery, @@ -190,7 +191,7 @@ pub(super) async fn handle_task_attempt_watchdog( ExecutionAttemptWatchdogResponseOutcome::RetryDelivery, )); }; - let receipt = checkpoint_task_hands_shared(workflow, ctx, &attempt_request, &started).await?; + let receipt = checkpoint_task_hands_shared(ctx, &attempt_request, &started).await?; let disposition = classify_stale_attempt(task_effect_idempotency(&started)); let outcome = match disposition { StaleTaskAttemptDisposition::Retry => ExecutionTaskOutcome { diff --git a/crates/moa-orchestrator/src/workflows/execution_task_attempt/yielding.rs b/crates/moa-orchestrator/src/workflows/execution_task_attempt/yielding.rs index 33420b889..420d70c6d 100644 --- a/crates/moa-orchestrator/src/workflows/execution_task_attempt/yielding.rs +++ b/crates/moa-orchestrator/src/workflows/execution_task_attempt/yielding.rs @@ -3,7 +3,9 @@ use chrono::{Duration, Utc}; use moa_artifacts::execution_plan::{ExecutionTaskOutcome, ExecutionTaskResult}; use moa_core::types::action_policy::{ActionReviewOwner, ExecutionTaskOrigin}; -use moa_core::types::sandbox_workspace::ExecutionHandReleaseReceipt; +use moa_core::types::sandbox_workspace::{ + ExecutionHandContinuationDisposition, ExecutionHandReleaseReceipt, +}; use moa_execution::{ repository::{ ExecutionAttemptState, ExecutionScope, @@ -28,9 +30,14 @@ use crate::{ services::{ action_reviews::{AcknowledgeExecutionActionReviewRequest, ActionReviewsClient}, llm_gateway::{LLMCompletionOwner, cancel_completion_owner}, - tool_executor::{CheckpointAndReleaseExecutionHandsRequest, ToolExecutorClient}, + tool_executor::{ + CheckpointAndReleaseExecutionHandsRequest, + CheckpointExecutionHandsRetainingComputeRequest, ToolExecutorClient, + }, }, workflows::{ + attempt_slice::durable_utc_now_shared, + durable_utc_now, errors::execution_error_to_handler_error, execution_task_attempt::{ ExecutionTaskAttemptImpl, @@ -74,8 +81,7 @@ pub(super) async fn park_review( else { return Ok(()); }; - let workspace_release_receipt = - checkpoint_task_hands_workflow(workflow, ctx, request, &started).await?; + let workspace_release_receipt = checkpoint_task_hands_workflow(ctx, request, &started).await?; let review_uid = continuation .pending_review_uid() .ok_or_else(|| TerminalError::new("review continuation is missing its review UID"))?; @@ -98,7 +104,7 @@ pub(super) async fn park_review( schema_version: continuation.schema_version, payload, workspace_release_receipt, - created_at: journal_now_workflow(ctx, "task_review_checkpointed_at").await?, + created_at: durable_utc_now(ctx, "task_review_checkpointed_at").await?, }; let parked = ctx .run(|| async move { @@ -165,8 +171,24 @@ pub(super) async fn yield_continuation( else { return Ok(()); }; - let workspace_release_receipt = - checkpoint_task_hands_workflow(workflow, ctx, request, &started).await?; + // A continuation is not a wait: the yield below marks the task ready and enqueues + // its run activation immediately, so the next slice is admitted in seconds. The + // durable checkpoint head still advances here, and the sandbox is kept — suspended + // where the provider can genuinely release compute, briefly hot where it cannot — + // instead of destroyed and re-provisioned across a zero-length yield. Every genuine + // park — review, input, external job, cancel, pause — still releases unconditionally. + let disposition = continue_task_hands_workflow(ctx, request, &started).await?; + // A failed suspension leaves the hand running with no owner willing to bet on it, + // so it finishes the ordinary checkpoint-and-destroy path and fences the resulting + // receipt into this checkpoint exactly as a genuine park would. + let workspace_release_receipt = match disposition { + ExecutionHandContinuationDisposition::SuspendFailed => { + checkpoint_task_hands_workflow(ctx, request, &started).await? + } + ExecutionHandContinuationDisposition::NoComputeOwned + | ExecutionHandContinuationDisposition::Suspended + | ExecutionHandContinuationDisposition::RetainedHot => None, + }; let payload = continuation.to_bounded_json().map_err(TerminalError::new)?; let repository = workflow.repository.clone(); let checkpoint = NewTaskAttemptCheckpoint { @@ -176,7 +198,7 @@ pub(super) async fn yield_continuation( schema_version: continuation.schema_version, payload, workspace_release_receipt, - created_at: journal_now_workflow(ctx, "task_agent_continuation_checkpointed_at").await?, + created_at: durable_utc_now(ctx, "task_agent_continuation_checkpointed_at").await?, }; ctx.run(|| async move { repository @@ -223,10 +245,9 @@ pub(super) async fn park_input( else { return Ok(()); }; - let workspace_release_receipt = - checkpoint_task_hands_workflow(workflow, ctx, request, &started).await?; + let workspace_release_receipt = checkpoint_task_hands_workflow(ctx, request, &started).await?; let payload = continuation.to_bounded_json().map_err(TerminalError::new)?; - let settled_at = journal_now_workflow(ctx, "task_input_checkpointed_at").await?; + let settled_at = durable_utc_now(ctx, "task_input_checkpointed_at").await?; let checkpoint = NewTaskAttemptCheckpoint { fence: task_attempt_fence(request), task_generation: started.task.generation, @@ -317,7 +338,7 @@ pub(super) async fn cancel_task_attempt( if task.status == ExecutionTaskStatus::Dispatching && task.attempt_state == ExecutionAttemptState::Cancelling { - let now = journal_now_shared(ctx, "unstarted_task_cancel_settled_at").await?; + let now = durable_utc_now_shared(ctx, "unstarted_task_cancel_settled_at").await?; let attempt_deadline_at = task.attempt_deadline_at.ok_or_else(|| { TerminalError::new("unstarted cancelled task is missing its attempt deadline") })?; @@ -387,8 +408,8 @@ pub(super) async fn cancel_task_attempt( // Re-running the ordinary release claim here would compare those generations, // return `Stale`, and acknowledge the cancel without draining active capacity. let started = TaskAttemptRecord { run, task }; - let receipt = checkpoint_task_hands_shared(workflow, ctx, &attempt_request, &started).await?; - let now = journal_now_shared(ctx, "task_cancel_settled_at").await?; + let receipt = checkpoint_task_hands_shared(ctx, &attempt_request, &started).await?; + let now = durable_utc_now_shared(ctx, "task_cancel_settled_at").await?; let repository = workflow.repository.clone(); let config = workflow.config.clone(); let fence = task_attempt_fence(&attempt_request); @@ -443,6 +464,7 @@ pub(super) async fn cancel_task_attempt( Ok(()) } +/// Claims exclusive release ownership of one exact attempt before any teardown. pub(super) async fn begin_release_workflow( workflow: &ExecutionTaskAttemptImpl, ctx: &WorkflowContext<'_>, @@ -450,7 +472,7 @@ pub(super) async fn begin_release_workflow( task_generation: u64, reason: &'static str, ) -> Result, HandlerError> { - let claimed_at = journal_now_workflow(ctx, "task_attempt_release_claimed_at").await?; + let claimed_at = durable_utc_now(ctx, "task_attempt_release_claimed_at").await?; let repository = workflow.repository.clone(); let fence = task_attempt_fence(request); let outcome = ctx @@ -464,15 +486,10 @@ pub(super) async fn begin_release_workflow( .name("begin_task_attempt_release") .await? .into_inner(); - Ok(match outcome { - TaskAttemptReleaseClaimOutcome::Applied(record) - | TaskAttemptReleaseClaimOutcome::Replayed(record) => Some(*record), - TaskAttemptReleaseClaimOutcome::NotFound - | TaskAttemptReleaseClaimOutcome::Stale - | TaskAttemptReleaseClaimOutcome::InvalidState => None, - }) + Ok(release_claim(outcome)) } +/// Claims exclusive release ownership from a shared watchdog or cancellation handler. pub(super) async fn begin_release_shared( workflow: &ExecutionTaskAttemptImpl, ctx: &SharedWorkflowContext<'_>, @@ -480,7 +497,7 @@ pub(super) async fn begin_release_shared( task_generation: u64, reason: &'static str, ) -> Result, HandlerError> { - let claimed_at = journal_now_shared(ctx, "task_attempt_release_claimed_at").await?; + let claimed_at = durable_utc_now_shared(ctx, "task_attempt_release_claimed_at").await?; let repository = workflow.repository.clone(); let fence = task_attempt_fence(request); let outcome = ctx @@ -494,22 +511,65 @@ pub(super) async fn begin_release_shared( .name("begin_task_attempt_release") .await? .into_inner(); - Ok(match outcome { + Ok(release_claim(outcome)) +} + +/// Keeps only a claim that actually owns the exact attempt's release. +fn release_claim(outcome: TaskAttemptReleaseClaimOutcome) -> Option { + match outcome { TaskAttemptReleaseClaimOutcome::Applied(record) | TaskAttemptReleaseClaimOutcome::Replayed(record) => Some(*record), TaskAttemptReleaseClaimOutcome::NotFound | TaskAttemptReleaseClaimOutcome::Stale | TaskAttemptReleaseClaimOutcome::InvalidState => None, - }) + } +} + +/// Publishes the attempt's checkpoint and keeps its sandbox for the next slice. +/// +/// The published checkpoint is what makes keeping the sandbox safe at all: whether +/// it is suspended or held hot, losing it before the next slice arrives costs a +/// restore from this exact head and nothing else. Which of the two happens is a +/// provider capability question, so it is decided in the hands layer and reported +/// back rather than guessed here. +pub(super) async fn continue_task_hands_workflow( + ctx: &WorkflowContext<'_>, + request: &ExecutionTaskAttemptRequest, + started: &TaskAttemptRecord, +) -> Result { + let publish_started_at = durable_utc_now(ctx, "task_hand_continuation_started_at").await?; + Ok(crate::restate_identity::replay_safe_request( + ctx.service_client::() + .checkpoint_execution_hands_retaining_compute(Json::from( + CheckpointExecutionHandsRetainingComputeRequest { + tenant_id: request.tenant_id, + session_id: started.run.session_id, + run_uid: request.run_uid, + task_id: moa_core::types::identifiers::ExecutionTaskScopeId( + request.task_id.as_uuid(), + ), + logical_generation: started.task.generation, + attempt_generation: request.attempt_generation, + publish_deadline_at: task_hand_release_deadline(publish_started_at), + retention_deadline_at: task_hand_retention_deadline( + publish_started_at, + request.attempt_deadline_at, + ), + }, + )), + ) + .call() + .await? + .into_inner()) } +/// Obtains provider-verified checkpoint and release proof for the attempt's hands. pub(super) async fn checkpoint_task_hands_workflow( - _workflow: &ExecutionTaskAttemptImpl, ctx: &WorkflowContext<'_>, request: &ExecutionTaskAttemptRequest, started: &TaskAttemptRecord, ) -> Result, HandlerError> { - let release_started_at = journal_now_workflow(ctx, "task_hand_release_started_at").await?; + let release_started_at = durable_utc_now(ctx, "task_hand_release_started_at").await?; let receipt = crate::restate_identity::replay_safe_request( ctx.service_client::() .checkpoint_and_release_execution_hands(Json::from(checkpoint_request( @@ -524,13 +584,13 @@ pub(super) async fn checkpoint_task_hands_workflow( Ok(Some(receipt)) } +/// Obtains the same release proof from a shared watchdog or cancellation handler. pub(super) async fn checkpoint_task_hands_shared( - _workflow: &ExecutionTaskAttemptImpl, ctx: &SharedWorkflowContext<'_>, request: &ExecutionTaskAttemptRequest, started: &TaskAttemptRecord, ) -> Result, HandlerError> { - let release_started_at = journal_now_shared(ctx, "task_hand_release_started_at").await?; + let release_started_at = durable_utc_now_shared(ctx, "task_hand_release_started_at").await?; let receipt = crate::restate_identity::replay_safe_request( ctx.service_client::() .checkpoint_and_release_execution_hands(Json::from(checkpoint_request( @@ -567,34 +627,44 @@ fn task_hand_release_deadline(release_started_at: chrono::DateTime) -> chro release_started_at + Duration::minutes(5) } -async fn journal_now_workflow( - ctx: &WorkflowContext<'_>, - name: &'static str, -) -> Result, HandlerError> { - Ok(ctx - .run(|| async { Ok::<_, HandlerError>(Json::from(Utc::now())) }) - .name(name) - .await? - .into_inner()) -} +/// How long a continuation boundary may keep an unsuspendable sandbox hot. +/// +/// This bound only applies to providers that cannot actually release compute, where +/// the sandbox stays fully billed and keeps its `ActiveHands` admission slot for the +/// whole window. That is a bet that the next slice arrives before the window closes, +/// and it only pays off when it arrives *fast*: a longer window does not raise the +/// odds, it just extends the loss when the bet fails — hot idle compute plus a slot +/// withheld from runnable work plus, in the end, the full restore anyway. +/// +/// One reaper interval is therefore the whole budget. It matches +/// `HandLeaseReaperConfig::interval` (30s by default, +/// `crates/moa-hands/src/core/reaper.rs`), which is also the granularity at which the +/// deadline can actually be enforced — a shorter value would not be observed sooner, +/// and a longer one buys nothing the fast path needs. Providers with real suspension +/// never reach this constant: they release compute in the yield path instead. +const TASK_CONTINUATION_HAND_RETENTION: Duration = Duration::seconds(30); -pub(super) async fn journal_now_shared( - ctx: &SharedWorkflowContext<'_>, - name: &'static str, -) -> Result, HandlerError> { - Ok(ctx - .run(|| async { Ok::<_, HandlerError>(Json::from(Utc::now())) }) - .name(name) - .await? - .into_inner()) +/// Bounds retention by both the retention window and the attempt's own deadline. +/// +/// The sandbox was admitted under this attempt's compute deadline, so retention never +/// carries it past that instant even when the window would allow it. +fn task_hand_retention_deadline( + published_at: chrono::DateTime, + attempt_deadline_at: chrono::DateTime, +) -> chrono::DateTime { + (published_at + TASK_CONTINUATION_HAND_RETENTION).min(attempt_deadline_at) } #[cfg(test)] mod tests { use chrono::{Duration, TimeZone, Utc}; use moa_execution::wire::ExecutionAttemptCancelReason; + use moa_hands::core::reaper::HandLeaseReaperConfig; - use super::{TaskCancelSettlement, task_cancel_settlement, task_hand_release_deadline}; + use super::{ + TASK_CONTINUATION_HAND_RETENTION, TaskCancelSettlement, task_cancel_settlement, + task_hand_release_deadline, task_hand_retention_deadline, + }; #[test] fn pause_cancel_uses_nonterminal_release_finalizer() { @@ -621,6 +691,44 @@ mod tests { } } + #[test] + fn continuation_retention_never_outlives_the_attempt_that_was_admitted_for_it() { + // Pins: a retained continuation hand is bounded by the retention window and, when + // the attempt ends sooner, by the attempt deadline it was admitted under. Losing + // either bound would let one task hold a fleet active-hands slot indefinitely. + let published_at = Utc + .with_ymd_and_hms(2026, 8, 12, 9, 0, 0) + .single() + .expect("fixture timestamp is valid"); + + let roomy_attempt_deadline = published_at + Duration::hours(1); + assert_eq!( + task_hand_retention_deadline(published_at, roomy_attempt_deadline), + published_at + TASK_CONTINUATION_HAND_RETENTION, + ); + + let expiring_attempt_deadline = published_at + Duration::seconds(10); + assert_eq!( + task_hand_retention_deadline(published_at, expiring_attempt_deadline), + expiring_attempt_deadline, + ); + assert!(TASK_CONTINUATION_HAND_RETENTION < Duration::minutes(10)); + } + + #[test] + fn hot_retention_never_outlives_one_reaper_sweep() { + // Pins: hot retention on an unsuspendable provider is one reaper interval and no + // more. The window is fully billed compute that also withholds an admission slot + // from runnable work, so a longer bet does not improve the odds of the next slice + // arriving — it only enlarges the loss when the bet fails. The bound is the + // 30s `HandLeaseReaperConfig::interval` default that actually enforces it. + assert_eq!( + TASK_CONTINUATION_HAND_RETENTION, + Duration::from_std(HandLeaseReaperConfig::default().interval) + .expect("the reaper interval fits a chrono duration"), + ); + } + #[test] fn overdue_watchdog_gets_a_fresh_bounded_sandbox_release_deadline() { // Pins: an expired compute deadline still permits one bounded checkpoint/destroy cycle; diff --git a/crates/moa-orchestrator/src/workflows/mod.rs b/crates/moa-orchestrator/src/workflows/mod.rs index 7c66badfe..dca5a4fcc 100644 --- a/crates/moa-orchestrator/src/workflows/mod.rs +++ b/crates/moa-orchestrator/src/workflows/mod.rs @@ -4,6 +4,7 @@ use chrono::{DateTime, Utc}; use restate_sdk::prelude::*; pub mod artifact_release_evaluation; +pub(crate) mod attempt_slice; pub(crate) mod child_invocation; pub mod consolidate; pub(crate) mod errors; diff --git a/crates/moa-orchestrator/tests/coordinator_worker_behavior_provider_e2e.rs b/crates/moa-orchestrator/tests/coordinator_worker_behavior_provider_e2e.rs index 74ffd2c77..ca3a76445 100644 --- a/crates/moa-orchestrator/tests/coordinator_worker_behavior_provider_e2e.rs +++ b/crates/moa-orchestrator/tests/coordinator_worker_behavior_provider_e2e.rs @@ -3082,7 +3082,7 @@ fn recovery_matrix_execution_candidate( expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { delay_seconds: 86_400, }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, }, input_schema: json!({"type": "object", "additionalProperties": false}), output_schema: output_schema.clone(), diff --git a/crates/moa-orchestrator/tests/execution_run_service_e2e.rs b/crates/moa-orchestrator/tests/execution_run_service_e2e.rs index a5132f85a..524322d1f 100644 --- a/crates/moa-orchestrator/tests/execution_run_service_e2e.rs +++ b/crates/moa-orchestrator/tests/execution_run_service_e2e.rs @@ -136,7 +136,7 @@ async fn output_only_run_is_durable_detached_and_reaches_terminal_state() -> Res expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { delay_seconds: 86_400, }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, }, input_schema: json!({"type": "object", "additionalProperties": false}), output_schema: json!({ @@ -355,7 +355,7 @@ async fn cancellation_preserves_preconfirmation_null_and_postqueue_timestamp() - expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { delay_seconds: 86_400, }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, }, input_schema: json!({"type": "object"}), output_schema: json!({"type": "object"}), diff --git a/crates/moa-orchestrator/tests/execution_run_service_e2e/admission_replay.rs b/crates/moa-orchestrator/tests/execution_run_service_e2e/admission_replay.rs index e42adbb6f..7facb524a 100644 --- a/crates/moa-orchestrator/tests/execution_run_service_e2e/admission_replay.rs +++ b/crates/moa-orchestrator/tests/execution_run_service_e2e/admission_replay.rs @@ -358,7 +358,7 @@ fn template_skill_source() -> String { expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { delay_seconds: 86_400, }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, }, input_schema: io_schema.clone(), output_schema: io_schema.clone(), diff --git a/crates/moa-orchestrator/tests/execution_run_service_e2e/bulk_and_recovery.rs b/crates/moa-orchestrator/tests/execution_run_service_e2e/bulk_and_recovery.rs index 5783eab6a..8582a114e 100644 --- a/crates/moa-orchestrator/tests/execution_run_service_e2e/bulk_and_recovery.rs +++ b/crates/moa-orchestrator/tests/execution_run_service_e2e/bulk_and_recovery.rs @@ -814,7 +814,7 @@ fn bulk_candidate( expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { delay_seconds: 86_400, }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, }, input_schema: json!({"type": "object", "additionalProperties": false}), output_schema: report_schema.clone(), diff --git a/crates/moa-orchestrator/tests/execution_run_service_e2e/compensation_recovery.rs b/crates/moa-orchestrator/tests/execution_run_service_e2e/compensation_recovery.rs index 180c1d13c..d5fd401b4 100644 --- a/crates/moa-orchestrator/tests/execution_run_service_e2e/compensation_recovery.rs +++ b/crates/moa-orchestrator/tests/execution_run_service_e2e/compensation_recovery.rs @@ -631,7 +631,7 @@ fn compensated_plan( expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { delay_seconds: 60, }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, }, input_schema: json!({ "type": "object", diff --git a/crates/moa-orchestrator/tests/execution_run_service_e2e/observability.rs b/crates/moa-orchestrator/tests/execution_run_service_e2e/observability.rs index 2d39d0590..38465849e 100644 --- a/crates/moa-orchestrator/tests/execution_run_service_e2e/observability.rs +++ b/crates/moa-orchestrator/tests/execution_run_service_e2e/observability.rs @@ -96,7 +96,7 @@ async fn execution_observability_exports_stable_identity_and_replay_safe_service expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { delay_seconds: 86_400, }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, }, input_schema: json!({"type": "object", "additionalProperties": false}), output_schema: json!({ diff --git a/crates/moa-orchestrator/tests/execution_run_service_e2e/replan_and_completion.rs b/crates/moa-orchestrator/tests/execution_run_service_e2e/replan_and_completion.rs index 53615a934..0d66c2d8c 100644 --- a/crates/moa-orchestrator/tests/execution_run_service_e2e/replan_and_completion.rs +++ b/crates/moa-orchestrator/tests/execution_run_service_e2e/replan_and_completion.rs @@ -1099,8 +1099,21 @@ async fn completion_gate_missing_citation_service_e2e() -> Result<()> { #[tokio::test] #[ignore = "requires the local Restate/Postgres/OpenFGA/Redis service fixture"] async fn completion_gate_missing_deliverable_service_e2e() -> Result<()> { - // Pins: useful terminal output cannot hide a skipped required node or missing report pointer. - let fixture = replan_fixture(default_script(), FixtureCapabilityOptions::default()).await?; + // Pins: every declared completion check passing is not enough to complete a run. The + // required node runs and passes, the terminal output validates against both declared + // schemas, and the run must still end Partial because the goal's `report` deliverable + // has no value at its declared output pointer. + let fixture = replan_fixture( + replan_script_with_text_agents( + &[], + &[( + "BUILD_PARTIAL_REPORT", + serde_json::to_string(&json!({"body": "drafted"}))?, + )], + )?, + FixtureCapabilityOptions::default(), + ) + .await?; let test = fixture.isolated().await; let started = start_compiled_run( &fixture, @@ -1113,16 +1126,11 @@ async fn completion_gate_missing_deliverable_service_e2e() -> Result<()> { .await?; let terminal = await_execution_terminal(test.client(), &started.run).await?; - assert_completion_partial(&terminal, 1, 2, 1); + assert_completion_partial(&terminal, 2, 2, 2); assert_eq!( terminal.output, Some(json!({"summary": "useful but incomplete"})) ); - assert!( - terminal - .gaps - .contains(&"completion check deliverable_report failed".to_string()) - ); assert!( terminal .gaps @@ -1130,11 +1138,12 @@ async fn completion_gate_missing_deliverable_service_e2e() -> Result<()> { ); let evidence = synthesis_evidence(test.client(), &started).await?; let check = completion_result(&evidence, "deliverable_report")?; - assert!(!check.passed); - assert_eq!( - check.evidence, - json!({"incomplete_node_ids": ["report_builder"]}) + assert!( + check.passed, + "the required node completed, so its check must pass and the deliverable must be \ + the only thing standing between this run and completion" ); + assert_eq!(check.evidence, json!({"incomplete_node_ids": []})); assert_execution_eval_case( &fixture, test.client(), @@ -1146,9 +1155,6 @@ async fn completion_gate_missing_deliverable_service_e2e() -> Result<()> { ExecutionInvariantSpec::TerminalStatusIn { statuses: vec![ExecutionRunStatus::Partial], }, - ExecutionInvariantSpec::CompletionCheckFailed { - check_id: "deliverable_report".to_string(), - }, ExecutionInvariantSpec::TerminalGapContains { text: "deliverable report is missing".to_string(), }, @@ -1163,90 +1169,84 @@ async fn completion_gate_missing_deliverable_service_e2e() -> Result<()> { #[tokio::test] #[ignore = "requires the local Restate/Postgres/OpenFGA/Redis service fixture"] -async fn execution_eval_declared_contradiction_check_prevents_completion_service_e2e() -> Result<()> -{ - // Pins: contradiction is enforced only because this goal declares a named conflict verifier; - // two useful but opposing source outputs cannot be reported as complete when it is skipped. - let fixture = replan_fixture( - replan_script_with_text_agents( - &[], - &[ - ( - "CONTRADICTION_SOURCE_A", - serde_json::to_string(&json!({"position": "raise"}))?, - ), - ( - "CONTRADICTION_SOURCE_B", - serde_json::to_string(&json!({"position": "cut"}))?, - ), - ], - )?, - FixtureCapabilityOptions::default(), - ) - .await?; +async fn declared_contradiction_contract_is_rejected_against_the_service_catalog_service_e2e() +-> Result<()> { + // Pins: this goal declares `conflict_verifier` as a required node while the plan makes it + // conditional, so a false condition would skip the very node completion requires. That is + // now refused at compile time, against the catalog, authorization envelope, and budget the + // running service actually issues — not merely against a hand-built offline fixture. + // + // This scenario used to assert the runtime behavior instead: the verifier was skipped, the + // check counted it as failed, and the run ended Partial while both source outputs survived. + // That end state is unreachable without a skip. Every non-output node must be an ancestor + // of the output node (`validate_terminal_output`), and any node that ends neither + // `completed` nor `skipped` cancels its descendants, so a plan cannot both leave a + // required node unpassed and still emit terminal output. Rejecting the contract at compile + // time removes the state rather than reporting it late, which is the stronger contract. + let fixture = replan_fixture(default_script(), FixtureCapabilityOptions::default()).await?; let test = fixture.isolated().await; - let started = start_compiled_run( - &fixture, - &test, - "declared-contradiction-check", - "compare two sources and explicitly resolve any contradiction", - None, - |_| Ok(declared_contradiction_contract()), - ) - .await?; + let objective = "compare two sources and explicitly resolve any contradiction"; + let session_id = test.create_session("declared-contradiction-check").await?; + let session = test.client().get_session(session_id).await?; + let originating_user_sequence_num = test + .client() + .append_event( + session_id, + Event::UserMessage { + text: objective.to_string(), + attachments: Vec::new(), + }, + ) + .await?; + let planning: ExecutionPlanningContextResponse = test + .client() + .post_call( + "/Execution/planning_context", + &ExecutionPlanningContextRequest { + tenant_id: session.tenant_id, + contact_id: None, + session_id, + originating_user_sequence_num, + deadline_at: chrono::Utc::now() + chrono::TimeDelta::days(1), + requested_template: None, + }, + ) + .await?; + + let (mut goal, plan) = declared_contradiction_contract(); + goal.objective = objective.to_string(); + let outcome = compile(CompileExecutionRequest { + goal, + plan, + run_input: run_input_for_objective(objective), + catalog: planning.snapshot.catalog.clone(), + authorization: planning.snapshot.authorization.clone(), + approved_budget: planning.snapshot.budget.clone(), + config: ExecutionConfig::default(), + now: moa_test_support::fixtures::pg_now(), + }); - let terminal = await_execution_terminal(test.client(), &started.run).await?; - assert_completion_partial(&terminal, 3, 4, 3); - assert_eq!( - terminal.output, - Some(json!({"summary": "sources disagree; conflict remains unresolved"})) - ); assert!( - terminal - .gaps - .contains(&"completion check declared_conflict_check failed".to_string()) - ); - let tasks = list_execution_tasks(test.client(), started.run.clone()).await?; - assert_eq!( - task_by_node(&tasks.tasks, "source_a")?.status, - ExecutionTaskStatus::Completed + outcome.compiled.is_none(), + "a plan whose required node can be skipped must not compile" ); - assert_eq!( - task_by_node(&tasks.tasks, "source_b")?.status, - ExecutionTaskStatus::Completed - ); - assert_eq!( - tasks - .tasks - .iter() - .filter(|task| task.node_id == "conflict_verifier") - .count(), - 0, - "a false condition must not materialize the declared verifier" + let issue = outcome + .report + .issues + .iter() + .find(|issue| issue.code == "conditional_required_node") + .with_context(|| { + format!( + "expected a conditional_required_node rejection, got {:?}", + outcome.report.issues + ) + })?; + assert_eq!(issue.path, "plan.nodes[2].when"); + assert!( + issue.message.contains("declared_conflict_check"), + "the rejection must name the completion check that made the node required: {}", + issue.message ); - assert_execution_eval_case( - &fixture, - test.client(), - &started.run, - None, - "declared-contradiction-check-prevents-completion", - &[ - ExecutionInvariantSpec::MustNotComplete, - ExecutionInvariantSpec::TerminalStatusIn { - statuses: vec![ExecutionRunStatus::Partial], - }, - ExecutionInvariantSpec::CompletionCheckFailed { - check_id: "declared_conflict_check".to_string(), - }, - ExecutionInvariantSpec::TerminalGapContains { - text: "completion check declared_conflict_check failed".to_string(), - }, - ExecutionInvariantSpec::BudgetWithinApproved, - ExecutionInvariantSpec::ProgressMatchesTasks, - ExecutionInvariantSpec::NoRawTaskOutputEvents, - ], - ) - .await?; Ok(()) } @@ -1727,12 +1727,8 @@ where }) } -fn run_input_for_objective(objective: &str) -> Value { - if objective == "produce the required report deliverable" { - json!({"build_report": false}) - } else { - json!({}) - } +fn run_input_for_objective(_objective: &str) -> Value { + json!({}) } async fn await_waiting_replan( @@ -2048,7 +2044,7 @@ fn useful_replan_contract( expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { delay_seconds: 86_400, }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, }, input_schema: empty_input_schema(), output_schema: output_schema.clone(), @@ -2306,7 +2302,7 @@ fn map_then_output_plan(spec: MapThenOutputPlan<'_>) -> ExecutionPlanDefinition expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { delay_seconds: 86_400, }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, }, input_schema: empty_input_schema(), output_schema: output_schema.clone(), @@ -2375,36 +2371,31 @@ fn missing_deliverable_contract() -> (ExecutionGoalContract, ExecutionPlanDefini output_schema_check("summary_schema", "summary"), ], }, + // The deliverable is unsatisfiable by construction rather than by accident: the + // plan's terminal `output_schema` is `report_schema()`, which forbids additional + // properties, so no terminal output this plan can legally emit carries a value at + // the deliverable's `/report` pointer. Every node completes, every check passes, + // and the deliverable is the sole reason the run must not report completion. ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { delay_seconds: 86_400, }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, }, - input_schema: json!({ - "type": "object", - "additionalProperties": false, - "required": ["build_report"], - "properties": {"build_report": {"type": "boolean"}} - }), + input_schema: empty_input_schema(), output_schema: output_schema.clone(), nodes: vec![ ExecutionNode { id: "report_builder".to_string(), requirement_ids: vec!["report_body".to_string()], depends_on: Vec::new(), - when: Some(moa_artifacts::execution_plan::ExecutionCondition::Equals { - reference: ExecutionReference { - path: "$.input.build_report".to_string(), - }, - value: json!(true), - }), + when: None, input: json!({}), output_schema: json!({"type": "object"}), operation: ExecutionOperation::Agent { - instructions: "BUILD_REPORT_ONLY_WHEN_ENABLED".to_string(), + instructions: "BUILD_PARTIAL_REPORT".to_string(), skill_refs: Vec::new(), capability_refs: Vec::new(), max_turns: 1, @@ -2489,7 +2480,7 @@ fn declared_contradiction_contract() -> (ExecutionGoalContract, ExecutionPlanDef expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { delay_seconds: 86_400, }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, }, input_schema: empty_input_schema(), output_schema: report_schema.clone(), @@ -2598,7 +2589,7 @@ fn injected_content_contract( expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { delay_seconds: 86_400, }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, }, input_schema: empty_input_schema(), output_schema: output_schema.clone(), diff --git a/crates/moa-orchestrator/tests/execution_run_service_e2e/routing.rs b/crates/moa-orchestrator/tests/execution_run_service_e2e/routing.rs index d955a8bf7..cf9d168bd 100644 --- a/crates/moa-orchestrator/tests/execution_run_service_e2e/routing.rs +++ b/crates/moa-orchestrator/tests/execution_run_service_e2e/routing.rs @@ -1303,7 +1303,7 @@ fn research_candidate( expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { delay_seconds: 86_400, }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, }, input_schema: empty_input_schema(), output_schema: output_schema.clone(), @@ -1410,7 +1410,7 @@ fn template_skill_source() -> String { expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { delay_seconds: 86_400, }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, }, input_schema: template_io_schema(), output_schema: template_io_schema(), diff --git a/crates/moa-orchestrator/tests/execution_run_service_e2e/task_lifecycle.rs b/crates/moa-orchestrator/tests/execution_run_service_e2e/task_lifecycle.rs index 1a6f32684..5253f3294 100644 --- a/crates/moa-orchestrator/tests/execution_run_service_e2e/task_lifecycle.rs +++ b/crates/moa-orchestrator/tests/execution_run_service_e2e/task_lifecycle.rs @@ -1992,7 +1992,7 @@ fn recompile_as_agent( expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { delay_seconds: 86_400, }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, }, input_schema: json!({"type": "object", "additionalProperties": false}), output_schema: output_schema.clone(), @@ -2063,7 +2063,7 @@ fn recompile_as_external_wait( expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { delay_seconds: 86_400, }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, }, }, ExternalWaitKind::Signal => ExecutionOperation::WaitSignal { @@ -2072,7 +2072,7 @@ fn recompile_as_external_wait( expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { delay_seconds: 86_400, }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, }, }, }; @@ -2082,7 +2082,7 @@ fn recompile_as_external_wait( expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { delay_seconds: 86_400, }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, }, input_schema: json!({"type": "object", "additionalProperties": false}), output_schema: output_schema.clone(), @@ -2304,7 +2304,7 @@ fn lifecycle_plan_with_output_schema( expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { delay_seconds: 86_400, }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, }, input_schema: json!({"type": "object", "additionalProperties": false}), output_schema: output_schema.clone(), diff --git a/crates/moa-orchestrator/tests/execution_run_service_e2e/terminal_matrix.rs b/crates/moa-orchestrator/tests/execution_run_service_e2e/terminal_matrix.rs index 6a64a7979..9201d2f63 100644 --- a/crates/moa-orchestrator/tests/execution_run_service_e2e/terminal_matrix.rs +++ b/crates/moa-orchestrator/tests/execution_run_service_e2e/terminal_matrix.rs @@ -895,7 +895,7 @@ fn output_candidate( expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { delay_seconds: 86_400, }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, }, input_schema: empty_object_schema(), output_schema: schema.clone(), @@ -930,7 +930,7 @@ fn replan_candidate(objective: &str) -> GeneratedExecutionCandidate { expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { delay_seconds: 86_400, }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, }, input_schema: empty_object_schema(), output_schema: schema.clone(), diff --git a/crates/moa-orchestrator/tests/integration/action_policy_flow_e2e.rs b/crates/moa-orchestrator/tests/integration/action_policy_flow_e2e.rs index 316599874..6df4c0f82 100644 --- a/crates/moa-orchestrator/tests/integration/action_policy_flow_e2e.rs +++ b/crates/moa-orchestrator/tests/integration/action_policy_flow_e2e.rs @@ -1050,7 +1050,7 @@ async fn insert_execution_review_task( expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::At { at: chrono::Utc::now() + chrono::TimeDelta::hours(1), }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, }, input_schema: json!({ "type": "object" }), output_schema: json!({ "type": "object" }), diff --git a/crates/moa-orchestrator/tests/long_horizon_execution_canary_live.rs b/crates/moa-orchestrator/tests/long_horizon_execution_canary_live.rs index c45f16b76..3d506f4ba 100644 --- a/crates/moa-orchestrator/tests/long_horizon_execution_canary_live.rs +++ b/crates/moa-orchestrator/tests/long_horizon_execution_canary_live.rs @@ -18,6 +18,9 @@ async fn deployed_long_horizon_invariants_hold_for_24_hours_live() -> Result<()> // Pins: an explicitly selected external deployment is sampled for a full // 24 hours; an instantaneous healthy sample cannot satisfy this canary. if !canary_selected("24h")? { + eprintln!( + "SKIPPED long-horizon canary 24h: MOA_LONG_HORIZON_CANARY_WINDOW selects the other window" + ); return Ok(()); } run_canary(Duration::from_secs(24 * 60 * 60)).await @@ -29,6 +32,9 @@ async fn deployed_long_horizon_invariants_hold_for_seven_days_live() -> Result<( // Pins: the seven-day deployment soak continuously rejects overdue runs, // parked compute ownership, and still-live attempt invocations. if !canary_selected("7d")? { + eprintln!( + "SKIPPED long-horizon canary 7d: MOA_LONG_HORIZON_CANARY_WINDOW selects the other window" + ); return Ok(()); } run_canary(Duration::from_secs(7 * 24 * 60 * 60)).await @@ -36,7 +42,14 @@ async fn deployed_long_horizon_invariants_hold_for_seven_days_live() -> Result<( fn canary_selected(expected: &str) -> Result { if std::env::var("MOA_RUN_LONG_HORIZON_CANARY").as_deref() != Ok("1") { - return Ok(false); + // Both cases are `#[ignore]`d, so reaching this point means the binary was + // explicitly selected with `--run-ignored`. Returning `Ok(false)` here used to + // report a green 24h/7d soak that sampled nothing, making an unauthorized sweep + // indistinguishable from a real deployment canary in CI logs. + bail!( + "long-horizon canary was explicitly selected without MOA_RUN_LONG_HORIZON_CANARY=1; \ + refusing to report a passing soak that sampled nothing" + ); } let selected = std::env::var("MOA_LONG_HORIZON_CANARY_WINDOW").context( "MOA_RUN_LONG_HORIZON_CANARY=1 requires MOA_LONG_HORIZON_CANARY_WINDOW=24h or 7d", diff --git a/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e.rs b/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e.rs index 5a3a71bb1..04ed060d2 100644 --- a/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e.rs +++ b/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e.rs @@ -226,7 +226,7 @@ fn fixture_input_wait_policy( expiry: ExecutionTemporalTarget::After { delay_seconds: remaining_seconds / 2, }, - on_expiry: ExecutionWaitExpiryAction::FailRun, + on_expiry: ExecutionWaitExpiryAction::FailTask, }) } diff --git a/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/burst_admission.rs b/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/burst_admission.rs index d5e4c532f..a1c2067cb 100644 --- a/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/burst_admission.rs +++ b/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/burst_admission.rs @@ -15,7 +15,22 @@ async fn one_thousand_common_wakes_bound_capacity_invocations_and_oldest_ready_a const RUN_COUNT: usize = 1_000; const FLEET_CAP: usize = 32; const ADMISSION_CONCURRENCY: usize = 8; - const BURST_TIMEOUT: Duration = Duration::from_secs(360); + // Seconds between admission and the shared absolute wake, and the margin by + // which every run must already be parked before that wake arrives. + const PRE_WAKE_SECONDS: i64 = 180; + const PRE_WAKE_MARGIN_SECONDS: i64 = 30; + // Absolute execution deadline handed to each admitted run. It has to outlast + // the pre-wake window plus the whole fleet-capped drain, so it is + // deliberately far larger than any phase bound below: a run that expires + // here is a product deadline defect, not a slow observation. + const RUN_DEADLINE: Duration = Duration::from_secs(900); + // Bound on any single observation phase. Each phase reports its own + // diagnostic well inside the lane's per-case ceiling rather than being + // terminated by nextest with no attribution. + const PHASE_TIMEOUT: Duration = Duration::from_secs(90); + // Total bound on the ~32-wave drain. Bounding the whole drain instead of + // each wave keeps the worst case additive rather than multiplicative. + const DRAIN_BUDGET: Duration = Duration::from_secs(240); let tool_name = "long_horizon_thousand_wake_probe"; let fixture = execution_fixture_with_tools( vec![FixtureCapabilityTool { @@ -70,7 +85,7 @@ async fn one_thousand_common_wakes_bound_capacity_invocations_and_oldest_ready_a let capability_name = moa_hands::mcp_tool_reference("fixture-capability", tool_name); allow_fixture_capability(&fixture, tenant_id, &capability_name, "thousand-wake").await?; let test = fixture.isolated().await; - let common_wake = Utc::now() + TimeDelta::seconds(180); + let common_wake = Utc::now() + TimeDelta::seconds(PRE_WAKE_SECONDS); let runs = stream::iter(0..RUN_COUNT) .map(|index| { let test = &test; @@ -94,7 +109,7 @@ async fn one_thousand_common_wakes_bound_capacity_invocations_and_oldest_ready_a capability, output_node(&["burst-capability"], json!({"completed": true})), ], - BURST_TIMEOUT, + RUN_DEADLINE, false, ) .await @@ -107,18 +122,18 @@ async fn one_thousand_common_wakes_bound_capacity_invocations_and_oldest_ready_a .await?; assert_eq!(runs.len(), RUN_COUNT); let pool = PgPool::connect(&fixture.postgres_url).await?; - await_tenant_run_count(&pool, tenant_id, "waiting_timer", RUN_COUNT, BURST_TIMEOUT).await?; + await_tenant_run_count(&pool, tenant_id, "waiting_timer", RUN_COUNT, PHASE_TIMEOUT).await?; await_capacity_quantity_before( &pool, tenant_id, "parked_runs", RUN_COUNT, - common_wake - TimeDelta::seconds(30), + common_wake - TimeDelta::seconds(PRE_WAKE_MARGIN_SECONDS), ) .await?; let admission_margin = common_wake.signed_duration_since(Utc::now()); assert!( - admission_margin > TimeDelta::seconds(30), + admission_margin > TimeDelta::seconds(PRE_WAKE_MARGIN_SECONDS), "1,000 runs were not fully parked before the shared wake: {admission_margin:?}" ); fixture.otlp_capture()?.clear().await; @@ -126,7 +141,7 @@ async fn one_thousand_common_wakes_bound_capacity_invocations_and_oldest_ready_a let controller = fixture .fixture_capability() .context("thousand-wake fixture omitted capability controller")?; - controller.wait_for_calls(FLEET_CAP, BURST_TIMEOUT).await?; + controller.wait_for_calls(FLEET_CAP, PHASE_TIMEOUT).await?; tokio::time::sleep(Duration::from_millis(500)).await; assert_eq!(controller.calls().len(), FLEET_CAP); let active: i64 = sqlx::query_scalar( @@ -146,7 +161,7 @@ async fn one_thousand_common_wakes_bound_capacity_invocations_and_oldest_ready_a assert!(invocations.len() <= FLEET_CAP); let dispatch_metric = fixture .otlp_capture()? - .wait_for_metric(BURST_TIMEOUT, |metric| { + .wait_for_metric(PHASE_TIMEOUT, |metric| { metric.name() == "moa_execution_dispatch_batch_size" && metric.data_points().iter().any(|point| { point.count() > 0 @@ -163,7 +178,7 @@ async fn one_thousand_common_wakes_bound_capacity_invocations_and_oldest_ready_a })); let oldest_ready_metric = fixture .otlp_capture()? - .wait_for_metric(BURST_TIMEOUT, |metric| { + .wait_for_metric(PHASE_TIMEOUT, |metric| { metric.name() == "moa_execution_oldest_ready_age_seconds" && metric .data_points() @@ -181,9 +196,16 @@ async fn one_thousand_common_wakes_bound_capacity_invocations_and_oldest_ready_a let mut released = 0; let mut maximum_oldest_ready_seconds = 0.0_f64; + let drain_deadline = Instant::now() + DRAIN_BUDGET; while released < RUN_COUNT { let next = (released + FLEET_CAP).min(RUN_COUNT); - controller.wait_for_calls(next, BURST_TIMEOUT).await?; + let remaining = drain_deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + bail!( + "fleet-capped drain released {released}/{RUN_COUNT} runs within {DRAIN_BUDGET:?}" + ); + } + controller.wait_for_calls(next, remaining).await?; let wave_active: i64 = sqlx::query_scalar( "SELECT COALESCE(SUM(quantity), 0)::BIGINT FROM moa.execution_capacity_reservation \ WHERE tenant_id = $1 AND resource_dimension = 'active_tasks' AND state <> 'released'", @@ -207,7 +229,7 @@ async fn one_thousand_common_wakes_bound_capacity_invocations_and_oldest_ready_a maximum_oldest_ready_seconds <= 60.0, "oldest ready task exceeded bounded age: {maximum_oldest_ready_seconds}s" ); - await_tenant_run_count(&pool, tenant_id, "completed", RUN_COUNT, BURST_TIMEOUT).await?; + await_tenant_run_count(&pool, tenant_id, "completed", RUN_COUNT, PHASE_TIMEOUT).await?; let first = status(&test, &runs[0]).await?; let last = status(&test, &runs[RUN_COUNT - 1]).await?; assert_eq!(first.output, Some(json!({"completed": true}))); diff --git a/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/deadline_and_waits.rs b/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/deadline_and_waits.rs index 6f1d7d753..1fc06d8fd 100644 --- a/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/deadline_and_waits.rs +++ b/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/deadline_and_waits.rs @@ -130,7 +130,7 @@ async fn exact_wait_expiry_fails_once_and_late_delivery_cannot_revive_run_servic signal_name: "never-arrives".to_string(), wait_policy: ExecutionWaitPolicy { expiry: after_logical_days(2), - on_expiry: ExecutionWaitExpiryAction::FailRun, + on_expiry: ExecutionWaitExpiryAction::FailTask, }, }, json!({"type": "object"}), diff --git a/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/pause_and_external.rs b/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/pause_and_external.rs index b4c48ca91..9f9f39a00 100644 --- a/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/pause_and_external.rs +++ b/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/pause_and_external.rs @@ -948,7 +948,7 @@ async fn sandbox_hand_releases_during_signal_wait_and_reacquires_after_resume_se signal_name: "resume-sandbox".to_string(), wait_policy: ExecutionWaitPolicy { expiry: after_logical_days(7), - on_expiry: ExecutionWaitExpiryAction::FailRun, + on_expiry: ExecutionWaitExpiryAction::FailTask, }, }, json!({"type": "object"}), @@ -1583,7 +1583,7 @@ async fn parked_signal_rejects_wrong_generation_and_replays_duplicate_after_valk signal_name: "provider-complete".to_string(), wait_policy: ExecutionWaitPolicy { expiry: after_logical_days(5), - on_expiry: ExecutionWaitExpiryAction::FailRun, + on_expiry: ExecutionWaitExpiryAction::FailTask, }, }, json!({"type": "object"}), diff --git a/crates/moa-orchestrator/tests/orchestrator_db/action_reviews_reaper_db.rs b/crates/moa-orchestrator/tests/orchestrator_db/action_reviews_reaper_db.rs index c0dfe35e5..91f8afda2 100644 --- a/crates/moa-orchestrator/tests/orchestrator_db/action_reviews_reaper_db.rs +++ b/crates/moa-orchestrator/tests/orchestrator_db/action_reviews_reaper_db.rs @@ -910,7 +910,7 @@ async fn insert_execution_task(pool: &PgPool, tenant_id: TenantId) -> ExecutionT expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::At { at: chrono::Utc::now() + chrono::TimeDelta::hours(1), }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, }, input_schema: serde_json::json!({ "type": "object" }), output_schema: serde_json::json!({ "type": "object" }), diff --git a/crates/moa-orchestrator/tests/orchestrator_db/analytics_export_db.rs b/crates/moa-orchestrator/tests/orchestrator_db/analytics_export_db.rs index b9bd5ea08..b6adcc042 100644 --- a/crates/moa-orchestrator/tests/orchestrator_db/analytics_export_db.rs +++ b/crates/moa-orchestrator/tests/orchestrator_db/analytics_export_db.rs @@ -230,7 +230,7 @@ async fn seed_execution_analytics_fixture( expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::At { at: chrono::Utc::now() + chrono::TimeDelta::hours(1), }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, }, input_schema: json!({ "type": "object" }), output_schema: json!({ "type": "object" }), diff --git a/crates/moa-orchestrator/tests/orchestrator_db/execution_service_db.rs b/crates/moa-orchestrator/tests/orchestrator_db/execution_service_db.rs index 0629f1deb..5f9b84468 100644 --- a/crates/moa-orchestrator/tests/orchestrator_db/execution_service_db.rs +++ b/crates/moa-orchestrator/tests/orchestrator_db/execution_service_db.rs @@ -228,7 +228,7 @@ async fn execution_task_citation_lineage_survives_reload_and_terminal_summary_db expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::At { at: chrono::Utc::now() + chrono::TimeDelta::hours(1), }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, }, input_schema: json!({"type": "object"}), output_schema: json!({"type": "object"}), @@ -520,7 +520,7 @@ async fn execution_service_rows_require_parent_session_and_keep_authorization_im expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::At { at: chrono::Utc::now() + chrono::TimeDelta::hours(1), }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailRun, + on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, }, input_schema: json!({ "type": "object" }), output_schema: json!({ "type": "object" }), diff --git a/crates/xtask/src/check_architecture_boundaries/budgets.rs b/crates/xtask/src/check_architecture_boundaries/budgets.rs index 551c3d7a1..27e6d6b6d 100644 --- a/crates/xtask/src/check_architecture_boundaries/budgets.rs +++ b/crates/xtask/src/check_architecture_boundaries/budgets.rs @@ -152,11 +152,11 @@ const LOC_BUDGETS: &[LocBudget] = &[ reason: "compiler validation is the largest focused compiler module after estimate, amendment, and test extraction", }, LocBudget { - label: "execution interpreter terminal decisions", - path: "crates/moa-execution/src/interpreter/terminal.rs", + label: "execution interpreter capability catalog validation", + path: "crates/moa-execution/src/interpreter/catalog.rs", scope: LocScope::File, max_lines: 450, - reason: "terminal interpretation stays separate from projection, reservation, materialization, and aggregation", + reason: "catalog validation stays separate from materialization and reservation; the whole-plan scheduler that owned terminal/aggregate/projection interpretation was deleted with the bounded-activation rewrite", }, LocBudget { label: "artifact validation shell", diff --git a/crates/xtask/src/execution_trace_manifest.rs b/crates/xtask/src/execution_trace_manifest.rs index cef1fbb58..37f230837 100644 --- a/crates/xtask/src/execution_trace_manifest.rs +++ b/crates/xtask/src/execution_trace_manifest.rs @@ -1070,6 +1070,13 @@ const SENDERS: &[SenderManifestEntry] = &[ "ToolExecutorClient", "checkpoint_and_release_execution_hands" ), + sender!( + "crates/moa-orchestrator/src/workflows/execution_task_attempt/yielding.rs", + "continue_task_hands_workflow", + TRACE_HELPER, + "ToolExecutorClient", + "checkpoint_execution_hands_retaining_compute" + ), sender!( "crates/moa-orchestrator/src/workflows/execution_task_attempt/yielding.rs", "park_review", diff --git a/docs/01-architecture-overview.md b/docs/01-architecture-overview.md index e330f9dd1..8d1a16564 100644 --- a/docs/01-architecture-overview.md +++ b/docs/01-architecture-overview.md @@ -173,7 +173,7 @@ its operation enum has exactly eight variants: 8. `Output { value }` resolves and validates the terminal output. Every wait expiry uses `ExecutionWaitPolicy { expiry, on_expiry }`, where -`on_expiry` is `FailTask`, `FailRun`, or `ContinueWith { output }`. +`on_expiry` is `FailTask`, `FailTask`, or `ContinueWith { output }`. `ExecutionTemporalTarget::At { at }` is an exact UTC instant and is allowed in one-off compiled plans. `After { delay_seconds }` is nonzero and resolves from the instant the task actually enters its wait, not from planning or run diff --git a/docs/12-restate-architecture.md b/docs/12-restate-architecture.md index ecb96e94f..97fd87201 100644 --- a/docs/12-restate-architecture.md +++ b/docs/12-restate-architecture.md @@ -391,7 +391,7 @@ one-off plan. Nonzero `After { delay_seconds }` is resolved from the instant the task enters the wait. Reusable templates reject `At` and use `After`, so earlier dependency duration cannot make a template timer stale. Entering any wait persists `due_at`, releases attempt and hand capacity, and schedules an immutable -trigger; expiry follows `FailTask`, `FailRun`, or `ContinueWith { output }`. +trigger; expiry follows `FailTask`, `FailTask`, or `ContinueWith { output }`. Before dispatch, the repository atomically reserves worst-case microusd, tokens, tasks, tool calls, retrieved bytes, deadline allowance, and tenant/fleet diff --git a/docs/17-observability.md b/docs/17-observability.md index 61b5fd221..87c1bb78a 100644 --- a/docs/17-observability.md +++ b/docs/17-observability.md @@ -102,13 +102,19 @@ bounded Restate activation state, compact session events, and trace attributes. Pending and every waiting phase are storage-only; only admitted attempts may own active capacity or hands. -Fleet health uses bounded labels only. `moa_execution_runs{phase}` separates -active, input/review/signal/timer/external, pause, and compensation phases. -Oldest-ready age, overdue deadlines, trigger/outbox lag and dead letters, -oldest active-attempt/external-job age, admission utilization, tenant maximum -share, durable reconciliation and retention last-success ages, parked tasks retaining -hands, and old Restate deployment age/replica-hours carry no tenant, run, task, -deployment, or provider account identifier. IDs belong in traces and Postgres drilldown. +Fleet health uses bounded labels only, and every exported series backs an alert: +oldest-ready age, overdue deadlines, trigger/outbox lag and dead letters, oldest +active-attempt age, admission utilization by resource and fleet/tenant-peak scope, +durable reconciliation and retention last-success ages, and parked tasks retaining +hands. None carry a tenant, run, task, deployment, or provider account identifier. +IDs belong in traces and Postgres drilldown. + +A metric that no alert consumes is not exported. Per-phase run census, tenant +maximum share, oldest external-job age, and old Restate deployment age/replica-hours +were removed rather than left as series nothing reads. +`k8s/scripts/validate-observability.sh` enforces the invariant directly: every +`pub fn record_*` in `runtime_metrics.rs` must have a caller outside that file and +outside `tests/`, so a recorder can never again be declared without being wired. Reconciliation and retention expose separate durable health receipts. Trigger/outbox repair drives `moa_execution_maintenance_*`; terminal-evidence retention drives diff --git a/docs/examples/artifacts/damaged-food-order.skill.yaml b/docs/examples/artifacts/damaged-food-order.skill.yaml index 44ed3746f..75cbc71b4 100644 --- a/docs/examples/artifacts/damaged-food-order.skill.yaml +++ b/docs/examples/artifacts/damaged-food-order.skill.yaml @@ -41,7 +41,7 @@ definition: kind: after delay_seconds: 86400 on_expiry: - kind: fail_run + kind: fail_task input_schema: type: object properties: @@ -62,6 +62,10 @@ definition: $ref: $.input.evidence_summary output_schema: type: object + required: [evidence_sufficient] + properties: + evidence_sufficient: + type: boolean operation: kind: capability reference: @@ -72,6 +76,16 @@ definition: max_attempts: 2 initial_backoff_ms: 100 max_backoff_ms: 1000 + # The two nodes below are mutually exclusive branches on the same verified + # value. Exactly one of them runs; the other is committed as `skipped`, which + # releases its dependents for ordering without producing an output. + # + # A conditional node is an effectful leaf: nothing may read its output, because + # a skipped branch has none. That is why the terminal output below reports the + # verified order rather than whichever branch happened to fire, and why + # `req_damaged_order` is also served by two unconditional nodes — a requirement + # served only by conditional nodes could end a run with nothing eligible to + # satisfy it. - id: reorder_confirmation requirement_ids: [req_damaged_order] depends_on: [verify_order] @@ -122,17 +136,15 @@ definition: max_backoff_ms: 1000 - id: output requirement_ids: [req_damaged_order] - depends_on: [reorder_confirmation, notify_restaurant] + depends_on: [verify_order, reorder_confirmation, notify_restaurant] input: {} output_schema: type: object operation: kind: output value: - reorder_confirmation: - $ref: $.nodes.reorder_confirmation.output - restaurant_notification: - $ref: $.nodes.notify_restaurant.output + verified_order: + $ref: $.nodes.verify_order.output compensation: null retry: max_attempts: 1 diff --git a/docs/examples/artifacts/patterns/custom-logic.skill.yaml b/docs/examples/artifacts/patterns/custom-logic.skill.yaml index 069a60378..83d3fe916 100644 --- a/docs/examples/artifacts/patterns/custom-logic.skill.yaml +++ b/docs/examples/artifacts/patterns/custom-logic.skill.yaml @@ -41,9 +41,10 @@ definition: kind: after delay_seconds: 86400 on_expiry: - kind: fail_run + kind: fail_task input_schema: type: object + required: [priority] properties: priority: type: string @@ -52,6 +53,10 @@ definition: output_schema: type: object nodes: + # Explicit routing: each branch declares the exact value it runs for, so a run + # takes one of them and the other is committed as `skipped`. The branches are + # effectful leaves — the terminal output reports the route that was chosen, not + # a branch's output, because a skipped branch produces none. - id: escalate requirement_ids: [req_route] depends_on: [] @@ -111,10 +116,8 @@ definition: operation: kind: output value: - escalation: - $ref: $.nodes.escalate.output - standard_response: - $ref: $.nodes.standard_response.output + routed_priority: + $ref: $.input.priority compensation: null retry: max_attempts: 1 diff --git a/docs/examples/artifacts/patterns/human-approval.skill.yaml b/docs/examples/artifacts/patterns/human-approval.skill.yaml index 6b0d58645..b57501e38 100644 --- a/docs/examples/artifacts/patterns/human-approval.skill.yaml +++ b/docs/examples/artifacts/patterns/human-approval.skill.yaml @@ -34,7 +34,7 @@ definition: kind: after delay_seconds: 86400 on_expiry: - kind: fail_run + kind: fail_task input_schema: type: object output_schema: @@ -72,7 +72,7 @@ definition: kind: after delay_seconds: 86400 on_expiry: - kind: fail_run + kind: fail_task compensation: null retry: max_attempts: 1 diff --git a/docs/examples/artifacts/patterns/parallel-review.skill.yaml b/docs/examples/artifacts/patterns/parallel-review.skill.yaml index 92f1d2c69..729f136c0 100644 --- a/docs/examples/artifacts/patterns/parallel-review.skill.yaml +++ b/docs/examples/artifacts/patterns/parallel-review.skill.yaml @@ -35,7 +35,7 @@ definition: kind: after delay_seconds: 86400 on_expiry: - kind: fail_run + kind: fail_task input_schema: type: object output_schema: @@ -93,7 +93,7 @@ definition: kind: after delay_seconds: 86400 on_expiry: - kind: fail_run + kind: fail_task compensation: null retry: max_attempts: 1 diff --git a/docs/examples/artifacts/patterns/react-agent.skill.yaml b/docs/examples/artifacts/patterns/react-agent.skill.yaml index f25d724c2..a6cd35bef 100644 --- a/docs/examples/artifacts/patterns/react-agent.skill.yaml +++ b/docs/examples/artifacts/patterns/react-agent.skill.yaml @@ -35,7 +35,7 @@ definition: kind: after delay_seconds: 86400 on_expiry: - kind: fail_run + kind: fail_task input_schema: type: object output_schema: diff --git a/docs/examples/artifacts/patterns/sequential.skill.yaml b/docs/examples/artifacts/patterns/sequential.skill.yaml index e1e712be3..651e73b45 100644 --- a/docs/examples/artifacts/patterns/sequential.skill.yaml +++ b/docs/examples/artifacts/patterns/sequential.skill.yaml @@ -39,7 +39,7 @@ definition: kind: after delay_seconds: 86400 on_expiry: - kind: fail_run + kind: fail_task input_schema: type: object properties: diff --git a/docs/schemas/moa-skill-v1.schema.json b/docs/schemas/moa-skill-v1.schema.json index a3b4648b3..5a9676196 100644 --- a/docs/schemas/moa-skill-v1.schema.json +++ b/docs/schemas/moa-skill-v1.schema.json @@ -7,48 +7,97 @@ "instructions": { "type": "object", "properties": { - "path": { "type": "string", "default": "SKILL.md" } + "path": { + "type": "string", + "default": "SKILL.md" + } } }, - "inputs": { "type": "object" }, - "outputs": { "type": "object" }, + "inputs": { + "type": "object" + }, + "outputs": { + "type": "object" + }, "connectors": { "type": "array", - "items": { "type": "string", "pattern": "^connector://.+" } + "items": { + "type": "string", + "pattern": "^connector://.+" + } }, "allowed_tools": { "type": "array", - "items": { "type": "string" } + "items": { + "type": "string" + } }, "actions": { "type": "array", "items": { "type": "object", - "required": ["id", "kind"], + "required": [ + "id", + "kind" + ], "properties": { - "id": { "type": "string", "minLength": 1 }, - "description": { "type": "string" }, - "kind": { "enum": ["connector_action", "tool", "code"] }, - "ref": { "type": "string" }, - "runtime": { "type": "string" }, - "entrypoint": { "type": "string" }, - "input_schema": { "type": "object" }, - "output_schema": { "type": "object" }, - "ui": { "type": "object" } + "id": { + "type": "string", + "minLength": 1 + }, + "description": { + "type": "string" + }, + "kind": { + "enum": [ + "connector_action", + "tool", + "code" + ] + }, + "ref": { + "type": "string" + }, + "runtime": { + "type": "string" + }, + "entrypoint": { + "type": "string" + }, + "input_schema": { + "type": "object" + }, + "output_schema": { + "type": "object" + }, + "ui": { + "type": "object" + } } } }, - "execution_plan": { "$ref": "#/$defs/ExecutionPlanTemplate" }, - "ui": { "type": "object" } + "execution_plan": { + "$ref": "#/$defs/ExecutionPlanTemplate" + }, + "ui": { + "type": "object" + } }, "$defs": { "ExecutionPlanTemplate": { "type": "object", "additionalProperties": false, - "required": ["goal", "plan"], + "required": [ + "goal", + "plan" + ], "properties": { - "goal": { "$ref": "#/$defs/ExecutionGoalTemplate" }, - "plan": { "$ref": "#/$defs/ExecutionPlanDefinition" } + "goal": { + "$ref": "#/$defs/ExecutionGoalTemplate" + }, + "plan": { + "$ref": "#/$defs/ExecutionPlanDefinition" + } } }, "ExecutionGoalTemplate": { @@ -64,43 +113,71 @@ "properties": { "requirements": { "type": "array", - "items": { "$ref": "#/$defs/ExecutionRequirement" } + "items": { + "$ref": "#/$defs/ExecutionRequirement" + } }, "deliverables": { "type": "array", - "items": { "$ref": "#/$defs/ExecutionDeliverable" } + "items": { + "$ref": "#/$defs/ExecutionDeliverable" + } }, "coverage": { "type": "array", - "items": { "$ref": "#/$defs/CoverageRequirement" } + "items": { + "$ref": "#/$defs/CoverageRequirement" + } }, "constraints": { "type": "array", - "items": { "$ref": "#/$defs/ExecutionConstraint" } + "items": { + "$ref": "#/$defs/ExecutionConstraint" + } }, "completion_checks": { "type": "array", - "items": { "$ref": "#/$defs/CompletionCheck" } + "items": { + "$ref": "#/$defs/CompletionCheck" + } } } }, "ExecutionRequirement": { "type": "object", "additionalProperties": false, - "required": ["id", "description"], + "required": [ + "id", + "description" + ], "properties": { - "id": { "type": "string" }, - "description": { "type": "string" } + "id": { + "type": "string" + }, + "description": { + "type": "string" + } } }, "ExecutionDeliverable": { "type": "object", "additionalProperties": false, - "required": ["id", "description", "output_pointer", "schema"], + "required": [ + "id", + "description", + "output_pointer", + "schema" + ], "properties": { - "id": { "type": "string" }, - "description": { "type": "string" }, - "output_pointer": { "type": "string" }, + "id": { + "type": "string" + }, + "description": { + "type": "string" + }, + "output_pointer": { + "type": "string" + }, "schema": {} } }, @@ -115,20 +192,35 @@ "require_all" ], "properties": { - "id": { "type": "string" }, - "description": { "type": "string" }, - "map_node_id": { "type": "string" }, + "id": { + "type": "string" + }, + "description": { + "type": "string" + }, + "map_node_id": { + "type": "string" + }, "expected_items": {}, - "require_all": { "type": "boolean" } + "require_all": { + "type": "boolean" + } } }, "ExecutionConstraint": { "type": "object", "additionalProperties": false, - "required": ["id", "description"], + "required": [ + "id", + "description" + ], "properties": { - "id": { "type": "string" }, - "description": { "type": "string" } + "id": { + "type": "string" + }, + "description": { + "type": "string" + } } }, "CompletionCheck": { @@ -142,17 +234,27 @@ "kind" ], "properties": { - "id": { "type": "string" }, - "description": { "type": "string" }, + "id": { + "type": "string" + }, + "description": { + "type": "string" + }, "requirement_ids": { "type": "array", - "items": { "type": "string" } + "items": { + "type": "string" + } }, "constraint_ids": { "type": "array", - "items": { "type": "string" } + "items": { + "type": "string" + } }, - "kind": { "$ref": "#/$defs/CompletionCheckKind" } + "kind": { + "$ref": "#/$defs/CompletionCheckKind" + } } }, "CompletionCheckKind": { @@ -160,41 +262,67 @@ { "type": "object", "additionalProperties": false, - "required": ["kind"], + "required": [ + "kind" + ], "properties": { - "kind": { "const": "output_schema" } + "kind": { + "const": "output_schema" + } } }, { "type": "object", "additionalProperties": false, - "required": ["kind", "node_ids"], + "required": [ + "kind", + "node_ids" + ], "properties": { - "kind": { "const": "required_nodes" }, + "kind": { + "const": "required_nodes" + }, "node_ids": { "type": "array", - "items": { "type": "string" } + "items": { + "type": "string" + } } } }, { "type": "object", "additionalProperties": false, - "required": ["kind", "map_node_id"], + "required": [ + "kind", + "map_node_id" + ], "properties": { - "kind": { "const": "map_coverage" }, - "map_node_id": { "type": "string" } + "kind": { + "const": "map_coverage" + }, + "map_node_id": { + "type": "string" + } } }, { "type": "object", "additionalProperties": false, - "required": ["kind", "node_ids", "min_per_task"], + "required": [ + "kind", + "node_ids", + "min_per_task" + ], "properties": { - "kind": { "const": "citations" }, + "kind": { + "const": "citations" + }, "node_ids": { "type": "array", - "items": { "type": "string" } + "items": { + "type": "string" + } }, "min_per_task": { "type": "integer", @@ -206,10 +334,18 @@ { "type": "object", "additionalProperties": false, - "required": ["kind", "instructions", "max_turns"], + "required": [ + "kind", + "instructions", + "max_turns" + ], "properties": { - "kind": { "const": "agent_verifier" }, - "instructions": { "type": "string" }, + "kind": { + "const": "agent_verifier" + }, + "instructions": { + "type": "string" + }, "max_turns": { "type": "integer", "format": "uint32", @@ -230,13 +366,19 @@ "nodes" ], "properties": { - "cancel_policy": { "$ref": "#/$defs/ExecutionCancelPolicy" }, - "input_wait_policy": { "$ref": "#/$defs/ExecutionWaitPolicy" }, + "cancel_policy": { + "$ref": "#/$defs/ExecutionCancelPolicy" + }, + "input_wait_policy": { + "$ref": "#/$defs/ExecutionInputWaitPolicy" + }, "input_schema": {}, "output_schema": {}, "nodes": { "type": "array", - "items": { "$ref": "#/$defs/ExecutionNode" } + "items": { + "$ref": "#/$defs/ExecutionNode" + } } } }, @@ -254,135 +396,219 @@ "retry" ], "properties": { - "id": { "type": "string" }, + "id": { + "type": "string" + }, "requirement_ids": { "type": "array", - "items": { "type": "string" } + "items": { + "type": "string" + } }, "depends_on": { "type": "array", - "items": { "type": "string" } + "items": { + "type": "string" + } }, "when": { + "description": "Optional condition evaluated once, before the node is activated. A false condition skips the node: it commits a skipped node aggregate with a null output, and its dependents are released for ordering only. A conditional node's output cannot be read, it cannot be the terminal output node, and it cannot be the only node serving a requirement.", "anyOf": [ - { "$ref": "#/$defs/ExecutionCondition" }, - { "type": "null" } + { + "$ref": "#/$defs/ExecutionCondition" + }, + { + "type": "null" + } ] }, "input": {}, "output_schema": {}, - "operation": { "$ref": "#/$defs/ExecutionOperation" }, + "operation": { + "$ref": "#/$defs/ExecutionOperation" + }, "compensation": { "anyOf": [ - { "$ref": "#/$defs/ExecutionCompensation" }, - { "type": "null" } + { + "$ref": "#/$defs/ExecutionCompensation" + }, + { + "type": "null" + } ] }, - "retry": { "$ref": "#/$defs/RetryPolicy" }, + "retry": { + "$ref": "#/$defs/RetryPolicy" + }, "budget": { "anyOf": [ - { "$ref": "#/$defs/ExecutionBudgetLimit" }, - { "type": "null" } + { + "$ref": "#/$defs/ExecutionBudgetLimit" + }, + { + "type": "null" + } ] } } }, + "ExecutionCondition": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "reference" + ], + "properties": { + "kind": { + "const": "exists" + }, + "reference": { + "$ref": "#/$defs/ExecutionReference" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "reference", + "value" + ], + "properties": { + "kind": { + "const": "equals" + }, + "reference": { + "$ref": "#/$defs/ExecutionReference" + }, + "value": {} + } + } + ] + }, + "ExecutionReference": { + "type": "object", + "additionalProperties": false, + "required": [ + "$ref" + ], + "properties": { + "$ref": { + "type": "string" + } + } + }, "ExecutionCancelPolicy": { "type": "string", - "enum": ["retain_effects", "compensate_committed"] + "enum": [ + "retain_effects", + "compensate_committed" + ] }, "ExecutionCompensation": { "type": "object", "additionalProperties": false, - "required": ["compensator", "input_mapping"], + "required": [ + "compensator", + "input_mapping" + ], "properties": { - "compensator": { "$ref": "#/$defs/CapabilityReference" }, - "input_mapping": { "$ref": "#/$defs/CompensationInputMapping" } + "compensator": { + "$ref": "#/$defs/CapabilityReference" + }, + "input_mapping": { + "$ref": "#/$defs/CompensationInputMapping" + } } }, "CompensationInputMapping": { "type": "object", "additionalProperties": false, - "required": ["bindings"], + "required": [ + "bindings" + ], "properties": { "bindings": { "type": "array", "maxItems": 64, - "items": { "$ref": "#/$defs/CompensationInputBinding" } + "items": { + "$ref": "#/$defs/CompensationInputBinding" + } } } }, "CompensationInputBinding": { "type": "object", "additionalProperties": false, - "required": ["target_pointer", "source"], + "required": [ + "target_pointer", + "source" + ], "properties": { - "target_pointer": { "type": "string" }, - "source": { "$ref": "#/$defs/CompensationValueSource" } - } - }, - "CompensationValueSource": { - "oneOf": [ - { - "type": "object", - "additionalProperties": false, - "required": ["kind", "pointer"], - "properties": { - "kind": { "const": "original_input" }, - "pointer": { "type": "string" } - } + "target_pointer": { + "type": "string" }, - { - "type": "object", - "additionalProperties": false, - "required": ["kind", "pointer"], - "properties": { - "kind": { "const": "original_output" }, - "pointer": { "type": "string" } - } + "source": { + "$ref": "#/$defs/CompensationValueSource" } - ] + } }, - "ExecutionCondition": { + "CompensationValueSource": { "oneOf": [ { "type": "object", "additionalProperties": false, - "required": ["kind", "reference"], + "required": [ + "kind", + "pointer" + ], "properties": { - "kind": { "const": "exists" }, - "reference": { "$ref": "#/$defs/ExecutionReference" } + "kind": { + "const": "original_input" + }, + "pointer": { + "type": "string" + } } }, { "type": "object", "additionalProperties": false, - "required": ["kind", "reference", "value"], + "required": [ + "kind", + "pointer" + ], "properties": { - "kind": { "const": "equals" }, - "reference": { "$ref": "#/$defs/ExecutionReference" }, - "value": {} + "kind": { + "const": "original_output" + }, + "pointer": { + "type": "string" + } } } ] }, - "ExecutionReference": { - "type": "object", - "additionalProperties": false, - "required": ["$ref"], - "properties": { - "$ref": { "type": "string" } - } - }, "ExecutionOperation": { "oneOf": [ { "type": "object", "additionalProperties": false, - "required": ["kind", "reference"], + "required": [ + "kind", + "reference" + ], "properties": { - "kind": { "const": "capability" }, - "reference": { "$ref": "#/$defs/CapabilityReference" } + "kind": { + "const": "capability" + }, + "reference": { + "$ref": "#/$defs/CapabilityReference" + } } }, { @@ -396,15 +622,23 @@ "max_turns" ], "properties": { - "kind": { "const": "agent" }, - "instructions": { "type": "string" }, + "kind": { + "const": "agent" + }, + "instructions": { + "type": "string" + }, "skill_refs": { "type": "array", - "items": { "$ref": "#/$defs/ArtifactRef" } + "items": { + "$ref": "#/$defs/ArtifactRef" + } }, "capability_refs": { "type": "array", - "items": { "$ref": "#/$defs/CapabilityReference" } + "items": { + "$ref": "#/$defs/CapabilityReference" + } }, "max_turns": { "type": "integer", @@ -425,16 +659,22 @@ "task" ], "properties": { - "kind": { "const": "map" }, + "kind": { + "const": "map" + }, "items": {}, - "item_key": { "type": "string" }, + "item_key": { + "type": "string" + }, "max_items": { "type": "integer", "format": "uint64", "minimum": 0 }, "item_output_schema": {}, - "task": { "$ref": "#/$defs/MapTask" } + "task": { + "$ref": "#/$defs/MapTask" + } } }, { @@ -448,14 +688,18 @@ "batch_size" ], "properties": { - "kind": { "const": "reduce" }, + "kind": { + "const": "reduce" + }, "items": {}, "max_items": { "type": "integer", "format": "uint64", "minimum": 0 }, - "reducer": { "$ref": "#/$defs/ExecutionReducer" }, + "reducer": { + "$ref": "#/$defs/ExecutionReducer" + }, "batch_size": { "type": "integer", "format": "uint32", @@ -466,39 +710,72 @@ { "type": "object", "additionalProperties": false, - "required": ["kind", "prompt", "wait_policy"], + "required": [ + "kind", + "prompt", + "wait_policy" + ], "properties": { - "kind": { "const": "review" }, - "prompt": { "type": "string" }, - "wait_policy": { "$ref": "#/$defs/ExecutionWaitPolicy" } + "kind": { + "const": "review" + }, + "prompt": { + "type": "string" + }, + "wait_policy": { + "$ref": "#/$defs/ExecutionWaitPolicy" + } } }, { "type": "object", "additionalProperties": false, - "required": ["kind", "signal_name", "wait_policy"], + "required": [ + "kind", + "signal_name", + "wait_policy" + ], "properties": { - "kind": { "const": "wait_signal" }, - "signal_name": { "type": "string" }, - "wait_policy": { "$ref": "#/$defs/ExecutionWaitPolicy" } + "kind": { + "const": "wait_signal" + }, + "signal_name": { + "type": "string" + }, + "wait_policy": { + "$ref": "#/$defs/ExecutionWaitPolicy" + } } }, { "type": "object", "additionalProperties": false, - "required": ["kind", "wake", "result"], + "required": [ + "kind", + "wake", + "result" + ], "properties": { - "kind": { "const": "wait_until" }, - "wake": { "$ref": "#/$defs/ExecutionTemporalTarget" }, + "kind": { + "const": "wait_until" + }, + "wake": { + "$ref": "#/$defs/ExecutionTemporalTarget" + }, "result": {} } }, { "type": "object", "additionalProperties": false, - "required": ["kind", "value"], + "required": [ + "kind", + "value" + ], "properties": { - "kind": { "const": "output" }, + "kind": { + "const": "output" + }, "value": {} } } @@ -507,62 +784,92 @@ "ExecutionWaitPolicy": { "type": "object", "additionalProperties": false, - "required": ["expiry", "on_expiry"], + "required": [ + "expiry", + "on_expiry" + ], "properties": { - "expiry": { "$ref": "#/$defs/ExecutionTemporalTarget" }, - "on_expiry": { "$ref": "#/$defs/ExecutionWaitExpiryAction" } + "expiry": { + "$ref": "#/$defs/ExecutionTemporalTarget" + }, + "on_expiry": { + "$ref": "#/$defs/ExecutionWaitExpiryAction" + } + } + }, + "ExecutionInputWaitPolicy": { + "description": "Plan-level expiry policy for runtime input requests. It settles whichever task returned NeedsInput, so a declared continue_with output has no node output_schema to validate against and is rejected.", + "type": "object", + "additionalProperties": false, + "required": [ + "expiry", + "on_expiry" + ], + "properties": { + "expiry": { + "$ref": "#/$defs/ExecutionTemporalTarget" + }, + "on_expiry": { + "$ref": "#/$defs/ExecutionInputWaitExpiryAction" + } + } + }, + "ExecutionInputWaitExpiryAction": { + "type": "object", + "additionalProperties": false, + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "fail_task" + } } }, "ExecutionTemporalTarget": { - "oneOf": [ - { - "type": "object", - "additionalProperties": false, - "required": ["kind", "at"], - "properties": { - "kind": { "const": "at" }, - "at": { "type": "string", "format": "date-time" } - } + "description": "Skills are reusable templates, so only wait-entry-relative targets are accepted; the canonical Rust enum also carries an absolute `at` branch that skill validation always rejects.", + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "delay_seconds" + ], + "properties": { + "kind": { + "const": "after" }, - { - "type": "object", - "additionalProperties": false, - "required": ["kind", "delay_seconds"], - "properties": { - "kind": { "const": "after" }, - "delay_seconds": { - "type": "integer", - "format": "uint64", - "minimum": 1 - } - } + "delay_seconds": { + "type": "integer", + "format": "uint64", + "minimum": 1 } - ] + } }, "ExecutionWaitExpiryAction": { "oneOf": [ { "type": "object", "additionalProperties": false, - "required": ["kind"], - "properties": { - "kind": { "const": "fail_task" } - } - }, - { - "type": "object", - "additionalProperties": false, - "required": ["kind"], + "required": [ + "kind" + ], "properties": { - "kind": { "const": "fail_run" } + "kind": { + "const": "fail_task" + } } }, { "type": "object", "additionalProperties": false, - "required": ["kind", "output"], + "required": [ + "kind", + "output" + ], "properties": { - "kind": { "const": "continue_with" }, + "kind": { + "const": "continue_with" + }, "output": {} } } @@ -571,10 +878,17 @@ "CapabilityReference": { "type": "object", "additionalProperties": false, - "required": ["name", "version"], + "required": [ + "name", + "version" + ], "properties": { - "name": { "type": "string" }, - "version": { "type": "string" } + "name": { + "type": "string" + }, + "version": { + "type": "string" + } } }, "ArtifactRef": { @@ -599,10 +913,17 @@ { "type": "object", "additionalProperties": false, - "required": ["kind", "reference"], + "required": [ + "kind", + "reference" + ], "properties": { - "kind": { "const": "capability" }, - "reference": { "$ref": "#/$defs/CapabilityReference" } + "kind": { + "const": "capability" + }, + "reference": { + "$ref": "#/$defs/CapabilityReference" + } } }, { @@ -616,15 +937,23 @@ "max_turns" ], "properties": { - "kind": { "const": "agent" }, - "instructions": { "type": "string" }, + "kind": { + "const": "agent" + }, + "instructions": { + "type": "string" + }, "skill_refs": { "type": "array", - "items": { "$ref": "#/$defs/ArtifactRef" } + "items": { + "$ref": "#/$defs/ArtifactRef" + } }, "capability_refs": { "type": "array", - "items": { "$ref": "#/$defs/CapabilityReference" } + "items": { + "$ref": "#/$defs/CapabilityReference" + } }, "max_turns": { "type": "integer", @@ -640,10 +969,17 @@ { "type": "object", "additionalProperties": false, - "required": ["kind", "reference"], + "required": [ + "kind", + "reference" + ], "properties": { - "kind": { "const": "capability" }, - "reference": { "$ref": "#/$defs/CapabilityReference" } + "kind": { + "const": "capability" + }, + "reference": { + "$ref": "#/$defs/CapabilityReference" + } } }, { @@ -657,15 +993,23 @@ "max_turns" ], "properties": { - "kind": { "const": "agent" }, - "instructions": { "type": "string" }, + "kind": { + "const": "agent" + }, + "instructions": { + "type": "string" + }, "skill_refs": { "type": "array", - "items": { "$ref": "#/$defs/ArtifactRef" } + "items": { + "$ref": "#/$defs/ArtifactRef" + } }, "capability_refs": { "type": "array", - "items": { "$ref": "#/$defs/CapabilityReference" } + "items": { + "$ref": "#/$defs/CapabilityReference" + } }, "max_turns": { "type": "integer", @@ -707,32 +1051,50 @@ "additionalProperties": false, "properties": { "max_cost_microusd": { - "type": ["integer", "null"], + "type": [ + "integer", + "null" + ], "format": "uint64", "minimum": 0 }, "max_tokens": { - "type": ["integer", "null"], + "type": [ + "integer", + "null" + ], "format": "uint64", "minimum": 0 }, "max_tasks": { - "type": ["integer", "null"], + "type": [ + "integer", + "null" + ], "format": "uint64", "minimum": 0 }, "max_tool_calls": { - "type": ["integer", "null"], + "type": [ + "integer", + "null" + ], "format": "uint64", "minimum": 0 }, "max_retrieved_bytes": { - "type": ["integer", "null"], + "type": [ + "integer", + "null" + ], "format": "uint64", "minimum": 0 }, "deadline_at": { - "type": ["string", "null"], + "type": [ + "string", + "null" + ], "format": "date-time" } } diff --git a/k8s/scripts/validate-observability.sh b/k8s/scripts/validate-observability.sh index 0c5807a6b..b567a9dbe 100755 --- a/k8s/scripts/validate-observability.sh +++ b/k8s/scripts/validate-observability.sh @@ -29,6 +29,7 @@ LOCAL_LGTM="${REPO_ROOT}/k8s/overlays/local/lgtm.yaml" LOCAL_LGTM_RESTATE="${REPO_ROOT}/k8s/overlays/local/otelcol-local-restate.yaml" LOCAL_RESTATE_PATCH="${REPO_ROOT}/k8s/overlays/local/patches/restate-cluster.yaml" PRODUCTION_RESTATE_PATCH="${REPO_ROOT}/k8s/overlays/production/patches/restate-observability.yaml" +RUNTIME_METRICS_RS="${REPO_ROOT}/crates/moa-observability/src/runtime_metrics.rs" # Pinned tool versions. A validator that accepts whatever binary happens to be on # PATH cannot tell "this config is valid" from "this version of the checker did @@ -57,7 +58,6 @@ EXPECTED_ALERTS=( MOAExecutionOverdueDeadlines MOAExecutionQueueSampleSaturated MOAExecutionRetentionStale - MOAExecutionTriggerDeadLetters MOAExecutionTriggerLagHigh MOALLMFailoverElevated MOALineageDeadLettering @@ -631,9 +631,11 @@ PY done echo "Cross-checking alert metric names against the Rust source..." -# The check nothing else performs. A metric renamed in Rust leaves the alert -# expression parsing perfectly and matching no series forever, and neither -# promtool nor kubeconform nor the compiler can see it. +# A metric renamed in Rust leaves the alert expression parsing perfectly and +# matching no series forever, and neither promtool nor kubeconform nor the +# compiler can see it. This proves only that the NAME still exists in Rust. It +# proves nothing about emission - see the recorder-caller check below, which is +# the half this check used to claim and never performed. REFERENCED_METRICS="${WORK_DIR}/referenced-metrics.txt" [[ -s "${REFERENCED_METRICS}" ]] \ || die "no metric names were extracted from any alert expression" @@ -654,9 +656,34 @@ while read -r metric; do fi done < <(sort -u "${REFERENCED_METRICS}") if [[ "${#MISSING_METRICS[@]}" -gt 0 ]]; then - die "alert rules reference metrics that no crate emits: ${MISSING_METRICS[*]}" + die "alert rules reference metrics that no crate names: ${MISSING_METRICS[*]}" fi -echo " $(sort -u "${REFERENCED_METRICS}" | wc -l | tr -d ' ') referenced metrics all emitted" +echo " $(sort -u "${REFERENCED_METRICS}" | wc -l | tr -d ' ') referenced metric names resolve in crates/" + +echo "Checking that every metric recorder has a production caller..." +# The check nothing else performs. Name presence is not emission: a `gauge!()` +# inside a function nobody invokes satisfies every check above, which is how +# eleven metrics and the six alerts reading them shipped permanently dead - three +# of them critical, including the only guards on the parked-hand and exact-deadline +# invariants. A recorder emits only if non-test code outside its own definition +# file calls it, so `tests/` binaries and the `#[cfg(test)]` module inside +# runtime_metrics.rs are deliberately not counted as callers. +# +# A recorder restored ahead of its call site fails here on purpose: the failure is +# the work item, and it clears when the owning path starts recording. Silencing it +# by exempting a name, by counting a test caller, or by deleting the recorder again +# reintroduces exactly the blind spot this check exists to close. +UNCALLED_RECORDERS=() +while read -r recorder; do + grep -rl --include='*.rs' -E "${recorder}[[:space:]]*\(" "${REPO_ROOT}/crates" \ + | grep -qvE "(runtime_metrics\.rs|/tests/)" \ + || UNCALLED_RECORDERS+=("${recorder}") +done < <(grep -oE '^pub fn record_[a-z0-9_]+' "${RUNTIME_METRICS_RS}" | awk '{print $3}') +if [[ "${#UNCALLED_RECORDERS[@]}" -gt 0 ]]; then + die "$(printf 'runtime_metrics.rs declares recorders that only its own file or a test binary calls, so their metrics never exist in production and any alert on them can never fire:\n %s\nWire each one into its owning path, or delete the recorder, its describe/bucket registration and its alert together.' \ + "${UNCALLED_RECORDERS[*]}")" +fi +echo " $(grep -cE '^pub fn record_[a-z0-9_]+' "${RUNTIME_METRICS_RS}") metric recorders all have production callers" echo "Checking the declared alert set..." DECLARED_ALERTS="$(grep -ohE '^ *- alert: [A-Za-z0-9]+' "${FILTERED_RULE_FILES[@]}" \ @@ -815,12 +842,10 @@ expected = { "moa_execution_overdue_deadlines", "moa_execution_runs", "moa_execution_tenant_max_share_ratio", - "moa_execution_trigger_dead_letters", "moa_execution_trigger_due", "moa_execution_trigger_lag_seconds", + "moa_restate_draining_deployment_blocking_invocations", "moa_restate_draining_deployment_oldest_age_seconds", - "moa_restate_draining_deployment_replica_hours", - "moa_restate_draining_deployment_replicas", "moa_restate_draining_deployments", "moa_sandbox_workspace_active_hands", "moa_sandbox_workspace_parked_tasks_with_active_hands", @@ -829,7 +854,7 @@ expected = { } for metric in expected: if f'"{metric}"' not in runtime_metrics: - raise SystemExit(f"long-horizon metric {metric} is not emitted by runtime_metrics.rs") + raise SystemExit(f"long-horizon metric {metric} is not declared by runtime_metrics.rs") execution_source = runtime_metrics[ runtime_metrics.index("pub fn record_execution_run_phase") : @@ -863,8 +888,8 @@ rules = [ for group in alerts.get("spec", {}).get("groups") or [] for rule in group.get("rules") or [] ] -if len(rules) != 12: - raise SystemExit(f"{alerts_path} must contain exactly twelve actionable alerts") +if len(rules) != 11: + raise SystemExit(f"{alerts_path} must contain exactly eleven actionable alerts") alert_expressions = "\n".join(rule.get("expr", "") for rule in rules) required_alert_metrics = { "moa_execution_active_attempt_oldest_age_seconds", @@ -879,7 +904,6 @@ required_alert_metrics = { "moa_execution_outbox_lag_seconds", "moa_execution_queue_sample_saturated", "moa_execution_overdue_deadlines", - "moa_execution_trigger_dead_letters", "moa_execution_trigger_lag_seconds", } observed_alert_metrics = set(re.findall(r"moa_execution_[a-z0-9_]+", alert_expressions)) @@ -904,6 +928,23 @@ for required_clause in ( "MOAExecutionRetentionStale must alert on missing, unready, and older-than-two-hour receipts" ) +# A bare `max(metric) > N` evaluates to *no data* when nothing emits the series, so +# a guard that loses its producer stops guarding instead of firing. Every unlabeled +# scalar rule must therefore carry an `absent()` clause. The two `by (...)` rules are +# exempt: absent() yields a label-less vector that would render their own +# annotations empty, and MOAExecutionMaintenanceReconcileStale - emitted by the same +# handler - is their absence guard. +labelled_alerts = {"MOAExecutionAdmissionSaturated", "MOAExecutionQueueSampleSaturated"} +for rule in rules: + name = rule.get("alert") + if name in labelled_alerts: + continue + if "absent(" not in rule.get("expr", ""): + raise SystemExit( + f"{name} has no absent() clause, so a missing producer evaluates to no " + "data and the alert silently stops guarding instead of firing" + ) + kustomization = (root / "ops/prometheus/alerts/kustomization.yaml").read_text( encoding="utf-8" ) @@ -913,7 +954,8 @@ if kustomization.count("moa-long-horizon-execution.yaml") != 1: ) print( - f" OK {len(expected)} low-cardinality metrics back {len(rules)} long-horizon alerts" + f" OK {len(expected)} low-cardinality metrics back {len(rules)} long-horizon alerts, " + "each with an absence guard" ) PY echo "Checking the sandbox workspace metrics/dashboard/alert contract..." @@ -1043,6 +1085,23 @@ rules = [ ] if len(rules) != 8: raise SystemExit(f"{alerts_path} must contain exactly eight actionable alerts") + +# The parked-hand rule is the only automated guard on the invariant that a parked +# run owns no sandbox. Without absent() a missing producer reads as zero violations, +# which is how it shipped inert. +parked_hand = [ + rule for rule in rules if rule.get("alert") == "MOASandboxParkedTaskRetainsActiveHand" +] +if len(parked_hand) != 1: + raise SystemExit("sandbox alerts must define MOASandboxParkedTaskRetainsActiveHand exactly once") +if "absent(moa_sandbox_workspace_parked_tasks_with_active_hands)" not in parked_hand[0].get( + "expr", "" +): + raise SystemExit( + "MOASandboxParkedTaskRetainsActiveHand must fire when nothing publishes the " + "parked-hand invariant series, not treat absence as zero violations" + ) + for rule in rules: expression = rule.get("expr", "") annotations = rule.get("annotations") or {} diff --git a/ops/prometheus/alerts/moa-long-horizon-execution.yaml b/ops/prometheus/alerts/moa-long-horizon-execution.yaml index 2b261c6b1..cf6f52ae7 100644 --- a/ops/prometheus/alerts/moa-long-horizon-execution.yaml +++ b/ops/prometheus/alerts/moa-long-horizon-execution.yaml @@ -1,4 +1,18 @@ # Long-horizon execution and singleton-maintenance alerts. +# +# Every unlabeled scalar guard below is written `absent(metric) or max(metric) > N`. +# A bare `max(metric) > N` evaluates to *no data* when nothing emits the series, so +# the alert silently stops guarding instead of firing - which is exactly how the +# deadline and stuck-attempt guards shipped inert. The two `by (...)` alerts keep the +# bare form on purpose: `absent()` returns a label-less vector, which would fire an +# alert whose own description renders empty labels. Their producer is the same handler +# that emits `moa_execution_maintenance_ready`, so MOAExecutionMaintenanceReconcileStale +# is their absence guard. +# +# Every metric here is recorded unconditionally from the bounded fleet snapshot, +# including the healthy zero. A gauge written only while work exists would make +# `absent()` fire on a quiet fleet and would keep its last value forever after a +# drain; both are treated as producer bugs, not as reasons to weaken these alerts. apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule metadata: @@ -14,7 +28,7 @@ spec: interval: 30s rules: - alert: MOAExecutionOverdueDeadlines - expr: max(moa_execution_overdue_deadlines) > 0 + expr: absent(moa_execution_overdue_deadlines) or max(moa_execution_overdue_deadlines) > 0 for: 2m labels: severity: critical @@ -23,7 +37,7 @@ spec: description: "{{ $value }} execution runs are still nonterminal after their absolute deadline. Verify deadline triggers, controller activation, and terminal projection delivery." - alert: MOAExecutionOldestReadySLO - expr: max(moa_execution_oldest_ready_age_seconds) > 120 + expr: absent(moa_execution_oldest_ready_age_seconds) or max(moa_execution_oldest_ready_age_seconds) > 120 for: 5m labels: severity: warning @@ -32,7 +46,7 @@ spec: description: "The oldest ready task has waited {{ $value }} seconds (threshold 120s). Check admission capacity, dispatcher ownership, and outbox delivery." - alert: MOAExecutionActiveAttemptStuck - expr: max(moa_execution_active_attempt_oldest_age_seconds) > 900 + expr: absent(moa_execution_active_attempt_oldest_age_seconds) or max(moa_execution_active_attempt_oldest_age_seconds) > 900 for: 5m labels: severity: critical @@ -41,7 +55,7 @@ spec: description: "The oldest active task attempt is {{ $value }} seconds old (threshold 900s). Verify watchdog delivery and ambiguous-effect reconciliation before replaying work." - alert: MOAExecutionTriggerLagHigh - expr: max(moa_execution_trigger_lag_seconds) > 120 + expr: absent(moa_execution_trigger_lag_seconds) or max(moa_execution_trigger_lag_seconds) > 120 for: 5m labels: severity: warning @@ -49,17 +63,8 @@ spec: summary: Due execution triggers are delayed description: "The oldest due undelivered trigger is {{ $value }} seconds late (threshold 120s). Check the maintenance owner, Restate ingress, and trigger claim expiry." - - alert: MOAExecutionTriggerDeadLetters - expr: max(moa_execution_trigger_dead_letters) > 0 - for: 2m - labels: - severity: critical - annotations: - summary: Execution triggers entered dead-letter state - description: "{{ $value }} execution triggers require operator repair. Inspect immutable trigger IDs and generation fences before redelivery." - - alert: MOAExecutionOutboxLagHigh - expr: max(moa_execution_outbox_lag_seconds) > 120 + expr: absent(moa_execution_outbox_lag_seconds) or max(moa_execution_outbox_lag_seconds) > 120 for: 5m labels: severity: warning @@ -68,7 +73,7 @@ spec: description: "The oldest undispatched execution outbox row is {{ $value }} seconds old (threshold 120s). Check claim ownership and Restate ingress availability." - alert: MOAExecutionOutboxDeadLetters - expr: max(moa_execution_outbox_dead_letters) > 0 + expr: absent(moa_execution_outbox_dead_letters) or max(moa_execution_outbox_dead_letters) > 0 for: 2m labels: severity: critical @@ -113,7 +118,7 @@ spec: description: "Execution {{ $labels.resource }} utilization at {{ $labels.scope }} scope is {{ $value | humanizePercentage }} (threshold 90%). Confirm fair queueing before increasing the durable limit." - alert: MOAExecutionExternalJobStuck - expr: max(moa_execution_external_job_oldest_age_seconds) > 86400 + expr: absent(moa_execution_external_job_oldest_age_seconds) or max(moa_execution_external_job_oldest_age_seconds) > 86400 for: 30m labels: severity: warning diff --git a/ops/prometheus/alerts/moa-restate.yaml b/ops/prometheus/alerts/moa-restate.yaml index 6955abb3d..9f5277722 100644 --- a/ops/prometheus/alerts/moa-restate.yaml +++ b/ops/prometheus/alerts/moa-restate.yaml @@ -128,8 +128,13 @@ spec: summary: Restate has a partition without an effective leader description: "Only {{ $value }} effective partition leaders are visible. Compare with restate_num_partitions and inspect cluster membership before changing placement." + # The only MOA-owned gauge in this file. `absent()` is load-bearing: the + # maintenance owner records the whole draining snapshot on every pass, + # including the fully drained zero, so a missing series means nobody is + # watching the drain rather than that no revision is draining. A bare + # `max(...) > 3600` would evaluate to no data and stop guarding silently. - alert: MOARestateOldDeploymentDrainAge - expr: max(moa_restate_draining_deployment_oldest_age_seconds) > 3600 + expr: absent(moa_restate_draining_deployment_oldest_age_seconds) or max(moa_restate_draining_deployment_oldest_age_seconds) > 3600 for: 15m labels: severity: warning diff --git a/ops/prometheus/alerts/sandbox-workspaces.yaml b/ops/prometheus/alerts/sandbox-workspaces.yaml index b264e36c9..9c954376c 100644 --- a/ops/prometheus/alerts/sandbox-workspaces.yaml +++ b/ops/prometheus/alerts/sandbox-workspaces.yaml @@ -83,8 +83,13 @@ spec: description: "{{ $value }} unresolved {{ $labels.classification }} findings exist for {{ $labels.provider_kind }}. Findings are quarantined and must never be auto-deleted; prove ownership and absence before resolving them." runbook_url: https://github.com/hwuiwon/moa/blob/main/docs/19-data-operations.md#provider-inventory-drift + # `absent()` is load-bearing: this is the only automated guard on the + # invariant that a parked run owns no sandbox, and a bare `max(...) > 0` + # reads as healthy whenever nothing emits the series at all. The producer + # records the healthy zero on every fleet snapshot, so absence means the + # invariant is unobserved, not satisfied. - alert: MOASandboxParkedTaskRetainsActiveHand - expr: max(moa_sandbox_workspace_parked_tasks_with_active_hands) > 0 + expr: absent(moa_sandbox_workspace_parked_tasks_with_active_hands) or max(moa_sandbox_workspace_parked_tasks_with_active_hands) > 0 for: 1m labels: severity: critical diff --git a/scripts/cutover-long-horizon-execution.sh b/scripts/cutover-long-horizon-execution.sh index 9c1eb9b23..ce2df4543 100755 --- a/scripts/cutover-long-horizon-execution.sh +++ b/scripts/cutover-long-horizon-execution.sh @@ -10,6 +10,7 @@ RESTATE_INGRESS_URL="" OLD_DEPLOYMENT_ID="" NEW_DEPLOYMENT_URI="" ARCHIVE_DIR="" +SESSION_EVENTS_SCHEMA="" CONFIRMED=0 readonly OLD_SERVICES=(ExecutionRun ExecutionTask ExecutionCompensation) @@ -50,7 +51,8 @@ Usage: scripts/cutover-long-horizon-execution.sh \ --restate-ingress-url URL \ --old-deployment-id ID \ --new-deployment-uri URI \ - --archive-dir ABSOLUTE_EMPTY_DIRECTORY \ + --archive-dir ABSOLUTE_DIRECTORY \ + --session-events-schema NAME \ [--confirm-destructive-cutover] Hard-cuts the retired ExecutionRun, ExecutionTask, and ExecutionCompensation @@ -58,9 +60,30 @@ Restate runtime to bounded execution activations. The script always performs and prints its read-only Postgres and Restate preflight before considering any mutation. Without --confirm-destructive-cutover it exits after preflight. -The archive directory must already exist, be absolute, and be empty. The -database URL, Restate Admin/ingress URLs, old deployment ID, and new immutable -deployment URI are mandatory; no destructive target is inferred. +The archive directory must already exist, be absolute, and be either empty or +hold this exact cutover's own cutover-manifest.txt from an interrupted earlier +invocation. The database URL, Restate Admin/ingress URLs, old deployment ID, +new immutable deployment URI, and session-events schema are mandatory; no +destructive target is inferred. + +--session-events-schema names the Postgres schema, in the same database as +--database-admin-url, that holds the session store's `events` table. Historical +`ExecutionProgress` payloads predate this change set's added required fields +and can no longer decode; a single stale row fails an entire session history +replay, so the cutover deletes them after archiving them as CSV. + +That schema is usually `public`: `config.database.schema` is an Option that is +None by default, so the session store resolves `events` through `search_path`. +It is deliberately NOT defaulted here — an unset value must never become a +destructive target. Pass the value your deployment actually uses. + +Archived history is out of scope. `session_event_archives` payloads are +compressed blobs that SQL cannot rewrite, and dropping whole archive rows would +destroy unrelated history for the same session. The final step names that +residue explicitly and states the remedy; it does not act on it. + +Every stage is idempotent and the read-only preflight runs on every invocation, +so an interrupted cutover is resumed by rerunning the identical command. USAGE } @@ -111,6 +134,11 @@ while [[ $# -gt 0 ]]; do ARCHIVE_DIR="${2%/}" shift 2 ;; + --session-events-schema) + require_value "$1" "${2:-}" + SESSION_EVENTS_SCHEMA="$2" + shift 2 + ;; --confirm-destructive-cutover) CONFIRMED=1 shift @@ -131,8 +159,9 @@ require_value --restate-ingress-url "${RESTATE_INGRESS_URL}" require_value --old-deployment-id "${OLD_DEPLOYMENT_ID}" require_value --new-deployment-uri "${NEW_DEPLOYMENT_URI}" require_value --archive-dir "${ARCHIVE_DIR}" +require_value --session-events-schema "${SESSION_EVENTS_SCHEMA}" -for command in cargo curl find jq pg_dump psql restate seq tee tr wc; do +for command in cargo curl find grep jq pg_dump psql restate seq tee tr wc; do require_cmd "${command}" done @@ -149,8 +178,24 @@ done [[ "${ARCHIVE_DIR}" == /* && "${ARCHIVE_DIR}" != / && "${ARCHIVE_DIR}" != *".."* ]] \ || die "--archive-dir must be an explicit absolute non-root path without '..'" [[ -d "${ARCHIVE_DIR}" ]] || die "--archive-dir must already exist" -[[ -z "$(find "${ARCHIVE_DIR}" -mindepth 1 -maxdepth 1 -print -quit)" ]] \ - || die "--archive-dir must be empty" +[[ "${SESSION_EVENTS_SCHEMA}" =~ ^[A-Za-z_][A-Za-z0-9_]{0,62}$ ]] \ + || die "--session-events-schema must be a plain unquoted Postgres identifier" + +# Every failure-prone stage of this cutover runs after the schema change, so a +# rerun has to be able to resume rather than trip over its own earlier output. +# A non-empty archive directory is accepted only when it holds this exact +# cutover's manifest; anything else is still an unrelated directory and refused. +ARCHIVE_MANIFEST="${ARCHIVE_DIR}/cutover-manifest.txt" +ARCHIVE_STAGE="pending" +if [[ -n "$(find "${ARCHIVE_DIR}" -mindepth 1 -maxdepth 1 -print -quit)" ]]; then + [[ -f "${ARCHIVE_MANIFEST}" ]] \ + || die "--archive-dir must be empty or hold this cutover's cutover-manifest.txt" + grep -qxF "old_deployment_id=${OLD_DEPLOYMENT_ID}" "${ARCHIVE_MANIFEST}" \ + || die "the existing cutover-manifest.txt names a different --old-deployment-id" + grep -qxF "new_deployment_uri=${NEW_DEPLOYMENT_URI}" "${ARCHIVE_MANIFEST}" \ + || die "the existing cutover-manifest.txt names a different --new-deployment-uri" + ARCHIVE_STAGE="complete" +fi restate_cli() { RESTATE_ADMIN_URL="${RESTATE_ADMIN_URL}" \ @@ -159,6 +204,16 @@ restate_cli() { restate -e local "$@" } +# The exact central-migration identities this cutover applies. Comparing names +# as well as versions is what lets a resumed invocation treat V000060 as "the +# migration stage already ran" instead of "some other chain reached 60". +readonly APPLIED_MIGRATIONS_EXPECTED=$'59|long_horizon_execution\n60|sandbox_active_compute_capacity' + +applied_migration_identities() { + psql -X "${DATABASE_ADMIN_URL}" --set=ON_ERROR_STOP=1 --tuples-only --no-align \ + --command "SELECT version, name FROM public.refinery_schema_history WHERE version IN (59, 60) ORDER BY version;" +} + restate_query() { local query="$1" curl -fsS \ @@ -189,8 +244,6 @@ OLD_DEPLOYMENT_MATCHES="$( jq -r --arg deployment_id "${OLD_DEPLOYMENT_ID}" \ '[.deployments[] | select(.id == $deployment_id)] | length' <<<"${DEPLOYMENTS_JSON}" )" -[[ "${OLD_DEPLOYMENT_MATCHES}" == 1 ]] \ - || die "old deployment ID must resolve to exactly one registered deployment" echo "== Read-only preflight: retired service deployments ==" jq ' [.deployments[] @@ -199,20 +252,40 @@ jq ' or .name == "ExecutionCompensation")) | {id, uri, services: [.services[]?.name]}] ' <<<"${DEPLOYMENTS_JSON}" -OLD_SERVICE_DEPLOYMENTS_VALID="$( - jq -r --arg deployment_id "${OLD_DEPLOYMENT_ID}" ' +OLD_SERVICE_DEPLOYMENTS="$( + jq -r ' [.deployments[] | select(any(.services[]?; .name == "ExecutionRun" or .name == "ExecutionTask" - or .name == "ExecutionCompensation"))] as $old - | ($old | length) == 1 - and ($old[0].id == $deployment_id) - and (["ExecutionRun", "ExecutionTask", "ExecutionCompensation"] - | all(. as $service | any($old[0].services[]?; .name == $service))) + or .name == "ExecutionCompensation"))] + | length ' <<<"${DEPLOYMENTS_JSON}" )" -[[ "${OLD_SERVICE_DEPLOYMENTS_VALID}" == true ]] \ - || die "the exact old deployment must be the sole deployment containing all retired execution services" +if [[ "${OLD_DEPLOYMENT_MATCHES}" == 1 ]]; then + OLD_SERVICE_DEPLOYMENTS_VALID="$( + jq -r --arg deployment_id "${OLD_DEPLOYMENT_ID}" ' + [.deployments[] + | select(any(.services[]?; .name == "ExecutionRun" + or .name == "ExecutionTask" + or .name == "ExecutionCompensation"))] as $old + | ($old | length) == 1 + and ($old[0].id == $deployment_id) + and (["ExecutionRun", "ExecutionTask", "ExecutionCompensation"] + | all(. as $service | any($old[0].services[]?; .name == $service))) + ' <<<"${DEPLOYMENTS_JSON}" + )" + [[ "${OLD_SERVICE_DEPLOYMENTS_VALID}" == true ]] \ + || die "the exact old deployment must be the sole deployment containing all retired execution services" + DEPLOYMENT_STAGE="pending" +elif [[ "${OLD_DEPLOYMENT_MATCHES}" == 0 && "${OLD_SERVICE_DEPLOYMENTS}" == 0 ]]; then + # The only accepted resume shape: an earlier invocation already removed the + # exact old deployment, and no other deployment exposes a retired execution + # service. Both halves are required — a missing ID with a retired service + # still registered elsewhere means something other than this script acted. + DEPLOYMENT_STAGE="complete" +else + die "old deployment ID must resolve to exactly one registered deployment, or already be removed with no retired execution service registered anywhere" +fi readonly OLD_INVOCATIONS_SQL=" SELECT id, status, target_service_name, target_handler_name, @@ -243,107 +316,206 @@ MIGRATION_POSITION="$( )" printf 'latest central migration: V%06d\n' "${MIGRATION_POSITION}" -printf 'preflight totals: nonterminal_runs=%s old_service_or_deployment_invocations=%s\n' \ - "${NONTERMINAL_RUNS}" "${OLD_INVOCATIONS}" +case "${MIGRATION_POSITION}" in + 58) + MIGRATION_STAGE="pending" + ;; + 60) + [[ "$(applied_migration_identities)" == "${APPLIED_MIGRATIONS_EXPECTED}" ]] \ + || die "the database reports V000060 without the exact V59/V60 identities this cutover applies" + MIGRATION_STAGE="complete" + ;; + *) + die "the database must be at exactly V000058 before, or exactly V000060 after, the V59/V60 hard cut" + ;; +esac + +echo "== Read-only preflight: undecodable historical ExecutionProgress events ==" +SESSION_EVENTS_TABLE_PRESENT="$( + psql -X "${DATABASE_ADMIN_URL}" --set=ON_ERROR_STOP=1 --tuples-only --no-align \ + --command "SELECT to_regclass('${SESSION_EVENTS_SCHEMA}.events') IS NOT NULL;" +)" +[[ "${SESSION_EVENTS_TABLE_PRESENT}" == t ]] \ + || die "--session-events-schema must name a schema holding the session store's events table in this database" +STALE_PROGRESS_EVENTS="$( + psql -X "${DATABASE_ADMIN_URL}" --set=ON_ERROR_STOP=1 --tuples-only --no-align \ + --command "SELECT count(*) FROM ${SESSION_EVENTS_SCHEMA}.events WHERE event_type = 'ExecutionProgress';" +)" + +printf 'preflight totals: nonterminal_runs=%s old_service_or_deployment_invocations=%s stale_execution_progress_events=%s\n' \ + "${NONTERMINAL_RUNS}" "${OLD_INVOCATIONS}" "${STALE_PROGRESS_EVENTS}" +printf 'resumable stage state: archive=%s old_deployment=%s migration=%s\n' \ + "${ARCHIVE_STAGE}" "${DEPLOYMENT_STAGE}" "${MIGRATION_STAGE}" [[ "${NONTERMINAL_RUNS}" == 0 ]] \ || die "terminalize or cancel every legacy execution through the old product runtime" [[ "${OLD_INVOCATIONS}" == 0 ]] \ || die "retired execution services or the exact old deployment still own nonterminal invocations" -[[ "${MIGRATION_POSITION}" == 58 ]] \ - || die "the database must be at exactly V000058 before applying the V59/V60 hard cut" + +# Stages complete in a fixed order: evidence is archived, then the retired +# deployment is removed, then the schema is cut. A later stage complete over an +# earlier one that is not means something other than this script changed the +# target, so refuse rather than resume onto an unknown state. +[[ "${DEPLOYMENT_STAGE}" != "complete" || "${ARCHIVE_STAGE}" == "complete" ]] \ + || die "the retired deployment is already removed but --archive-dir holds no matching cutover manifest" +[[ "${MIGRATION_STAGE}" != "complete" || "${DEPLOYMENT_STAGE}" == "complete" ]] \ + || die "V59/V60 are already applied while a retired execution deployment is still registered" echo "Read-only preflight passed. No state has been changed." [[ "${CONFIRMED}" == 1 ]] \ || die "rerun with --confirm-destructive-cutover after reviewing the printed evidence" echo "== Archive terminal execution evidence ==" -ARCHIVE_FILE="${ARCHIVE_DIR}/terminal-execution-before-v59.sql" -printf '%s\n' "${DEPLOYMENTS_JSON}" >"${ARCHIVE_DIR}/restate-deployments-before-cutover.json" -readonly TERMINAL_INVOCATIONS_SQL=" +if [[ "${ARCHIVE_STAGE}" == "complete" ]]; then + echo "cutover-manifest.txt already records this cutover's evidence; skipping archive" +else + ARCHIVE_FILE="${ARCHIVE_DIR}/terminal-execution-before-v59.sql" + printf '%s\n' "${DEPLOYMENTS_JSON}" >"${ARCHIVE_DIR}/restate-deployments-before-cutover.json" + TERMINAL_INVOCATIONS_SQL=" SELECT id, status, target_service_name, target_handler_name, pinned_deployment_id, last_attempt_deployment_id FROM sys_invocation WHERE target_service_name IN ('ExecutionRun', 'ExecutionTask', 'ExecutionCompensation') AND status IN ('completed', 'killed') ORDER BY id;" -restate_query "${TERMINAL_INVOCATIONS_SQL}" \ - >"${ARCHIVE_DIR}/terminal-restate-invocations-before-cutover.json" -pg_dump \ - --dbname "${DATABASE_ADMIN_URL}" \ - --data-only \ - --no-owner \ - --no-privileges \ - --table moa.execution_run \ - --table moa.execution_task \ - --table moa.execution_compensation \ - --file "${ARCHIVE_FILE}" -[[ -s "${ARCHIVE_FILE}" ]] || die "terminal execution archive is empty" -{ - printf 'old_deployment_id=%s\n' "${OLD_DEPLOYMENT_ID}" - printf 'new_deployment_uri=%s\n' "${NEW_DEPLOYMENT_URI}" - printf 'nonterminal_runs=%s\n' "${NONTERMINAL_RUNS}" - printf 'old_deployment_invocations=%s\n' "${OLD_INVOCATIONS}" - printf 'archive_bytes=%s\n' "$(wc -c <"${ARCHIVE_FILE}" | tr -d ' ')" -} >"${ARCHIVE_DIR}/cutover-manifest.txt" + restate_query "${TERMINAL_INVOCATIONS_SQL}" \ + >"${ARCHIVE_DIR}/terminal-restate-invocations-before-cutover.json" + pg_dump \ + --dbname "${DATABASE_ADMIN_URL}" \ + --data-only \ + --no-owner \ + --no-privileges \ + --table moa.execution_run \ + --table moa.execution_task \ + --table moa.execution_compensation \ + --file "${ARCHIVE_FILE}" + [[ -s "${ARCHIVE_FILE}" ]] || die "terminal execution archive is empty" + psql -X "${DATABASE_ADMIN_URL}" --set=ON_ERROR_STOP=1 --pset=pager=off \ + --command "\\copy (SELECT * FROM ${SESSION_EVENTS_SCHEMA}.events WHERE event_type = 'ExecutionProgress') TO '${ARCHIVE_DIR}/undecodable-execution-progress-events.csv' WITH (FORMAT csv, HEADER true)" + # The manifest is the resume marker, so it is written last: a directory that + # holds it has every preceding artifact in this stage. + { + printf 'old_deployment_id=%s\n' "${OLD_DEPLOYMENT_ID}" + printf 'new_deployment_uri=%s\n' "${NEW_DEPLOYMENT_URI}" + printf 'nonterminal_runs=%s\n' "${NONTERMINAL_RUNS}" + printf 'old_deployment_invocations=%s\n' "${OLD_INVOCATIONS}" + printf 'stale_execution_progress_events=%s\n' "${STALE_PROGRESS_EVENTS}" + printf 'session_events_schema=%s\n' "${SESSION_EVENTS_SCHEMA}" + printf 'archive_bytes=%s\n' "$(wc -c <"${ARCHIVE_FILE}" | tr -d ' ')" + } >"${ARCHIVE_MANIFEST}" +fi -echo "== Apply repository-owned V59/V60 migration chain ==" -( - cd -- "${REPO_ROOT}" - MOA_DATABASE_URL="${DATABASE_ADMIN_URL}" \ - MOA_DATABASE_ADMIN_URL="${DATABASE_ADMIN_URL}" \ - cargo run -p moa-orchestrator --bin moa-orchestrator-bin --locked -- migrate -) -APPLIED_MIGRATIONS="$( - psql -X "${DATABASE_ADMIN_URL}" --set=ON_ERROR_STOP=1 --tuples-only --no-align \ - --command "SELECT version, name FROM public.refinery_schema_history WHERE version IN (59, 60) ORDER BY version;" \ - | tee "${ARCHIVE_DIR}/applied-migrations.txt" -)" -[[ "${APPLIED_MIGRATIONS}" == $'59|long_horizon_execution\n60|sandbox_active_compute_capacity' ]] \ - || die "the repository runner did not record the exact V59/V60 identities" - -echo "== Reset only retired execution-service state and completed journals ==" -for service in "${OLD_SERVICES[@]}"; do - restate_cli --yes state clear "${service}" - for _attempt in $(seq 1 1000); do - TERMINAL_SERVICE_COUNT_JSON="$(restate_query " +# The retired deployment is removed before the schema is cut. Preflight proves +# current counts are zero, not that new work cannot arrive: while the old +# deployment stays registered, `Execution/start` and the retired +# ExecutionRun/ExecutionTask/ExecutionCompensation handlers remain routable, and +# after V59 they would be routable against a schema they cannot read. The +# journal purge runs first because it addresses those services by name and they +# stop existing the moment the deployment is gone. +echo "== Reset retired execution-service state and remove the exact old deployment ==" +if [[ "${DEPLOYMENT_STAGE}" == "complete" ]]; then + echo "retired deployment is already removed; skipping state reset and removal" +else + for service in "${OLD_SERVICES[@]}"; do + restate_cli --yes state clear "${service}" + for _attempt in $(seq 1 1000); do + TERMINAL_SERVICE_COUNT_JSON="$(restate_query " SELECT count(*) AS invocation_count FROM sys_invocation WHERE target_service_name = '${service}' AND status IN ('completed', 'killed');")" - TERMINAL_SERVICE_COUNT="$(jq -er '.rows[0].invocation_count | tonumber' \ - <<<"${TERMINAL_SERVICE_COUNT_JSON}")" - [[ "${TERMINAL_SERVICE_COUNT}" == 0 ]] && break - restate_cli --yes invocations purge --limit 500 "${service}" \ - >>"${ARCHIVE_DIR}/restate-invocation-purge.log" + TERMINAL_SERVICE_COUNT="$(jq -er '.rows[0].invocation_count | tonumber' \ + <<<"${TERMINAL_SERVICE_COUNT_JSON}")" + [[ "${TERMINAL_SERVICE_COUNT}" == 0 ]] && break + restate_cli --yes invocations purge --limit 500 "${service}" \ + >>"${ARCHIVE_DIR}/restate-invocation-purge.log" + done + [[ "${TERMINAL_SERVICE_COUNT}" == 0 ]] \ + || die "retired ${service} invocation history exceeded the bounded purge loop" done - [[ "${TERMINAL_SERVICE_COUNT}" == 0 ]] \ - || die "retired ${service} invocation history exceeded the bounded purge loop" -done -echo "== Remove exact old deployment and register the new immutable endpoint ==" -curl -fsS \ - -X DELETE "${RESTATE_ADMIN_URL}/deployments/${OLD_DEPLOYMENT_ID}?force=true" \ - -o "${ARCHIVE_DIR}/old-deployment-removal.json" -for _attempt in $(seq 1 60); do - DEPLOYMENTS_JSON="$(curl -fsS "${RESTATE_ADMIN_URL}/deployments")" - if ! jq -e --arg deployment_id "${OLD_DEPLOYMENT_ID}" \ + curl -fsS \ + -X DELETE "${RESTATE_ADMIN_URL}/deployments/${OLD_DEPLOYMENT_ID}?force=true" \ + -o "${ARCHIVE_DIR}/old-deployment-removal.json" + for _attempt in $(seq 1 60); do + DEPLOYMENTS_JSON="$(curl -fsS "${RESTATE_ADMIN_URL}/deployments")" + if ! jq -e --arg deployment_id "${OLD_DEPLOYMENT_ID}" \ + '.deployments[] | select(.id == $deployment_id)' \ + <<<"${DEPLOYMENTS_JSON}" >/dev/null; then + break + fi + sleep 2 + done + if jq -e --arg deployment_id "${OLD_DEPLOYMENT_ID}" \ '.deployments[] | select(.id == $deployment_id)' \ <<<"${DEPLOYMENTS_JSON}" >/dev/null; then - break + die "old deployment remained registered after the bounded removal wait" fi - sleep 2 -done -if jq -e --arg deployment_id "${OLD_DEPLOYMENT_ID}" \ - '.deployments[] | select(.id == $deployment_id)' \ - <<<"${DEPLOYMENTS_JSON}" >/dev/null; then - die "old deployment remained registered after the bounded removal wait" fi -curl -fsS \ - -X POST "${RESTATE_ADMIN_URL}/deployments" \ - -H "content-type: application/json" \ - --data-binary "$(jq -cn --arg uri "${NEW_DEPLOYMENT_URI}" '{uri: $uri}')" \ - -o "${ARCHIVE_DIR}/new-deployment-registration.json" +echo "== Apply repository-owned V59/V60 migration chain ==" +if [[ "${MIGRATION_STAGE}" == "complete" ]]; then + echo "V59/V60 already recorded with their exact identities; skipping migration" + applied_migration_identities >"${ARCHIVE_DIR}/applied-migrations.txt" +else + ( + cd -- "${REPO_ROOT}" + MOA_DATABASE_URL="${DATABASE_ADMIN_URL}" \ + MOA_DATABASE_ADMIN_URL="${DATABASE_ADMIN_URL}" \ + cargo run -p moa-orchestrator --bin moa-orchestrator-bin --locked -- migrate + ) + APPLIED_MIGRATIONS="$(applied_migration_identities | tee "${ARCHIVE_DIR}/applied-migrations.txt")" + [[ "${APPLIED_MIGRATIONS}" == "${APPLIED_MIGRATIONS_EXPECTED}" ]] \ + || die "the repository runner did not record the exact V59/V60 identities" +fi + +# `ExecutionProgress` gained ten fields, six of them required, under +# `deny_unknown_fields`. Decoding propagates rather than skips, so one historical +# row fails an entire session history replay, dashboard page, or archive read. +# Every run is already terminal here, so the progress trail has no surviving +# reader; the rows are archived as CSV above and deleted. Delete-where is +# naturally idempotent, so this stage needs no resume marker of its own. +echo "== Retire undecodable historical ExecutionProgress session events ==" +psql -X "${DATABASE_ADMIN_URL}" --set=ON_ERROR_STOP=1 --pset=pager=off \ + --command "DELETE FROM ${SESSION_EVENTS_SCHEMA}.events WHERE event_type = 'ExecutionProgress';" \ + | tee "${ARCHIVE_DIR}/retired-execution-progress-events.txt" +REMAINING_PROGRESS_EVENTS="$( + psql -X "${DATABASE_ADMIN_URL}" --set=ON_ERROR_STOP=1 --tuples-only --no-align \ + --command "SELECT count(*) FROM ${SESSION_EVENTS_SCHEMA}.events WHERE event_type = 'ExecutionProgress';" +)" +[[ "${REMAINING_PROGRESS_EVENTS}" == 0 ]] \ + || die "undecodable ExecutionProgress events remain after the retirement delete" +cat >&2 < Date: Wed, 12 Aug 2026 16:28:31 -0400 Subject: [PATCH 03/21] fix --- crates/moa-config/src/env_overlay/mod.rs | 2 + crates/moa-config/src/env_overlay/tests.rs | 4 + crates/moa-config/src/execution.rs | 36 ++ crates/moa-core/src/events.rs | 116 ++++++ crates/moa-edge/src/routes/session_stream.rs | 2 + crates/moa-execution/src/compiler/mod.rs | 17 + crates/moa-execution/src/compiler/tests.rs | 168 ++++++++ .../moa-execution/src/compiler/validation.rs | 1 + .../compiler/validation/wait_feasibility.rs | 256 ++++++++++++ .../moa-execution/src/repository/capacity.rs | 14 +- crates/moa-execution/src/repository/task.rs | 145 ++++++- .../moa-execution/src/repository/trigger.rs | 125 ++++++ crates/moa-execution/src/wire.rs | 78 +++- .../execution_db/long_horizon_state_db.rs | 366 ++++++++++++++++++ .../tests/execution_db/trigger_outbox_db.rs | 1 + .../src/objects/session/execution_runs.rs | 9 + .../src/objects/session/state.rs | 8 + .../src/services/execution_trigger.rs | 35 +- .../execution_task_attempt/active.rs | 59 +++ .../execution_task_attempt/watchdog.rs | 109 +++++- .../tests/orchestrator_offline/session_vo.rs | 25 ++ .../tests/session_db/execution_events_db.rs | 9 + crates/moa-wire/src/turn.rs | 9 + docs/23-environment-variables.md | 1 + 24 files changed, 1565 insertions(+), 30 deletions(-) create mode 100644 crates/moa-execution/src/compiler/validation/wait_feasibility.rs diff --git a/crates/moa-config/src/env_overlay/mod.rs b/crates/moa-config/src/env_overlay/mod.rs index 14f9c5783..819c79a14 100644 --- a/crates/moa-config/src/env_overlay/mod.rs +++ b/crates/moa-config/src/env_overlay/mod.rs @@ -738,6 +738,8 @@ pub struct EnvOverlay { pub execution_dispatch_batch_size: Option, /// `MOA_EXECUTION_ACTIVE_ATTEMPT_TIMEOUT_SECONDS`. pub execution_active_attempt_timeout_seconds: Option, + /// `MOA_EXECUTION_ATTEMPT_HEARTBEAT_STALENESS_SECONDS`. + pub execution_attempt_heartbeat_staleness_seconds: Option, /// `MOA_EXECUTION_MAX_TENANT_ACTIVE_RUNS`. pub execution_max_tenant_active_runs: Option, /// `MOA_EXECUTION_MAX_FLEET_ACTIVE_RUNS`. diff --git a/crates/moa-config/src/env_overlay/tests.rs b/crates/moa-config/src/env_overlay/tests.rs index b03822c59..6f84884ab 100644 --- a/crates/moa-config/src/env_overlay/tests.rs +++ b/crates/moa-config/src/env_overlay/tests.rs @@ -570,6 +570,7 @@ fn from_iter_applies_every_execution_resource_override() { ("MOA_EXECUTION_MAXIMUM_ACTIVATION_STEPS", "192"), ("MOA_EXECUTION_DISPATCH_BATCH_SIZE", "48"), ("MOA_EXECUTION_ACTIVE_ATTEMPT_TIMEOUT_SECONDS", "900"), + ("MOA_EXECUTION_ATTEMPT_HEARTBEAT_STALENESS_SECONDS", "180"), ("MOA_EXECUTION_MAX_TENANT_ACTIVE_RUNS", "120"), ("MOA_EXECUTION_MAX_FLEET_ACTIVE_RUNS", "1200"), ("MOA_EXECUTION_MAX_TENANT_ACTIVE_TASKS", "384"), @@ -611,6 +612,7 @@ fn from_iter_applies_every_execution_resource_override() { assert_eq!(config.execution.maximum_activation_steps, 192); assert_eq!(config.execution.dispatch_batch_size, 48); assert_eq!(config.execution.active_attempt_timeout_seconds, 900); + assert_eq!(config.execution.attempt_heartbeat_staleness_seconds, 180); assert_eq!(config.execution.max_tenant_active_runs, 120); assert_eq!(config.execution.max_fleet_active_runs, 1_200); assert_eq!(config.execution.max_tenant_active_tasks, 384); @@ -650,6 +652,7 @@ fn from_iter_rejects_invalid_values_for_every_execution_override() { "MOA_EXECUTION_MAXIMUM_ACTIVATION_STEPS", "MOA_EXECUTION_DISPATCH_BATCH_SIZE", "MOA_EXECUTION_ACTIVE_ATTEMPT_TIMEOUT_SECONDS", + "MOA_EXECUTION_ATTEMPT_HEARTBEAT_STALENESS_SECONDS", "MOA_EXECUTION_MAX_TENANT_ACTIVE_RUNS", "MOA_EXECUTION_MAX_FLEET_ACTIVE_RUNS", "MOA_EXECUTION_MAX_TENANT_ACTIVE_TASKS", @@ -712,6 +715,7 @@ fn execution_long_horizon_overlay_rejects_zero_and_inconsistent_limits() { "MOA_EXECUTION_MAXIMUM_ACTIVATION_STEPS", "MOA_EXECUTION_DISPATCH_BATCH_SIZE", "MOA_EXECUTION_ACTIVE_ATTEMPT_TIMEOUT_SECONDS", + "MOA_EXECUTION_ATTEMPT_HEARTBEAT_STALENESS_SECONDS", "MOA_EXECUTION_MAX_TENANT_ACTIVE_RUNS", "MOA_EXECUTION_MAX_FLEET_ACTIVE_RUNS", "MOA_EXECUTION_MAX_TENANT_ACTIVE_TASKS", diff --git a/crates/moa-config/src/execution.rs b/crates/moa-config/src/execution.rs index a77af8c5d..d7ca87375 100644 --- a/crates/moa-config/src/execution.rs +++ b/crates/moa-config/src/execution.rs @@ -27,6 +27,9 @@ pub struct ExecutionConfig { pub dispatch_batch_size: usize, /// Maximum duration of one active task attempt, in seconds. pub active_attempt_timeout_seconds: u64, + /// Interval without durable attempt progress after which an active attempt is stalled, + /// in seconds. + pub attempt_heartbeat_staleness_seconds: u64, /// Maximum non-parked execution runs admitted for one tenant. pub max_tenant_active_runs: u32, /// Maximum non-parked execution runs admitted across the fleet. @@ -91,6 +94,7 @@ impl Default for ExecutionConfig { maximum_activation_steps: 128, dispatch_batch_size: DEFAULT_MAX_IN_FLIGHT_TASKS, active_attempt_timeout_seconds: 10 * 60, + attempt_heartbeat_staleness_seconds: 2 * 60, max_tenant_active_runs: 100, max_fleet_active_runs: 1_000, max_tenant_active_tasks: 256, @@ -152,6 +156,10 @@ impl ExecutionConfig { "execution.active_attempt_timeout_seconds", self.active_attempt_timeout_seconds, ), + ( + "execution.attempt_heartbeat_staleness_seconds", + self.attempt_heartbeat_staleness_seconds, + ), ( "execution.max_tenant_active_runs", u64::from(self.max_tenant_active_runs), @@ -253,6 +261,12 @@ impl ExecutionConfig { .to_string(), )); } + if self.attempt_heartbeat_staleness_seconds >= self.active_attempt_timeout_seconds { + return Err(MoaError::ConfigError( + "execution.attempt_heartbeat_staleness_seconds must be less than execution.active_attempt_timeout_seconds because a staleness window at or beyond the attempt deadline can never classify a stall before the deadline does" + .to_string(), + )); + } if self.trigger_reconciliation_cadence_seconds > self.active_attempt_timeout_seconds { return Err(MoaError::ConfigError( "execution.trigger_reconciliation_cadence_seconds must not exceed execution.active_attempt_timeout_seconds" @@ -337,6 +351,7 @@ mod tests { maximum_activation_steps: 128, dispatch_batch_size: DEFAULT_MAX_IN_FLIGHT_TASKS, active_attempt_timeout_seconds: 10 * 60, + attempt_heartbeat_staleness_seconds: 2 * 60, max_tenant_active_runs: 100, max_fleet_active_runs: 1_000, max_tenant_active_tasks: 256, @@ -403,6 +418,27 @@ mod tests { timeout.active_attempt_timeout_seconds = timeout.maximum_horizon_seconds + 1; assert!(timeout.validate().is_err()); + let zero_heartbeat = ExecutionConfig { + attempt_heartbeat_staleness_seconds: 0, + ..ExecutionConfig::default() + }; + assert!(zero_heartbeat.validate().is_err()); + + let unreachable_heartbeat = ExecutionConfig { + attempt_heartbeat_staleness_seconds: ExecutionConfig::default() + .active_attempt_timeout_seconds, + ..ExecutionConfig::default() + }; + assert!( + unreachable_heartbeat + .validate() + .expect_err("a staleness window at the attempt deadline can never fire first") + .to_string() + .contains( + "attempt_heartbeat_staleness_seconds must be less than execution.active_attempt_timeout_seconds" + ) + ); + let mut reconciliation = ExecutionConfig::default(); reconciliation.trigger_reconciliation_cadence_seconds = reconciliation.active_attempt_timeout_seconds + 1; diff --git a/crates/moa-core/src/events.rs b/crates/moa-core/src/events.rs index fec800eb1..b1a38aa58 100644 --- a/crates/moa-core/src/events.rs +++ b/crates/moa-core/src/events.rs @@ -86,6 +86,36 @@ pub struct ExecutionRemainingBudget { pub deadline_at: Option>, } +/// Cumulative spend paired with declared goal advancement for one execution run. +/// +/// `remaining_budget` alone cannot separate a run that is spending and advancing from +/// one that is spending and stuck: it is `None` on every uncapped dimension, and a +/// reader without the approved limit cannot recover consumption from it. Reporting the +/// consumed side next to the requirement denominator makes cost-per-advance readable +/// from a single progress event. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ExecutionProgressEconomics { + /// Cumulative reconciled billed cost in integer micro-US-dollars. + pub consumed_cost_microusd: u64, + /// Cumulative reconciled model tokens. + pub consumed_tokens: u64, + /// Cumulative reconciled terminal logical tasks. + pub consumed_tasks: u64, + /// Cumulative reconciled governed tool or capability calls. + pub consumed_tool_calls: u64, + /// Cumulative reconciled bytes retrieved from external or memory sources. + pub consumed_retrieved_bytes: u64, + /// Number of requirements declared by the run's immutable goal contract. + pub requirements_total: u64, + /// Requirements evidenced as satisfied, present only once terminal evaluation has run. + /// + /// Requirement satisfaction is a whole-plan predicate over per-node terminal state, so + /// it is durable only in terminal evidence. Mid-run, the advancement denominator is + /// `requirements_total` against the event's own logical-task counts. + pub requirements_satisfied: Option, +} + /// Compact aggregate progress for one detached execution run. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -118,6 +148,12 @@ pub struct ExecutionProgress { pub blocker_audience: Option, /// Budget remaining after cumulative consumption and live reservations. pub remaining_budget: ExecutionRemainingBudget, + /// Cumulative spend paired with the declared goal-requirement denominator. + /// + /// Absent on progress events persisted before progress-per-cost was reported; a + /// reader must treat `None` as unreported rather than as zero spend. + #[serde(default)] + pub economics: Option, /// Number of materialized logical tasks. pub total: u64, /// Number of successfully completed logical tasks. @@ -1872,6 +1908,77 @@ mod tests { assert_eq!(decoded, event); } + #[test] + fn execution_progress_reports_spend_against_requirements_and_still_decodes_historical_rows() { + // Pins: one progress event carries consumed spend next to the goal-requirement + // denominator, so a reader separates "spending and advancing" from "spending and + // stuck" without joining another event; and progress rows persisted before that + // field existed still decode, as `None`, with no cutover of the session event log. + let progress = ExecutionProgress { + run_uid: Uuid::from_u128(43), + originating_user_sequence_num: 11, + plan_revision: 1, + status: "running".to_string(), + phase: ExecutionProgressPhase::Running, + waiting_since: None, + next_wake_at: None, + last_progress_at: Utc::now(), + external_job_uid: None, + ready_tasks: 2, + active_tasks: 1, + parked_tasks: 0, + blocker_audience: None, + remaining_budget: ExecutionRemainingBudget { + cost_microusd: None, + tokens: None, + tasks: None, + tool_calls: None, + retrieved_bytes: None, + deadline_at: None, + }, + economics: Some(ExecutionProgressEconomics { + consumed_cost_microusd: 7_500, + consumed_tokens: 900_000, + consumed_tasks: 6, + consumed_tool_calls: 14, + consumed_retrieved_bytes: 40_000, + requirements_total: 5, + requirements_satisfied: None, + }), + total: 12, + completed: 6, + failed: 0, + cancelled: 0, + }; + + // Spend is readable even though every remaining-budget dimension is uncapped, which + // is exactly the case where `remaining_budget` alone reports nothing. + let encoded = serde_json::to_value(&progress).expect("serialize execution progress"); + assert_eq!(encoded["economics"]["consumed_cost_microusd"], 7_500); + assert_eq!(encoded["economics"]["requirements_total"], 5); + assert_eq!(encoded["remaining_budget"]["cost_microusd"], Value::Null); + + let mut historical = encoded.clone(); + historical + .as_object_mut() + .expect("progress object") + .remove("economics") + .expect("economics field present"); + let decoded = serde_json::from_value::(historical) + .expect("historical progress row without economics must decode"); + assert_eq!(decoded.economics, None); + assert_eq!(decoded.completed, progress.completed); + + // The strict contract is otherwise unchanged: genuinely unknown fields still fail. + let mut unknown = encoded; + unknown + .as_object_mut() + .expect("progress object") + .insert("progress_per_dollar".to_string(), Value::from(1)); + serde_json::from_value::(unknown) + .expect_err("unknown progress fields must still be rejected"); + } + #[test] fn execution_run_delivery_events_round_trip_with_exact_processing_effects() { // Pins: compact execution delivery remains typed across session-event replay. @@ -1910,6 +2017,15 @@ mod tests { retrieved_bytes: Some(8_000), deadline_at: None, }, + economics: Some(ExecutionProgressEconomics { + consumed_cost_microusd: 20, + consumed_tokens: 200, + consumed_tasks: 3, + consumed_tool_calls: 6, + consumed_retrieved_bytes: 2_000, + requirements_total: 3, + requirements_satisfied: None, + }), total: 4, completed: 2, failed: 1, diff --git a/crates/moa-edge/src/routes/session_stream.rs b/crates/moa-edge/src/routes/session_stream.rs index 6a2a73c64..1abdfa557 100644 --- a/crates/moa-edge/src/routes/session_stream.rs +++ b/crates/moa-edge/src/routes/session_stream.rs @@ -764,6 +764,7 @@ mod tests { retrieved_bytes: Some(1_000), deadline_at: None, }, + economics: None, total: 1, completed: 0, failed: 0, @@ -841,6 +842,7 @@ mod tests { retrieved_bytes: Some(5_000), deadline_at: None, }, + economics: None, total: 9, completed: 4, failed: 1, diff --git a/crates/moa-execution/src/compiler/mod.rs b/crates/moa-execution/src/compiler/mod.rs index d84b81996..58c0c7e80 100644 --- a/crates/moa-execution/src/compiler/mod.rs +++ b/crates/moa-execution/src/compiler/mod.rs @@ -10,6 +10,7 @@ use validation::activation_bounds::{ validate_completion_activation_bounds, validate_plan_activation_bound, }; use validation::schema_references::{validate_declared_reference_paths, validate_schemas}; +use validation::wait_feasibility::validate_declared_wait_feasibility; use validation::{ append_artifact_reports, append_error, validate_amendment_reference_narrowing, validate_authorization, validate_catalog, validate_goal_plan_links, validate_plan_references, @@ -229,6 +230,14 @@ pub fn compile(request: CompileExecutionRequest) -> CompileExecutionOutcome { "approved_budget.deadline_at", &mut report, ); + validate_declared_wait_feasibility( + &request.plan, + None, + request.approved_budget.deadline_at, + request.now, + "approved_budget.deadline_at", + &mut report, + ); let estimate = estimate_plan( &request.goal, @@ -368,6 +377,14 @@ pub fn validate_amendment(request: ValidateAmendmentRequest) -> AmendmentValidat "remaining_budget.deadline_at", &mut report, ); + validate_declared_wait_feasibility( + &definition, + Some(&request.projection), + request.remaining_budget.deadline_at, + request.now, + "remaining_budget.deadline_at", + &mut report, + ); let full_estimate = estimate_plan( &request.goal, diff --git a/crates/moa-execution/src/compiler/tests.rs b/crates/moa-execution/src/compiler/tests.rs index 2d2e7412a..d1cb7d38c 100644 --- a/crates/moa-execution/src/compiler/tests.rs +++ b/crates/moa-execution/src/compiler/tests.rs @@ -221,6 +221,174 @@ fn execution_planning_compiler_rejects_verifiers_over_dispatch_batch() { ); } +#[test] +fn execution_planning_compiler_admits_wait_chain_inside_deadline() { + // Pins: declared-wait feasibility rejects only chains that cannot fit; a plan whose + // sequential waits sum to less than the horizon still compiles. + let request = wait_chain_compile_request(&[2, 2, 2], 7); + + let outcome = compile(request); + + assert!( + outcome.report.issues.is_empty(), + "feasible wait chain should compile cleanly: {:?}", + outcome.report.issues + ); + assert!(outcome.compiled.is_some()); +} + +#[test] +fn execution_planning_compiler_rejects_sequential_waits_beyond_deadline() { + // Pins: three sequential three-day waits inside a seven-day run each fit the horizon + // individually while the chain they form needs nine days; only summing along the path + // catches it before the run burns six days and dies with partial output. + let request = wait_chain_compile_request(&[3, 3, 3], 7); + + let outcome = compile(request); + + assert!(outcome.compiled.is_none()); + let issue = outcome + .report + .issues + .iter() + .find(|issue| issue.code == "declared_waits_exceed_deadline") + .expect("sequential wait chain should be rejected as infeasible"); + assert_eq!(issue.path, "plan.nodes[2]"); + assert!( + issue.message.contains("`wait_0` -> `wait_1` -> `wait_2`"), + "message must name the offending chain: {}", + issue.message + ); + assert!( + issue.message.contains("777600 seconds"), + "message must state the chain total: {}", + issue.message + ); + assert!( + outcome + .report + .issues + .iter() + .all(|issue| issue.code != "temporal_target_after_deadline"), + "each individual wait fits the horizon, so only the path sum may reject this plan" + ); +} + +#[test] +fn execution_planning_compiler_rejects_single_wait_beyond_deadline() { + // Pins: the per-wait horizon rule survives the path check; one oversized wait is still + // rejected at the wait that declares it. + let request = wait_chain_compile_request(&[9], 7); + + let outcome = compile(request); + + assert!(outcome.compiled.is_none()); + assert!( + outcome + .report + .issues + .iter() + .any(|issue| issue.code == "temporal_target_after_deadline" + && issue.path == "plan.nodes[0].operation.wake") + ); +} + +#[test] +fn execution_planning_amendment_rejects_added_wait_beyond_deadline() { + // Pins: an amendment cannot append a wait that makes the remaining chain overrun the + // narrowed deadline, and waits already served by completed nodes are not recharged. + let request = wait_chain_compile_request(&[3, 3], 9); + let compiled = compile(request.clone()) + .compiled + .expect("feasible wait chain should compile"); + let now = request.now; + let outcome = validate_amendment(ValidateAmendmentRequest { + goal: compiled.goal, + active_plan: compiled.plan, + amendment: PlanAmendment { + base_plan_revision: 1, + reason: "Await one more downstream confirmation".to_string(), + evidence: json!({ "failure": "none" }), + operations: vec![PlanAmendmentOperation::AddNode { + node: wait_node("wait_2", Some("wait_1"), 3), + }], + }, + projection: ExecutionAmendmentProjection { + plan_revision: 1, + node_statuses: BTreeMap::from([ + ("wait_0".to_string(), ExecutionNodeStatus::Completed), + ("wait_1".to_string(), ExecutionNodeStatus::Pending), + ("output".to_string(), ExecutionNodeStatus::Pending), + ]), + started_node_ids: BTreeSet::from(["wait_0".to_string()]), + replan_tasks: Vec::new(), + }, + catalog: request.catalog, + authorization: request.authorization, + remaining_budget: ExecutionBudgetLimit { + deadline_at: Some(now + chrono::Duration::days(5)), + ..generous_budget() + }, + config: ExecutionConfig::default(), + now, + }); + + assert!(outcome.plan.is_none()); + let issue = outcome + .report + .issues + .iter() + .find(|issue| issue.code == "declared_waits_exceed_deadline") + .expect("amendment must not introduce an infeasible wait chain"); + assert!( + issue.message.contains("518400 seconds"), + "the completed wait must not be recharged to the remaining chain: {}", + issue.message + ); +} + +fn wait_chain_compile_request(delay_days: &[u64], horizon_days: i64) -> CompileExecutionRequest { + let mut request = output_only_compile_request(); + let mut nodes = Vec::with_capacity(delay_days.len() + 1); + let mut previous = None; + for (index, delay) in delay_days.iter().enumerate() { + let id = format!("wait_{index}"); + nodes.push(wait_node(&id, previous.as_deref(), *delay)); + previous = Some(id); + } + let mut output = request.plan.nodes[0].clone(); + output.depends_on = previous.into_iter().collect(); + nodes.push(output); + + request.plan.nodes = nodes; + request.approved_budget.deadline_at = Some(request.now + chrono::Duration::days(horizon_days)); + request +} + +fn wait_node(id: &str, depends_on: Option<&str>, delay_days: u64) -> ExecutionNode { + ExecutionNode { + id: id.to_string(), + requirement_ids: vec!["req_report".to_string()], + depends_on: depends_on.map(ToString::to_string).into_iter().collect(), + when: None, + input: json!({}), + output_schema: json!({ "type": "object" }), + operation: ExecutionOperation::WaitUntil { + wake: ExecutionTemporalTarget::After { + delay_seconds: delay_days * 24 * 60 * 60, + }, + result: json!({}), + }, + compensation: None, + retry: RetryPolicy { + max_attempts: 1, + initial_backoff_ms: 0, + max_backoff_ms: 0, + }, + budget: None, + } +} + fn output_only_compile_request() -> CompileExecutionRequest { let catalog = ExecutionCapabilityCatalog::build(Vec::new()) .expect("empty capability catalog should be valid"); diff --git a/crates/moa-execution/src/compiler/validation.rs b/crates/moa-execution/src/compiler/validation.rs index 114d42fbd..d5e45f4cb 100644 --- a/crates/moa-execution/src/compiler/validation.rs +++ b/crates/moa-execution/src/compiler/validation.rs @@ -2,6 +2,7 @@ pub(super) mod activation_bounds; pub(super) mod schema_references; +pub(super) mod wait_feasibility; use self::schema_references::validate_one_schema; use super::*; diff --git a/crates/moa-execution/src/compiler/validation/wait_feasibility.rs b/crates/moa-execution/src/compiler/validation/wait_feasibility.rs new file mode 100644 index 000000000..552138cf7 --- /dev/null +++ b/crates/moa-execution/src/compiler/validation/wait_feasibility.rs @@ -0,0 +1,256 @@ +//! Longest declared-wait path feasibility against the run deadline. +//! +//! `validate_temporal_target` checks each wait *individually* against the whole +//! remaining horizon, which is a necessary condition and nothing more. Three +//! sequential three-day waits inside a seven-day run each pass that check while the +//! chain they form needs nine days: the run is admitted, burns six days, and dies at +//! `deadline_at` with partial output. This pass closes that gap by relaxing the +//! declared waits along the plan DAG and rejecting the plan when the longest chain +//! cannot fit inside the remaining horizon. + +use std::collections::{HashMap, HashSet}; + +use chrono::{DateTime, Utc}; +use moa_artifacts::execution_plan::{ + ExecutionNode, ExecutionOperation, ExecutionPlanDefinition, ExecutionTemporalTarget, +}; + +use crate::{ + compiler::ExecutionValidationReport, + state::{ExecutionAmendmentProjection, ExecutionNodeStatus}, +}; + +/// Rejects a plan whose longest chain of declared waits cannot fit before `deadline_at`. +/// +/// Only *declared* waiting time is summed, because only declared waiting time is exact: +/// +/// - `WaitUntil { wake: After { delay_seconds } }` is resolved against the clock at wait +/// entry, so it adds `delay_seconds` to whenever the node becomes ready; +/// - `WaitUntil { wake: At { at } }` fires at an absolute instant, so it does not add to +/// its predecessors at all — it pins the chain to `at`, whichever is later; +/// - a `Review` or `WaitSignal` contributes its `wait_policy.expiry`, the worst case +/// before the wait settles itself. Feasibility has to hold when nobody responds, and it +/// holds the same way whether expiry fails the task or continues with a declared output. +/// +/// Every other operation contributes zero. Active work has no duration input anywhere in +/// the compiler — the capability catalog carries no latency metadata — so estimating it +/// would encode a guess as an admission gate. The bound this pass computes is therefore a +/// lower bound on elapsed time, and it rejects only plans that provably cannot finish. +/// +/// Two further contingent waits are deliberately excluded, because counting them would +/// reject plans that are feasible on every execution that does not hit them: +/// +/// - `plan.input_wait_policy.expiry`, which settles whichever task returned `NeedsInput`. +/// It applies to no node in particular and to every node in principle, so charging it +/// per node would inflate the path by the node count on plans that never ask for input. +/// - `RetryPolicy` backoff, which is millisecond-scale, contingent on failure, and +/// already multiplied into the resource estimate rather than the schedule. +/// +/// `Map` and `Reduce` need no special handling: `MapTask` and `ExecutionReducer` admit +/// only capability and agent work, so the DSL cannot express a wait inside a map item or +/// a reducer batch. Their declared wait is zero by construction, not by approximation, +/// and the concurrency of map items never has to be reasoned about here. +/// +/// A conditional node counts toward the path even though a false condition would skip it. +/// The taken branch really does have to wait, so a plan admitted on the assumption the +/// branch is skipped can still overrun; and the existing per-wait rule already validates +/// conditional nodes without consulting `when`, so excluding them here would make the two +/// rules disagree about the same wait. The cost of counting them is a plan rejected at +/// compile time with the offending chain named, which the author can restructure. The +/// cost of not counting them is the failure this pass exists to prevent. +pub(in crate::compiler) fn validate_declared_wait_feasibility( + plan: &ExecutionPlanDefinition, + projection: Option<&ExecutionAmendmentProjection>, + deadline_at: Option>, + now: DateTime, + deadline_path: &str, + report: &mut ExecutionValidationReport, +) { + // A missing, elapsed, or out-of-horizon deadline is already reported by + // `validate_temporal_contract`, and no remaining horizon exists to measure against. + let Some(deadline_at) = deadline_at else { + return; + }; + let Ok(remaining) = deadline_at.signed_duration_since(now).to_std() else { + return; + }; + let remaining_seconds = remaining.as_secs(); + + let Some(offsets) = relax_declared_waits(plan, projection, now) else { + return; + }; + + // Report the chain at the node that *declares* its last wait rather than at a + // downstream node that merely inherits the same offset: that is where an author has + // something to change. Authoring order breaks the remaining ties deterministically. + let mut critical: Option<(usize, &ExecutionNode, u64)> = None; + for (index, node) in plan.nodes.iter().enumerate() { + let Some(offset) = offsets.get(node.id.as_str()) else { + continue; + }; + let key = (offset.finish_seconds, declared_wait(node).is_some()); + if critical.is_none_or(|(_, best, seconds)| key > (seconds, declared_wait(best).is_some())) + { + critical = Some((index, node, offset.finish_seconds)); + } + } + + let Some((index, node, finish_seconds)) = + critical.filter(|(_, _, seconds)| *seconds >= remaining_seconds) + else { + return; + }; + report.error( + "declared_waits_exceed_deadline", + format!("plan.nodes[{index}]"), + format!( + "declared waits along `{}` total {finish_seconds} seconds, which does not fit the \ + {remaining_seconds} seconds remaining before `{deadline_path}`", + render_chain(&offsets, node.id.as_str()), + ), + ); +} + +/// Earliest instant, in seconds after `now`, at which one node's declared waits can be over. +struct WaitOffset<'a> { + /// Seconds after `now` before this node's own declared wait can have elapsed. + finish_seconds: u64, + /// Dependency that forced this node's start, used to reconstruct the chain. + predecessor: Option<&'a str>, +} + +/// Relaxes declared waits over the plan DAG in topological order. +/// +/// Returns `None` when the dependencies are cyclic, which +/// [`moa_artifacts::validation::validate_execution_plan_definition`] already reports as a +/// structural error; this pass has nothing to add to it. +fn relax_declared_waits<'a>( + plan: &'a ExecutionPlanDefinition, + projection: Option<&ExecutionAmendmentProjection>, + now: DateTime, +) -> Option>> { + let node_ids = plan + .nodes + .iter() + .map(|node| node.id.as_str()) + .collect::>(); + let mut offsets = HashMap::<&str, WaitOffset<'_>>::with_capacity(plan.nodes.len()); + let mut pending = plan.nodes.iter().collect::>(); + + // Kahn ordering by repeated sweeps: `maximum_activation_steps` caps the node count at + // a small constant, so the quadratic worst case costs less than building an index. + while !pending.is_empty() { + let mut progressed = false; + let mut deferred = Vec::with_capacity(pending.len()); + for node in pending { + let ready = node + .depends_on + .iter() + .all(|id| !node_ids.contains(id.as_str()) || offsets.contains_key(id.as_str())); + if !ready { + deferred.push(node); + continue; + } + progressed = true; + let offset = node_offset(node, &offsets, projection, now); + offsets.insert(node.id.as_str(), offset); + } + if !progressed { + return None; + } + pending = deferred; + } + + Some(offsets) +} + +/// Computes one node's finish offset from its dependencies and its own declared wait. +fn node_offset<'a>( + node: &'a ExecutionNode, + offsets: &HashMap<&'a str, WaitOffset<'a>>, + projection: Option<&ExecutionAmendmentProjection>, + now: DateTime, +) -> WaitOffset<'a> { + let mut start_seconds = 0_u64; + let mut predecessor = None; + for dependency in &node.depends_on { + let Some(offset) = offsets.get(dependency.as_str()) else { + continue; + }; + if offset.finish_seconds > start_seconds || predecessor.is_none() { + start_seconds = offset.finish_seconds; + predecessor = Some(dependency.as_str()); + } + } + + let finish_seconds = match declared_wait(node).filter(|_| !is_settled(projection, &node.id)) { + // A relative delay starts running when the node is entered, so it stacks on top of + // everything the node waited for first. + Some(ExecutionTemporalTarget::After { delay_seconds }) => { + start_seconds.saturating_add(*delay_seconds) + } + // An absolute instant does not stack: the timer is due at `at` however early the + // node became ready, and arriving after `at` costs nothing further. + Some(ExecutionTemporalTarget::At { at }) => start_seconds.max( + at.signed_duration_since(now) + .to_std() + .map(|duration| duration.as_secs()) + .unwrap_or_default(), + ), + None => start_seconds, + }; + + WaitOffset { + finish_seconds, + predecessor, + } +} + +/// Returns the temporal target one node declares as its own waiting time, if any. +fn declared_wait(node: &ExecutionNode) -> Option<&ExecutionTemporalTarget> { + match &node.operation { + ExecutionOperation::WaitUntil { wake, .. } => Some(wake), + ExecutionOperation::Review { wait_policy, .. } + | ExecutionOperation::WaitSignal { wait_policy, .. } => Some(&wait_policy.expiry), + ExecutionOperation::Capability { .. } + | ExecutionOperation::Agent { .. } + | ExecutionOperation::Map { .. } + | ExecutionOperation::Reduce { .. } + | ExecutionOperation::Output { .. } => None, + } +} + +/// Reports whether an amendment's projection shows a node's wait as already served. +/// +/// A node still `Running` or `Waiting` keeps its full declared wait, because the portion +/// already elapsed is not knowable from the projection and over-counting it is the safe +/// direction. Only terminal nodes drop out, exactly as they do from the remaining-resource +/// estimate. +fn is_settled(projection: Option<&ExecutionAmendmentProjection>, node_id: &str) -> bool { + projection.is_some_and(|projection| { + projection.node_statuses.get(node_id).is_some_and(|status| { + matches!( + status, + ExecutionNodeStatus::Completed + | ExecutionNodeStatus::Skipped + | ExecutionNodeStatus::Failed + | ExecutionNodeStatus::Cancelled + ) + }) + }) +} + +/// Renders the dependency chain that produced one node's offset, oldest node first. +fn render_chain(offsets: &HashMap<&str, WaitOffset<'_>>, terminal: &str) -> String { + let mut chain = vec![terminal]; + let mut current = terminal; + while let Some(previous) = offsets.get(current).and_then(|offset| offset.predecessor) { + if chain.contains(&previous) { + break; + } + chain.push(previous); + current = previous; + } + chain.reverse(); + chain.join("` -> `") +} diff --git a/crates/moa-execution/src/repository/capacity.rs b/crates/moa-execution/src/repository/capacity.rs index 36c3a9757..5a76a0e9e 100644 --- a/crates/moa-execution/src/repository/capacity.rs +++ b/crates/moa-execution/src/repository/capacity.rs @@ -13,6 +13,7 @@ use super::{ rows::*, run::active_run_capacity_request, sql::*, + task::attempt_heartbeat_staleness_window, trigger::{ExecutionTriggerKind, NewExecutionTrigger, create_trigger_with_dispatch_in_conn}, }; @@ -293,6 +294,17 @@ impl ExecutionRepository { .ok_or_else(|| Error::InvalidRepositoryInput { message: "active attempt deadline is not representable".to_string(), })?; + // The watchdog is armed at the first staleness observation rather than the deadline, so a + // wedged attempt is caught one staleness window after it stops committing durable steps + // instead of after its whole authorized window. `min` is load-bearing: the deadline stays + // the hard backstop and the watchdog can never be armed beyond it. + let watchdog_due_at = deadline.min( + now.checked_add_signed(attempt_heartbeat_staleness_window(config)?) + .ok_or_else(|| Error::InvalidRepositoryInput { + message: "active attempt heartbeat observation is not representable" + .to_string(), + })?, + ); let retry_after = now .checked_add_signed(Duration::seconds( i64::try_from(config.trigger_reconciliation_cadence_seconds).map_err(|_| { @@ -426,7 +438,7 @@ impl ExecutionRepository { compensation_attempt_generation: None, schedule_incarnation: None, occurrence_sequence: None, - due_at: deadline, + due_at: watchdog_due_at, payload: json!({}), }, ) diff --git a/crates/moa-execution/src/repository/task.rs b/crates/moa-execution/src/repository/task.rs index 160cb9aa3..12828fc3a 100644 --- a/crates/moa-execution/src/repository/task.rs +++ b/crates/moa-execution/src/repository/task.rs @@ -88,8 +88,82 @@ pub struct TaskAttemptRecord { pub task: ExecutionTaskRecord, } -/// Result of recording durable progress for one active attempt. +/// Liveness of one active attempt, observed from its persisted deadline and progress. +/// +/// The two failure classes are deliberately distinct. `DeadlineExceeded` means the attempt +/// consumed its whole authorized window; `Stalled` means the attempt is still inside that +/// window but has not committed a durable step within the configured heartbeat interval, so a +/// wedged model call or tool no longer has to burn the full window before it is observable. #[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ActiveAttemptLiveness { + /// The attempt is inside its deadline and reported durable progress recently. + Live, + /// The attempt is inside its deadline but has not reported progress within the window. + Stalled, + /// The attempt reached the absolute deadline frozen by admission. + DeadlineExceeded, +} + +impl ActiveAttemptLiveness { + /// Reports whether this observation must terminate the attempt. + #[must_use] + pub const fn is_expired(self) -> bool { + matches!(self, Self::Stalled | Self::DeadlineExceeded) + } + + /// Stable label for durable messages and telemetry. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Live => "live", + Self::Stalled => "stalled", + Self::DeadlineExceeded => "deadline_exceeded", + } + } +} + +/// Returns the configured heartbeat staleness window as a chrono duration. +/// +/// Shared by watchdog arming, watchdog deferral, and liveness classification so all three +/// derive the same window from one configured value. +pub fn attempt_heartbeat_staleness_window(config: &ExecutionConfig) -> Result { + i64::try_from(config.attempt_heartbeat_staleness_seconds) + .ok() + .and_then(chrono::TimeDelta::try_seconds) + .ok_or_else(|| Error::InvalidRepositoryInput { + message: "attempt heartbeat staleness window exceeds chrono duration".to_string(), + }) +} + +/// Classifies one active attempt from its admission deadline and last durable progress. +/// +/// The deadline is evaluated first so the pre-existing absolute-deadline behaviour is +/// unchanged; heartbeat staleness only ever classifies an attempt that is still inside its +/// deadline. A staleness window is only meaningful when it is shorter than the attempt +/// timeout, which [`moa_config::ExecutionConfig::validate`] enforces. +#[must_use] +pub fn classify_active_attempt_liveness( + config: &ExecutionConfig, + attempt_deadline_at: DateTime, + last_progress_at: DateTime, + observed_at: DateTime, +) -> ActiveAttemptLiveness { + if attempt_deadline_at <= observed_at { + return ActiveAttemptLiveness::DeadlineExceeded; + } + let Ok(staleness) = attempt_heartbeat_staleness_window(config) else { + // An unrepresentable window can never elapse, so the deadline stays the only authority. + return ActiveAttemptLiveness::Live; + }; + match last_progress_at.checked_add_signed(staleness) { + Some(stale_at) if stale_at <= observed_at => ActiveAttemptLiveness::Stalled, + Some(_) => ActiveAttemptLiveness::Live, + None => ActiveAttemptLiveness::Live, + } +} + +/// Result of recording durable progress for one active attempt. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] pub enum TaskAttemptProgressOutcome { /// The exact active attempt advanced its progress timestamp. Applied, @@ -5154,12 +5228,14 @@ impl ExecutionRepository { #[cfg(test)] mod tests { use super::{ - TaskAttemptCheckpointKind, TaskAttemptFence, UnstartedTaskAttemptDisposition, - append_agent_resume_input, external_start_checkpoint_payload_is_provisional, + ActiveAttemptLiveness, TaskAttemptCheckpointKind, TaskAttemptFence, + UnstartedTaskAttemptDisposition, append_agent_resume_input, + classify_active_attempt_liveness, external_start_checkpoint_payload_is_provisional, paused_task_attempt_release_history_matches, task_release_receipt_is_verified_absence, unstarted_task_attempt_history_matches, }; - use chrono::Utc; + use chrono::{Duration, Utc}; + use moa_config::ExecutionConfig; use moa_core::types::context::ContextMessage; use moa_core::types::{ identifiers::{ExecutionRunScopeId, ExecutionTaskScopeId, TenantId}, @@ -5415,4 +5491,65 @@ mod tests { receipt.writer_epoch = Some(1); assert!(!task_release_receipt_is_verified_absence(&receipt)); } + + #[test] + fn attempt_liveness_separates_a_stall_from_a_slow_but_progressing_attempt_offline() { + // Pins: an attempt that keeps committing durable steps stays live for its whole + // authorized window, a wedged attempt inside that window is classified stalled well + // before the deadline, and the absolute deadline still outranks the heartbeat window. + let config = ExecutionConfig { + attempt_heartbeat_staleness_seconds: 60, + active_attempt_timeout_seconds: 600, + ..ExecutionConfig::default() + }; + let started_at = Utc::now(); + let deadline = started_at + Duration::seconds(600); + + // Nine minutes in, having reported progress thirty seconds ago. + let observed_at = started_at + Duration::seconds(540); + assert_eq!( + classify_active_attempt_liveness( + &config, + deadline, + observed_at - Duration::seconds(30), + observed_at, + ), + ActiveAttemptLiveness::Live + ); + + // Same instant, but the attempt has committed nothing since it started. + assert_eq!( + classify_active_attempt_liveness(&config, deadline, started_at, observed_at), + ActiveAttemptLiveness::Stalled + ); + + // The stall is visible nine minutes before the deadline would have exposed it. + assert_eq!( + classify_active_attempt_liveness( + &config, + deadline, + started_at, + started_at + Duration::seconds(60), + ), + ActiveAttemptLiveness::Stalled + ); + assert_eq!( + classify_active_attempt_liveness( + &config, + deadline, + started_at, + started_at + Duration::seconds(59), + ), + ActiveAttemptLiveness::Live + ); + + // A progressing attempt that reaches its deadline is deadline-exceeded, never stalled. + assert_eq!( + classify_active_attempt_liveness(&config, deadline, deadline, deadline), + ActiveAttemptLiveness::DeadlineExceeded + ); + assert!(!ActiveAttemptLiveness::Live.is_expired()); + assert!(ActiveAttemptLiveness::Stalled.is_expired()); + assert!(ActiveAttemptLiveness::DeadlineExceeded.is_expired()); + } } diff --git a/crates/moa-execution/src/repository/trigger.rs b/crates/moa-execution/src/repository/trigger.rs index 61f8634ed..887a297d1 100644 --- a/crates/moa-execution/src/repository/trigger.rs +++ b/crates/moa-execution/src/repository/trigger.rs @@ -275,6 +275,19 @@ pub enum ExecutionWatchdogTriggerOutcome { NoOp(ExecutionTriggerNoOp), } +/// Result of rearming one live task watchdog for its next staleness observation. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ExecutionWatchdogDeferOutcome { + /// The same immutable watchdog was rearmed for a strictly later observation. + Deferred { + /// New absolute observation time, never beyond the attempt deadline. + next_due_at: DateTime, + }, + /// The watchdog must stay due: the attempt is already stale, at its deadline, superseded, + /// or not a live task attempt. + NotDeferred, +} + #[derive(Deserialize)] #[serde(deny_unknown_fields)] struct ExternalReconcileTriggerPayload { @@ -1073,6 +1086,118 @@ impl ExecutionRepository { )) } + /// Rearms one live task watchdog for its next heartbeat-staleness observation. + /// + /// The watchdog is armed at a staleness window rather than the attempt deadline, so a live + /// attempt is observed repeatedly and must be pushed forward each time it proves progress. + /// The next observation is `min(attempt_deadline_at, last_progress_at + staleness)`, so the + /// deadline remains the hard backstop and deferral can never postpone it. + /// + /// This is a rearm, not a supersede: the trigger stays `pending` and keeps its + /// `scheduled_triggers` capacity receipt, and the existing delivery row is rewritten in place + /// rather than released and recreated. Every failure to establish a strictly later, still + /// live observation returns [`ExecutionWatchdogDeferOutcome::NotDeferred`] and leaves the + /// watchdog due, which preserves the pre-existing retry behaviour exactly. + pub async fn defer_task_attempt_watchdog( + &self, + scope: ExecutionScope, + config: &ExecutionConfig, + trigger_uid: Uuid, + ) -> Result { + let staleness = crate::repository::task::attempt_heartbeat_staleness_window(config)?; + let mut conn = scope.begin(&self.pool).await?; + prelock_trigger_scheduled_capacity_in_conn(conn.as_mut(), trigger_uid).await?; + let row = + sqlx::query("SELECT * FROM moa.execution_trigger WHERE trigger_uid=$1 FOR UPDATE") + .bind(trigger_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(row) = row else { + conn.commit().await.map_err(storage_error)?; + return Ok(ExecutionWatchdogDeferOutcome::NotDeferred); + }; + let trigger = trigger_from_row(&row)?; + // A non-task watchdog is not an error here. Trigger delivery routes task and compensation + // watchdogs through one retry branch, and compensation attempts carry no heartbeat, so the + // safe answer for anything but a live task watchdog is to leave the trigger alone. + if trigger.kind != ExecutionTriggerKind::TaskWatchdog + || !matches!( + trigger.state, + ExecutionDeliveryState::Pending | ExecutionDeliveryState::Dispatching + ) + || !trigger_is_current(conn.as_mut(), &trigger).await? + { + conn.commit().await.map_err(storage_error)?; + return Ok(ExecutionWatchdogDeferOutcome::NotDeferred); + } + let progress = sqlx::query_as::<_, (Option>, DateTime, DateTime)>( + "SELECT attempt_deadline_at, last_progress_at, now() FROM moa.execution_task \ + WHERE tenant_id=$1 AND run_uid=$2 AND task_id=$3 AND attempt_generation=$4 \ + AND active_dispatch_uid IS NOT NULL", + ) + .bind(trigger.tenant_id.0) + .bind(trigger.run_uid) + .bind(trigger.task_id) + .bind(to_optional_i64( + trigger.attempt_generation, + "attempt generation", + )?) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some((Some(attempt_deadline_at), last_progress_at, observed_at)) = progress else { + conn.commit().await.map_err(storage_error)?; + return Ok(ExecutionWatchdogDeferOutcome::NotDeferred); + }; + let Some(next_due_at) = last_progress_at + .checked_add_signed(staleness) + .map(|stale_at| stale_at.min(attempt_deadline_at)) + else { + conn.commit().await.map_err(storage_error)?; + return Ok(ExecutionWatchdogDeferOutcome::NotDeferred); + }; + // Strictly forward only. An observation at or before now would re-fire immediately, and one + // at or before the current arm would not move detection at all. + if next_due_at <= observed_at || next_due_at <= trigger.due_at { + conn.commit().await.map_err(storage_error)?; + return Ok(ExecutionWatchdogDeferOutcome::NotDeferred); + } + // Restate journals trigger delivery by dispatch UID, so a rearm needs a new delivery + // identity or the next claim would replay the completed RetryDelivery invocation. + let next_dispatch_uid = rearmed_trigger_delivery_dispatch_uid(trigger_uid, next_due_at); + sqlx::query( + "UPDATE moa.execution_trigger SET state='pending', due_at=$2, delivered_at=NULL, \ + updated_at=now() WHERE trigger_uid=$1", + ) + .bind(trigger_uid) + .bind(next_due_at) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let rearmed = sqlx::query( + "UPDATE moa.execution_dispatch_outbox SET dispatch_uid=$3,state='pending', \ + not_before_at=$2,delivery_attempts=0, \ + claim_owner=NULL,claimed_at=NULL,claim_expires_at=NULL,delivered_at=NULL, \ + updated_at=now() \ + WHERE trigger_uid=$1 AND dispatch_kind='trigger_delivery' \ + AND state IN ('pending','dispatching','delivered')", + ) + .bind(trigger_uid) + .bind(next_due_at) + .bind(next_dispatch_uid) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if rearmed.rows_affected() != 1 { + return Err(Error::InvalidRepositoryData { + message: "task watchdog trigger is missing its exact delivery outbox".to_string(), + }); + } + conn.commit().await.map_err(storage_error)?; + Ok(ExecutionWatchdogDeferOutcome::Deferred { next_due_at }) + } + /// Supersedes one run deadline trigger after its bounded terminal fence is durable. pub async fn settle_run_deadline_trigger( &self, diff --git a/crates/moa-execution/src/wire.rs b/crates/moa-execution/src/wire.rs index cf2b74832..4acf1c227 100644 --- a/crates/moa-execution/src/wire.rs +++ b/crates/moa-execution/src/wire.rs @@ -11,8 +11,8 @@ use moa_artifacts::{ use moa_core::events::Event; use moa_core::events::{ ExecutionBlockerAudience, ExecutionFailureDisposition, ExecutionProgress, - ExecutionProgressPhase, ExecutionRemainingBudget, ExecutionTaskResultsRef, - ExecutionTerminalSummary, + ExecutionProgressEconomics, ExecutionProgressPhase, ExecutionRemainingBudget, + ExecutionTaskResultsRef, ExecutionTerminalSummary, }; use moa_core::types::{ contact::ContactId, @@ -27,7 +27,10 @@ use uuid::Uuid; use crate::{ Error, Result, budget::BudgetLedger, - capability::{ExecutionAuthorizationEnvelope, ExecutionCapabilityCatalog, ExecutionHash}, + capability::{ + ExecutionAuthorizationEnvelope, ExecutionCapabilityCatalog, ExecutionEstimate, + ExecutionHash, + }, compiler::CompiledExecution, state::{ CompensationId, ExecutionProjection, ExecutionRunStatus, ExecutionSourceKind, @@ -696,6 +699,11 @@ pub fn execution_progress_from_run( retrieved_bytes: remaining.max_retrieved_bytes, deadline_at: remaining.deadline_at, }, + economics: Some(execution_progress_economics( + &run.consumed, + run.goal.requirements.len(), + run.terminal_evidence.as_ref(), + )?), total: run.progress_total_tasks, completed: run.progress_completed_tasks, failed: run.progress_failed_tasks, @@ -703,6 +711,32 @@ pub fn execution_progress_from_run( }) } +/// Relates reconciled spend to the goal-requirement denominator for one progress event. +/// +/// Every input is already resident on the loaded run row, so the projection adds no query +/// to the progress hot path. `requirements_satisfied` is durable only in terminal evidence, +/// so it stays `None` for the whole active life of the run. +fn execution_progress_economics( + consumed: &ExecutionEstimate, + requirement_count: usize, + terminal_evidence: Option<&ExecutionTerminalEvidence>, +) -> Result { + Ok(ExecutionProgressEconomics { + consumed_cost_microusd: consumed.cost_microusd, + consumed_tokens: consumed.tokens, + consumed_tasks: consumed.tasks, + consumed_tool_calls: consumed.tool_calls, + consumed_retrieved_bytes: consumed.retrieved_bytes, + requirements_total: u64::try_from(requirement_count).map_err(|_| { + Error::ArithmeticOverflow { + context: "execution progress requirement count".to_string(), + } + })?, + requirements_satisfied: terminal_evidence + .map(|evidence| evidence.satisfied_requirement_count), + }) +} + fn execution_blocker_audience( run: &crate::repository::ExecutionRunRecord, ) -> Option { @@ -1537,6 +1571,44 @@ fn invalid_cursor(message: &str) -> Error { mod tests { use super::*; + #[test] + fn execution_progress_economics_reports_consumed_spend_against_requirements_offline() { + // Pins: progress projects reconciled spend and the goal-requirement denominator from + // the already-loaded run row, and only claims satisfied requirements once terminal + // evidence exists, so a mid-run reader is never told a stuck run satisfied anything. + let consumed = ExecutionEstimate { + cost_microusd: 12_500, + tokens: 1_400_000, + tool_calls: 31, + retrieved_bytes: 640_000, + tasks: 9, + }; + + let active = + execution_progress_economics(&consumed, 4, None).expect("project active economics"); + assert_eq!(active.consumed_cost_microusd, 12_500); + assert_eq!(active.consumed_tokens, 1_400_000); + assert_eq!(active.consumed_tasks, 9); + assert_eq!(active.consumed_tool_calls, 31); + assert_eq!(active.consumed_retrieved_bytes, 640_000); + assert_eq!(active.requirements_total, 4); + assert_eq!(active.requirements_satisfied, None); + + let evidence = ExecutionTerminalEvidence { + cause: crate::state::ExecutionTerminalCause::Completion { limit_stop: None }, + satisfied_requirement_count: 3, + requirement_count: 4, + }; + let terminal = execution_progress_economics(&consumed, 4, Some(&evidence)) + .expect("project terminal economics"); + assert_eq!(terminal.requirements_satisfied, Some(3)); + assert_eq!(terminal.requirements_total, 4); + assert_eq!( + terminal.consumed_cost_microusd, active.consumed_cost_microusd, + "terminal evaluation must not restate spend" + ); + } + #[test] fn execution_progress_phase_exhaustively_maps_aggregate_wait_and_pause_states_offline() { // Pins: run-only progress distinguishes every public storage-only wait and pause phase; diff --git a/crates/moa-execution/tests/execution_db/long_horizon_state_db.rs b/crates/moa-execution/tests/execution_db/long_horizon_state_db.rs index 1e7ffecb2..c08cfc955 100644 --- a/crates/moa-execution/tests/execution_db/long_horizon_state_db.rs +++ b/crates/moa-execution/tests/execution_db/long_horizon_state_db.rs @@ -1,5 +1,15 @@ //! Long-horizon execution state, identity, RLS, and generation-fence contracts. +use moa_artifacts::execution_plan::{ExecutionNode, ExecutionOperation}; +use moa_execution::repository::ready::{ReadyMaterializationOutcome, ReadyMaterializationRequest}; +use moa_execution::repository::task::{ + ActiveAttemptLiveness, TaskAttemptFence, TaskAttemptProgressOutcome, TaskAttemptStartOutcome, + classify_active_attempt_liveness, +}; +use moa_execution::repository::trigger::{ + ExecutionTriggerNoOp, ExecutionWatchdogDeferOutcome, ExecutionWatchdogTriggerOutcome, +}; + use super::support::*; #[tokio::test] @@ -255,3 +265,359 @@ async fn attempt_generation_and_long_horizon_guards_reject_stale_or_invalid_stat ); Ok(()) } + +/// Admits and starts one active attempt per item key, returning the run and each started fence. +async fn start_admitted_attempts( + repository: &ExecutionRepository, + tenant_id: TenantId, + key: &str, + config: &ExecutionConfig, + item_keys: &[&str], +) -> Result< + (Uuid, Vec<(TaskAttemptFence, chrono::DateTime)>), + Box, +> { + let scope = ExecutionScope::Tenant { tenant_id }; + let mut candidate = new_run( + tenant_id, + None, + key, + ExecutionRunStatus::Queued, + budget(item_keys.len() as u64 * 4), + ); + candidate.plan.definition.nodes = vec![ExecutionNode { + id: "work".to_string(), + requirement_ids: vec!["req".to_string()], + depends_on: Vec::new(), + when: None, + input: json!({}), + output_schema: json!({ "type": "object" }), + operation: ExecutionOperation::Output { value: json!({}) }, + compensation: None, + retry: RetryPolicy { + max_attempts: 1, + initial_backoff_ms: 1, + max_backoff_ms: 1, + }, + budget: None, + }]; + let run = create_run(repository, scope, candidate).await?; + repository + .initialize_scheduler_state(scope, run.run_uid) + .await?; + assert!(matches!( + repository + .materialize_ready_page( + scope, + &ExecutionConfig::default(), + ReadyMaterializationRequest { + run_uid: run.run_uid, + plan_revision: 1, + node_id: "work".to_string(), + expected_cursor: 0, + reduce_cursor: None, + source_exhausted: true, + terminal_output: None, + condition_skipped: false, + tasks: item_keys + .iter() + .map(|item| logical_task(run.run_uid, "work", item, estimate(1))) + .collect(), + }, + ) + .await?, + ReadyMaterializationOutcome::Applied { .. } + )); + let admitted = repository + .admit_ready_attempts(config, item_keys.len() as u32, Utc::now()) + .await? + .admitted; + assert_eq!( + admitted.len(), + item_keys.len(), + "every ready task must be admitted" + ); + let mut started = Vec::new(); + for admission in admitted { + let fence = TaskAttemptFence { + tenant_id: admission.tenant_id, + run_uid: admission.run_uid, + task_id: admission.task_id, + controller_generation: admission.controller_generation, + attempt_generation: admission.attempt_generation, + dispatch_uid: admission.dispatch_uid, + capacity_reservation_uid: admission.capacity_reservation_uid, + watchdog_trigger_uid: admission.watchdog_trigger_uid, + attempt_deadline_at: admission.attempt_deadline_at, + }; + let TaskAttemptStartOutcome::Started(record) = repository.start_task_attempt(fence).await? + else { + panic!("an exactly admitted dispatch must start"); + }; + started.push((fence, record.task.last_progress_at)); + } + Ok((run.run_uid, started)) +} + +#[tokio::test] +async fn attempt_heartbeat_keeps_a_progressing_attempt_live_while_a_wedged_one_stalls_db() +-> TestResult { + // Pins: the durable heartbeat an active slice writes at an in-slice step boundary is what + // keeps that attempt classified live. Two attempts admitted together and started together + // diverge only because one committed a step boundary; the attempt that committed nothing is + // classified stalled while its admission deadline is still eight minutes away, which is the + // detection latency the heartbeat exists to remove. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let repository = ExecutionRepository::new(test_db.store().pool().clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let config = ExecutionConfig { + attempt_heartbeat_staleness_seconds: 60, + active_attempt_timeout_seconds: 600, + ..ExecutionConfig::default() + }; + let (run_uid, started) = start_admitted_attempts( + &repository, + tenant_id, + "attempt-heartbeat-staleness", + &config, + &["progressing", "wedged"], + ) + .await?; + let ((progressing_fence, progressing_started_at), (wedged_fence, wedged_started_at)) = + (started[0], started[1]); + + // The progressing attempt commits one in-slice step boundary ninety seconds in; the wedged + // attempt commits nothing after its start. + let heartbeat_at = progressing_started_at + Duration::seconds(90); + assert_eq!( + repository + .record_task_attempt_progress(progressing_fence, heartbeat_at) + .await?, + TaskAttemptProgressOutcome::Applied + ); + + let observed_at = progressing_started_at.max(wedged_started_at) + Duration::seconds(120); + let progressing = repository + .load_task(scope, run_uid, progressing_fence.task_id) + .await? + .expect("the heartbeated attempt must remain visible"); + let wedged = repository + .load_task(scope, run_uid, wedged_fence.task_id) + .await? + .expect("the wedged attempt must remain visible"); + assert_eq!(progressing.last_progress_at, heartbeat_at); + assert_eq!(wedged.last_progress_at, wedged_started_at); + + assert_eq!( + classify_active_attempt_liveness( + &config, + progressing_fence.attempt_deadline_at, + progressing.last_progress_at, + observed_at, + ), + ActiveAttemptLiveness::Live, + "an attempt that committed a durable step boundary is not stalled" + ); + assert_eq!( + classify_active_attempt_liveness( + &config, + wedged_fence.attempt_deadline_at, + wedged.last_progress_at, + observed_at, + ), + ActiveAttemptLiveness::Stalled, + "an attempt that committed nothing past the staleness window is stalled" + ); + assert!( + wedged_fence.attempt_deadline_at - observed_at >= Duration::minutes(7), + "the stall must be observable long before the admission deadline exposes it" + ); + Ok(()) +} + +#[tokio::test] +async fn stalled_attempt_watchdog_becomes_deliverable_before_its_deadline_db() -> TestResult { + // Pins: the watchdog is armed one staleness window ahead of the attempt deadline, so a wedged + // attempt becomes deliverable minutes before the deadline would have exposed it, while an + // attempt that keeps committing durable steps is rearmed instead of terminated. This is the + // difference between detecting a stall and merely classifying one after the fact. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + // A two-second window with a ten-minute deadline keeps the test fast while leaving the two + // apart by two orders of magnitude, which is exactly the separation the arming change buys. + let config = ExecutionConfig { + attempt_heartbeat_staleness_seconds: 2, + active_attempt_timeout_seconds: 600, + ..ExecutionConfig::default() + }; + let (run_uid, started) = start_admitted_attempts( + &repository, + tenant_id, + "watchdog-staleness-delivery", + &config, + &["progressing", "wedged", "capped"], + ) + .await?; + let [(progressing_fence, _), (wedged_fence, _), (capped_fence, _)] = started[..] else { + panic!("three attempts must start"); + }; + + // The watchdog is armed at the staleness window, not the deadline. + let armed_due_at: chrono::DateTime = + sqlx::query_scalar("SELECT due_at FROM moa.execution_trigger WHERE trigger_uid=$1") + .bind(wedged_fence.watchdog_trigger_uid) + .fetch_one(&pool) + .await?; + assert!( + armed_due_at < wedged_fence.attempt_deadline_at, + "the watchdog must be armed strictly before the attempt deadline" + ); + assert!( + wedged_fence.attempt_deadline_at - armed_due_at >= Duration::minutes(9), + "arming at the staleness window must leave almost the whole deadline unspent" + ); + + // Let the staleness window actually elapse. `last_progress_at` is monotonic in Postgres by + // trigger, so an aged attempt cannot be simulated by rewinding it. + tokio::time::sleep(std::time::Duration::from_millis(4_000)).await; + + // The progressing attempt commits a durable step boundary; the wedged one commits nothing. + let heartbeat_at: chrono::DateTime = + sqlx::query_scalar("SELECT now()").fetch_one(&pool).await?; + for fence in [progressing_fence, capped_fence] { + assert_eq!( + repository + .record_task_attempt_progress(fence, heartbeat_at) + .await?, + TaskAttemptProgressOutcome::Applied + ); + } + + // The wedged attempt's watchdog is now genuinely deliverable, and it is a stall rather than a + // consumed deadline: the deadline is still more than nine minutes away. + let prepared = repository + .prepare_watchdog_trigger(scope, wedged_fence.watchdog_trigger_uid) + .await?; + let ExecutionWatchdogTriggerOutcome::Task(request) = prepared else { + panic!("a stalled attempt's watchdog must be deliverable before its deadline"); + }; + assert_eq!(request.task_id, wedged_fence.task_id); + let wedged = repository + .load_task(scope, run_uid, wedged_fence.task_id) + .await? + .expect("the wedged attempt must remain visible"); + let observed_at: chrono::DateTime = + sqlx::query_scalar("SELECT now()").fetch_one(&pool).await?; + assert_eq!( + classify_active_attempt_liveness( + &config, + wedged_fence.attempt_deadline_at, + wedged.last_progress_at, + observed_at, + ), + ActiveAttemptLiveness::Stalled + ); + assert!( + wedged_fence.attempt_deadline_at - observed_at >= Duration::minutes(9), + "the stall is detected with the whole deadline still unspent" + ); + // A stalled attempt is never rearmed; its watchdog stays due and terminates it. + assert_eq!( + repository + .defer_task_attempt_watchdog(scope, &config, wedged_fence.watchdog_trigger_uid) + .await?, + ExecutionWatchdogDeferOutcome::NotDeferred + ); + assert!(matches!( + repository + .prepare_watchdog_trigger(scope, wedged_fence.watchdog_trigger_uid) + .await?, + ExecutionWatchdogTriggerOutcome::Task(_) + )); + + // The progressing attempt is rearmed for its next observation instead of terminated. + let ExecutionWatchdogDeferOutcome::Deferred { next_due_at } = repository + .defer_task_attempt_watchdog(scope, &config, progressing_fence.watchdog_trigger_uid) + .await? + else { + panic!("an attempt that proved progress must be rearmed, not terminated"); + }; + assert_eq!(next_due_at, heartbeat_at + Duration::seconds(2)); + assert!(next_due_at < progressing_fence.attempt_deadline_at); + assert_eq!( + repository + .prepare_watchdog_trigger(scope, progressing_fence.watchdog_trigger_uid) + .await?, + ExecutionWatchdogTriggerOutcome::NoOp(ExecutionTriggerNoOp::NotDue), + "a rearmed watchdog must stop firing until its next observation" + ); + let (trigger_state, trigger_due_at, dispatch_state, not_before_at): ( + String, + chrono::DateTime, + String, + chrono::DateTime, + ) = sqlx::query_as( + "SELECT trigger.state, trigger.due_at, dispatch.state, dispatch.not_before_at \ + FROM moa.execution_trigger AS trigger \ + JOIN moa.execution_dispatch_outbox AS dispatch \ + ON dispatch.trigger_uid = trigger.trigger_uid \ + AND dispatch.dispatch_kind = 'trigger_delivery' \ + WHERE trigger.trigger_uid = $1", + ) + .bind(progressing_fence.watchdog_trigger_uid) + .fetch_one(&pool) + .await?; + assert_eq!( + (trigger_state.as_str(), dispatch_state.as_str()), + ("pending", "pending"), + "a rearm keeps the trigger pending and reuses its one delivery row" + ); + assert_eq!((trigger_due_at, not_before_at), (next_due_at, next_due_at)); + // A rearm is not a supersede, so the trigger keeps its scheduled_triggers receipt. + let receipts: i64 = sqlx::query_scalar( + "SELECT count(*) FROM moa.execution_dispatch_outbox \ + WHERE trigger_uid=$1 AND dispatch_kind='trigger_delivery'", + ) + .bind(progressing_fence.watchdog_trigger_uid) + .fetch_one(&pool) + .await?; + assert_eq!(receipts, 1, "a rearm must not orphan a second delivery row"); + + // The deadline is the hard backstop: an attempt whose next observation would land past the + // deadline is rearmed only as far as the deadline itself. + sqlx::query( + "UPDATE moa.execution_task SET attempt_deadline_at=$2 WHERE run_uid=$1 AND task_id=$3", + ) + .bind(run_uid) + .bind(heartbeat_at + Duration::milliseconds(1_500)) + .bind(capped_fence.task_id.as_uuid()) + .execute(&pool) + .await?; + assert_eq!( + repository + .defer_task_attempt_watchdog(scope, &config, capped_fence.watchdog_trigger_uid) + .await?, + ExecutionWatchdogDeferOutcome::Deferred { + next_due_at: heartbeat_at + Duration::milliseconds(1_500) + }, + "deferral must never push an observation past the attempt deadline" + ); + + // A superseded controller generation cannot be rearmed. + sqlx::query("UPDATE moa.execution_run SET controller_generation=controller_generation+1 WHERE run_uid=$1") + .bind(run_uid) + .execute(&pool) + .await?; + assert_eq!( + repository + .defer_task_attempt_watchdog(scope, &config, progressing_fence.watchdog_trigger_uid) + .await?, + ExecutionWatchdogDeferOutcome::NotDeferred, + "a generation-stale watchdog must never be rearmed" + ); + Ok(()) +} diff --git a/crates/moa-execution/tests/execution_db/trigger_outbox_db.rs b/crates/moa-execution/tests/execution_db/trigger_outbox_db.rs index b61e91ad6..68420ca17 100644 --- a/crates/moa-execution/tests/execution_db/trigger_outbox_db.rs +++ b/crates/moa-execution/tests/execution_db/trigger_outbox_db.rs @@ -3166,6 +3166,7 @@ fn execution_capacity_config() -> ExecutionConfig { maximum_activation_steps: 128, dispatch_batch_size: 32, active_attempt_timeout_seconds: 10 * 60, + attempt_heartbeat_staleness_seconds: 2 * 60, max_tenant_active_runs: 100, max_fleet_active_runs: 1_000, max_tenant_active_tasks: 256, diff --git a/crates/moa-orchestrator/src/objects/session/execution_runs.rs b/crates/moa-orchestrator/src/objects/session/execution_runs.rs index d2d9abc1c..714faf4d1 100644 --- a/crates/moa-orchestrator/src/objects/session/execution_runs.rs +++ b/crates/moa-orchestrator/src/objects/session/execution_runs.rs @@ -912,6 +912,15 @@ mod tests { retrieved_bytes: Some(2_000), deadline_at: Some(waiting_since + chrono::TimeDelta::hours(4)), }, + economics: Some(moa_core::events::ExecutionProgressEconomics { + consumed_cost_microusd: 30, + consumed_tokens: 300, + consumed_tasks: 2, + consumed_tool_calls: 5, + consumed_retrieved_bytes: 1_500, + requirements_total: 2, + requirements_satisfied: None, + }), total: 5, completed: 2, failed: 0, diff --git a/crates/moa-orchestrator/src/objects/session/state.rs b/crates/moa-orchestrator/src/objects/session/state.rs index 3b24a3b59..1f8591512 100644 --- a/crates/moa-orchestrator/src/objects/session/state.rs +++ b/crates/moa-orchestrator/src/objects/session/state.rs @@ -189,6 +189,13 @@ pub struct ExecutionProgressSignature { pub blocker_audience: Option, /// Exact unconsumed and unreserved execution budget. pub remaining_budget: moa_core::events::ExecutionRemainingBudget, + /// Cumulative spend against the goal-requirement denominator. + /// + /// Reconciling a task's actuals can raise consumed spend while every other signature + /// field holds, which is exactly the run that is spending without advancing. Gating on + /// the projection without this field would suppress that publication. + #[serde(default)] + pub economics: Option, /// Materialized logical task count. pub total: u64, /// Successfully completed logical task count. @@ -214,6 +221,7 @@ impl From<&moa_core::events::ExecutionProgress> for ExecutionProgressSignature { parked_tasks: progress.parked_tasks, blocker_audience: progress.blocker_audience, remaining_budget: progress.remaining_budget.clone(), + economics: progress.economics.clone(), total: progress.total, completed: progress.completed, failed: progress.failed, diff --git a/crates/moa-orchestrator/src/services/execution_trigger.rs b/crates/moa-orchestrator/src/services/execution_trigger.rs index 41d6f49b4..a71925f80 100644 --- a/crates/moa-orchestrator/src/services/execution_trigger.rs +++ b/crates/moa-orchestrator/src/services/execution_trigger.rs @@ -8,7 +8,7 @@ use moa_execution::repository::{ trigger::{ ExecutionExternalReconcileTriggerOutcome, ExecutionExternalStartRecoveryTriggerOutcome, ExecutionRunDeadlineTriggerOutcome, ExecutionTriggerFireOutcome, ExecutionTriggerKind, - ExecutionTriggerNoOp, ExecutionWatchdogTriggerOutcome, + ExecutionTriggerNoOp, ExecutionWatchdogDeferOutcome, ExecutionWatchdogTriggerOutcome, }, }; use moa_execution::wire::{ @@ -329,6 +329,39 @@ impl ExecutionTrigger for ExecutionTriggerImpl { WatchdogRoute::NoOp { response } => return Ok(Json::from(response)), }; if watchdog_outcome == ExecutionAttemptWatchdogResponseOutcome::RetryDelivery { + // A task watchdog is armed one heartbeat-staleness window ahead of the attempt + // deadline, so a live attempt that is still committing durable steps is observed + // repeatedly and legitimately answers RetryDelivery. Rearm it for its next + // observation and complete this delivery rather than spinning revalidation until + // the deadline. The repository caps the new observation at the attempt deadline and + // declines anything that is not a strictly later, still-current task watchdog, so a + // stalled attempt, a superseded generation, and a compensation watchdog all fall + // through to the pre-existing revalidation path unchanged. + let repository = self.repository.clone(); + let config = self.config.clone(); + let deferred = ctx + .run(|| async move { + repository + .defer_task_attempt_watchdog( + ExecutionScope::Tenant { tenant_id }, + &config, + trigger_uid, + ) + .await + .map(|outcome| { + Json::from(matches!( + outcome, + ExecutionWatchdogDeferOutcome::Deferred { .. } + )) + }) + .map_err(execution_error_to_handler_error) + }) + .name(format!("execution_watchdog_defer_{trigger_uid}")) + .await? + .into_inner(); + if deferred { + return Ok(Json::from(ExecutionTriggerFireResponse::NotDue)); + } // The receiver can race a different durable owner transition. Revalidate after // its response so a watchdog superseded by that transition completes as a stale // delivery instead of blocking the fleet-serialized drain behind endless retry. diff --git a/crates/moa-orchestrator/src/workflows/execution_task_attempt/active.rs b/crates/moa-orchestrator/src/workflows/execution_task_attempt/active.rs index 3e63e3f76..5910168ca 100644 --- a/crates/moa-orchestrator/src/workflows/execution_task_attempt/active.rs +++ b/crates/moa-orchestrator/src/workflows/execution_task_attempt/active.rs @@ -341,6 +341,62 @@ pub(super) async fn execute_task_attempt( } } +/// Durable step boundary at which an active attempt reports progress. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum AttemptHeartbeat { + /// One model turn returned, so the following tool dispatch starts its own stall window. + ModelTurn, + /// One governed tool invocation returned, so sandbox release and continuation persistence + /// start their own stall window. + ToolCall, +} + +impl AttemptHeartbeat { + /// Deterministic journal step name for this boundary. + const fn observation_step(self) -> &'static str { + match self { + Self::ModelTurn => "task_attempt_model_turn_progress_at", + Self::ToolCall => "task_attempt_tool_call_progress_at", + } + } + + /// Deterministic journal step name for the persisted heartbeat. + const fn write_step(self) -> &'static str { + match self { + Self::ModelTurn => "record_task_attempt_model_turn_progress", + Self::ToolCall => "record_task_attempt_tool_call_progress", + } + } +} + +/// Advances the active attempt's durable progress clock at one completed step boundary. +/// +/// Only an attempt that currently owns active capacity is heartbeated; the repository call is +/// fenced on the exact dispatch and rejects a parked, superseded, or already-settled attempt, +/// so a waiting task can never appear to make progress. The observation timestamp is journaled +/// through `durable_utc_now` so replay reuses the recorded instant instead of a fresh clock +/// read, and the repository write itself is monotonic. +async fn record_attempt_heartbeat( + workflow: &ExecutionTaskAttemptImpl, + ctx: &WorkflowContext<'_>, + request: &ExecutionTaskAttemptRequest, + boundary: AttemptHeartbeat, +) -> Result<(), HandlerError> { + let observed_at = durable_utc_now(ctx, boundary.observation_step()).await?; + let repository = workflow.repository.clone(); + let fence = task_attempt_fence(request); + ctx.run(|| async move { + repository + .record_task_attempt_progress(fence, observed_at) + .await + .map(Json::from) + .map_err(crate::workflows::errors::execution_error_to_handler_error) + }) + .name(boundary.write_step()) + .await?; + Ok(()) +} + async fn persist_external_start_checkpoint( workflow: &ExecutionTaskAttemptImpl, ctx: &WorkflowContext<'_>, @@ -504,6 +560,7 @@ async fn execute_direct_capability( workflow.channel_adapters.as_ref(), ) .await?; + record_attempt_heartbeat(workflow, ctx, request, AttemptHeartbeat::ToolCall).await?; usage.tool_calls = usage.tool_calls.saturating_add(1); classify_capability_outcome(capability, governed, usage) } @@ -1064,6 +1121,7 @@ async fn execute_agent_turn( .call() .await? .into_inner(); + record_attempt_heartbeat(workflow, ctx, request, AttemptHeartbeat::ModelTurn).await?; usage.tokens = usage .tokens .saturating_add(response.usage.total_input_tokens() as u64) @@ -1239,6 +1297,7 @@ async fn execute_agent_turn( workflow.channel_adapters.as_ref(), ) .await?; + record_attempt_heartbeat(workflow, ctx, request, AttemptHeartbeat::ToolCall).await?; usage.tool_calls = usage.tool_calls.saturating_add(1); match governed { GovernedInvocationOutcome::Completed(result) diff --git a/crates/moa-orchestrator/src/workflows/execution_task_attempt/watchdog.rs b/crates/moa-orchestrator/src/workflows/execution_task_attempt/watchdog.rs index 5609682b7..a47d7d954 100644 --- a/crates/moa-orchestrator/src/workflows/execution_task_attempt/watchdog.rs +++ b/crates/moa-orchestrator/src/workflows/execution_task_attempt/watchdog.rs @@ -10,8 +10,9 @@ use moa_execution::{ repository::{ ExecutionAttemptState, ExecutionScope, task::{ - TaskAttemptFence, TaskAttemptRecord, TaskAttemptSettlementOutcome, - UnstartedTaskAttemptDisposition, + ActiveAttemptLiveness, TaskAttemptFence, TaskAttemptRecord, + TaskAttemptSettlementOutcome, UnstartedTaskAttemptDisposition, + classify_active_attempt_liveness, }, }, state::{ExecutionTaskStatus, LogicalTaskKind, exhaust_retry_outcome, retry_delay_ms}, @@ -108,7 +109,9 @@ pub(super) async fn handle_task_attempt_watchdog( )); } let now = durable_utc_now_shared(ctx, "task_watchdog_observed_at").await?; - if deadline > now { + let liveness = + classify_active_attempt_liveness(&workflow.config, deadline, task.last_progress_at, now); + if !liveness.is_expired() { return Ok(watchdog_result( ExecutionAttemptWatchdogResponseOutcome::RetryDelivery, )); @@ -193,24 +196,7 @@ pub(super) async fn handle_task_attempt_watchdog( }; let receipt = checkpoint_task_hands_shared(ctx, &attempt_request, &started).await?; let disposition = classify_stale_attempt(task_effect_idempotency(&started)); - let outcome = match disposition { - StaleTaskAttemptDisposition::Retry => ExecutionTaskOutcome { - schema_version: 1, - usage: started.task.actual.clone(), - result: ExecutionTaskResult::Failed { - class: ExecutionFailureClass::Retryable, - message: "task attempt watchdog expired before durable settlement".to_string(), - }, - }, - StaleTaskAttemptDisposition::UnknownOutcome => ExecutionTaskOutcome { - schema_version: 1, - usage: started.task.actual.clone(), - result: ExecutionTaskResult::UnknownOutcome { - message: "non-idempotent task attempt exceeded its watchdog after possible commit" - .to_string(), - }, - }, - }; + let outcome = expired_attempt_outcome(disposition, liveness, started.task.actual.clone()); let outcome = exhaust_retry_outcome(started.task.attempt, &started.task.retry, outcome); let retry_at = matches!( outcome.result, @@ -245,6 +231,48 @@ pub(super) async fn handle_task_attempt_watchdog( Ok(watchdog_result(settlement)) } +/// Builds the durable outcome for one attempt the watchdog observed as expired. +/// +/// The persisted message names why the attempt was terminated, because a stall inside the +/// authorized window and a fully consumed window call for different operator responses even +/// though both settle through the same retry or reconciliation path. +fn expired_attempt_outcome( + disposition: StaleTaskAttemptDisposition, + liveness: ActiveAttemptLiveness, + usage: moa_artifacts::execution_plan::ExecutionUsage, +) -> ExecutionTaskOutcome { + let reason = match liveness { + ActiveAttemptLiveness::Live => "watchdog fired on a live attempt", + ActiveAttemptLiveness::Stalled => { + "reported no durable progress within the heartbeat staleness window" + } + ActiveAttemptLiveness::DeadlineExceeded => "exceeded its active attempt deadline", + }; + match disposition { + StaleTaskAttemptDisposition::Retry => ExecutionTaskOutcome { + schema_version: 1, + usage, + result: ExecutionTaskResult::Failed { + class: ExecutionFailureClass::Retryable, + message: format!( + "task attempt {reason} before durable settlement ({})", + liveness.as_str() + ), + }, + }, + StaleTaskAttemptDisposition::UnknownOutcome => ExecutionTaskOutcome { + schema_version: 1, + usage, + result: ExecutionTaskResult::UnknownOutcome { + message: format!( + "non-idempotent task attempt {reason} after possible commit ({})", + liveness.as_str() + ), + }, + }, + } +} + fn watchdog_result(outcome: ExecutionAttemptWatchdogResponseOutcome) -> TaskAttemptWatchdogResult { TaskAttemptWatchdogResult { outcome } } @@ -314,4 +342,43 @@ mod tests { StaleTaskAttemptDisposition::UnknownOutcome ); } + + // Pins: the watchdog only terminates an attempt its liveness classification calls expired, + // and the durable message distinguishes a heartbeat stall from a consumed deadline so the + // two failures are separable in the task record without new event plumbing. + #[test] + fn watchdog_records_why_an_expired_attempt_was_terminated_offline() { + assert!(!ActiveAttemptLiveness::Live.is_expired()); + assert!(ActiveAttemptLiveness::Stalled.is_expired()); + assert!(ActiveAttemptLiveness::DeadlineExceeded.is_expired()); + + let usage = moa_artifacts::execution_plan::ExecutionUsage { + cost_microusd: 7, + tokens: 11, + tool_calls: 1, + retrieved_bytes: 13, + }; + let stalled = expired_attempt_outcome( + StaleTaskAttemptDisposition::Retry, + ActiveAttemptLiveness::Stalled, + usage.clone(), + ); + let ExecutionTaskResult::Failed { class, message } = stalled.result else { + panic!("an idempotent expired attempt must stay retryable"); + }; + assert_eq!(class, ExecutionFailureClass::Retryable); + assert!(message.contains("heartbeat staleness window"), "{message}"); + assert!(message.contains("(stalled)"), "{message}"); + + let overdue = expired_attempt_outcome( + StaleTaskAttemptDisposition::UnknownOutcome, + ActiveAttemptLiveness::DeadlineExceeded, + usage, + ); + let ExecutionTaskResult::UnknownOutcome { message } = overdue.result else { + panic!("a non-idempotent expired attempt must stay ambiguous"); + }; + assert!(message.contains("active attempt deadline"), "{message}"); + assert!(message.contains("(deadline_exceeded)"), "{message}"); + } } diff --git a/crates/moa-orchestrator/tests/orchestrator_offline/session_vo.rs b/crates/moa-orchestrator/tests/orchestrator_offline/session_vo.rs index e5e43786d..139aece7f 100644 --- a/crates/moa-orchestrator/tests/orchestrator_offline/session_vo.rs +++ b/crates/moa-orchestrator/tests/orchestrator_offline/session_vo.rs @@ -155,6 +155,15 @@ fn execution_progress(run_uid: Uuid) -> moa_core::events::ExecutionProgress { retrieved_bytes: Some(7_000), deadline_at: Some(now + chrono::Duration::hours(2)), }, + economics: Some(moa_core::events::ExecutionProgressEconomics { + consumed_cost_microusd: 30, + consumed_tokens: 300, + consumed_tasks: 3, + consumed_tool_calls: 4, + consumed_retrieved_bytes: 3_000, + requirements_total: 3, + requirements_satisfied: None, + }), total: 8, completed: 2, failed: 1, @@ -190,6 +199,15 @@ fn session_progress_projects_exact_persisted_active_execution_values() { retrieved_bytes: Some(4_000), deadline_at: None, }, + economics: Some(moa_core::events::ExecutionProgressEconomics { + consumed_cost_microusd: 60, + consumed_tokens: 600, + consumed_tasks: 12, + consumed_tool_calls: 8, + consumed_retrieved_bytes: 6_000, + requirements_total: 3, + requirements_satisfied: None, + }), total: 13, completed: 8, failed: 3, @@ -271,6 +289,12 @@ fn execution_progress_requires_cadence_and_changed_exact_public_projection() { Some(moa_core::events::ExecutionBlockerAudience::External); let mut remaining_budget_changed = baseline.clone(); remaining_budget_changed.remaining_budget.tasks = Some(5); + let mut economics_changed = baseline.clone(); + economics_changed + .economics + .as_mut() + .expect("baseline reports economics") + .consumed_cost_microusd += 1; let mut total_changed = baseline.clone(); total_changed.total += 1; let mut completed_changed = baseline.clone(); @@ -292,6 +316,7 @@ fn execution_progress_requires_cadence_and_changed_exact_public_projection() { ("parked_tasks", parked_tasks_changed), ("blocker_audience", blocker_audience_changed), ("remaining_budget", remaining_budget_changed), + ("economics", economics_changed), ("total", total_changed), ("completed", completed_changed), ("failed", failed_changed), diff --git a/crates/moa-session/tests/session_db/execution_events_db.rs b/crates/moa-session/tests/session_db/execution_events_db.rs index 3b796e146..eac636972 100644 --- a/crates/moa-session/tests/session_db/execution_events_db.rs +++ b/crates/moa-session/tests/session_db/execution_events_db.rs @@ -69,6 +69,15 @@ async fn execution_events_db_round_trip_compact_payloads_without_task_output_cop retrieved_bytes: Some(10_000), deadline_at: None, }, + economics: Some(moa_core::events::ExecutionProgressEconomics { + consumed_cost_microusd: 40, + consumed_tokens: 400, + consumed_tasks: 4, + consumed_tool_calls: 3, + consumed_retrieved_bytes: 2_500, + requirements_total: 2, + requirements_satisfied: None, + }), total: 6, completed: 3, failed: 1, diff --git a/crates/moa-wire/src/turn.rs b/crates/moa-wire/src/turn.rs index 8f1fbb7f6..112beae2b 100644 --- a/crates/moa-wire/src/turn.rs +++ b/crates/moa-wire/src/turn.rs @@ -893,6 +893,15 @@ mod tests { retrieved_bytes: Some(10_000), deadline_at: None, }, + economics: Some(moa_core::events::ExecutionProgressEconomics { + consumed_cost_microusd: 25, + consumed_tokens: 250, + consumed_tasks: 10, + consumed_tool_calls: 2, + consumed_retrieved_bytes: 1_000, + requirements_total: 4, + requirements_satisfied: None, + }), total: 11, completed: 7, failed: 2, diff --git a/docs/23-environment-variables.md b/docs/23-environment-variables.md index 3d4990106..28eb82bf0 100644 --- a/docs/23-environment-variables.md +++ b/docs/23-environment-variables.md @@ -84,6 +84,7 @@ Grouped by top-level config section. `_unset_`/`_none_` means the field is | `MOA_EXECUTION_AGENT_TURN_RETRIEVED_BYTES` | `execution.agent_turn_retrieved_bytes` | 10000000 | Worst-case retrieved-byte estimate for one agent turn | | `MOA_EXECUTION_AGENT_TURN_TOKENS` | `execution.agent_turn_tokens` | 8000 | Worst-case token estimate for one agent turn | | `MOA_EXECUTION_AGENT_TURN_TOOL_CALLS` | `execution.agent_turn_tool_calls` | 8 | Worst-case governed tool-call estimate for one agent turn | +| `MOA_EXECUTION_ATTEMPT_HEARTBEAT_STALENESS_SECONDS` | `execution.attempt_heartbeat_staleness_seconds` | 120 | Interval without durable attempt progress after which an active attempt is classified stalled; must be less than the active attempt timeout | | `MOA_EXECUTION_DISPATCH_BATCH_SIZE` | `execution.dispatch_batch_size` | 64 | Maximum ready task attempts dispatched by one controller activation | | `MOA_EXECUTION_MAX_COST_MICROUSD` | `execution.max_cost_microusd` | 100000000 | Default run cost limit in integer micro-USD | | `MOA_EXECUTION_MAX_FLEET_ACTIVE_RUNS` | `execution.max_fleet_active_runs` | 1000 | Fleet ceiling for admitted non-parked execution runs | From 7e229e761099403505351a4cfb361ffb23ed48be Mon Sep 17 00:00:00 2001 From: Hwuiwon Kim Date: Thu, 13 Aug 2026 13:50:33 -0400 Subject: [PATCH 04/21] fix audited long-horizon execution issues --- Cargo.lock | 3 + Dockerfile | 14 + Makefile | 2 + .../src/validation/execution_plan.rs | 30 +- .../artifacts_offline/definition_roundtrip.rs | 55 + .../execution_plan_validation.rs | 11 +- .../moa-brain/src/execution_planning/mod.rs | 122 +- .../src/execution_planning/request.rs | 13 +- .../src/execution_planning/response.rs | 2 + .../src/execution_planning/routing.rs | 2 +- .../src/prompts/execution_planner.md | 2 +- crates/moa-brain/tests/brain_turn_offline.rs | 112 +- crates/moa-brain/tests/query_rewrite_live.rs | 2 +- .../moa-brain/tests/turbopuffer_news_live.rs | 2 +- crates/moa-config/src/env_overlay/mod.rs | 1 + crates/moa-config/src/env_overlay/tests.rs | 8 + crates/moa-config/src/execution.rs | 17 +- .../moa-core/src/types/execution_planning.rs | 60 + .../moa-core/src/types/sandbox_workspace.rs | 30 - .../scenarios/execution/manifest.toml | 2 +- .../tests/eval_offline/execution_snapshot.rs | 3 + crates/moa-execution/src/repository/audit.rs | 20 + .../src/repository/audit_codec.rs | 11 + .../moa-execution/src/repository/capacity.rs | 63 +- .../src/repository/compensation.rs | 138 ++- .../src/repository/completion.rs | 2 +- .../src/repository/external_job.rs | 132 ++- crates/moa-execution/src/repository/mod.rs | 17 + .../src/repository/planning_budget.rs | 495 ++++++++ .../moa-execution/src/repository/retention.rs | 18 +- crates/moa-execution/src/repository/rows.rs | 5 + crates/moa-execution/src/repository/sql.rs | 35 +- crates/moa-execution/src/repository/task.rs | 189 ++- .../moa-execution/src/repository/trigger.rs | 59 +- crates/moa-execution/src/wire.rs | 2 +- .../execution_db/completion_projection_db.rs | 14 + .../controller_wake_recovery_db.rs | 5 + .../execution_db/long_horizon_state_db.rs | 254 +++- .../execution_db/planning_and_audit_db.rs | 221 +++- .../tests/execution_db/trigger_outbox_db.rs | 53 +- .../execution_db/wait_entry_deadline_db.rs | 103 +- crates/moa-hands/Cargo.toml | 2 + crates/moa-hands/src/adapters/daytona/mod.rs | 287 +++-- .../moa-hands/src/adapters/daytona/tests.rs | 108 +- crates/moa-hands/src/adapters/e2b/mod.rs | 47 +- crates/moa-hands/src/adapters/e2b/storage.rs | 15 +- crates/moa-hands/src/adapters/e2b/tests.rs | 80 +- crates/moa-hands/src/adapters/local/mod.rs | 2 +- crates/moa-hands/src/core/dispatch.rs | 30 +- crates/moa-hands/src/core/leases.rs | 7 +- crates/moa-hands/src/core/lifecycle.rs | 75 +- crates/moa-hands/src/core/mod.rs | 4 +- crates/moa-hands/src/core/reaper.rs | 20 +- .../src/core/sandbox_workspace/capacity.rs | 204 +--- .../src/core/sandbox_workspace/lifecycle.rs | 470 +++----- .../core/sandbox_workspace/maintenance/mod.rs | 80 +- .../src/core/sandbox_workspace/operations.rs | 165 ++- crates/moa-hands/src/core/telemetry.rs | 2 +- crates/moa-hands/src/lib.rs | 8 +- crates/moa-hands/src/tools/bash.rs | 54 + crates/moa-hands/tests/daytona_live.rs | 15 + .../tests/hands_db/hand_lease_reaper_db.rs | 131 ++ .../hands_db/sandbox_workspace/dispatch_db.rs | 6 + .../sandbox_workspace/lifecycle_db.rs | 1054 +++-------------- .../sandbox_workspace/maintenance_db.rs | 136 +++ .../hands_db/sandbox_workspace/purge_db.rs | 6 + .../sandbox_workspace/storage_resources_db.rs | 9 +- crates/moa-migrations/Cargo.toml | 4 +- crates/moa-migrations/README.md | 11 +- crates/moa-migrations/build.rs | 17 + .../moa-migrations/migration-ownership.toml | 14 + .../V000059__long_horizon_execution.sql | 147 ++- ...00060__sandbox_active_compute_capacity.sql | 10 +- crates/moa-migrations/src/lib.rs | 10 +- .../tests/run_idempotency_db/connectors.rs | 67 +- .../execution_and_security_catalog.rs | 57 +- .../execution_compensation.rs | 7 +- .../tests/run_idempotency_db/protocol.rs | 23 +- .../run_idempotency_db/session_status.rs | 29 +- .../tests/run_idempotency_db/support.rs | 17 +- .../tests/run_idempotency_db/tenant_purge.rs | 78 +- .../moa-observability/src/runtime_metrics.rs | 9 +- .../services/execution_amendment_planner.rs | 278 ++++- .../execution_amendment_planner/tests.rs | 43 +- .../src/services/tool_executor.rs | 99 +- .../execution_task_attempt/active.rs | 294 ++++- .../execution_task_attempt/watchdog.rs | 221 +++- .../execution_task_attempt/yielding.rs | 141 +-- .../long_horizon_execution_canary_live.rs | 86 +- .../burst_admission.rs | 19 +- .../deadline_and_waits.rs | 7 + .../disaster_recovery.rs | 20 +- .../pause_and_external.rs | 8 +- crates/xtask/src/execution_trace_manifest.rs | 7 - docker-compose.yml | 7 +- docs/01-architecture-overview.md | 10 +- docs/02-brain-orchestration.md | 4 + docs/10-technology-stack.md | 7 +- docs/12-restate-architecture.md | 21 +- docs/17-observability.md | 18 +- docs/22-load-and-chaos-testing.md | 6 + docs/23-environment-variables.md | 3 +- .../patterns/custom-logic.skill.yaml | 5 +- docs/operations/restate-operations.md | 3 +- k8s/base/10-restate-cluster.yaml | 16 + k8s/scripts/validate-observability.sh | 41 + ops/prometheus/alerts/sandbox-workspaces.yaml | 4 +- scripts/cutover-long-horizon-execution.sh | 23 +- scripts/run-clean-e2e.sh | 7 +- 109 files changed, 4927 insertions(+), 2317 deletions(-) create mode 100644 crates/moa-execution/src/repository/planning_budget.rs create mode 100644 crates/moa-migrations/build.rs diff --git a/Cargo.lock b/Cargo.lock index a7f395c4a..9dc0c1b90 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3843,6 +3843,8 @@ dependencies = [ "hex", "ignore", "jsonschema", + "metrics", + "metrics-exporter-prometheus", "moa-artifacts", "moa-auth-providers", "moa-config", @@ -5422,6 +5424,7 @@ dependencies = [ "bytes", "fallible-iterator", "postgres-protocol", + "uuid", ] [[package]] diff --git a/Dockerfile b/Dockerfile index 5a8014488..80b6710bd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,6 +5,20 @@ WORKDIR /build COPY . . ARG MOA_ORCHESTRATOR_FEATURES="" +# `.cargo/config.toml` adds `-C prefer-dynamic` to speed up local dev builds, but the release +# profile enables thin LTO and rustc rejects that combination outright ("cannot prefer dynamic +# linking when performing LTO"). Only release builds hit it, so it breaks image builds while +# leaving every local `cargo build` working, which is why it can go unnoticed. +# +# `RUSTFLAGS` is the only setting that reliably wins here: it sits above `build.rustflags` in +# cargo's precedence order, whereas `CARGO_BUILD_RUSTFLAGS` is merely the environment spelling +# of that same config key and does not displace it. +# +# It replaces the flags wholesale rather than appending, so `tokio_unstable` has to be restated: +# the runtime metrics hooks the code compiles against are gated on it, and dropping it fails the +# build on missing cfg items. A statically linked release binary is also what the runtime stage +# needs, since it copies the binary into a slim image without the toolchain's shared objects. +ENV RUSTFLAGS="--cfg tokio_unstable" RUN if [ -n "${MOA_ORCHESTRATOR_FEATURES}" ]; then \ cargo build --locked --release -p moa-orchestrator --bin moa-orchestrator-bin --features "${MOA_ORCHESTRATOR_FEATURES}"; \ else \ diff --git a/Makefile b/Makefile index ab1afa778..22372b0d8 100644 --- a/Makefile +++ b/Makefile @@ -109,6 +109,8 @@ test-long-horizon: -u MOA_RESTATE_ADMIN_URL \ -u RESTATE_ADMIN_URL \ -u MOA_RESTATE_DEPLOYMENT_URI \ + -u MOA_RUNTIME_CACHE_BACKEND \ + -u MOA_RUNTIME_CACHE_REDIS_URL \ -u MOA_ANTHROPIC_API_KEY \ -u MOA_OPENAI_API_KEY \ -u MOA_GOOGLE_API_KEY \ diff --git a/crates/moa-artifacts/src/validation/execution_plan.rs b/crates/moa-artifacts/src/validation/execution_plan.rs index b5819f791..83d4f0b8b 100644 --- a/crates/moa-artifacts/src/validation/execution_plan.rs +++ b/crates/moa-artifacts/src/validation/execution_plan.rs @@ -296,10 +296,11 @@ mod tests { // so a `continue_with` output has nothing to validate against. Before this check // it compiled cleanly and the schema violation surfaced at run materialization as // a non-retryable infrastructure error against whichever task happened to ask for - // input. Both directions are asserted so the rejection is provably about - // `continue_with` and not about the surrounding fixture. + // input. The accepted direction proves the rejection is about `continue_with` + // and not the surrounding fixture; the removed `fail_run` wire spelling is + // rejected explicitly instead of pretending it remains a second valid action. #[test] - fn input_wait_policy_rejects_continue_with_but_accepts_a_failing_action() { + fn input_wait_policy_accepts_fail_task_and_rejects_other_settlements() { let continued = plan(ExecutionWaitExpiryAction::ContinueWith { output: json!({ "approved": true }), }); @@ -314,16 +315,19 @@ mod tests { "continue_with must be refused for the plan-level input wait policy: {report:?}" ); - for action in [ - ExecutionWaitExpiryAction::FailTask, - ExecutionWaitExpiryAction::FailTask, - ] { - let report = validate_execution_plan_definition(&plan(action)); - assert!( - report.errors.is_empty(), - "a failing input wait expiry must still validate: {report:?}" - ); - } + let report = validate_execution_plan_definition(&plan(ExecutionWaitExpiryAction::FailTask)); + assert!( + report.errors.is_empty(), + "fail_task must remain the valid input-wait expiry: {report:?}" + ); + + assert!( + serde_json::from_value::(json!({ + "kind": "fail_run" + })) + .is_err(), + "the removed fail_run wire spelling must fail closed" + ); } fn plan(on_expiry: ExecutionWaitExpiryAction) -> ExecutionPlanDefinition { diff --git a/crates/moa-artifacts/tests/artifacts_offline/definition_roundtrip.rs b/crates/moa-artifacts/tests/artifacts_offline/definition_roundtrip.rs index 15ccfa170..0e83adc8c 100644 --- a/crates/moa-artifacts/tests/artifacts_offline/definition_roundtrip.rs +++ b/crates/moa-artifacts/tests/artifacts_offline/definition_roundtrip.rs @@ -812,6 +812,61 @@ fn prompt_examples_parse_as_skill_execution_plans() { ); } +#[test] +fn custom_logic_example_totalizes_priority_branches_and_requires_retry_input() { + // Pins: every input accepted by the documented custom-logic schema selects one + // declared branch, and the high-priority branch cannot reference an omitted retry value. + let skill = parse_skill_example( + "patterns/custom-logic", + include_str!("../../../../docs/examples/artifacts/patterns/custom-logic.skill.yaml"), + ); + assert_eq!( + skill.inputs["required"], + serde_json::json!(["priority", "retry"]) + ); + assert_eq!( + skill.inputs["properties"]["priority"]["enum"], + serde_json::json!(["high", "standard"]) + ); + + let template = skill + .execution_plan + .expect("custom-logic example should declare an execution plan"); + assert_eq!( + template.plan.input_schema["required"], + serde_json::json!(["priority", "retry"]) + ); + assert_eq!( + template.plan.input_schema["properties"]["priority"]["enum"], + serde_json::json!(["high", "standard"]) + ); + let branches = template + .plan + .nodes + .iter() + .filter_map(|node| match &node.when { + Some(moa_artifacts::execution_plan::ExecutionCondition::Equals { + reference, + value, + }) => Some((reference.path.as_str(), value.as_str())), + Some(moa_artifacts::execution_plan::ExecutionCondition::Exists { .. }) | None => None, + }) + .collect::>(); + assert_eq!( + branches, + vec![ + ("$.input.priority", Some("high")), + ("$.input.priority", Some("standard")), + ] + ); + assert_eq!( + template.plan.nodes[0].input, + serde_json::json!({ + "retry_requested": { "$ref": "$.input.retry" } + }) + ); +} + fn parse_skill_example(name: &str, yaml: &str) -> moa_artifacts::skill::SkillDefinition { let document = ArtifactDocument::from_yaml(yaml) .unwrap_or_else(|error| panic!("example {name} should parse: {error}")); diff --git a/crates/moa-artifacts/tests/artifacts_offline/execution_plan_validation.rs b/crates/moa-artifacts/tests/artifacts_offline/execution_plan_validation.rs index 3e5852dd3..8f7b0e820 100644 --- a/crates/moa-artifacts/tests/artifacts_offline/execution_plan_validation.rs +++ b/crates/moa-artifacts/tests/artifacts_offline/execution_plan_validation.rs @@ -168,10 +168,6 @@ fn all_eight_execution_operations_round_trip_exact_json_and_yaml() { fn wait_expiry_actions_round_trip_with_canonical_tagged_shapes() { // Pins: wait expiry is explicit and every settlement action has one stable wire shape. let cases = [ - ( - ExecutionWaitExpiryAction::FailTask, - json!({ "kind": "fail_task" }), - ), ( ExecutionWaitExpiryAction::FailTask, json!({ "kind": "fail_task" }), @@ -211,6 +207,13 @@ fn wait_expiry_actions_round_trip_with_canonical_tagged_shapes() { unknown_action.is_err(), "undeclared wait expiry actions must reject" ); + let removed_fail_run = serde_json::from_value::(json!({ + "kind": "fail_run" + })); + assert!( + removed_fail_run.is_err(), + "the removed fail_run wire spelling must reject rather than alias fail_task" + ); } #[test] diff --git a/crates/moa-brain/src/execution_planning/mod.rs b/crates/moa-brain/src/execution_planning/mod.rs index f972d8df4..eb60f3f21 100644 --- a/crates/moa-brain/src/execution_planning/mod.rs +++ b/crates/moa-brain/src/execution_planning/mod.rs @@ -15,8 +15,9 @@ use moa_core::{ types::execution_planning::{ ExecutionAuditReport, ExecutionAuditViolation, ExecutionCompileOutcome, ExecutionCompileSource, ExecutionPlannerCallKind, ExecutionPlannerOutcome, - ExecutionPlanningAuditEnvelope, ExecutionPlanningAuditPayload, ExecutionSourceProvenance, - GeneratedPlanPlannerProvenance, bounded_audit_report, execution_planning_hash, + ExecutionPlanningAuditEnvelope, ExecutionPlanningAuditPayload, ExecutionRouteUsage, + ExecutionSourceProvenance, GeneratedPlanPlannerProvenance, bounded_audit_report, + execution_planning_hash, }, }; use moa_execution::{ @@ -57,6 +58,24 @@ enum ClassifiedCompileOutcome { Rejected, } +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +struct PlannerCallMetering { + usage: ExecutionRouteUsage, + cost_microusd: u64, +} + +fn planner_call_metering( + response: &moa_core::types::completion::CompletionResponse, +) -> Result { + let usage = response.token_usage(); + Ok(PlannerCallMetering { + usage: routing::route_usage(usage)?, + cost_microusd: moa_providers::pricing_for_model(response.model.as_str()) + .map(|pricing| pricing.cost_micros(&usage)) + .unwrap_or_default(), + }) +} + /// Plans or instantiates one initial execution candidate over frozen session authority. pub async fn plan_execution( provider: &dyn LLMProvider, @@ -403,6 +422,7 @@ async fn call_initial_provider( )); } }; + let metering = planner_call_metering(&response)?; let raw = response.text; let duration = duration_micros(started); if raw.len() > EXECUTION_PLANNER_CANDIDATE_MAX_BYTES { @@ -431,6 +451,7 @@ async fn call_initial_provider( Some(raw_hash), None, Some(report_json), + metering, duration, ), }); @@ -465,6 +486,7 @@ async fn call_initial_provider( Some(raw_hash), None, Some(canonical_string(&report)?), + metering, duration, ), }); @@ -494,6 +516,7 @@ async fn call_initial_provider( Some(candidate_hash), Some(candidate_json), None, + metering, duration, ), }) @@ -518,6 +541,7 @@ fn provider_error_call( None, None, None, + PlannerCallMetering::default(), duration, ), } @@ -738,6 +762,7 @@ fn planner_audit( candidate_hash: Option, candidate_json: Option, compiler_report: Option, + metering: PlannerCallMetering, duration_micros: u64, ) -> ExecutionPlanningAuditEnvelope { ExecutionPlanningAuditEnvelope { @@ -754,6 +779,8 @@ fn planner_audit( outcome, provider_model, prompt_version: EXECUTION_PLANNER_PROMPT_VERSION.to_string(), + usage: metering.usage, + cost_microusd: metering.cost_microusd, candidate_hash, candidate_json, compiler_report, @@ -951,8 +978,12 @@ pub async fn plan_amendment( 0, ) .await?; - let mut audits = vec![first_call.audit.clone()]; - let (parsed, repair_attempts) = match first_call.parsed { + let mut audits = Vec::new(); + let first_parsed = match record_amendment_provider_call(first_call, &mut audits) { + Ok(parsed) => parsed, + Err(message) => return Ok(amendment_budget_exhausted(message, audits)), + }; + let (parsed, repair_attempts) = match first_parsed { ParsedAmendmentCall::SchemaRejected(_) if request.config.planner_repair_attempts > 0 => { let completion = request::amendment_schema_repair_completion_request(&request) .map_err(|error| MoaError::SerializationError(error.to_string()))?; @@ -964,8 +995,11 @@ pub async fn plan_amendment( 1, ) .await?; - audits.push(repair_call.audit.clone()); - (repair_call.parsed, 1) + let parsed = match record_amendment_provider_call(repair_call, &mut audits) { + Ok(parsed) => parsed, + Err(message) => return Ok(amendment_budget_exhausted(message, audits)), + }; + (parsed, 1) } parsed => (parsed, 0), }; @@ -1006,14 +1040,17 @@ pub async fn plan_amendment( 1, ) .await?; - audits.push(repair_call.audit.clone()); + let repair_parsed = match record_amendment_provider_call(repair_call, &mut audits) { + Ok(parsed) => parsed, + Err(message) => return Ok(amendment_budget_exhausted(message, audits)), + }; let ParsedAmendmentCall::Candidate { candidate: repaired, candidate_json: repaired_json, candidate_hash: repaired_hash, - } = repair_call.parsed + } = repair_parsed else { - return Ok(amendment_terminal_provider(repair_call.parsed, audits)); + return Ok(amendment_terminal_provider(repair_parsed, audits)); }; let second = compile_amendment_candidate(&request, &repaired)?; let repair_audit = audits.last_mut().ok_or_else(|| { @@ -1028,9 +1065,12 @@ pub async fn plan_amendment( } } -struct AmendmentProviderCall { - parsed: ParsedAmendmentCall, - audit: ExecutionPlanningAuditEnvelope, +enum AmendmentProviderCall { + Audited { + parsed: ParsedAmendmentCall, + audit: Box, + }, + BudgetExhausted(String), } enum ParsedAmendmentCall { @@ -1047,6 +1087,19 @@ enum ParsedAmendmentCall { ProviderFailure(String), } +fn record_amendment_provider_call( + call: AmendmentProviderCall, + audits: &mut Vec, +) -> std::result::Result { + match call { + AmendmentProviderCall::Audited { parsed, audit } => { + audits.push(*audit); + Ok(parsed) + } + AmendmentProviderCall::BudgetExhausted(message) => Err(message), + } +} + async fn call_amendment_provider( provider: &dyn LLMProvider, request: &ExecutionAmendmentPlanningRequest, @@ -1071,6 +1124,9 @@ async fn call_amendment_provider( } }, Err(MoaError::Cancelled) => return Err(MoaError::Cancelled), + Err(MoaError::BudgetExhausted(message)) => { + return Ok(AmendmentProviderCall::BudgetExhausted(message)); + } Err(error) => { return Ok(amendment_provider_error( request, @@ -1082,6 +1138,7 @@ async fn call_amendment_provider( )); } }; + let metering = planner_call_metering(&response)?; let raw = response.text; let duration = duration_micros(started); if raw.len() > EXECUTION_PLANNER_CANDIDATE_MAX_BYTES { @@ -1097,11 +1154,11 @@ async fn call_amendment_provider( raw.as_bytes(), ), }; - return Ok(AmendmentProviderCall { + return Ok(AmendmentProviderCall::Audited { parsed: ParsedAmendmentCall::Unsupported( "amendment response exceeded its byte cap".to_string(), ), - audit: amendment_planner_audit( + audit: Box::new(amendment_planner_audit( request, call_kind, ordinal, @@ -1110,8 +1167,9 @@ async fn call_amendment_provider( Some(raw_hash), None, Some(canonical_string(&report)?), + metering, duration, - ), + )), }); } let candidate = match serde_json::from_str::(&raw) { @@ -1129,11 +1187,11 @@ async fn call_amendment_provider( }], ) .map_err(contract_error)?; - return Ok(AmendmentProviderCall { + return Ok(AmendmentProviderCall::Audited { parsed: ParsedAmendmentCall::SchemaRejected( "amendment response failed the strict response schema".to_string(), ), - audit: amendment_planner_audit( + audit: Box::new(amendment_planner_audit( request, call_kind, ordinal, @@ -1145,21 +1203,22 @@ async fn call_amendment_provider( )), None, Some(canonical_string(&report)?), + metering, duration, - ), + )), }); } }; let candidate_json = canonical_string(&candidate)?; let candidate_hash = execution_planning_hash("moa.execution.planner-candidate", candidate_json.as_bytes()); - Ok(AmendmentProviderCall { + Ok(AmendmentProviderCall::Audited { parsed: ParsedAmendmentCall::Candidate { candidate, candidate_json: candidate_json.clone(), candidate_hash: candidate_hash.clone(), }, - audit: amendment_planner_audit( + audit: Box::new(amendment_planner_audit( request, call_kind, ordinal, @@ -1168,8 +1227,9 @@ async fn call_amendment_provider( Some(candidate_hash), Some(candidate_json), None, + metering, duration, - ), + )), }) } @@ -1181,9 +1241,9 @@ fn amendment_provider_error( duration: u64, message: String, ) -> AmendmentProviderCall { - AmendmentProviderCall { + AmendmentProviderCall::Audited { parsed: ParsedAmendmentCall::ProviderFailure(message), - audit: amendment_planner_audit( + audit: Box::new(amendment_planner_audit( request, call_kind, ordinal, @@ -1192,8 +1252,9 @@ fn amendment_provider_error( None, None, None, + PlannerCallMetering::default(), duration, - ), + )), } } @@ -1266,6 +1327,7 @@ fn amendment_planner_audit( candidate_hash: Option, candidate_json: Option, compiler_report: Option, + metering: PlannerCallMetering, duration_micros: u64, ) -> ExecutionPlanningAuditEnvelope { ExecutionPlanningAuditEnvelope { @@ -1282,6 +1344,8 @@ fn amendment_planner_audit( outcome, provider_model, prompt_version: EXECUTION_PLANNER_PROMPT_VERSION.to_string(), + usage: metering.usage, + cost_microusd: metering.cost_microusd, candidate_hash, candidate_json, compiler_report, @@ -1391,6 +1455,16 @@ fn amendment_terminal_provider( } } +fn amendment_budget_exhausted( + message: String, + audits: Vec, +) -> ExecutionAmendmentPlanningResult { + ExecutionAmendmentPlanningResult { + kind: ExecutionAmendmentPlanningResultKind::BudgetExhausted { message }, + audits, + } +} + fn amendment_classified_terminal( classification: ClassifiedCompileOutcome, audits: Vec, diff --git a/crates/moa-brain/src/execution_planning/request.rs b/crates/moa-brain/src/execution_planning/request.rs index e5b9541d6..775c126fb 100644 --- a/crates/moa-brain/src/execution_planning/request.rs +++ b/crates/moa-brain/src/execution_planning/request.rs @@ -19,7 +19,7 @@ use serde::{Deserialize, Serialize}; use serde_json::Value; /// Stable execution-planner prompt identifier. -pub const EXECUTION_PLANNER_PROMPT_VERSION: &str = "execution-planner-v7"; +pub const EXECUTION_PLANNER_PROMPT_VERSION: &str = "execution-planner-v8"; /// Fixed maximum collected planner output tokens. pub const EXECUTION_PLANNER_MAX_OUTPUT_TOKENS: usize = 32_768; const EXECUTION_PLANNER_PROMPT: &str = include_str!("../prompts/execution_planner.md"); @@ -295,10 +295,10 @@ mod tests { use super::*; #[test] - fn execution_planner_prompt_v7_pins_long_horizon_and_compiler_invariants() { + fn execution_planner_prompt_v8_pins_long_horizon_and_compiler_invariants() { // Pins: the current emitted prompt version and its compiler-facing guidance // change together so live planner provenance identifies this exact contract. - assert_eq!(EXECUTION_PLANNER_PROMPT_VERSION, "execution-planner-v7"); + assert_eq!(EXECUTION_PLANNER_PROMPT_VERSION, "execution-planner-v8"); assert_eq!( EXECUTION_PLANNER_PROMPT, concat!( @@ -308,15 +308,16 @@ mod tests { "- Set `goal.objective` to the frozen `objective` byte-for-byte.\n", "- Choose exactly one explicit `plan.cancel_policy`: `retain_effects` or `compensate_committed`.\n", "- Treat the frozen `budget.deadline_at` as the absolute Durable-run deadline. Never emit a wait, retry window, or active task whose bound reaches or exceeds it.\n", - "- Use `WaitUntil` for an absolute calendar-time delay. Its `wake_at` must be before the run deadline, and its declared `result` is the structured value made available to downstream nodes after the timer fires.\n", - "- Give every `Review` and `WaitSignal` an explicit wait policy. Represent human and external waits only with these storage-backed wait operations; never keep an `Agent` or `Capability` active while waiting for a person, callback, schedule, or retry time.\n", + "- Use `WaitUntil` for a calendar-time delay. Its `wake` is a tagged temporal target, either `{\"kind\":\"at\",\"at\":\"\"}` for an exact instant or `{\"kind\":\"after\",\"delay_seconds\":}` for a delay measured from the moment the node starts waiting. Emit exactly those fields for the chosen shape and nothing else. The resolved wake time must land before the run deadline. Its declared `result` is the structured value made available to downstream nodes after the timer fires.\n", + "- Give every `Review` and `WaitSignal` an explicit `wait_policy` of `{\"expiry\": , \"on_expiry\": }`, using the same two temporal-target shapes. An expiry action is either `{\"kind\":\"fail_task\"}` or `{\"kind\":\"continue_with\",\"output\":}`. Represent human and external waits only with these storage-backed wait operations; never keep an `Agent` or `Capability` active while waiting for a person, callback, schedule, or retry time.\n", + "- Always set `plan.input_wait_policy`. It is required, and it governs every task that pauses for runtime input rather than one named node, so its `on_expiry` accepts only `{\"kind\":\"fail_task\"}` — never `continue_with`.\n", "- Decompose long work into bounded active tasks separated by durable nodes. Never plan a continuously running multi-hour or multi-day model call, tool call, shell process, network connection, or sandbox; use a registered asynchronous capability when the catalog explicitly provides one.\n", "- Set every node's `compensation` explicitly. Use `null` unless the node is a direct side-effecting `Capability` whose exact catalog entry advertises the same compensator and bounded input mapping, and that compensator has `requires_sandbox=false`. Never add compensation to reads, agents, maps, reduces, reviews, signals, or outputs, never use a sandbox-backed compensator, and never invent rollback authority.\n", "- An amendment must preserve compensation for work that is running or committed and must not weaken the run's cancellation policy.\n", "- Every goal-entry ID, completion-check ID, execution-node ID, and every ID referenced from those structures must match `[a-z][a-z0-9_-]{0,63}`.\n", "- Link every requirement and every constraint to at least one completion check via `requirement_ids` and `constraint_ids`.\n", "- Put every goal requirement ID in at least one completion check's `requirement_ids`. If the plan has only one completion check, it must list every requirement ID. For a simple `Agent`-to-`Output` plan, prefer one `OutputSchema` check listing all requirement IDs.\n", - "- Use only whole-value execution references: exactly `$.input` or `$.nodes..output`; never append field paths.\n", + "- Use only whole-value binding objects of exactly `{\"$ref\":\"\"}` with no sibling keys or string interpolation. A reference path may select the complete `$.input` or `$.nodes..output` value, or append dot-separated object fields such as `$.input.query` and `$.nodes.lookup.output.items`; node references may read only declared dependencies. Never use bracket/index syntax.\n", "- When an `output` operation forwards another node's output, set the output node's `input` to `{}` and put `{\"$ref\":\"$.nodes..output\"}` directly in `operation.value`.\n", ) ); diff --git a/crates/moa-brain/src/execution_planning/response.rs b/crates/moa-brain/src/execution_planning/response.rs index a4043d0d2..fddda193f 100644 --- a/crates/moa-brain/src/execution_planning/response.rs +++ b/crates/moa-brain/src/execution_planning/response.rs @@ -67,6 +67,8 @@ pub enum ExecutionAmendmentPlanningResultKind { /// /// The `message` is planner-authored and safe to surface to the caller. Unsupported { message: String }, + /// The durable run budget denied the next automatic planner call before provider I/O. + BudgetExhausted { message: String }, /// The amendment planner provider or transport failed, so no verdict exists. /// /// The `message` carries raw provider detail for durable diagnostics only and must diff --git a/crates/moa-brain/src/execution_planning/routing.rs b/crates/moa-brain/src/execution_planning/routing.rs index 8f8a258db..b2426a2e0 100644 --- a/crates/moa-brain/src/execution_planning/routing.rs +++ b/crates/moa-brain/src/execution_planning/routing.rs @@ -484,7 +484,7 @@ fn decision_missing_input_count(decision: &ExecutionRouteDecision) -> Result } } -fn route_usage(usage: TokenUsage) -> Result { +pub(super) fn route_usage(usage: TokenUsage) -> Result { Ok(ExecutionRouteUsage { input_tokens_uncached: u64::try_from(usage.input_tokens_uncached).map_err(|_| { MoaError::ValidationError("route uncached input usage exceeds u64".to_string()) diff --git a/crates/moa-brain/src/prompts/execution_planner.md b/crates/moa-brain/src/prompts/execution_planner.md index b16ac15aa..23b214675 100644 --- a/crates/moa-brain/src/prompts/execution_planner.md +++ b/crates/moa-brain/src/prompts/execution_planner.md @@ -15,5 +15,5 @@ Compiler invariants: - Every goal-entry ID, completion-check ID, execution-node ID, and every ID referenced from those structures must match `[a-z][a-z0-9_-]{0,63}`. - Link every requirement and every constraint to at least one completion check via `requirement_ids` and `constraint_ids`. - Put every goal requirement ID in at least one completion check's `requirement_ids`. If the plan has only one completion check, it must list every requirement ID. For a simple `Agent`-to-`Output` plan, prefer one `OutputSchema` check listing all requirement IDs. -- Use only whole-value execution references: exactly `$.input` or `$.nodes..output`; never append field paths. +- Use only whole-value binding objects of exactly `{"$ref":""}` with no sibling keys or string interpolation. A reference path may select the complete `$.input` or `$.nodes..output` value, or append dot-separated object fields such as `$.input.query` and `$.nodes.lookup.output.items`; node references may read only declared dependencies. Never use bracket/index syntax. - When an `output` operation forwards another node's output, set the output node's `input` to `{}` and put `{"$ref":"$.nodes..output"}` directly in `operation.value`. diff --git a/crates/moa-brain/tests/brain_turn_offline.rs b/crates/moa-brain/tests/brain_turn_offline.rs index 0a14ef378..1f999740f 100644 --- a/crates/moa-brain/tests/brain_turn_offline.rs +++ b/crates/moa-brain/tests/brain_turn_offline.rs @@ -19,6 +19,28 @@ use wiremock::MockServer; use offline_session_store::{MockSessionStore, session_meta}; use openai_wiremock::{captured_json_bodies, mount_openai_text}; +struct BudgetDeniedPlannerProvider; + +#[async_trait::async_trait] +impl moa_core::traits::LLMProvider for BudgetDeniedPlannerProvider { + fn name(&self) -> &'static str { + "budget-denied-planner" + } + + fn capabilities(&self) -> moa_core::types::model::ModelCapabilities { + MockLlmProvider.capabilities() + } + + async fn complete( + &self, + _request: moa_core::types::completion::SharedCompletionRequest, + ) -> moa_core::error::Result { + Err(moa_core::error::MoaError::BudgetExhausted( + "automatic amendment planning exhausted the approved run budget".to_string(), + )) + } +} + fn fixture_worker_workspace_scope( session: &SessionMeta, ) -> moa_core::types::sandbox_workspace::SandboxWorkspaceScope { @@ -512,6 +534,67 @@ async fn execution_planning_amendment_schema_rejection_regenerates_once_without_ ); } +#[tokio::test] +async fn execution_planning_amendment_audits_attribute_each_repair_call_usage_and_cost() { + // Pins: malformed amendment output and its one repair are two paid provider calls. Each audit + // must retain its own normalized token counters and exact model-priced cost rather than + // collapsing repair spend into an unattributed planner total. + let first_usage = moa_core::types::completion::TokenUsage { + input_tokens_uncached: 1_000, + input_tokens_cache_write: 200, + input_tokens_cache_read: 300, + output_tokens: 0, + }; + let repair_usage = moa_core::types::completion::TokenUsage { + input_tokens_uncached: 2_000, + input_tokens_cache_write: 400, + input_tokens_cache_read: 600, + output_tokens: 0, + }; + let provider = ScriptedProvider::new(MockLlmProvider.capabilities()) + .push_response(ScriptedResponse::text("INVALID_AMENDMENT").with_usage(first_usage)) + .push_response( + ScriptedResponse::text(execution_amendment_candidate(7, true)).with_usage(repair_usage), + ); + + let result = moa_brain::execution_planning::plan_amendment( + &provider, + execution_amendment_planning_request(), + ) + .await + .expect("one metered amendment repair should succeed"); + + let calls = result + .audits + .iter() + .filter_map(|audit| match &audit.payload { + moa_core::types::execution_planning::ExecutionPlanningAuditPayload::PlannerCall { + call_ordinal, + usage, + cost_microusd, + .. + } => Some((*call_ordinal, *usage, *cost_microusd)), + _ => None, + }) + .collect::>(); + assert_eq!(calls.len(), 2); + assert_eq!(calls[0].0, 0); + assert_eq!(calls[0].1.input_tokens_uncached, 1_000); + assert_eq!(calls[0].1.input_tokens_cache_write, 200); + assert_eq!(calls[0].1.input_tokens_cache_read, 300); + assert!(calls[0].1.output_tokens > 0); + assert!(calls[0].2 > 0, "the initial call must retain priced spend"); + assert_eq!(calls[1].0, 1); + assert_eq!(calls[1].1.input_tokens_uncached, 2_000); + assert_eq!(calls[1].1.input_tokens_cache_write, 400); + assert_eq!(calls[1].1.input_tokens_cache_read, 600); + assert!(calls[1].1.output_tokens > calls[0].1.output_tokens); + assert!( + calls[1].2 > calls[0].2, + "repair spend must be attributed separately" + ); +} + #[tokio::test] async fn execution_planning_second_amendment_schema_rejection_stops_without_third_call() { // Pins: a malformed amendment schema-regeneration response is terminal and cannot recurse. @@ -623,6 +706,27 @@ async fn execution_planning_amendment_provider_failure_is_distinct_from_unsuppor assert_eq!(provider.recorded_requests().len(), 1); } +#[tokio::test] +async fn execution_planning_amendment_budget_denial_is_not_a_provider_call_audit() { + // Pins: durable reserve-before-dispatch denial is a typed budget stop, not a provider failure + // and not an invented paid-call audit; the provider implementation has made no gateway call. + let result = moa_brain::execution_planning::plan_amendment( + &BudgetDeniedPlannerProvider, + execution_amendment_planning_request(), + ) + .await + .expect("budget denial should remain a typed amendment result"); + + assert!(matches!( + result.kind, + moa_brain::execution_planning::ExecutionAmendmentPlanningResultKind::BudgetExhausted { .. } + )); + assert!( + result.audits.is_empty(), + "a call denied before provider I/O must not persist a planner-call audit" + ); +} + #[tokio::test] async fn execution_routing_respond_execute_use_classifier_while_pinned_template_skips_planner() { // Pins: ordinary routes use one strict classifier response while a pinned template remains a @@ -768,7 +872,9 @@ fn execution_planning_request( max_tasks: Some(100), max_tool_calls: Some(100), max_retrieved_bytes: Some(1_000_000), - deadline_at: None, + deadline_at: Some( + moa_test_support::fixtures::pg_now() + chrono::TimeDelta::hours(2), + ), }, }, execution_template: None, @@ -800,6 +906,10 @@ fn execution_planning_candidate(objective: &str, max_attempts: u32) -> String { }, "plan": { "cancel_policy": "retain_effects", + "input_wait_policy": { + "expiry": {"kind": "after", "delay_seconds": 3600}, + "on_expiry": {"kind": "fail_task"} + }, "input_schema": { "type": "object" }, "output_schema": { "type": "object" }, "nodes": [{ diff --git a/crates/moa-brain/tests/query_rewrite_live.rs b/crates/moa-brain/tests/query_rewrite_live.rs index dfc8e008f..03ec90ef0 100644 --- a/crates/moa-brain/tests/query_rewrite_live.rs +++ b/crates/moa-brain/tests/query_rewrite_live.rs @@ -1,4 +1,4 @@ -// Live counterpart: see query_rewrite_offline.rs for the wiremock version that runs in PR CI. +// Live counterpart: see brain_offline/query_rewrite_offline.rs in the brain_offline harness for the wiremock PR-CI coverage. //! Ignored live smoke test for query rewrite gate behavior. diff --git a/crates/moa-brain/tests/turbopuffer_news_live.rs b/crates/moa-brain/tests/turbopuffer_news_live.rs index 540502839..02174c40f 100644 --- a/crates/moa-brain/tests/turbopuffer_news_live.rs +++ b/crates/moa-brain/tests/turbopuffer_news_live.rs @@ -1,4 +1,4 @@ -// Live counterpart: see turbopuffer_news_offline.rs for the wiremock version that runs in PR CI. +// Live counterpart: see brain_offline/turbopuffer_news_offline.rs in the brain_offline harness for the wiremock PR-CI coverage. //! Live end-to-end Turbopuffer promotion and retrieval test. diff --git a/crates/moa-config/src/env_overlay/mod.rs b/crates/moa-config/src/env_overlay/mod.rs index 819c79a14..60ddbd399 100644 --- a/crates/moa-config/src/env_overlay/mod.rs +++ b/crates/moa-config/src/env_overlay/mod.rs @@ -1237,6 +1237,7 @@ const ALLOWLIST_EXACT: &[&str] = &[ "MOA_ORCHESTRATOR_FEATURES", "MOA_PERSIST_TURN_METRICS", "MOA_PROVIDERS_OVERRIDE", + "MOA_RESTATE_ADMIN_URL", // maintenance-only drain observer override "MOA_SCIM_BASE_URL", "MOA_SKIP_FGA", "MOA_TOXIPROXY_URL", diff --git a/crates/moa-config/src/env_overlay/tests.rs b/crates/moa-config/src/env_overlay/tests.rs index 6f84884ab..afb5ddd73 100644 --- a/crates/moa-config/src/env_overlay/tests.rs +++ b/crates/moa-config/src/env_overlay/tests.rs @@ -88,6 +88,14 @@ fn registry_accepts_known_field_and_allowlisted_specials() { assert!(clean.is_empty(), "expected no unknown vars, got {clean:?}"); } +#[test] +fn strict_registry_accepts_maintenance_restate_admin_override() { + // Pins: the singleton maintenance drain observer may use its direct Admin + // endpoint override without strict environment auditing rejecting startup. + EnvOverlay::audit_env_registry(names(&["MOA_RESTATE_ADMIN_URL"]), true) + .expect("maintenance Restate Admin override must be an exact approved special variable"); +} + #[test] fn registry_ignores_non_moa_and_lowercase_prefix_boundary() { // Pins: only `MOA_`-prefixed names are considered; `MOALITE` and unrelated diff --git a/crates/moa-config/src/execution.rs b/crates/moa-config/src/execution.rs index d7ca87375..32616a27f 100644 --- a/crates/moa-config/src/execution.rs +++ b/crates/moa-config/src/execution.rs @@ -27,8 +27,18 @@ pub struct ExecutionConfig { pub dispatch_batch_size: usize, /// Maximum duration of one active task attempt, in seconds. pub active_attempt_timeout_seconds: u64, - /// Interval without durable attempt progress after which an active attempt is stalled, - /// in seconds. + /// Floor of the window without durable attempt progress after which an attempt is + /// stalled, in seconds. + /// + /// This is a floor, not the whole window. The heartbeat is written at step boundaries + /// and not while a step runs, so the effective window is the larger of this value and + /// the bound the in-flight step declared plus + /// `moa_execution::repository::ATTEMPT_STEP_BOUND_MARGIN_SECONDS`. A step that declares + /// a long timeout therefore widens only its own window, instead of forcing every + /// attempt to wait out the slowest step the platform allows. + /// + /// It must stay strictly below `active_attempt_timeout_seconds`: a floor at or beyond + /// the attempt deadline can never classify a stall before the deadline does. pub attempt_heartbeat_staleness_seconds: u64, /// Maximum non-parked execution runs admitted for one tenant. pub max_tenant_active_runs: u32, @@ -94,6 +104,9 @@ impl Default for ExecutionConfig { maximum_activation_steps: 128, dispatch_batch_size: DEFAULT_MAX_IN_FLIGHT_TASKS, active_attempt_timeout_seconds: 10 * 60, + // Covers a step that declares no bound of its own: a model turn, and any tool + // call that does not ask for longer than the default sandbox command timeout. + // A step that declares more widens its own window instead of this floor. attempt_heartbeat_staleness_seconds: 2 * 60, max_tenant_active_runs: 100, max_fleet_active_runs: 1_000, diff --git a/crates/moa-core/src/types/execution_planning.rs b/crates/moa-core/src/types/execution_planning.rs index 7bbc0795d..7975449ce 100644 --- a/crates/moa-core/src/types/execution_planning.rs +++ b/crates/moa-core/src/types/execution_planning.rs @@ -574,6 +574,10 @@ pub enum ExecutionPlanningAuditPayload { provider_model: String, /// Stable planner prompt version. prompt_version: String, + /// Normalized provider token usage for this exact planner call. + usage: ExecutionRouteUsage, + /// Provider cost attributed to this exact planner call in integer micro-US-dollars. + cost_microusd: u64, /// Candidate or raw-response hash when required by the outcome. candidate_hash: Option, /// Canonical strict candidate JSON when required by the outcome. @@ -1420,6 +1424,8 @@ pub fn planning_audit_semantically_equal( outcome: left_outcome, provider_model: left_model, prompt_version: left_prompt, + usage: left_usage, + cost_microusd: left_cost, candidate_hash: left_hash, candidate_json: left_candidate, compiler_report: left_report, @@ -1433,6 +1439,8 @@ pub fn planning_audit_semantically_equal( outcome: right_outcome, provider_model: right_model, prompt_version: right_prompt, + usage: right_usage, + cost_microusd: right_cost, candidate_hash: right_hash, candidate_json: right_candidate, compiler_report: right_report, @@ -1447,6 +1455,8 @@ pub fn planning_audit_semantically_equal( left_outcome, left_model, left_prompt, + left_usage, + left_cost, left_hash, left_candidate, left_report, @@ -1458,6 +1468,8 @@ pub fn planning_audit_semantically_equal( right_outcome, right_model, right_prompt, + right_usage, + right_cost, right_hash, right_candidate, right_report, @@ -1552,6 +1564,8 @@ pub fn validate_planning_audit_envelope( outcome, provider_model, prompt_version, + usage, + cost_microusd, candidate_hash, candidate_json, compiler_report, @@ -1587,6 +1601,16 @@ pub fn validate_planning_audit_envelope( } ensure_nonempty_bytes("payload.provider_model", provider_model, 128)?; ensure_nonempty_bytes("payload.prompt_version", prompt_version, 64)?; + if matches!(outcome, ExecutionPlannerOutcome::ProviderError) + && (!usage.is_zero() || *cost_microusd != 0) + { + return Err(ExecutionPlanningContractError::InvalidField { + field: "payload.usage".to_string(), + message: + "planner calls without a collected response cannot carry usage or cost" + .to_string(), + }); + } let requires_candidate = !matches!(outcome, ExecutionPlannerOutcome::ProviderError); if requires_candidate != candidate_hash.is_some() { return Err(ExecutionPlanningContractError::InvalidField { @@ -2436,6 +2460,8 @@ mod tests { outcome, provider_model: "planner-model".to_string(), prompt_version: "execution-planner".to_string(), + usage: ExecutionRouteUsage::default(), + cost_microusd: 0, candidate_hash, candidate_json, compiler_report, @@ -2869,6 +2895,16 @@ mod tests { &accepted, &accepted_replay )); + if let ExecutionPlanningAuditPayload::PlannerCall { usage, .. } = + &mut accepted_replay.payload + { + usage.output_tokens = 1; + } + assert!( + !planning_audit_semantically_equal(&accepted, &accepted_replay), + "planner usage is billed replay evidence, not an ignorable measurement" + ); + accepted_replay = accepted.clone(); if let ExecutionPlanningAuditPayload::PlannerCall { compiler_report, .. } = &mut accepted_replay.payload @@ -2891,6 +2927,30 @@ mod tests { )); } + #[test] + fn provider_error_planner_audit_rejects_unattributed_usage_offline() { + // Pins: a provider failure without a collected response cannot invent billed token or cost + // attribution; only calls carrying an authoritative response may report those counters. + let mut audit = + planner_call_envelope(ExecutionPlannerOutcome::ProviderError, None, None, None); + assert_eq!(validate_planning_audit_envelope(&audit), Ok(())); + let ExecutionPlanningAuditPayload::PlannerCall { + usage, + cost_microusd, + .. + } = &mut audit.payload + else { + panic!("fixture must be a planner call"); + }; + usage.input_tokens_uncached = 1; + *cost_microusd = 1; + assert!(matches!( + validate_planning_audit_envelope(&audit), + Err(ExecutionPlanningContractError::InvalidField { field, .. }) + if field == "payload.usage" + )); + } + #[test] fn execution_planning_audit_report_sorts_truncates_and_hashes_full_input() { // Pins: bounded audit evidence remains deterministic without dropping full-report identity. diff --git a/crates/moa-core/src/types/sandbox_workspace.rs b/crates/moa-core/src/types/sandbox_workspace.rs index 78c21837d..10898bc7b 100644 --- a/crates/moa-core/src/types/sandbox_workspace.rs +++ b/crates/moa-core/src/types/sandbox_workspace.rs @@ -244,36 +244,6 @@ pub enum ExecutionHandReleaseOwner { }, } -/// What a continuation boundary actually did with the attempt's sandbox compute. -/// -/// A continuation is not a wait: the next slice is enqueued immediately, so the -/// boundary tries to keep the sandbox rather than destroy and re-provision it -/// milliseconds later. The three keep-or-not outcomes have materially different -/// cost and different follow-up work for the caller, so they are reported -/// explicitly instead of being flattened into success. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "disposition", rename_all = "snake_case", deny_unknown_fields)] -pub enum ExecutionHandContinuationDisposition { - /// The attempt owned no durable workspace or no live lease; nothing was kept. - NoComputeOwned, - /// Compute was released in-path and the filesystem kept for the next slice. - /// - /// The lease stays live so the next slice reattaches, and the active-compute - /// capacity charge went back to the fleet. - Suspended, - /// The provider cannot release compute, so the hand stays hot on a short bound. - /// - /// The reaper owns the deadline; a slice that does not arrive inside it loses - /// the sandbox and restores from the checkpoint published here. - RetainedHot, - /// Suspension was attempted and failed; the caller must fall back to release. - /// - /// The checkpoint is published either way, so the caller finishes the - /// ordinary checkpoint-and-destroy path rather than leaving a hand hot on a - /// bet that already lost. - SuspendFailed, -} - /// Durable proof that one exact execution attempt released its sandbox compute. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] diff --git a/crates/moa-eval/scenarios/execution/manifest.toml b/crates/moa-eval/scenarios/execution/manifest.toml index 4d87de89b..a4079d483 100644 --- a/crates/moa-eval/scenarios/execution/manifest.toml +++ b/crates/moa-eval/scenarios/execution/manifest.toml @@ -7,7 +7,7 @@ count = 328 [contract] path = "contract-recorded.jsonl" -sha256 = "a7d2f680db94d508f8265ac8551fcfd46cdc137605b1efd18aa4e45c000e110a" +sha256 = "3099323222930134e78f6945fe288015c3704c3b09c2f6b7d18def0a2035de5a" count = 80 [task_quality] diff --git a/crates/moa-eval/tests/eval_offline/execution_snapshot.rs b/crates/moa-eval/tests/eval_offline/execution_snapshot.rs index 3d96f8d4a..ee7ce2b9b 100644 --- a/crates/moa-eval/tests/eval_offline/execution_snapshot.rs +++ b/crates/moa-eval/tests/eval_offline/execution_snapshot.rs @@ -317,6 +317,8 @@ fn runtime_parts( outcome: ExecutionPlannerOutcome::Accepted, provider_model: "scripted-planner".to_string(), prompt_version: "execution-planner".to_string(), + usage: Default::default(), + cost_microusd: 0, candidate_hash: Some("a".repeat(64)), candidate_json: Some(RAW_AUDIT_SECRET.to_string()), compiler_report: Some("raw compiler report".to_string()), @@ -404,6 +406,7 @@ fn task_record(run_uid: Uuid, item_key: &str, status: ExecutionTaskStatus) -> Ex }, attempt_started_at: Some(now), last_progress_at: now, + progress_step_bound_seconds: None, attempt_deadline_at: None, waiting_since: None, ready_at: None, diff --git a/crates/moa-execution/src/repository/audit.rs b/crates/moa-execution/src/repository/audit.rs index fc8e2d949..47eb4a312 100644 --- a/crates/moa-execution/src/repository/audit.rs +++ b/crates/moa-execution/src/repository/audit.rs @@ -75,6 +75,10 @@ pub struct PlannerCallAuditEvidence { pub outcome: ExecutionPlannerOutcome, /// First persisted measured duration. pub duration_micros: u64, + /// Normalized provider token usage attributed to this call. + pub usage: ExecutionRouteUsage, + /// Provider cost attributed to this call in micro-US-dollars. + pub cost_microusd: u64, /// Candidate hash when required by the outcome. pub candidate_hash: Option, } @@ -457,6 +461,8 @@ impl ExecutionRepository { outcome, provider_model, prompt_version, + usage, + cost_microusd, candidate_hash, candidate_json, compiler_report, @@ -508,6 +514,20 @@ impl ExecutionRepository { .bind(candidate_hash) .bind(candidate_json) .bind(compiler_report) + .bind(to_i64( + usage.input_tokens_uncached, + "planner uncached input tokens", + )?) + .bind(to_i64( + usage.input_tokens_cache_write, + "planner cache-write tokens", + )?) + .bind(to_i64( + usage.input_tokens_cache_read, + "planner cache-read tokens", + )?) + .bind(to_i64(usage.output_tokens, "planner output tokens")?) + .bind(to_i64(*cost_microusd, "planner cost")?) .bind(duration_micros_db) .bind(*created_at) .fetch_optional(conn.as_mut()) diff --git a/crates/moa-execution/src/repository/audit_codec.rs b/crates/moa-execution/src/repository/audit_codec.rs index 6f6366076..e0ba0170e 100644 --- a/crates/moa-execution/src/repository/audit_codec.rs +++ b/crates/moa-execution/src/repository/audit_codec.rs @@ -94,6 +94,8 @@ impl PersistedPlannerAudit { outcome, provider_model, prompt_version, + usage, + cost_microusd, candidate_hash, candidate_json, compiler_report, @@ -110,6 +112,8 @@ impl PersistedPlannerAudit { && self.evidence.outcome == *outcome && self.provider_model == *provider_model && self.prompt_version == *prompt_version + && self.evidence.usage == *usage + && self.evidence.cost_microusd == *cost_microusd && self.evidence.candidate_hash == *candidate_hash && self.candidate_json == *candidate_json && self.compiler_report == *compiler_report @@ -342,6 +346,13 @@ pub(super) fn planner_audit_from_row(row: &PgRow) -> Result>>()?; let processed_task_count = - u32::try_from(task_rows.len()).map_err(|_| Error::InvalidRepositoryData { + u32::try_from(tasks.len()).map_err(|_| Error::InvalidRepositoryData { message: "terminal drain task page exceeds u32".to_string(), })?; + let storage_task_ids = tasks + .iter() + .filter(|task| { + task.status != ExecutionTaskStatus::WaitingExternal + && !matches!( + task.attempt_state, + ExecutionAttemptState::Dispatching | ExecutionAttemptState::Running + ) + }) + .map(|task| task.task_id.as_uuid()) + .collect::>(); + supersede_storage_task_waits(&mut conn, run.run_uid, run.tenant_id.0, &storage_task_ids) + .await?; let mut settled_task_count = 0_u64; - let mut cancellation_dispatches = Vec::with_capacity(task_rows.len()); - for row in task_rows { - let task = task_from_row(&row)?; + let mut cancellation_dispatches = Vec::with_capacity(tasks.len()); + for task in tasks { if task.status == ExecutionTaskStatus::WaitingExternal { let external_job_uid = task.external_job_uid @@ -4346,7 +4356,6 @@ async fn advance_pending_terminal_page_in_conn( ); continue; } - supersede_storage_task_waits(&mut conn, &task).await?; let original_status = task.status; match record_task_outcome_in_conn( &mut conn, @@ -5140,30 +5149,101 @@ fn compensation_outcome_from_review_resolution( async fn supersede_storage_task_waits( conn: &mut ScopedConn<'_>, - task: &ExecutionTaskRecord, + run_uid: Uuid, + tenant_id: Uuid, + task_ids: &[Uuid], ) -> Result<()> { + if task_ids.is_empty() { + return Ok(()); + } let trigger_uids = sqlx::query_scalar::<_, Uuid>( "UPDATE moa.execution_trigger SET state='superseded', updated_at=NOW() \ - WHERE run_uid=$1 AND task_id=$2 \ + WHERE run_uid=$1 AND task_id = ANY($2::UUID[]) \ AND trigger_kind <> 'task_watchdog' AND state = 'pending' \ RETURNING trigger_uid", ) - .bind(task.run_uid) - .bind(task.task_id.as_uuid()) + .bind(run_uid) + .bind(task_ids) .fetch_all(conn.as_mut()) .await .map_err(sqlx_error)?; - if !trigger_uids.is_empty() { - sqlx::query( - "UPDATE moa.execution_dispatch_outbox SET state='superseded', claim_owner=NULL, \ - claimed_at=NULL, claim_expires_at=NULL, updated_at=NOW() \ - WHERE trigger_uid=ANY($1::UUID[]) \ - AND state IN ('pending','dispatching')", - ) - .bind(&trigger_uids) - .execute(conn.as_mut()) - .await - .map_err(sqlx_error)?; + if trigger_uids.is_empty() { + return Ok(()); + } + sqlx::query( + "UPDATE moa.execution_dispatch_outbox \ + SET state='cancelled', claim_owner=NULL, claimed_at=NULL, claim_expires_at=NULL, \ + updated_at=NOW() \ + WHERE trigger_uid = ANY($1::UUID[]) AND state IN ('pending','dispatching')", + ) + .bind(&trigger_uids) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let receipt_count: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM moa.execution_capacity_reservation AS reservation \ + JOIN moa.execution_trigger AS trigger \ + ON trigger.trigger_uid=reservation.trigger_uid \ + AND trigger.tenant_id=reservation.tenant_id \ + AND trigger.run_uid IS NOT DISTINCT FROM reservation.run_uid \ + AND trigger.controller_generation IS NOT DISTINCT FROM reservation.controller_generation \ + WHERE reservation.trigger_uid = ANY($1::UUID[]) \ + AND reservation.tenant_id=$2 AND reservation.run_uid=$3 \ + AND reservation.resource_dimension='scheduled_triggers'", + ) + .bind(&trigger_uids) + .bind(tenant_id) + .bind(run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if usize::try_from(receipt_count).ok() != Some(trigger_uids.len()) { + return Err(Error::InvalidRepositoryData { + message: "storage-wait trigger capacity receipts do not match their exact owners" + .to_string(), + }); + } + let released_quantities = sqlx::query_scalar::<_, i64>( + "UPDATE moa.execution_capacity_reservation \ + SET state='released', released_at=NOW(), updated_at=NOW() \ + WHERE trigger_uid = ANY($1::UUID[]) AND tenant_id=$2 AND run_uid=$3 \ + AND resource_dimension='scheduled_triggers' \ + AND state IN ('reserved','reconciling') AND released_at IS NULL \ + RETURNING quantity", + ) + .bind(&trigger_uids) + .bind(tenant_id) + .bind(run_uid) + .fetch_all(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let released_quantity = released_quantities + .into_iter() + .try_fold(0_i64, i64::checked_add) + .ok_or_else(|| Error::InvalidRepositoryData { + message: "storage-wait trigger capacity quantity overflowed PostgreSQL BIGINT" + .to_string(), + })?; + if released_quantity == 0 { + return Ok(()); + } + let buckets = sqlx::query( + "UPDATE moa.execution_capacity_bucket \ + SET reserved_quantity=reserved_quantity-$2, version=version+1, updated_at=NOW() \ + WHERE resource_dimension='scheduled_triggers' AND reserved_quantity >= $2 \ + AND ((scope_kind='fleet' AND tenant_id IS NULL) \ + OR (scope_kind='tenant' AND tenant_id=$1))", + ) + .bind(tenant_id) + .bind(released_quantity) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if buckets.rows_affected() != 2 { + return Err(Error::InvalidRepositoryData { + message: "storage-wait trigger release did not decrement both capacity buckets" + .to_string(), + }); } Ok(()) } @@ -5226,7 +5306,7 @@ async fn checkpoint_pending_terminal_wake( let row = sqlx::query( "UPDATE moa.execution_run SET status=$4, activation_state='idle', \ next_wake_at=NULL, waiting_since=NULL, ready_task_count=$5, \ - active_task_count=$6, processed_wake_epoch=$3, \ + active_task_count=$6, processed_wake_epoch=$3, activation_failure_count=0, \ last_progress_at=GREATEST(last_progress_at,$7), updated_at=NOW() \ WHERE run_uid=$1 AND controller_generation=$2 AND wake_epoch >= $3 \ AND processed_wake_epoch < $3 \ @@ -5363,7 +5443,7 @@ async fn finalize_pending_terminal_exact( waiting_input_user_task_count=0, waiting_input_tenant_admin_task_count=0, \ waiting_input_external_task_count=0, waiting_reasons_truncated=FALSE, \ waiting_since=NULL, ready_task_count=0, active_task_count=0, \ - processed_wake_epoch=$3, completed_at=$12, \ + processed_wake_epoch=$3, activation_failure_count=0, completed_at=$12, \ last_progress_at=GREATEST(last_progress_at,$12), updated_at=NOW() \ WHERE run_uid=$1 AND controller_generation=$2 AND wake_epoch >= $3 \ AND processed_wake_epoch < $3 AND pending_terminal_cause=$13 \ diff --git a/crates/moa-execution/src/repository/completion.rs b/crates/moa-execution/src/repository/completion.rs index b1a7223fb..79811bc1a 100644 --- a/crates/moa-execution/src/repository/completion.rs +++ b/crates/moa-execution/src/repository/completion.rs @@ -673,7 +673,7 @@ async fn commit_completion_page( }; let acknowledged = sqlx::query( "UPDATE moa.execution_run SET processed_wake_epoch=$3, \ - activation_state='idle',updated_at=NOW() \ + activation_state='idle',activation_failure_count=0,updated_at=NOW() \ WHERE run_uid=$1 AND controller_generation=$2 AND wake_epoch=$3 \ AND processed_wake_epoch<$3 AND activation_state='advancing' \ RETURNING tenant_id", diff --git a/crates/moa-execution/src/repository/external_job.rs b/crates/moa-execution/src/repository/external_job.rs index 7f1447860..215faa423 100644 --- a/crates/moa-execution/src/repository/external_job.rs +++ b/crates/moa-execution/src/repository/external_job.rs @@ -38,8 +38,9 @@ use super::{ run::enqueue_run_activation_in_conn, sqlx_error, storage_error, task::{ - ExternalJobTaskSettlementOutcome, TaskAttemptExternalOutcome, - TaskExternalStartRetryOutcome, + ExternalJobTaskSettlementOutcome, TaskAttemptCheckpointKind, TaskAttemptExternalOutcome, + TaskAttemptFence, TaskExternalStartRetryOutcome, + external_start_checkpoint_payload_is_provisional, settle_external_job_terminal_in_conn as settle_task_external_job_terminal_in_conn, }, to_i64, @@ -464,6 +465,133 @@ impl ExecutionRepository { Ok(record) } + /// Loads the exact current unbound provider-start recovery that owns a running task attempt. + /// + /// A due task watchdog must defer to this intent: only the provider adapter's crash-safe + /// `recover_start` result can decide whether the ambiguous start created external work. + /// Returning `None` means no complete current recovery authority exists for the supplied + /// attempt fence; malformed matched durable state is reported as repository corruption. + pub async fn load_current_task_external_start_recovery( + &self, + fence: TaskAttemptFence, + ) -> Result> { + let mut conn = ExecutionScope::ControlPlane.begin(&self.pool).await?; + let row = sqlx::query( + r#" + SELECT job.*, recovery.trigger_uid, recovery.payload AS recovery_payload, + checkpoint.checkpoint_kind, checkpoint.payload AS checkpoint_payload + FROM moa.execution_task AS task + JOIN moa.execution_run AS run + ON run.run_uid=task.run_uid AND run.tenant_id=task.tenant_id + JOIN moa.execution_capacity_reservation AS active_capacity + ON active_capacity.reservation_uid=$7 + AND active_capacity.tenant_id=task.tenant_id + AND active_capacity.run_uid=task.run_uid + AND active_capacity.task_id=task.task_id + AND active_capacity.controller_generation=$4 + AND active_capacity.attempt_generation=$3 + AND active_capacity.resource_dimension='active_tasks' + AND active_capacity.state IN ('reserved','reconciling') + AND active_capacity.released_at IS NULL + JOIN moa.execution_trigger AS watchdog + ON watchdog.trigger_uid=$6 AND watchdog.tenant_id=task.tenant_id + AND watchdog.run_uid=task.run_uid AND watchdog.task_id=task.task_id + AND watchdog.controller_generation=$4 AND watchdog.attempt_generation=$3 + AND watchdog.trigger_kind='task_watchdog' AND watchdog.state='pending' + JOIN moa.execution_task_checkpoint AS checkpoint + ON checkpoint.tenant_id=task.tenant_id AND checkpoint.run_uid=task.run_uid + AND checkpoint.task_id=task.task_id AND checkpoint.controller_generation=$4 + AND checkpoint.attempt_generation=$3 AND checkpoint.dispatch_uid=$5 + AND checkpoint.superseded_at IS NULL + JOIN moa.execution_external_job AS job + ON job.tenant_id=task.tenant_id AND job.run_uid=task.run_uid + AND job.task_id=task.task_id AND job.attempt_generation=$3 + AND job.state='unbound' + JOIN moa.execution_capacity_reservation AS job_capacity + ON job_capacity.tenant_id=job.tenant_id + AND job_capacity.external_job_uid=job.external_job_uid + AND job_capacity.resource_dimension='external_jobs' + AND job_capacity.state='reserved' AND job_capacity.released_at IS NULL + JOIN moa.execution_trigger AS recovery + ON recovery.tenant_id=job.tenant_id AND recovery.run_uid=job.run_uid + AND recovery.task_id=job.task_id AND recovery.attempt_generation=$3 + AND recovery.controller_generation=$4 + AND recovery.trigger_kind='external_start_recovery' + AND recovery.state='pending' + WHERE task.tenant_id=$1 AND task.run_uid=$2 AND task.task_id=$8 + AND task.attempt_generation=$3 AND task.active_dispatch_uid=$5 + AND task.status='running' AND task.attempt_state='running' + AND run.controller_generation=$4 AND run.pending_terminal_status IS NULL + "#, + ) + .bind(fence.tenant_id.0) + .bind(fence.run_uid) + .bind(to_i64(fence.attempt_generation, "attempt generation")?) + .bind(to_i64( + fence.controller_generation, + "controller generation", + )?) + .bind(fence.dispatch_uid) + .bind(fence.watchdog_trigger_uid) + .bind(fence.capacity_reservation_uid) + .bind(fence.task_id.as_uuid()) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(row) = row else { + conn.commit().await.map_err(storage_error)?; + return Ok(None); + }; + let job = external_job_from_row(&row)?; + let checkpoint_kind = TaskAttemptCheckpointKind::parse( + &row.try_get::("checkpoint_kind") + .map_err(super::row_error)?, + )?; + let checkpoint_payload = row + .try_get::("checkpoint_payload") + .map_err(super::row_error)?; + if !external_start_checkpoint_payload_is_provisional(checkpoint_kind, &checkpoint_payload) { + return Err(Error::InvalidRepositoryData { + message: "unbound task external job lost its provisional start checkpoint" + .to_string(), + }); + } + let trigger_uid = row + .try_get::("trigger_uid") + .map_err(super::row_error)?; + let request = ExecutionExternalJobStartRecoveryRequest { + tenant_id: job.tenant_id, + run_uid: job.run_uid, + owner: ExecutionExternalJobStartRecoveryOwner::Task { + task_id: fence.task_id.as_uuid(), + attempt_generation: fence.attempt_generation, + }, + external_job_uid: job.external_job_uid, + job_generation: job.job_generation, + provider: job.declared_provider, + idempotency_key: job.idempotency_key, + trigger_uid, + }; + let recovery_payload = row + .try_get::("recovery_payload") + .map_err(super::row_error)?; + if recovery_payload + != json!({ + "external_job_uid": request.external_job_uid, + "job_generation": request.job_generation, + "declared_provider": request.provider, + "idempotency_key": request.idempotency_key, + }) + { + return Err(Error::InvalidRepositoryData { + message: "unbound task external job recovery trigger payload is inconsistent" + .to_string(), + }); + } + conn.commit().await.map_err(storage_error)?; + Ok(Some(request)) + } + /// Reserves one exact unbound external-job intent before provider dispatch. pub async fn reserve_external_job_intent( &self, diff --git a/crates/moa-execution/src/repository/mod.rs b/crates/moa-execution/src/repository/mod.rs index f8918ba29..46fb1595f 100644 --- a/crates/moa-execution/src/repository/mod.rs +++ b/crates/moa-execution/src/repository/mod.rs @@ -13,6 +13,8 @@ mod materialize; pub mod outbox; mod outcome; mod outcome_support; +/// Durable amendment-planner provider-call budget reservations and attribution. +pub mod planning_budget; mod projection; pub mod ready; /// Durable bounded replan-stop intent handoff. @@ -459,6 +461,11 @@ pub struct ExecutionTaskRecord { pub attempt_started_at: Option>, /// Latest durable progress timestamp for this logical task. pub last_progress_at: DateTime, + /// Upper bound declared by the durable step currently in flight. + /// + /// `None` when the attempt sits between steps or runs a step that declares no bound, + /// in which case the configured staleness floor is the whole window. + pub progress_step_bound_seconds: Option, /// Absolute watchdog deadline for the current active attempt. pub attempt_deadline_at: Option>, /// Time at which the task entered its current storage-only wait. @@ -1161,6 +1168,16 @@ fn to_u32(value: i32, field: &str) -> Result { }) } +fn to_positive_u32(value: i32, field: &str) -> Result { + let value = to_u32(value, field)?; + if value == 0 { + return Err(Error::InvalidRepositoryData { + message: format!("{field} must be positive"), + }); + } + Ok(value) +} + fn storage_error(error: moa_core::error::MoaError) -> Error { match error { moa_core::error::MoaError::StorageUnavailable(message) => { diff --git a/crates/moa-execution/src/repository/planning_budget.rs b/crates/moa-execution/src/repository/planning_budget.rs new file mode 100644 index 000000000..f1c11d7c9 --- /dev/null +++ b/crates/moa-execution/src/repository/planning_budget.rs @@ -0,0 +1,495 @@ +//! Durable run-budget authorization and attribution for amendment-planner provider calls. + +use super::*; +use crate::capability::ExecutionEstimate; + +const AMENDMENT_PLANNING_RESERVATION_NAMESPACE: Uuid = + Uuid::from_u128(0xc6e4_a4f8_a46e_581f_bf54_6d89_31ea_251f); +const AMENDMENT_PLANNING_SETTLEMENT_NAMESPACE: Uuid = + Uuid::from_u128(0x0630_e60f_f390_5eb8_a266_e6e9_3ddd_f753); + +/// Cost and token capacity reserved for one amendment-planner provider call. +#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct AmendmentPlanningCallReservation { + /// Reserved provider cost in micro-US-dollars. + pub cost_microusd: u64, + /// Reserved provider tokens. + pub tokens: u64, +} + +/// Actual cost and token usage attributed to one amendment-planner provider call. +#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct PlanningUsage { + /// Provider cost in micro-US-dollars. + pub cost_microusd: u64, + /// Provider tokens. + pub tokens: u64, +} + +/// One exact automatic amendment-planner provider-call reservation request. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct AmendmentPlanningCallReservationRequest { + /// Owning execution run. + pub run_uid: Uuid, + /// Exact active plan revision being amended. + pub base_plan_revision: u64, + /// Zero-based provider-call ordinal within this amendment attempt. + pub call_ordinal: u8, + /// Conservative provider-call reservation. + pub reservation: AmendmentPlanningCallReservation, + /// Journaled authorization time. + pub now: DateTime, +} + +/// Why a new amendment-planner call was not authorized. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AmendmentPlanningCallDenial { + /// The run or plan revision is no longer at its amendment boundary. + StaleRevision, + /// A terminal intent already fences new provider work. + PendingTerminal, + /// An earlier overrun fail-closes new reservations. + BudgetOverrun, + /// The approved run deadline has elapsed. + DeadlineExceeded, + /// Cost or token capacity is exhausted. + BudgetExceeded, +} + +/// Append-preserved evidence for one amendment-planner call reservation and settlement. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct AmendmentPlanningCallRecord { + /// Deterministic reservation identity. + pub reservation_uid: Uuid, + /// Owning tenant. + pub tenant_id: TenantId, + /// Optional contact scope inherited from the run. + pub contact_id: Option, + /// Owning run. + pub run_uid: Uuid, + /// Exact amended plan revision. + pub base_plan_revision: u64, + /// Provider-call ordinal. + pub call_ordinal: u8, + /// Authorized conservative reservation. + pub reserved: AmendmentPlanningCallReservation, + /// Reconciled provider usage when settled. + pub actual: Option, + /// Whether this settlement exceeded its reservation or approved run budget. + pub budget_overrun: bool, + /// First authorization time. + pub created_at: DateTime, + /// Immutable settlement time. + pub settled_at: Option>, +} + +/// Idempotent outcome of reserving one amendment-planner provider call. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AmendmentPlanningCallReservationOutcome { + /// A new open reservation was committed. + Granted(AmendmentPlanningCallRecord), + /// The identical open authorization already exists; the journaled caller may proceed. + ReplayedOpen(AmendmentPlanningCallRecord), + /// The call was already reconciled; the caller may replay the same gateway idempotency key. + AlreadySettled(AmendmentPlanningCallRecord), + /// Canonical run state denies new provider work. + Denied(AmendmentPlanningCallDenial), + /// The logical call identity exists with another reservation. + Conflict, + /// No run is visible in the supplied scope. + NotFound, +} + +/// One exact reconciliation of an authorized amendment-planner provider call. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct AmendmentPlanningCallReconcileRequest { + /// Owning run. + pub run_uid: Uuid, + /// Reserved base plan revision. + pub base_plan_revision: u64, + /// Reserved provider-call ordinal. + pub call_ordinal: u8, + /// Actual billed cost and tokens. + pub actual: PlanningUsage, + /// Journaled settlement time. + pub settled_at: DateTime, +} + +/// Idempotent outcome of reconciling one amendment-planner provider call. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AmendmentPlanningCallReconcileOutcome { + /// First settlement and run-budget reconciliation committed. + Applied(AmendmentPlanningCallRecord), + /// The identical immutable settlement already exists. + Replayed(AmendmentPlanningCallRecord), + /// The logical call was settled with different actual usage. + Conflict, + /// No reservation is visible in the supplied scope. + NotFound, +} + +impl ExecutionRepository { + /// Reserves one exact automatic amendment-planner provider call against the live run budget. + pub async fn reserve_amendment_planning_call( + &self, + scope: ExecutionScope, + request: AmendmentPlanningCallReservationRequest, + ) -> Result { + let mut conn = scope.begin(&self.pool).await?; + let Some(row) = sqlx::query( + "SELECT *,now() AS observed_at FROM moa.execution_run WHERE run_uid=$1 FOR UPDATE", + ) + .bind(request.run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + conn.commit().await.map_err(storage_error)?; + return Ok(AmendmentPlanningCallReservationOutcome::NotFound); + }; + let run = rows::run_from_row(&row)?; + let observed_at = row + .try_get::, _>("observed_at") + .map_err(row_error)?; + if let Some(existing) = load_planning_call_in_conn( + conn.as_mut(), + request.run_uid, + request.base_plan_revision, + request.call_ordinal, + ) + .await? + { + conn.commit().await.map_err(storage_error)?; + if existing.reserved != request.reservation { + return Ok(AmendmentPlanningCallReservationOutcome::Conflict); + } + return Ok(if existing.actual.is_some() { + AmendmentPlanningCallReservationOutcome::AlreadySettled(existing) + } else { + AmendmentPlanningCallReservationOutcome::ReplayedOpen(existing) + }); + } + let denial = if run.plan_revision != request.base_plan_revision + || run.status != ExecutionRunStatus::WaitingReplan + { + Some(AmendmentPlanningCallDenial::StaleRevision) + } else if run.pending_terminal.is_some() { + Some(AmendmentPlanningCallDenial::PendingTerminal) + } else if run.budget_overrun { + Some(AmendmentPlanningCallDenial::BudgetOverrun) + } else if run + .approved_budget + .deadline_at + .is_some_and(|deadline| deadline <= observed_at) + { + Some(AmendmentPlanningCallDenial::DeadlineExceeded) + } else { + let mut ledger = projection::budget_ledger(&run); + ledger + .try_reserve(ExecutionEstimate { + cost_microusd: request.reservation.cost_microusd, + tokens: request.reservation.tokens, + tasks: 0, + tool_calls: 0, + retrieved_bytes: 0, + }) + .err() + .map(|_| AmendmentPlanningCallDenial::BudgetExceeded) + }; + if let Some(denial) = denial { + conn.commit().await.map_err(storage_error)?; + return Ok(AmendmentPlanningCallReservationOutcome::Denied(denial)); + } + let reservation_uid = amendment_planning_reservation_uid( + request.run_uid, + request.base_plan_revision, + request.call_ordinal, + ); + sqlx::query( + "UPDATE moa.execution_run SET reserved_cost_microusd=reserved_cost_microusd+$2, \ + reserved_tokens=reserved_tokens+$3,updated_at=NOW() WHERE run_uid=$1", + ) + .bind(request.run_uid) + .bind(to_i64( + request.reservation.cost_microusd, + "planning reserved cost", + )?) + .bind(to_i64( + request.reservation.tokens, + "planning reserved tokens", + )?) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let created_at: DateTime = sqlx::query_scalar( + "INSERT INTO moa.execution_amendment_planning_reservation \ + (reservation_uid,tenant_id,contact_id,run_uid,base_plan_revision,call_ordinal, \ + reserved_cost_microusd,reserved_tokens,created_at) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) RETURNING created_at", + ) + .bind(reservation_uid) + .bind(run.tenant_id.0) + .bind(run.contact_id.map(|contact| contact.0)) + .bind(run.run_uid) + .bind(to_i64( + request.base_plan_revision, + "planning base revision", + )?) + .bind(i16::from(request.call_ordinal)) + .bind(to_i64( + request.reservation.cost_microusd, + "planning reserved cost", + )?) + .bind(to_i64( + request.reservation.tokens, + "planning reserved tokens", + )?) + .bind(request.now) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let record = AmendmentPlanningCallRecord { + reservation_uid, + tenant_id: run.tenant_id, + contact_id: run.contact_id, + run_uid: run.run_uid, + base_plan_revision: request.base_plan_revision, + call_ordinal: request.call_ordinal, + reserved: request.reservation, + actual: None, + budget_overrun: false, + created_at, + settled_at: None, + }; + conn.commit().await.map_err(storage_error)?; + Ok(AmendmentPlanningCallReservationOutcome::Granted(record)) + } + + /// Reconciles one exact amendment-planner call and releases its reservation exactly once. + pub async fn reconcile_amendment_planning_call( + &self, + scope: ExecutionScope, + request: AmendmentPlanningCallReconcileRequest, + ) -> Result { + let mut conn = scope.begin(&self.pool).await?; + let Some(run_row) = + sqlx::query("SELECT * FROM moa.execution_run WHERE run_uid=$1 FOR UPDATE") + .bind(request.run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + conn.commit().await.map_err(storage_error)?; + return Ok(AmendmentPlanningCallReconcileOutcome::NotFound); + }; + let run = rows::run_from_row(&run_row)?; + let Some(record) = load_planning_call_in_conn( + conn.as_mut(), + request.run_uid, + request.base_plan_revision, + request.call_ordinal, + ) + .await? + else { + conn.commit().await.map_err(storage_error)?; + return Ok(AmendmentPlanningCallReconcileOutcome::NotFound); + }; + if let Some(actual) = &record.actual { + conn.commit().await.map_err(storage_error)?; + return Ok(if actual == &request.actual { + AmendmentPlanningCallReconcileOutcome::Replayed(record) + } else { + AmendmentPlanningCallReconcileOutcome::Conflict + }); + } + let ceiling = i64::MAX as u64; + let next_cost = run + .consumed + .cost_microusd + .saturating_add(request.actual.cost_microusd) + .min(ceiling); + let next_tokens = run + .consumed + .tokens + .saturating_add(request.actual.tokens) + .min(ceiling); + let budget_overrun = run.budget_overrun + || request.actual.cost_microusd > record.reserved.cost_microusd + || request.actual.tokens > record.reserved.tokens + || run + .approved_budget + .max_cost_microusd + .is_some_and(|limit| next_cost > limit) + || run + .approved_budget + .max_tokens + .is_some_and(|limit| next_tokens > limit) + || run.consumed.cost_microusd > ceiling - request.actual.cost_microusd.min(ceiling) + || run.consumed.tokens > ceiling - request.actual.tokens.min(ceiling); + let remaining_cost = run + .reserved + .cost_microusd + .checked_sub(record.reserved.cost_microusd) + .ok_or_else(|| Error::InvalidRepositoryData { + message: "amendment planning cost reservation is absent from run ledger" + .to_string(), + })?; + let remaining_tokens = run + .reserved + .tokens + .checked_sub(record.reserved.tokens) + .ok_or_else(|| Error::InvalidRepositoryData { + message: "amendment planning token reservation is absent from run ledger" + .to_string(), + })?; + sqlx::query( + "UPDATE moa.execution_run SET reserved_cost_microusd=$2,reserved_tokens=$3, \ + consumed_cost_microusd=$4,consumed_tokens=$5,budget_overrun=$6,updated_at=NOW() \ + WHERE run_uid=$1", + ) + .bind(run.run_uid) + .bind(to_i64( + remaining_cost, + "remaining planning cost reservation", + )?) + .bind(to_i64( + remaining_tokens, + "remaining planning token reservation", + )?) + .bind(to_i64(next_cost, "planning consumed cost")?) + .bind(to_i64(next_tokens, "planning consumed tokens")?) + .bind(budget_overrun) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let settlement_uid = Uuid::new_v5( + &AMENDMENT_PLANNING_SETTLEMENT_NAMESPACE, + record.reservation_uid.as_bytes(), + ); + let settled_at: DateTime = sqlx::query_scalar( + "INSERT INTO moa.execution_amendment_planning_settlement \ + (settlement_uid,reservation_uid,tenant_id,contact_id,run_uid, \ + actual_cost_microusd,actual_tokens,budget_overrun,settled_at) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) RETURNING settled_at", + ) + .bind(settlement_uid) + .bind(record.reservation_uid) + .bind(record.tenant_id.0) + .bind(record.contact_id.map(|contact| contact.0)) + .bind(record.run_uid) + .bind(to_i64( + request.actual.cost_microusd, + "planning actual cost", + )?) + .bind(to_i64(request.actual.tokens, "planning actual tokens")?) + .bind(budget_overrun) + .bind(request.settled_at) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let settled = AmendmentPlanningCallRecord { + actual: Some(request.actual), + budget_overrun, + settled_at: Some(settled_at), + ..record + }; + conn.commit().await.map_err(storage_error)?; + Ok(AmendmentPlanningCallReconcileOutcome::Applied(settled)) + } + + /// Loads one visible amendment-planner call attribution record. + pub async fn load_amendment_planning_call( + &self, + scope: ExecutionScope, + run_uid: Uuid, + base_plan_revision: u64, + call_ordinal: u8, + ) -> Result> { + let mut conn = scope.begin(&self.pool).await?; + let record = + load_planning_call_in_conn(conn.as_mut(), run_uid, base_plan_revision, call_ordinal) + .await?; + conn.commit().await.map_err(storage_error)?; + Ok(record) + } +} + +async fn load_planning_call_in_conn( + conn: &mut PgConnection, + run_uid: Uuid, + base_plan_revision: u64, + call_ordinal: u8, +) -> Result> { + let row = sqlx::query( + "SELECT reservation.*,settlement.actual_cost_microusd,settlement.actual_tokens, \ + settlement.budget_overrun,settlement.settled_at \ + FROM moa.execution_amendment_planning_reservation AS reservation \ + LEFT JOIN moa.execution_amendment_planning_settlement AS settlement \ + ON settlement.reservation_uid=reservation.reservation_uid \ + WHERE reservation.run_uid=$1 AND reservation.base_plan_revision=$2 \ + AND reservation.call_ordinal=$3", + ) + .bind(run_uid) + .bind(to_i64(base_plan_revision, "planning base revision")?) + .bind(i16::from(call_ordinal)) + .fetch_optional(&mut *conn) + .await + .map_err(sqlx_error)?; + row.as_ref().map(planning_call_from_row).transpose() +} + +fn planning_call_from_row(row: &PgRow) -> Result { + let actual_cost = rows::optional_u64(row, "actual_cost_microusd")?; + let actual_tokens = rows::optional_u64(row, "actual_tokens")?; + if actual_cost.is_some() != actual_tokens.is_some() { + return Err(Error::InvalidRepositoryData { + message: "amendment planning settlement has partial actual usage".to_string(), + }); + } + let actual = actual_cost + .zip(actual_tokens) + .map(|(cost_microusd, tokens)| PlanningUsage { + cost_microusd, + tokens, + }); + Ok(AmendmentPlanningCallRecord { + reservation_uid: row.try_get("reservation_uid").map_err(row_error)?, + tenant_id: TenantId(row.try_get("tenant_id").map_err(row_error)?), + contact_id: row + .try_get::, _>("contact_id") + .map_err(row_error)? + .map(ContactId), + run_uid: row.try_get("run_uid").map_err(row_error)?, + base_plan_revision: rows::required_u64(row, "base_plan_revision")?, + call_ordinal: u8::try_from(row.try_get::("call_ordinal").map_err(row_error)?) + .map_err(|_| Error::InvalidRepositoryData { + message: "amendment planning call ordinal exceeds u8".to_string(), + })?, + reserved: AmendmentPlanningCallReservation { + cost_microusd: rows::required_u64(row, "reserved_cost_microusd")?, + tokens: rows::required_u64(row, "reserved_tokens")?, + }, + actual, + budget_overrun: row + .try_get::, _>("budget_overrun") + .map_err(row_error)? + .unwrap_or(false), + created_at: row.try_get("created_at").map_err(row_error)?, + settled_at: row.try_get("settled_at").map_err(row_error)?, + }) +} + +fn amendment_planning_reservation_uid(run_uid: Uuid, revision: u64, ordinal: u8) -> Uuid { + Uuid::new_v5( + &AMENDMENT_PLANNING_RESERVATION_NAMESPACE, + format!("{run_uid}:{revision}:{ordinal}").as_bytes(), + ) +} diff --git a/crates/moa-execution/src/repository/retention.rs b/crates/moa-execution/src/repository/retention.rs index 5a9bd2592..dae693848 100644 --- a/crates/moa-execution/src/repository/retention.rs +++ b/crates/moa-execution/src/repository/retention.rs @@ -170,6 +170,14 @@ const ARCHIVE_SOURCES: &[ArchiveSource] = &[ kind: "execution_amendment_receipt", select_page_sql: "SELECT to_jsonb(source) AS record, to_jsonb(base_plan_revision) AS cursor FROM moa.execution_amendment_receipt AS source WHERE tenant_id = $1 AND run_uid = $2 AND ($4::JSONB IS NULL OR base_plan_revision > (($4 #>> '{}')::BIGINT)) ORDER BY base_plan_revision LIMIT $3", }, + ArchiveSource { + kind: "execution_amendment_planning_settlement", + select_page_sql: "SELECT to_jsonb(source) AS record, to_jsonb(settlement_uid::TEXT) AS cursor FROM moa.execution_amendment_planning_settlement AS source WHERE tenant_id = $1 AND run_uid = $2 AND ($4::JSONB IS NULL OR settlement_uid > (($4 #>> '{}')::UUID)) ORDER BY settlement_uid LIMIT $3", + }, + ArchiveSource { + kind: "execution_amendment_planning_reservation", + select_page_sql: "SELECT to_jsonb(source) AS record, to_jsonb(reservation_uid::TEXT) AS cursor FROM moa.execution_amendment_planning_reservation AS source WHERE tenant_id = $1 AND run_uid = $2 AND ($4::JSONB IS NULL OR reservation_uid > (($4 #>> '{}')::UUID)) ORDER BY reservation_uid LIMIT $3", + }, ArchiveSource { kind: "execution_node_materialization", select_page_sql: "SELECT to_jsonb(source) AS record, jsonb_build_array(plan_revision, node_id) AS cursor FROM moa.execution_node_materialization AS source WHERE tenant_id = $1 AND run_uid = $2 AND ($4::JSONB IS NULL OR (plan_revision, node_id) > (($4->>0)::BIGINT, $4->>1)) ORDER BY plan_revision, node_id LIMIT $3", @@ -1162,6 +1170,14 @@ async fn advance_deletion( "execution_amendment_receipt", "DELETE FROM moa.execution_amendment_receipt WHERE ctid IN (SELECT ctid FROM moa.execution_amendment_receipt WHERE tenant_id = $1 AND run_uid = $2 ORDER BY base_plan_revision LIMIT $3)", ), + ( + "execution_amendment_planning_settlement", + "DELETE FROM moa.execution_amendment_planning_settlement WHERE settlement_uid IN (SELECT settlement_uid FROM moa.execution_amendment_planning_settlement WHERE tenant_id = $1 AND run_uid = $2 ORDER BY settlement_uid LIMIT $3)", + ), + ( + "execution_amendment_planning_reservation", + "DELETE FROM moa.execution_amendment_planning_reservation WHERE reservation_uid IN (SELECT reservation_uid FROM moa.execution_amendment_planning_reservation WHERE tenant_id = $1 AND run_uid = $2 ORDER BY reservation_uid LIMIT $3)", + ), ( "execution_compensation", "DELETE FROM moa.execution_compensation WHERE compensation_id IN (SELECT compensation_id FROM moa.execution_compensation WHERE tenant_id = $1 AND run_uid = $2 ORDER BY compensation_id LIMIT $3)", @@ -1316,7 +1332,7 @@ mod tests { fn archive_sources_use_persisted_keyset_cursors_without_offsets() { // Pins: every archive source resumes strictly after its last committed key; restoring // OFFSET paging would make later pages increasingly expensive and replay-fragile. - assert_eq!(ARCHIVE_SOURCES.len(), 18); + assert_eq!(ARCHIVE_SOURCES.len(), 20); for source in ARCHIVE_SOURCES { assert!(source.select_page_sql.contains("$4"), "{}", source.kind); assert!( diff --git a/crates/moa-execution/src/repository/rows.rs b/crates/moa-execution/src/repository/rows.rs index cb0cbe0ae..088a805a1 100644 --- a/crates/moa-execution/src/repository/rows.rs +++ b/crates/moa-execution/src/repository/rows.rs @@ -309,6 +309,11 @@ pub(super) fn task_from_row(row: &PgRow) -> Result { )?, attempt_started_at: row.try_get("attempt_started_at").map_err(row_error)?, last_progress_at: row.try_get("last_progress_at").map_err(row_error)?, + progress_step_bound_seconds: row + .try_get::, _>("progress_step_bound_seconds") + .map_err(row_error)? + .map(|seconds| to_positive_u32(seconds, "progress step bound seconds")) + .transpose()?, attempt_deadline_at: row.try_get("attempt_deadline_at").map_err(row_error)?, waiting_since: row.try_get("waiting_since").map_err(row_error)?, ready_at: row.try_get("ready_at").map_err(row_error)?, diff --git a/crates/moa-execution/src/repository/sql.rs b/crates/moa-execution/src/repository/sql.rs index b1f60b555..ed82fcc22 100644 --- a/crates/moa-execution/src/repository/sql.rs +++ b/crates/moa-execution/src/repository/sql.rs @@ -86,11 +86,12 @@ pub(super) const INSERT_PLANNER_AUDIT_SQL: &str = r#" audit_uid, tenant_id, contact_id, session_id, originating_sequence, run_uid, plan_revision, call_kind, call_ordinal, outcome, provider_model, prompt_version, candidate_hash, candidate_json, - compiler_report, duration_micros, created_at + compiler_report, input_tokens_uncached, input_tokens_cache_write, + input_tokens_cache_read, output_tokens, cost_microusd, duration_micros, created_at ) VALUES ( $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, - $11, $12, $13, $14::JSON, $15::JSON, $16, $17 + $11, $12, $13, $14::JSON, $15::JSON, $16, $17, $18, $19, $20, $21, $22 ) ON CONFLICT DO NOTHING RETURNING @@ -98,7 +99,8 @@ pub(super) const INSERT_PLANNER_AUDIT_SQL: &str = r#" provider_model, prompt_version, candidate_hash, candidate_json::TEXT AS candidate_json, compiler_report::TEXT AS compiler_report, - duration_micros + input_tokens_uncached, input_tokens_cache_write, input_tokens_cache_read, + output_tokens, cost_microusd, duration_micros "#; pub(super) const LOAD_PLANNER_AUDIT_SQL: &str = r#" @@ -107,7 +109,8 @@ pub(super) const LOAD_PLANNER_AUDIT_SQL: &str = r#" provider_model, prompt_version, candidate_hash, candidate_json::TEXT AS candidate_json, compiler_report::TEXT AS compiler_report, - duration_micros + input_tokens_uncached, input_tokens_cache_write, input_tokens_cache_read, + output_tokens, cost_microusd, duration_micros FROM moa.execution_planner_call_audit WHERE tenant_id = $1 AND contact_id IS NOT DISTINCT FROM $2 @@ -363,7 +366,7 @@ pub(super) const LOAD_TASK_BATCH_SQL: &str = r#" task.actual_tool_calls, task.actual_retrieved_bytes, task.current_outcome, task.output, task.error, task.citations, task.generation_history, task.outcome_audit, task.attempt_generation, task.attempt_state, task.attempt_started_at, - task.last_progress_at, task.attempt_deadline_at, task.waiting_since, task.ready_at, + task.last_progress_at, task.progress_step_bound_seconds, task.attempt_deadline_at, task.waiting_since, task.ready_at, task.external_job_uid, task.active_dispatch_uid, task.dispatch_sequence, task.created_at, task.updated_at, task.reserved_at, task.started_at, task.completed_at @@ -388,7 +391,7 @@ pub(super) const LOAD_TASK_FOR_UPDATE_SQL: &str = r#" actual_cost_microusd, actual_tokens, actual_tasks, actual_tool_calls, actual_retrieved_bytes, current_outcome, output, error, citations, generation_history, outcome_audit, - attempt_generation, attempt_state, attempt_started_at, last_progress_at, + attempt_generation, attempt_state, attempt_started_at, last_progress_at, progress_step_bound_seconds, attempt_deadline_at, waiting_since, ready_at, external_job_uid, active_dispatch_uid, dispatch_sequence, created_at, updated_at, reserved_at, started_at, completed_at @@ -409,7 +412,7 @@ pub(super) const LOAD_TASK_SQL: &str = r#" actual_cost_microusd, actual_tokens, actual_tasks, actual_tool_calls, actual_retrieved_bytes, current_outcome, output, error, citations, generation_history, outcome_audit, - attempt_generation, attempt_state, attempt_started_at, last_progress_at, + attempt_generation, attempt_state, attempt_started_at, last_progress_at, progress_step_bound_seconds, attempt_deadline_at, waiting_since, ready_at, external_job_uid, active_dispatch_uid, dispatch_sequence, created_at, updated_at, reserved_at, started_at, completed_at @@ -485,7 +488,7 @@ pub(super) const RESERVE_TASK_SQL: &str = r#" actual_cost_microusd, actual_tokens, actual_tasks, actual_tool_calls, actual_retrieved_bytes, current_outcome, output, error, citations, generation_history, outcome_audit, - attempt_generation, attempt_state, attempt_started_at, last_progress_at, + attempt_generation, attempt_state, attempt_started_at, last_progress_at, progress_step_bound_seconds, attempt_deadline_at, waiting_since, ready_at, external_job_uid, active_dispatch_uid, dispatch_sequence, created_at, updated_at, reserved_at, started_at, completed_at @@ -513,7 +516,7 @@ pub(super) const MARK_TASK_RUNNING_SQL: &str = r#" actual_cost_microusd, actual_tokens, actual_tasks, actual_tool_calls, actual_retrieved_bytes, current_outcome, output, error, citations, generation_history, outcome_audit, - attempt_generation, attempt_state, attempt_started_at, last_progress_at, + attempt_generation, attempt_state, attempt_started_at, last_progress_at, progress_step_bound_seconds, attempt_deadline_at, waiting_since, ready_at, external_job_uid, active_dispatch_uid, dispatch_sequence, created_at, updated_at, reserved_at, started_at, completed_at @@ -555,7 +558,7 @@ pub(super) const RESUME_TASK_SQL: &str = r#" actual_cost_microusd, actual_tokens, actual_tasks, actual_tool_calls, actual_retrieved_bytes, current_outcome, output, error, citations, generation_history, outcome_audit, - attempt_generation, attempt_state, attempt_started_at, last_progress_at, + attempt_generation, attempt_state, attempt_started_at, last_progress_at, progress_step_bound_seconds, attempt_deadline_at, waiting_since, ready_at, external_job_uid, active_dispatch_uid, dispatch_sequence, created_at, updated_at, reserved_at, started_at, completed_at @@ -573,7 +576,7 @@ pub(super) const LIST_TASKS_SQL: &str = r#" actual_cost_microusd, actual_tokens, actual_tasks, actual_tool_calls, actual_retrieved_bytes, current_outcome, output, error, citations, generation_history, outcome_audit, - attempt_generation, attempt_state, attempt_started_at, last_progress_at, + attempt_generation, attempt_state, attempt_started_at, last_progress_at, progress_step_bound_seconds, attempt_deadline_at, waiting_since, ready_at, external_job_uid, active_dispatch_uid, dispatch_sequence, created_at, updated_at, reserved_at, started_at, completed_at @@ -628,6 +631,8 @@ pub(super) const RECORD_TASK_OUTCOME_SQL: &str = r#" ELSE NULL END, attempt_deadline_at = CASE WHEN $4 = 'running' THEN attempt_deadline_at ELSE NULL END, + progress_step_bound_seconds = CASE WHEN $4 = 'running' + THEN progress_step_bound_seconds ELSE NULL END, last_progress_at = GREATEST(last_progress_at, NOW()), reserved_cost_microusd = $5, reserved_tokens = $6, @@ -661,7 +666,7 @@ pub(super) const RECORD_TASK_OUTCOME_SQL: &str = r#" actual_cost_microusd, actual_tokens, actual_tasks, actual_tool_calls, actual_retrieved_bytes, current_outcome, output, error, citations, generation_history, outcome_audit, - attempt_generation, attempt_state, attempt_started_at, last_progress_at, + attempt_generation, attempt_state, attempt_started_at, last_progress_at, progress_step_bound_seconds, attempt_deadline_at, waiting_since, ready_at, external_job_uid, active_dispatch_uid, dispatch_sequence, created_at, updated_at, reserved_at, started_at, completed_at @@ -705,7 +710,7 @@ pub(super) const RECORD_RESERVATION_REJECTION_SQL: &str = r#" actual_cost_microusd, actual_tokens, actual_tasks, actual_tool_calls, actual_retrieved_bytes, current_outcome, output, error, citations, generation_history, outcome_audit, - attempt_generation, attempt_state, attempt_started_at, last_progress_at, + attempt_generation, attempt_state, attempt_started_at, last_progress_at, progress_step_bound_seconds, attempt_deadline_at, waiting_since, ready_at, external_job_uid, active_dispatch_uid, dispatch_sequence, created_at, updated_at, reserved_at, started_at, completed_at @@ -727,7 +732,7 @@ pub(super) const APPEND_TASK_OUTCOME_AUDIT_SQL: &str = r#" actual_cost_microusd, actual_tokens, actual_tasks, actual_tool_calls, actual_retrieved_bytes, current_outcome, output, error, citations, generation_history, outcome_audit, - attempt_generation, attempt_state, attempt_started_at, last_progress_at, + attempt_generation, attempt_state, attempt_started_at, last_progress_at, progress_step_bound_seconds, attempt_deadline_at, waiting_since, ready_at, external_job_uid, active_dispatch_uid, dispatch_sequence, created_at, updated_at, reserved_at, started_at, completed_at @@ -766,7 +771,7 @@ pub(super) const SUPERSEDE_REPLAN_TASK_SQL: &str = r#" actual_cost_microusd, actual_tokens, actual_tasks, actual_tool_calls, actual_retrieved_bytes, current_outcome, output, error, citations, generation_history, outcome_audit, - attempt_generation, attempt_state, attempt_started_at, last_progress_at, + attempt_generation, attempt_state, attempt_started_at, last_progress_at, progress_step_bound_seconds, attempt_deadline_at, waiting_since, ready_at, external_job_uid, active_dispatch_uid, dispatch_sequence, created_at, updated_at, reserved_at, started_at, completed_at diff --git a/crates/moa-execution/src/repository/task.rs b/crates/moa-execution/src/repository/task.rs index 12828fc3a..f06f3beb9 100644 --- a/crates/moa-execution/src/repository/task.rs +++ b/crates/moa-execution/src/repository/task.rs @@ -122,36 +122,76 @@ impl ActiveAttemptLiveness { } } -/// Returns the configured heartbeat staleness window as a chrono duration. +/// Returns the staleness window for an attempt whose in-flight step declared `step_bound`. /// /// Shared by watchdog arming, watchdog deferral, and liveness classification so all three -/// derive the same window from one configured value. -pub fn attempt_heartbeat_staleness_window(config: &ExecutionConfig) -> Result { - i64::try_from(config.attempt_heartbeat_staleness_seconds) +/// derive the same window from one configured value and one recorded step bound. +/// +/// The heartbeat is written at step boundaries and never while a step runs, so the window +/// has to outlast the step in flight or it classifies a working attempt as stalled. A +/// single global window would therefore have to clear the slowest step any attempt can +/// take, which pushes detection for every attempt out to the worst case. Taking the bound +/// from the step actually running keeps the common case tight: a step that declared +/// nothing gets the configured floor, and one that declared a long timeout gets exactly +/// that plus [`ATTEMPT_STEP_BOUND_MARGIN_SECONDS`]. +/// +/// The floor also applies to declared bounds shorter than it, because scheduling jitter +/// around a two-second step must not read as a stall. +pub fn attempt_heartbeat_staleness_window( + config: &ExecutionConfig, + step_bound: Option, +) -> Result { + let floor = i64::try_from(config.attempt_heartbeat_staleness_seconds) .ok() .and_then(chrono::TimeDelta::try_seconds) .ok_or_else(|| Error::InvalidRepositoryInput { message: "attempt heartbeat staleness window exceeds chrono duration".to_string(), - }) + })?; + let Some(bound) = step_bound else { + return Ok(floor); + }; + let margin = + chrono::TimeDelta::try_seconds(ATTEMPT_STEP_BOUND_MARGIN_SECONDS).ok_or_else(|| { + Error::InvalidRepositoryInput { + message: "attempt step bound margin exceeds chrono duration".to_string(), + } + })?; + let bounded = bound + .checked_add(&margin) + .ok_or_else(|| Error::InvalidRepositoryInput { + message: "attempt step bound plus margin exceeds chrono duration".to_string(), + })?; + Ok(floor.max(bounded)) } -/// Classifies one active attempt from its admission deadline and last durable progress. +/// Grace added to a declared step bound before the step is treated as stalled. +/// +/// A step that promised N seconds is not late at exactly N: the promise bounds the work, +/// not the dispatch, result write, and clock skew around it. Without this margin every +/// step that legitimately runs to its own limit races the watchdog. +pub const ATTEMPT_STEP_BOUND_MARGIN_SECONDS: i64 = 30; + +/// Classifies one active attempt from its deadline, last durable progress, and step bound. /// /// The deadline is evaluated first so the pre-existing absolute-deadline behaviour is /// unchanged; heartbeat staleness only ever classifies an attempt that is still inside its /// deadline. A staleness window is only meaningful when it is shorter than the attempt -/// timeout, which [`moa_config::ExecutionConfig::validate`] enforces. +/// timeout, which [`moa_config::ExecutionConfig::validate`] enforces for the floor. +/// +/// `step_bound` is the bound recorded for the step in flight, or `None` when the attempt is +/// between steps or running a step that declares no bound. #[must_use] pub fn classify_active_attempt_liveness( config: &ExecutionConfig, attempt_deadline_at: DateTime, last_progress_at: DateTime, + step_bound: Option, observed_at: DateTime, ) -> ActiveAttemptLiveness { if attempt_deadline_at <= observed_at { return ActiveAttemptLiveness::DeadlineExceeded; } - let Ok(staleness) = attempt_heartbeat_staleness_window(config) else { + let Ok(staleness) = attempt_heartbeat_staleness_window(config, step_bound) else { // An unrepresentable window can never elapse, so the deadline stays the only authority. return ActiveAttemptLiveness::Live; }; @@ -315,7 +355,7 @@ impl TaskAttemptCheckpointKind { } } - fn parse(value: &str) -> Result { + pub(super) fn parse(value: &str) -> Result { match value { "agent_continuation" => Ok(Self::AgentContinuation), "capability_review" => Ok(Self::CapabilityReview), @@ -725,6 +765,7 @@ impl ExecutionRepository { let row = sqlx::query( "UPDATE moa.execution_task SET status='ready', attempt_state='idle', \ attempt_generation=$5, waiting_since=NULL, ready_at=$6, \ + progress_step_bound_seconds=NULL, \ last_progress_at=GREATEST(last_progress_at,$6), \ generation_history=generation_history || jsonb_build_array(jsonb_build_object( \ 'kind','action_review_resolved','review_uid',$4::TEXT,'recorded_at',$6)), \ @@ -861,6 +902,7 @@ impl ExecutionRepository { } let row = sqlx::query( "UPDATE moa.execution_task SET attempt_state='cancelling', external_job_uid=$5, \ + progress_step_bound_seconds=NULL, \ last_progress_at=GREATEST(last_progress_at,$6), \ generation_history=generation_history || \ jsonb_build_array(jsonb_build_object( \ @@ -949,6 +991,7 @@ impl ExecutionRepository { } let row = sqlx::query( "UPDATE moa.execution_task SET attempt_state='cancelling', \ + progress_step_bound_seconds=NULL, \ last_progress_at=GREATEST(last_progress_at,$5), \ generation_history=generation_history || jsonb_build_array(jsonb_build_object( \ 'kind','bounded_attempt_release_claimed','dispatch_uid',$4::TEXT, \ @@ -1093,6 +1136,7 @@ impl ExecutionRepository { let row = sqlx::query( "UPDATE moa.execution_task SET status='waiting_review', attempt_state='waiting', \ waiting_since=$5, active_dispatch_uid=NULL, attempt_deadline_at=NULL, \ + progress_step_bound_seconds=NULL, \ last_progress_at=GREATEST(last_progress_at,$5), updated_at=NOW() \ WHERE run_uid=$1 AND task_id=$2 AND attempt_generation=$3 \ AND active_dispatch_uid=$4 AND status='running' RETURNING *", @@ -1260,6 +1304,7 @@ impl ExecutionRepository { let row = sqlx::query( "UPDATE moa.execution_task SET status='ready', attempt_state='idle', ready_at=$5, \ waiting_since=NULL, active_dispatch_uid=NULL, attempt_deadline_at=NULL, \ + progress_step_bound_seconds=NULL, \ attempt_generation=$6, last_progress_at=GREATEST(last_progress_at,$5), \ updated_at=NOW() \ WHERE run_uid=$1 AND task_id=$2 AND attempt_generation=$3 \ @@ -1368,6 +1413,7 @@ impl ExecutionRepository { SET status = 'running', attempt_state = 'running', \ attempt_started_at = COALESCE(attempt_started_at, NOW()), \ started_at = COALESCE(started_at, NOW()), last_progress_at = NOW(), \ + progress_step_bound_seconds = NULL, \ updated_at = NOW() \ WHERE run_uid = $1 AND task_id = $2 AND status = 'dispatching' \ AND attempt_generation = $3 AND active_dispatch_uid = $4 \ @@ -1577,7 +1623,8 @@ impl ExecutionRepository { }; let row = sqlx::query( "UPDATE moa.execution_task SET active_dispatch_uid=NULL, \ - attempt_deadline_at=NULL, generation_history=generation_history || \ + attempt_deadline_at=NULL, progress_step_bound_seconds=NULL, \ + generation_history=generation_history || \ jsonb_build_array($5::JSONB), \ last_progress_at=GREATEST(last_progress_at,$6), updated_at=NOW() \ WHERE run_uid=$1 AND task_id=$2 AND attempt_generation=$3 \ @@ -1618,6 +1665,7 @@ impl ExecutionRepository { "UPDATE moa.execution_task SET status='ready', attempt_state='idle', \ attempt_generation=$5, active_dispatch_uid=NULL, \ attempt_deadline_at=NULL, ready_at=$6, waiting_since=NULL, \ + progress_step_bound_seconds=NULL, \ generation_history=generation_history || jsonb_build_array($7::JSONB), \ last_progress_at=GREATEST(last_progress_at,$6), updated_at=NOW() \ WHERE run_uid=$1 AND task_id=$2 AND attempt_generation=$3 \ @@ -1679,11 +1727,29 @@ impl ExecutionRepository { } /// Records monotonic progress for one exact active task attempt. + /// + /// `step_bound_seconds` is the upper bound declared by the step the attempt is about to + /// enter, or `None` when the attempt is between steps. The watchdog widens its staleness + /// window to that bound, so a heartbeat written before a long declared step is what keeps + /// the step from being classified as stalled while it legitimately runs. pub async fn record_task_attempt_progress( &self, fence: TaskAttemptFence, observed_at: DateTime, + step_bound_seconds: Option, ) -> Result { + let step_bound_seconds_db = step_bound_seconds + .map(|seconds| { + if seconds == 0 { + return Err(Error::InvalidRepositoryInput { + message: "progress step bound seconds must be positive".to_string(), + }); + } + i32::try_from(seconds).map_err(|_| Error::InvalidRepositoryInput { + message: "progress step bound seconds exceeds PostgreSQL INTEGER".to_string(), + }) + }) + .transpose()?; let mut conn = ExecutionScope::ControlPlane.begin(&self.pool).await?; let Some(run_row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) .bind(fence.run_uid) @@ -1716,12 +1782,18 @@ impl ExecutionRepository { conn.commit().await.map_err(storage_error)?; return Ok(TaskAttemptProgressOutcome::InvalidState); } - if observed_at <= task.last_progress_at { - conn.commit().await.map_err(storage_error)?; - return Ok(TaskAttemptProgressOutcome::Replayed); - } + let outcome = if observed_at <= task.last_progress_at { + TaskAttemptProgressOutcome::Replayed + } else { + TaskAttemptProgressOutcome::Applied + }; + // The bound is overwritten rather than merged: it describes the step now in flight, + // so a stale wider bound from the previous step must not outlive it and keep the + // watchdog lenient after a long step has already finished. sqlx::query( - "UPDATE moa.execution_task SET last_progress_at = $5, updated_at = NOW() \ + "UPDATE moa.execution_task \ + SET last_progress_at = GREATEST(last_progress_at, $5), \ + progress_step_bound_seconds = $6, updated_at = NOW() \ WHERE run_uid = $1 AND task_id = $2 AND attempt_generation = $3 \ AND active_dispatch_uid = $4 AND attempt_state = 'running'", ) @@ -1730,9 +1802,14 @@ impl ExecutionRepository { .bind(to_i64(fence.attempt_generation, "attempt generation")?) .bind(fence.dispatch_uid) .bind(observed_at) + .bind(step_bound_seconds_db) .execute(conn.as_mut()) .await .map_err(sqlx_error)?; + if outcome == TaskAttemptProgressOutcome::Replayed { + conn.commit().await.map_err(storage_error)?; + return Ok(outcome); + } sqlx::query( "UPDATE moa.execution_run SET last_progress_at = GREATEST(last_progress_at, $2), \ updated_at = NOW() WHERE run_uid = $1", @@ -1743,7 +1820,7 @@ impl ExecutionRepository { .await .map_err(sqlx_error)?; conn.commit().await.map_err(storage_error)?; - Ok(TaskAttemptProgressOutcome::Applied) + Ok(outcome) } /// Persists one bounded task outcome, releases active capacity, and wakes its controller. @@ -2067,6 +2144,7 @@ impl ExecutionRepository { "UPDATE moa.execution_task SET status='ready', attempt_state='idle', \ attempt_generation=attempt_generation+1, active_dispatch_uid=NULL, \ attempt_deadline_at=NULL, ready_at=$5, waiting_since=NULL, \ + progress_step_bound_seconds=NULL, \ last_progress_at=GREATEST(last_progress_at,$5), \ generation_history=generation_history || jsonb_build_array(jsonb_build_object( \ 'kind','pause_release_finalized','dispatch_uid',$4::TEXT, \ @@ -2390,6 +2468,7 @@ impl ExecutionRepository { SET status = $3, attempt_state = $4, generation = $5, \ attempt_generation = $6, attempt = CASE WHEN $7::BOOLEAN THEN attempt + 1 ELSE attempt END, \ active_dispatch_uid = NULL, attempt_deadline_at = NULL, \ + progress_step_bound_seconds = NULL, \ waiting_since = CASE WHEN $4 = 'waiting' THEN $8 ELSE NULL END, \ ready_at = $9, last_progress_at = GREATEST(last_progress_at, $8), \ generation_history = generation_history || jsonb_build_array($11::JSONB), \ @@ -2671,6 +2750,7 @@ impl ExecutionRepository { let row = sqlx::query( "UPDATE moa.execution_task SET status='ready',attempt_state='idle', \ attempt_generation=$5,active_dispatch_uid=NULL,attempt_deadline_at=NULL, \ + progress_step_bound_seconds=NULL, \ waiting_since=NULL,ready_at=$6, \ last_progress_at=GREATEST(last_progress_at,$6), \ generation_history=generation_history || jsonb_build_array(jsonb_build_object( \ @@ -2834,6 +2914,7 @@ impl ExecutionRepository { "UPDATE moa.execution_task SET status='waiting_external',attempt_state='waiting', \ waiting_since=$5,external_job_uid=$6,active_dispatch_uid=NULL, \ attempt_deadline_at=NULL, \ + progress_step_bound_seconds=NULL, \ last_progress_at=GREATEST(last_progress_at,$5),updated_at=NOW() \ WHERE run_uid=$1 AND task_id=$2 AND attempt_generation=$3 \ AND active_dispatch_uid=$4 AND status='running' AND attempt_state='running' \ @@ -3050,7 +3131,7 @@ impl ExecutionRepository { "UPDATE moa.execution_task \ SET status = 'waiting_external', attempt_state = 'waiting', \ waiting_since = $5, external_job_uid = $6, active_dispatch_uid = NULL, \ - attempt_deadline_at = NULL, \ + attempt_deadline_at = NULL, progress_step_bound_seconds = NULL, \ last_progress_at = GREATEST(last_progress_at, $5), updated_at = NOW() \ WHERE run_uid = $1 AND task_id = $2 AND attempt_generation = $3 \ AND active_dispatch_uid = $4 RETURNING *", @@ -3275,6 +3356,7 @@ pub async fn settle_external_job_terminal_in_conn( let row = sqlx::query( "UPDATE moa.execution_task SET status='ready', attempt_state='idle', \ waiting_since=NULL, ready_at=$3, external_job_uid=NULL, \ + progress_step_bound_seconds=NULL, \ attempt_generation=$4, last_progress_at=GREATEST(last_progress_at,$3), \ updated_at=NOW() \ WHERE run_uid=$1 AND task_id=$2 AND status='waiting_external' \ @@ -3539,7 +3621,7 @@ async fn persist_task_attempt_checkpoint_for_state_in_conn( ))) } -fn external_start_checkpoint_payload_is_provisional( +pub(super) fn external_start_checkpoint_payload_is_provisional( kind: TaskAttemptCheckpointKind, payload: &Value, ) -> bool { @@ -3927,6 +4009,7 @@ pub(super) async fn settle_unstarted_task_attempt_in_conn( let row = sqlx::query( "UPDATE moa.execution_task SET status='ready', attempt_state='idle', \ attempt_generation=$5, active_dispatch_uid=NULL, attempt_deadline_at=NULL, \ + progress_step_bound_seconds=NULL, \ ready_at=$6, waiting_since=NULL, generation_history=generation_history || \ jsonb_build_array($7::JSONB), \ last_progress_at=GREATEST(last_progress_at,$6), updated_at=NOW() \ @@ -5228,8 +5311,8 @@ impl ExecutionRepository { #[cfg(test)] mod tests { use super::{ - ActiveAttemptLiveness, TaskAttemptCheckpointKind, TaskAttemptFence, - UnstartedTaskAttemptDisposition, append_agent_resume_input, + ATTEMPT_STEP_BOUND_MARGIN_SECONDS, ActiveAttemptLiveness, TaskAttemptCheckpointKind, + TaskAttemptFence, UnstartedTaskAttemptDisposition, append_agent_resume_input, classify_active_attempt_liveness, external_start_checkpoint_payload_is_provisional, paused_task_attempt_release_history_matches, task_release_receipt_is_verified_absence, unstarted_task_attempt_history_matches, @@ -5512,6 +5595,7 @@ mod tests { &config, deadline, observed_at - Duration::seconds(30), + None, observed_at, ), ActiveAttemptLiveness::Live @@ -5519,7 +5603,7 @@ mod tests { // Same instant, but the attempt has committed nothing since it started. assert_eq!( - classify_active_attempt_liveness(&config, deadline, started_at, observed_at), + classify_active_attempt_liveness(&config, deadline, started_at, None, observed_at), ActiveAttemptLiveness::Stalled ); @@ -5529,6 +5613,7 @@ mod tests { &config, deadline, started_at, + None, started_at + Duration::seconds(60), ), ActiveAttemptLiveness::Stalled @@ -5538,6 +5623,7 @@ mod tests { &config, deadline, started_at, + None, started_at + Duration::seconds(59), ), ActiveAttemptLiveness::Live @@ -5545,11 +5631,70 @@ mod tests { // A progressing attempt that reaches its deadline is deadline-exceeded, never stalled. assert_eq!( - classify_active_attempt_liveness(&config, deadline, deadline, deadline), + classify_active_attempt_liveness(&config, deadline, deadline, None, deadline), ActiveAttemptLiveness::DeadlineExceeded ); assert!(!ActiveAttemptLiveness::Live.is_expired()); assert!(ActiveAttemptLiveness::Stalled.is_expired()); assert!(ActiveAttemptLiveness::DeadlineExceeded.is_expired()); } + + #[test] + fn declared_step_bound_widens_only_its_own_stall_window() { + // Pins: a step that declared a bound longer than the configured floor is Live until + // that bound plus the margin elapses, while an attempt between steps at the very same + // instant is already Stalled. Without this, one long-running step would have to raise + // the floor for every attempt, which is what made the stall guard barely beat the + // deadline it is supposed to improve on. + let config = ExecutionConfig { + attempt_heartbeat_staleness_seconds: 120, + active_attempt_timeout_seconds: 600, + ..ExecutionConfig::default() + }; + let started_at = Utc::now(); + let deadline = started_at + Duration::seconds(600); + let bound = Some(Duration::seconds(300)); + + // Past the floor, inside the declared bound: the step is still working. + let inside = started_at + Duration::seconds(200); + assert_eq!( + classify_active_attempt_liveness(&config, deadline, started_at, bound, inside), + ActiveAttemptLiveness::Live + ); + // The same silence with no declared bound is a stall at the floor. + assert_eq!( + classify_active_attempt_liveness(&config, deadline, started_at, None, inside), + ActiveAttemptLiveness::Stalled + ); + + // The bound is not a grace-free cliff: it ends one margin after the declared bound. + let at_bound = started_at + Duration::seconds(300); + assert_eq!( + classify_active_attempt_liveness(&config, deadline, started_at, bound, at_bound), + ActiveAttemptLiveness::Live + ); + assert_eq!( + classify_active_attempt_liveness( + &config, + deadline, + started_at, + bound, + at_bound + Duration::seconds(ATTEMPT_STEP_BOUND_MARGIN_SECONDS), + ), + ActiveAttemptLiveness::Stalled + ); + + // A bound shorter than the floor never tightens the window below it, so jitter around + // a two-second step cannot read as a stall. + assert_eq!( + classify_active_attempt_liveness( + &config, + deadline, + started_at, + Some(Duration::seconds(2)), + started_at + Duration::seconds(119), + ), + ActiveAttemptLiveness::Live + ); + } } diff --git a/crates/moa-execution/src/repository/trigger.rs b/crates/moa-execution/src/repository/trigger.rs index 887a297d1..358e97d1e 100644 --- a/crates/moa-execution/src/repository/trigger.rs +++ b/crates/moa-execution/src/repository/trigger.rs @@ -27,7 +27,7 @@ use super::{ requeue_delivered_dispatch_in_conn, }, run::enqueue_run_activation_in_conn, - sqlx_error, storage_error, to_i64, to_optional_i64, + sqlx_error, storage_error, to_i64, to_optional_i64, to_positive_u32, }; const MAX_RECONCILE_BATCH_SIZE: u32 = 1_000; @@ -1104,7 +1104,6 @@ impl ExecutionRepository { config: &ExecutionConfig, trigger_uid: Uuid, ) -> Result { - let staleness = crate::repository::task::attempt_heartbeat_staleness_window(config)?; let mut conn = scope.begin(&self.pool).await?; prelock_trigger_scheduled_capacity_in_conn(conn.as_mut(), trigger_uid).await?; let row = @@ -1131,8 +1130,17 @@ impl ExecutionRepository { conn.commit().await.map_err(storage_error)?; return Ok(ExecutionWatchdogDeferOutcome::NotDeferred); } - let progress = sqlx::query_as::<_, (Option>, DateTime, DateTime)>( - "SELECT attempt_deadline_at, last_progress_at, now() FROM moa.execution_task \ + let progress = sqlx::query_as::< + _, + ( + Option>, + DateTime, + Option, + DateTime, + ), + >( + "SELECT attempt_deadline_at, last_progress_at, progress_step_bound_seconds, now() \ + FROM moa.execution_task \ WHERE tenant_id=$1 AND run_uid=$2 AND task_id=$3 AND attempt_generation=$4 \ AND active_dispatch_uid IS NOT NULL", ) @@ -1146,19 +1154,52 @@ impl ExecutionRepository { .fetch_optional(conn.as_mut()) .await .map_err(sqlx_error)?; - let Some((Some(attempt_deadline_at), last_progress_at, observed_at)) = progress else { + let Some((Some(attempt_deadline_at), last_progress_at, step_bound_seconds, observed_at)) = + progress + else { conn.commit().await.map_err(storage_error)?; return Ok(ExecutionWatchdogDeferOutcome::NotDeferred); }; - let Some(next_due_at) = last_progress_at + // Deferral must use the same window the classifier used. An attempt is Live only + // because its in-flight step declared a wider bound, so recomputing the next + // observation from the bare floor cannot advance past the due time that just fired: + // the deferral is refused, and the caller treats a still-current receiver as an error + // and retries the delivery forever. + let staleness = crate::repository::task::attempt_heartbeat_staleness_window( + config, + step_bound_seconds + .map(|seconds| { + let seconds = to_positive_u32(seconds, "progress step bound seconds")?; + chrono::TimeDelta::try_seconds(i64::from(seconds)).ok_or_else(|| { + Error::InvalidRepositoryData { + message: "progress step bound exceeds chrono duration".to_string(), + } + }) + }) + .transpose()?, + )?; + let Some(calculated_due_at) = last_progress_at .checked_add_signed(staleness) .map(|stale_at| stale_at.min(attempt_deadline_at)) else { conn.commit().await.map_err(storage_error)?; return Ok(ExecutionWatchdogDeferOutcome::NotDeferred); }; - // Strictly forward only. An observation at or before now would re-fire immediately, and one - // at or before the current arm would not move detection at all. + // The receiver and this transaction observe time independently. A recovered delivery can + // classify the attempt live immediately before its staleness boundary, then reach this + // transaction immediately after it. Give that already-journaled RetryDelivery one fresh + // identity instead of retrying the same memoized receiver response forever. + let next_due_at = if calculated_due_at <= observed_at || calculated_due_at <= trigger.due_at + { + observed_at + .checked_add_signed(chrono::TimeDelta::seconds(1)) + .map(|retry_at| retry_at.min(attempt_deadline_at)) + .unwrap_or(attempt_deadline_at) + } else { + calculated_due_at + }; + // Strictly forward only. Once the deadline itself is due, the receiver must settle the + // attempt (or yield to another exact recovery owner) rather than extending its authority. if next_due_at <= observed_at || next_due_at <= trigger.due_at { conn.commit().await.map_err(storage_error)?; return Ok(ExecutionWatchdogDeferOutcome::NotDeferred); @@ -1909,7 +1950,7 @@ fn trigger_capacity_request(trigger: &ExecutionTriggerRecord) -> ExecutionCapaci } } -async fn settle_trigger_dispatch( +pub(super) async fn settle_trigger_dispatch( conn: &mut PgConnection, trigger_uid: Uuid, state: ExecutionDeliveryState, diff --git a/crates/moa-execution/src/wire.rs b/crates/moa-execution/src/wire.rs index 4acf1c227..07807fc18 100644 --- a/crates/moa-execution/src/wire.rs +++ b/crates/moa-execution/src/wire.rs @@ -1690,7 +1690,7 @@ mod tests { #[test] fn execution_blocker_audience_uses_exact_scalar_priority_not_reason_samples_offline() { // Pins: truncated display samples cannot hide a higher-priority exact blocker; scalar - // audience counters always order User > TenantReviewer > External > Agent > System. + // audience counters always order User > TenantReviewer > External > System. assert_eq!( execution_blocker_audience_from_flags(true, true, true, true), Some(ExecutionBlockerAudience::User) diff --git a/crates/moa-execution/tests/execution_db/completion_projection_db.rs b/crates/moa-execution/tests/execution_db/completion_projection_db.rs index 9d29a3576..b47ec666d 100644 --- a/crates/moa-execution/tests/execution_db/completion_projection_db.rs +++ b/crates/moa-execution/tests/execution_db/completion_projection_db.rs @@ -580,6 +580,10 @@ async fn replan_stop_completion_pages_rebind_exact_wake_without_duplicate_verifi .load_replan_stop_intent(scope, run.run_uid, run.controller_generation, wake_epoch) .await? .expect("intent must follow its exact current wake"); + sqlx::query("UPDATE moa.execution_run SET activation_failure_count=5 WHERE run_uid=$1") + .bind(run.run_uid) + .execute(&pool) + .await?; match repository .advance_replan_stop_completion_projection( scope, @@ -605,6 +609,16 @@ async fn replan_stop_completion_pages_rebind_exact_wake_without_duplicate_verifi .wake_epoch .expect("ReplanStop continuation has an exact wake"); assert!(next_wake > wake_epoch); + let failure_count: i64 = sqlx::query_scalar( + "SELECT activation_failure_count FROM moa.execution_run WHERE run_uid=$1", + ) + .bind(run.run_uid) + .fetch_one(&pool) + .await?; + assert_eq!( + failure_count, 0, + "each successful ReplanStop page acknowledgement resets crash recovery" + ); let persisted_source = sqlx::query_scalar::<_, chrono::DateTime>( "SELECT source_progress_at FROM moa.execution_completion_scan WHERE run_uid=$1", ) diff --git a/crates/moa-execution/tests/execution_db/controller_wake_recovery_db.rs b/crates/moa-execution/tests/execution_db/controller_wake_recovery_db.rs index 0aa5d27ab..8fcf8119e 100644 --- a/crates/moa-execution/tests/execution_db/controller_wake_recovery_db.rs +++ b/crates/moa-execution/tests/execution_db/controller_wake_recovery_db.rs @@ -394,6 +394,11 @@ async fn resumed_recovery_fails_the_run_once_its_budget_is_exhausted_db() -> Tes "a wedged run with no outstanding work must finalize in its first bounded page" ); assert_eq!(commit.run.status, ExecutionRunStatus::Failed); + assert_eq!( + activation_failure_count(&pool, run.run_uid).await?, + 0, + "successful pending-terminal wake settlement resets the consecutive-failure budget" + ); assert_eq!( commit.run.terminal_reason, Some(ExecutionTerminalReason::InternalFailure) diff --git a/crates/moa-execution/tests/execution_db/long_horizon_state_db.rs b/crates/moa-execution/tests/execution_db/long_horizon_state_db.rs index c08cfc955..9abf93cff 100644 --- a/crates/moa-execution/tests/execution_db/long_horizon_state_db.rs +++ b/crates/moa-execution/tests/execution_db/long_horizon_state_db.rs @@ -3,8 +3,9 @@ use moa_artifacts::execution_plan::{ExecutionNode, ExecutionOperation}; use moa_execution::repository::ready::{ReadyMaterializationOutcome, ReadyMaterializationRequest}; use moa_execution::repository::task::{ - ActiveAttemptLiveness, TaskAttemptFence, TaskAttemptProgressOutcome, TaskAttemptStartOutcome, - classify_active_attempt_liveness, + ActiveAttemptLiveness, NewTaskAttemptCheckpoint, TaskAttemptCheckpointKind, + TaskAttemptContinuationYieldOutcome, TaskAttemptFence, TaskAttemptProgressOutcome, + TaskAttemptReleaseClaimOutcome, TaskAttemptStartOutcome, classify_active_attempt_liveness, }; use moa_execution::repository::trigger::{ ExecutionTriggerNoOp, ExecutionWatchdogDeferOutcome, ExecutionWatchdogTriggerOutcome, @@ -392,7 +393,7 @@ async fn attempt_heartbeat_keeps_a_progressing_attempt_live_while_a_wedged_one_s let heartbeat_at = progressing_started_at + Duration::seconds(90); assert_eq!( repository - .record_task_attempt_progress(progressing_fence, heartbeat_at) + .record_task_attempt_progress(progressing_fence, heartbeat_at, None) .await?, TaskAttemptProgressOutcome::Applied ); @@ -414,6 +415,7 @@ async fn attempt_heartbeat_keeps_a_progressing_attempt_live_while_a_wedged_one_s &config, progressing_fence.attempt_deadline_at, progressing.last_progress_at, + None, observed_at, ), ActiveAttemptLiveness::Live, @@ -424,6 +426,7 @@ async fn attempt_heartbeat_keeps_a_progressing_attempt_live_while_a_wedged_one_s &config, wedged_fence.attempt_deadline_at, wedged.last_progress_at, + None, observed_at, ), ActiveAttemptLiveness::Stalled, @@ -436,6 +439,248 @@ async fn attempt_heartbeat_keeps_a_progressing_attempt_live_while_a_wedged_one_s Ok(()) } +#[tokio::test] +async fn declared_step_bound_defers_the_watchdog_it_kept_alive_db() -> TestResult { + // Pins: when a step declares a bound wider than the staleness floor, the liveness + // classifier and the watchdog deferral must derive their window from that same bound. + // Deriving the deferral from the bare floor instead produces a state with no exit: the + // attempt classifies Live because of the wider bound, but the floor-derived next + // observation cannot advance past the due time that just fired, so the deferral refuses, + // the caller treats a still-current receiver as an error, and the delivery retries + // forever behind the fleet-serialized drain. Only a bound wider than the floor reaches + // this; an unbounded step defers normally and hides it. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let config = ExecutionConfig { + attempt_heartbeat_staleness_seconds: 2, + active_attempt_timeout_seconds: 600, + ..ExecutionConfig::default() + }; + let (_run_uid, started) = start_admitted_attempts( + &repository, + tenant_id, + "watchdog-step-bound-deferral", + &config, + &["bounded"], + ) + .await?; + let [(fence, _)] = started[..] else { + panic!("one attempt must start"); + }; + + let armed_due_at: chrono::DateTime = + sqlx::query_scalar("SELECT due_at FROM moa.execution_trigger WHERE trigger_uid=$1") + .bind(fence.watchdog_trigger_uid) + .fetch_one(&pool) + .await?; + + // The step declares far more than the two-second floor, as a long sandbox command or an + // external provider start does. + let bound_seconds = 120_u32; + let heartbeat_at: chrono::DateTime = + sqlx::query_scalar("SELECT now()").fetch_one(&pool).await?; + assert_eq!( + repository + .record_task_attempt_progress(fence, heartbeat_at, Some(bound_seconds)) + .await?, + TaskAttemptProgressOutcome::Applied + ); + + // Past the floor, well inside the declared bound: still working, not stalled. + let observed_at = heartbeat_at + Duration::seconds(30); + assert_eq!( + classify_active_attempt_liveness( + &config, + fence.attempt_deadline_at, + heartbeat_at, + Some(Duration::seconds(i64::from(bound_seconds))), + observed_at, + ), + ActiveAttemptLiveness::Live, + "a step inside its declared bound is live even past the staleness floor" + ); + + // The deferral must agree, and must move strictly forward. A floor-derived window would + // land at heartbeat + 2s, which is not past the already-armed due time, and refuse. + let ExecutionWatchdogDeferOutcome::Deferred { next_due_at } = repository + .defer_task_attempt_watchdog(scope, &config, fence.watchdog_trigger_uid) + .await? + else { + panic!( + "an attempt kept alive by its declared step bound must be rearmed on that same bound" + ); + }; + assert!( + next_due_at > armed_due_at, + "the rearmed observation must advance past the due time that just fired" + ); + assert!( + next_due_at >= heartbeat_at + Duration::seconds(i64::from(bound_seconds)), + "the rearmed observation must outlast the bound the step declared" + ); + assert_eq!( + repository + .prepare_watchdog_trigger(scope, fence.watchdog_trigger_uid) + .await?, + ExecutionWatchdogTriggerOutcome::NoOp(ExecutionTriggerNoOp::NotDue), + "a rearmed watchdog must stop firing until its next observation" + ); + + // A recovered delivery can journal Live just before the boundary while the deferral + // transaction observes the database clock just after it. The same delivery identity must + // still advance once; otherwise Restate memoizes RetryDelivery and revalidation loops forever. + let raced_progress_at: chrono::DateTime = + sqlx::query_scalar("SELECT now()").fetch_one(&pool).await?; + assert_eq!( + repository + .record_task_attempt_progress(fence, raced_progress_at, None) + .await?, + TaskAttemptProgressOutcome::Applied + ); + let raced_at = raced_progress_at + Duration::seconds(2); + sqlx::query("UPDATE moa.execution_trigger SET state='pending',due_at=$2 WHERE trigger_uid=$1") + .bind(fence.watchdog_trigger_uid) + .bind(raced_at) + .execute(&pool) + .await?; + sqlx::query( + "UPDATE moa.execution_dispatch_outbox SET state='pending',not_before_at=$2 \ + WHERE trigger_uid=$1 AND dispatch_kind='trigger_delivery'", + ) + .bind(fence.watchdog_trigger_uid) + .bind(raced_at) + .execute(&pool) + .await?; + tokio::time::sleep(std::time::Duration::from_millis(2_050)).await; + let ExecutionWatchdogDeferOutcome::Deferred { next_due_at } = repository + .defer_task_attempt_watchdog(scope, &config, fence.watchdog_trigger_uid) + .await? + else { + panic!("a boundary-crossing RetryDelivery must receive a fresh delivery identity"); + }; + assert!(next_due_at > raced_at); + Ok(()) +} + +#[tokio::test] +async fn three_agent_slices_keep_one_logical_task_reservation_and_reset_step_bound_db() -> TestResult +{ + // Pins: bounded continuations release only active capacity. Redispatching the same logical + // task must keep its original task-budget reservation, while each successor slice starts + // without inheriting the predecessor step's wider heartbeat bound. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let config = ExecutionConfig::default(); + let (run_uid, started) = start_admitted_attempts( + &repository, + tenant_id, + "three-slice-logical-reservation", + &config, + &["agent"], + ) + .await?; + let [(mut fence, _)] = started[..] else { + panic!("one first slice must start"); + }; + let initially_reserved = repository + .load_run(scope, run_uid) + .await? + .expect("run remains visible") + .reserved; + assert_eq!(initially_reserved.tasks, 1); + + for completed_slice in 1..=2 { + let heartbeat_at = moa_test_support::fixtures::pg_now(); + assert_eq!( + repository + .record_task_attempt_progress(fence, heartbeat_at, Some(120)) + .await?, + TaskAttemptProgressOutcome::Applied + ); + let active = listed_task(&repository, scope, run_uid, fence.task_id).await?; + assert_eq!(active.progress_step_bound_seconds, Some(120)); + let TaskAttemptReleaseClaimOutcome::Applied(releasing) = repository + .begin_task_attempt_release( + fence, + active.generation, + "bounded_agent_continuation", + heartbeat_at, + ) + .await? + else { + panic!("the active slice must enter its release boundary"); + }; + assert_eq!(releasing.task.progress_step_bound_seconds, None); + let TaskAttemptContinuationYieldOutcome::Applied { task: ready, .. } = repository + .yield_task_attempt_continuation(NewTaskAttemptCheckpoint { + fence, + task_generation: active.generation, + kind: TaskAttemptCheckpointKind::AgentContinuation, + schema_version: 1, + payload: json!({"state":{"kind":"agent","slice":completed_slice}}), + workspace_release_receipt: None, + created_at: heartbeat_at, + }) + .await? + else { + panic!("each bounded slice must checkpoint and return the task to ready storage"); + }; + assert_eq!(ready.progress_step_bound_seconds, None); + assert_eq!( + repository + .load_run(scope, run_uid) + .await? + .expect("run remains visible") + .reserved, + initially_reserved, + "yield must retain exactly one logical-task reservation" + ); + + let admission = repository + .admit_ready_attempts(&config, 1, Utc::now()) + .await? + .admitted + .into_iter() + .next() + .expect("successor bounded slice must be admitted"); + assert_eq!(admission.attempt_generation, fence.attempt_generation + 1); + fence = TaskAttemptFence { + tenant_id: admission.tenant_id, + run_uid: admission.run_uid, + task_id: admission.task_id, + controller_generation: admission.controller_generation, + attempt_generation: admission.attempt_generation, + dispatch_uid: admission.dispatch_uid, + capacity_reservation_uid: admission.capacity_reservation_uid, + watchdog_trigger_uid: admission.watchdog_trigger_uid, + attempt_deadline_at: admission.attempt_deadline_at, + }; + let TaskAttemptStartOutcome::Started(started) = + repository.start_task_attempt(fence).await? + else { + panic!("successor admission must start"); + }; + assert_eq!(started.task.progress_step_bound_seconds, None); + assert_eq!( + repository + .load_run(scope, run_uid) + .await? + .expect("run remains visible") + .reserved, + initially_reserved, + "redispatch must reuse rather than duplicate the logical-task reservation" + ); + } + assert_eq!(fence.attempt_generation, 3, "the third slice is active"); + Ok(()) +} + #[tokio::test] async fn stalled_attempt_watchdog_becomes_deliverable_before_its_deadline_db() -> TestResult { // Pins: the watchdog is armed one staleness window ahead of the attempt deadline, so a wedged @@ -491,7 +736,7 @@ async fn stalled_attempt_watchdog_becomes_deliverable_before_its_deadline_db() - for fence in [progressing_fence, capped_fence] { assert_eq!( repository - .record_task_attempt_progress(fence, heartbeat_at) + .record_task_attempt_progress(fence, heartbeat_at, None) .await?, TaskAttemptProgressOutcome::Applied ); @@ -517,6 +762,7 @@ async fn stalled_attempt_watchdog_becomes_deliverable_before_its_deadline_db() - &config, wedged_fence.attempt_deadline_at, wedged.last_progress_at, + None, observed_at, ), ActiveAttemptLiveness::Stalled diff --git a/crates/moa-execution/tests/execution_db/planning_and_audit_db.rs b/crates/moa-execution/tests/execution_db/planning_and_audit_db.rs index 295934a96..c1d65e241 100644 --- a/crates/moa-execution/tests/execution_db/planning_and_audit_db.rs +++ b/crates/moa-execution/tests/execution_db/planning_and_audit_db.rs @@ -1,6 +1,204 @@ //! Planning-context, normalized-audit, confirmation, and amendment persistence contracts. use super::support::*; +use moa_execution::repository::planning_budget::{ + AmendmentPlanningCallReconcileOutcome, AmendmentPlanningCallReconcileRequest, + AmendmentPlanningCallReservation, AmendmentPlanningCallReservationOutcome, + AmendmentPlanningCallReservationRequest, PlanningUsage, +}; + +#[tokio::test] +async fn amendment_planner_call_budget_is_reserved_and_reconciled_exactly_once_db() -> TestResult { + // Pins: one automatic amendment provider call has durable cost/token attribution, consumes + // no logical-task budget, and exact reserve/reconcile replays cannot double charge the run. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let run = create_run( + &repository, + scope, + new_run( + tenant_id, + None, + "amendment-planning-budget", + ExecutionRunStatus::Queued, + budget(10), + ), + ) + .await?; + sqlx::query("UPDATE moa.execution_run SET status='running' WHERE run_uid=$1") + .bind(run.run_uid) + .execute(&pool) + .await?; + sqlx::query("UPDATE moa.execution_run SET status='waiting_replan' WHERE run_uid=$1") + .bind(run.run_uid) + .execute(&pool) + .await?; + + let denied = AmendmentPlanningCallReservationRequest { + run_uid: run.run_uid, + base_plan_revision: 1, + call_ordinal: 9, + reservation: AmendmentPlanningCallReservation { + cost_microusd: 10_000, + tokens: 10_000, + }, + now: moa_test_support::fixtures::pg_now(), + }; + assert!(matches!( + repository + .reserve_amendment_planning_call(scope, denied) + .await?, + AmendmentPlanningCallReservationOutcome::Denied(_) + )); + let after_denial = repository + .load_run(scope, run.run_uid) + .await? + .expect("denied run remains visible"); + assert_eq!(after_denial.reserved, Default::default()); + assert_eq!(after_denial.consumed, Default::default()); + assert_eq!( + repository + .load_amendment_planning_call(scope, run.run_uid, 1, 9) + .await?, + None, + "denial must leave no attribution row for work that cannot run" + ); + + let request = AmendmentPlanningCallReservationRequest { + run_uid: run.run_uid, + base_plan_revision: 1, + call_ordinal: 2, + reservation: AmendmentPlanningCallReservation { + cost_microusd: 100, + tokens: 80, + }, + now: moa_test_support::fixtures::pg_now(), + }; + let AmendmentPlanningCallReservationOutcome::Granted(open) = repository + .reserve_amendment_planning_call(scope, request) + .await? + else { + panic!("the exact live amendment revision must authorize one provider call"); + }; + assert_eq!(open.reserved, request.reservation); + assert_eq!(open.actual, None); + assert_eq!( + repository + .reserve_amendment_planning_call(scope, request) + .await?, + AmendmentPlanningCallReservationOutcome::ReplayedOpen(open.clone()) + ); + let reserved_run = repository + .load_run(scope, run.run_uid) + .await? + .expect("run remains visible"); + assert_eq!(reserved_run.reserved.cost_microusd, 100); + assert_eq!(reserved_run.reserved.tokens, 80); + assert_eq!(reserved_run.reserved.tasks, 0); + + let reconcile = AmendmentPlanningCallReconcileRequest { + run_uid: run.run_uid, + base_plan_revision: 1, + call_ordinal: 2, + actual: PlanningUsage { + cost_microusd: 70, + tokens: 50, + }, + settled_at: moa_test_support::fixtures::pg_now(), + }; + let AmendmentPlanningCallReconcileOutcome::Applied(settled) = repository + .reconcile_amendment_planning_call(scope, reconcile.clone()) + .await? + else { + panic!("the first exact reconciliation must apply"); + }; + assert_eq!(settled.actual.as_ref(), Some(&reconcile.actual)); + assert_eq!( + repository + .reconcile_amendment_planning_call(scope, reconcile.clone()) + .await?, + AmendmentPlanningCallReconcileOutcome::Replayed(settled.clone()) + ); + assert_eq!( + repository + .reserve_amendment_planning_call(scope, request) + .await?, + AmendmentPlanningCallReservationOutcome::AlreadySettled(settled.clone()), + "post-reconcile Restate replay must recover the immutable settled authorization" + ); + let settled_run = repository + .load_run(scope, run.run_uid) + .await? + .expect("run remains visible"); + assert_eq!(settled_run.reserved.cost_microusd, 0); + assert_eq!(settled_run.reserved.tokens, 0); + assert_eq!(settled_run.consumed.cost_microusd, 70); + assert_eq!(settled_run.consumed.tokens, 50); + assert_eq!(settled_run.consumed.tasks, 0); + assert_eq!( + repository + .load_amendment_planning_call(scope, run.run_uid, 1, 2) + .await?, + Some(settled) + ); + + let mut conflicting = reconcile.clone(); + conflicting.actual.cost_microusd += 1; + assert_eq!( + repository + .reconcile_amendment_planning_call(scope, conflicting) + .await?, + AmendmentPlanningCallReconcileOutcome::Conflict + ); + + let repair = AmendmentPlanningCallReservationRequest { + call_ordinal: 3, + reservation: AmendmentPlanningCallReservation { + cost_microusd: 40, + tokens: 30, + }, + ..request + }; + assert!(matches!( + repository + .reserve_amendment_planning_call(scope, repair) + .await?, + AmendmentPlanningCallReservationOutcome::Granted(_) + )); + let repair_reconcile = AmendmentPlanningCallReconcileRequest { + call_ordinal: repair.call_ordinal, + actual: PlanningUsage { + cost_microusd: 20, + tokens: 10, + }, + ..reconcile + }; + let AmendmentPlanningCallReconcileOutcome::Applied(repair_settled) = repository + .reconcile_amendment_planning_call(scope, repair_reconcile.clone()) + .await? + else { + panic!("the repair call must reconcile independently"); + }; + assert_eq!(repair_settled.actual, Some(repair_reconcile.actual)); + assert_eq!( + repository + .reconcile_amendment_planning_call(scope, repair_reconcile) + .await?, + AmendmentPlanningCallReconcileOutcome::Replayed(repair_settled) + ); + let after_repair = repository + .load_run(scope, run.run_uid) + .await? + .expect("run remains visible"); + assert_eq!(after_repair.reserved, Default::default()); + assert_eq!(after_repair.consumed.cost_microusd, 90); + assert_eq!(after_repair.consumed.tokens, 60); + assert_eq!(after_repair.consumed.tasks, 0); + Ok(()) +} #[tokio::test] async fn planning_context_snapshot_is_immutable_and_exactly_replayed_db() -> TestResult { @@ -246,6 +444,11 @@ async fn normalized_planning_audits_return_first_measurements_and_conflict_db() RouteAuditWriteOutcome::Replayed(contact_evidence) ); + let planner_report = String::from_utf8(canonical_json_bytes(&ExecutionAuditReport::Schema { + violations: Vec::new(), + omitted_violations: 0, + full_report_hash: "d".repeat(64), + })?)?; let planner = ExecutionPlanningAuditEnvelope { schema_version: 1, tenant_id, @@ -257,12 +460,19 @@ async fn normalized_planning_audits_return_first_measurements_and_conflict_db() call_ordinal: 0, run_uid: None, plan_revision: None, - outcome: ExecutionPlannerOutcome::ProviderError, + outcome: ExecutionPlannerOutcome::SchemaRejected, provider_model: "planner-test".to_string(), prompt_version: "execution-planner".to_string(), - candidate_hash: None, + usage: ExecutionRouteUsage { + input_tokens_uncached: 21, + input_tokens_cache_write: 3, + input_tokens_cache_read: 5, + output_tokens: 8, + }, + cost_microusd: 29, + candidate_hash: Some("e".repeat(64)), candidate_json: None, - compiler_report: None, + compiler_report: Some(planner_report), duration_micros: 17, created_at: first_at, }, @@ -272,6 +482,11 @@ async fn normalized_planning_audits_return_first_measurements_and_conflict_db() else { panic!("first planner audit must apply"); }; + assert_eq!(planner_evidence.usage.input_tokens_uncached, 21); + assert_eq!(planner_evidence.usage.input_tokens_cache_write, 3); + assert_eq!(planner_evidence.usage.input_tokens_cache_read, 5); + assert_eq!(planner_evidence.usage.output_tokens, 8); + assert_eq!(planner_evidence.cost_microusd, 29); let mut planner_retry = planner.clone(); let ExecutionPlanningAuditPayload::PlannerCall { duration_micros, diff --git a/crates/moa-execution/tests/execution_db/trigger_outbox_db.rs b/crates/moa-execution/tests/execution_db/trigger_outbox_db.rs index 68420ca17..3a0f44a82 100644 --- a/crates/moa-execution/tests/execution_db/trigger_outbox_db.rs +++ b/crates/moa-execution/tests/execution_db/trigger_outbox_db.rs @@ -1483,11 +1483,9 @@ async fn task_external_start_recovery_adopts_started_not_started_and_replay_atom ); candidate.plan.definition.nodes = vec![watchdog_output_node()]; let run = create_run(&repository, scope, candidate).await?; - assert!( - repository - .initialize_scheduler_state(scope, run.run_uid) - .await? - ); + repository + .initialize_scheduler_state(scope, run.run_uid) + .await?; assert!(matches!( repository .materialize_ready_page( @@ -1600,17 +1598,38 @@ async fn task_external_start_recovery_adopts_started_not_started_and_replay_atom repository .reserve_external_job_intent(scope, &config, not_started_intent) .await?; + let Some(not_started_authority) = repository + .load_current_task_external_start_recovery(started[0].0) + .await? + else { + panic!("the exact current unbound external start must outrank watchdog teardown"); + }; + assert_eq!( + not_started_authority.external_job_uid, + not_started_recovery.external_job_uid + ); + assert_eq!( + not_started_authority.idempotency_key, + not_started_recovery.idempotency_key + ); assert!(matches!( repository - .recover_external_job_start_not_started(¬_started_recovery, Utc::now()) + .recover_external_job_start_not_started(¬_started_authority, Utc::now()) .await?, ExecutionExternalJobStartRecoveryAdoptionOutcome::Applied { compensation_release: None } )); + assert_eq!( + repository + .load_current_task_external_start_recovery(started[0].0) + .await?, + None, + "a settled external-start recovery must no longer defer the watchdog" + ); assert!(matches!( repository - .recover_external_job_start_not_started(¬_started_recovery, Utc::now()) + .recover_external_job_start_not_started(¬_started_authority, Utc::now()) .await?, ExecutionExternalJobStartRecoveryAdoptionOutcome::Replayed { compensation_release: None @@ -1640,6 +1659,16 @@ async fn task_external_start_recovery_adopts_started_not_started_and_replay_atom repository .reserve_external_job_intent(scope, &config, started_intent.clone()) .await?; + let Some(started_authority) = repository + .load_current_task_external_start_recovery(started[1].0) + .await? + else { + panic!("started fixture must expose its exact durable recovery authority"); + }; + assert_eq!( + started_authority.external_job_uid, + started_recovery.external_job_uid + ); sqlx::query( "UPDATE moa.execution_capacity_reservation \ SET expires_at=NOW() - INTERVAL '1 second' \ @@ -1668,7 +1697,7 @@ async fn task_external_start_recovery_adopts_started_not_started_and_replay_atom repository .recover_external_job_start_started( &config, - &started_recovery, + &started_authority, binding.clone(), Utc::now(), ) @@ -1677,9 +1706,15 @@ async fn task_external_start_recovery_adopts_started_not_started_and_replay_atom compensation_release: None } )); + assert_eq!( + repository + .load_current_task_external_start_recovery(started[1].0) + .await?, + None + ); assert!(matches!( repository - .recover_external_job_start_started(&config, &started_recovery, binding, Utc::now(),) + .recover_external_job_start_started(&config, &started_authority, binding, Utc::now(),) .await?, ExecutionExternalJobStartRecoveryAdoptionOutcome::Replayed { compensation_release: None diff --git a/crates/moa-execution/tests/execution_db/wait_entry_deadline_db.rs b/crates/moa-execution/tests/execution_db/wait_entry_deadline_db.rs index 109fc2523..0a24b7c0a 100644 --- a/crates/moa-execution/tests/execution_db/wait_entry_deadline_db.rs +++ b/crates/moa-execution/tests/execution_db/wait_entry_deadline_db.rs @@ -5,6 +5,10 @@ use moa_execution::repository::ready::{ReadyMaterializationOutcome, ReadyMateria use moa_execution::repository::task::{ TaskAttemptFence, TaskAttemptSettlementOutcome, TaskAttemptStartOutcome, }; +use moa_execution::repository::terminal::{ + PendingTerminalAdvanceOutcome, PendingTerminalAdvanceStage, +}; +use moa_execution::state::ExecutionTerminalEvidence; use super::support::*; @@ -140,7 +144,10 @@ async fn storage_wait_past_run_deadline_fails_its_node_instead_of_erroring_db() assert_eq!(failed_run.progress_failed_tasks, 1); assert_eq!(failed_run.waiting_task_count, 0); assert_eq!(failed_run.ready_task_count, 0); - assert_eq!(failed_run.next_wake_at, None); + assert_eq!( + failed_run.next_wake_at, failed_run.approved_budget.deadline_at, + "the run deadline remains the next durable wake when no task-local wait is parked" + ); assert!(failed_run.waiting_reasons.is_empty()); // A wait that still fits inside the same deadline keeps its ordinary parked projection. @@ -173,6 +180,100 @@ async fn storage_wait_past_run_deadline_fails_its_node_instead_of_erroring_db() }; assert_eq!(tasks[0].status, ExecutionTaskStatus::WaitingTimer); assert_eq!(triggers.len(), 1); + let trigger_uid = triggers[0].trigger_uid; + let bucket_before: Vec<(String, i64)> = sqlx::query_as( + "SELECT scope_kind,reserved_quantity FROM moa.execution_capacity_bucket \ + WHERE resource_dimension='scheduled_triggers' \ + AND (scope_kind='fleet' OR tenant_id=$1) ORDER BY scope_kind", + ) + .bind(tenant_id.0) + .fetch_all(&pool) + .await?; + assert_eq!(bucket_before.len(), 2); + let run_scheduled_before: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM moa.execution_capacity_reservation \ + WHERE run_uid=$1 AND resource_dimension='scheduled_triggers' \ + AND state IN ('reserved','reconciling')", + ) + .bind(run.run_uid) + .fetch_one(&pool) + .await?; + assert!(run_scheduled_before >= 1); + let current = repository + .load_run(scope, run.run_uid) + .await? + .expect("wait-owning run remains visible"); + + let PendingTerminalAdvanceOutcome::Applied(commit) = repository + .fence_completion_terminal_and_enqueue_settlement( + &config, + scope, + run.run_uid, + current.controller_generation, + current.wake_epoch, + PendingExecutionTerminal { + status: ExecutionRunStatus::Failed, + reason: ExecutionTerminalReason::InternalFailure, + terminal_evidence: ExecutionTerminalEvidence { + cause: ExecutionTerminalCause::InternalFailure, + satisfied_requirement_count: 0, + requirement_count: 1, + }, + completion_check_results: Vec::new(), + terminal_gaps: vec!["fixture terminal fence".to_string()], + output: None, + cancellation_reason: None, + }, + moa_test_support::fixtures::pg_now(), + 100, + ) + .await? + else { + panic!("terminal fencing must settle the storage-only wait"); + }; + assert_eq!(commit.stage, PendingTerminalAdvanceStage::Finalized); + let trigger_state: String = + sqlx::query_scalar("SELECT state FROM moa.execution_trigger WHERE trigger_uid=$1") + .bind(trigger_uid) + .fetch_one(&pool) + .await?; + assert_eq!(trigger_state, "superseded"); + let dispatch_state: String = + sqlx::query_scalar("SELECT state FROM moa.execution_dispatch_outbox WHERE trigger_uid=$1") + .bind(trigger_uid) + .fetch_one(&pool) + .await?; + assert_eq!( + dispatch_state, "cancelled", + "superseding a storage wait must terminally settle its pending outbox delivery" + ); + let capacity_state: (String, bool) = sqlx::query_as( + "SELECT state,released_at IS NOT NULL FROM moa.execution_capacity_reservation \ + WHERE trigger_uid=$1 AND resource_dimension='scheduled_triggers'", + ) + .bind(trigger_uid) + .fetch_one(&pool) + .await?; + assert_eq!(capacity_state, ("released".to_string(), true)); + let bucket_after: Vec<(String, i64)> = sqlx::query_as( + "SELECT scope_kind,reserved_quantity FROM moa.execution_capacity_bucket \ + WHERE resource_dimension='scheduled_triggers' \ + AND (scope_kind='fleet' OR tenant_id=$1) ORDER BY scope_kind", + ) + .bind(tenant_id.0) + .fetch_all(&pool) + .await?; + assert_eq!(bucket_after.len(), bucket_before.len()); + for ((before_scope, before), (after_scope, after)) in + bucket_before.into_iter().zip(bucket_after) + { + assert_eq!(after_scope, before_scope); + assert_eq!( + after, + before - run_scheduled_before, + "the wait plus any run-owned terminalized triggers must release every exact receipt" + ); + } Ok(()) } diff --git a/crates/moa-hands/Cargo.toml b/crates/moa-hands/Cargo.toml index af5618457..1d9d76ae0 100644 --- a/crates/moa-hands/Cargo.toml +++ b/crates/moa-hands/Cargo.toml @@ -48,6 +48,8 @@ zstd.workspace = true workspace-hack = { workspace = true } [dev-dependencies] +metrics.workspace = true +metrics-exporter-prometheus.workspace = true moa-auth-providers = { workspace = true } moa-memory-graph = { workspace = true } moa-migrations = { workspace = true } diff --git a/crates/moa-hands/src/adapters/daytona/mod.rs b/crates/moa-hands/src/adapters/daytona/mod.rs index e24139a78..c2d0b0da0 100644 --- a/crates/moa-hands/src/adapters/daytona/mod.rs +++ b/crates/moa-hands/src/adapters/daytona/mod.rs @@ -34,6 +34,7 @@ use moa_core::{ types::hands::SandboxTierCapabilities, types::hands::validate_sandbox_file_path, types::identifiers::{HandProvisioningOperationId, WorkspaceCheckpointId}, + types::resource::ResourceBudget, types::sandbox_workspace::{ ProviderAccountStorageInventory, ProviderInventoryResource, ProviderInventoryResourceKind, ProviderStorageRef, TenantStoragePurgeRequest, WorkspaceAttachRequest, @@ -70,7 +71,9 @@ use crate::tools::{bash, file_outline, file_read, grep}; const DAYTONA_SUPPORTED_CAPABILITIES: &[SandboxToolCapability] = &SandboxToolCapability::ALL; const DEFAULT_DAYTONA_IMAGE: &str = "daytonaio/workspace:latest"; -const DEFAULT_COMMAND_TIMEOUT: Duration = Duration::from_secs(300); +const DEFAULT_COMMAND_TIMEOUT: Duration = crate::tools::bash::DEFAULT_BASH_TIMEOUT; +const LIFECYCLE_TRANSITION_TIMEOUT: Duration = Duration::from_secs(60); +const LIFECYCLE_POLL_INTERVAL: Duration = Duration::from_secs(1); const DESTROY_RETRY_TIMEOUT: Duration = Duration::from_secs(30); const DESTROY_POLL_INTERVAL: Duration = Duration::from_secs(2); const PROVISION_RESOLVE_TIMEOUT: Duration = Duration::from_secs(30); @@ -574,43 +577,52 @@ impl DaytonaHandProvider { workspace_id: &str, command: &str, cwd: Option<&str>, - timeout: Option, + command_timeout: Option, ) -> Result { - let timeout_secs = timeout.map(|timeout| timeout.as_secs()); + let command_timeout = command_timeout.unwrap_or(DEFAULT_COMMAND_TIMEOUT); let started_at = Instant::now(); - let response = attempt - .client() - .post(format!( - "{}/toolbox/{}/process/execute", - attempt.origin(), - workspace_id + timeout(command_timeout, async { + let response = attempt + .client() + .post(format!( + "{}/toolbox/{}/process/execute", + attempt.origin(), + workspace_id + )) + .bearer_auth(attempt.credential()) + .json(&json!({ + "command": command, + "cwd": cwd, + "timeout": command_timeout.as_secs(), + })) + .send() + .await + .map_err(|error| { + MoaError::ProviderError(format!("failed to execute Daytona command: {error}")) + })?; + let value = expect_success_json(response, "Daytona").await?; + Ok(ToolOutput::from_process( + value + .get("result") + .and_then(Value::as_str) + .unwrap_or_default() + .to_string(), + String::new(), + value + .get("exitCode") + .or_else(|| value.get("code")) + .and_then(Value::as_i64) + .unwrap_or_default() as i32, + started_at.elapsed(), )) - .bearer_auth(attempt.credential()) - .json(&json!({ - "command": command, - "cwd": cwd, - "timeout": timeout_secs.unwrap_or(DEFAULT_COMMAND_TIMEOUT.as_secs()), - })) - .send() - .await - .map_err(|error| { - MoaError::ProviderError(format!("failed to execute Daytona command: {error}")) - })?; - let value = expect_success_json(response, "Daytona").await?; - Ok(ToolOutput::from_process( - value - .get("result") - .and_then(Value::as_str) - .unwrap_or_default() - .to_string(), - String::new(), - value - .get("exitCode") - .or_else(|| value.get("code")) - .and_then(Value::as_i64) - .unwrap_or_default() as i32, - started_at.elapsed(), - )) + }) + .await + .map_err(|_| { + MoaError::ToolError(format!( + "Daytona command timed out after {}s", + command_timeout.as_secs() + )) + })? } async fn prepare_mutable_root(&self, handle: &HandHandle) -> Result<()> { @@ -824,6 +836,7 @@ impl DaytonaHandProvider { tool: &str, input: &str, payload: &Value, + execution_timeout: Duration, ) -> Result { match supported_capability_for_tool(tool, DAYTONA_SUPPORTED_CAPABILITIES) { Some(SandboxToolCapability::Bash) => { @@ -842,7 +855,7 @@ impl DaytonaHandProvider { workspace_id, &command, None, - params.timeout_secs.map(|timeout| timeout.duration()), + Some(execution_timeout), ) .await?; if let Some(command) = trusted { @@ -852,8 +865,14 @@ impl DaytonaHandProvider { } Some(SandboxToolCapability::Grep) => { let command = grep::remote_shell_command(input, "/")?; - self.execute_command(attempt, workspace_id, &command, None, None) - .await + self.execute_command( + attempt, + workspace_id, + &command, + None, + Some(execution_timeout), + ) + .await } Some(SandboxToolCapability::FileOutline) => { let path = required_string_field(payload, "path")?; @@ -906,6 +925,108 @@ impl DaytonaHandProvider { None => Err(unsupported_tool("Daytona", tool)), } } + + async fn execute_bounded( + &self, + handle: &HandHandle, + tool: &str, + input: &str, + budget: ResourceBudget, + ) -> Result { + let execution_timeout = bash::effective_synchronous_timeout( + tool, + input, + DEFAULT_COMMAND_TIMEOUT, + budget.time_remaining(chrono::Utc::now()), + )?; + timeout(execution_timeout, async { + let workspace_id = handle.daytona_id()?; + let (account_id, account_generation) = cloud_account(handle, "Daytona")?; + let attempt = self + .attempt(account_id, account_generation, ProviderEndpoint::Toolbox) + .await?; + let payload: Value = serde_json::from_str(input)?; + match self + .dispatch_tool( + &attempt, + workspace_id, + tool, + input, + &payload, + execution_timeout, + ) + .await + { + Ok(output) => Ok(output), + Err(error) => match self.status(handle).await { + Ok(HandStatus::Stopped | HandStatus::Paused | HandStatus::Provisioning) => { + self.resume(handle).await?; + let attempt = self + .attempt(account_id, account_generation, ProviderEndpoint::Toolbox) + .await?; + self.dispatch_tool( + &attempt, + workspace_id, + tool, + input, + &payload, + execution_timeout, + ) + .await + } + _ => Err(error), + }, + } + }) + .await + .map_err(|_| { + MoaError::ToolError(format!( + "Daytona tool execution timed out after {}s", + execution_timeout.as_secs() + )) + })? + } + + async fn wait_for_status( + &self, + handle: &HandHandle, + expected: HandStatus, + operation: &'static str, + ) -> Result<()> { + let started_at = Instant::now(); + loop { + let status = self.status(handle).await?; + if status == expected { + return Ok(()); + } + if matches!(status, HandStatus::Destroyed | HandStatus::Failed) { + return Err(MoaError::ProviderError(format!( + "Daytona sandbox entered {status:?} while waiting to {operation}" + ))); + } + if started_at.elapsed() >= LIFECYCLE_TRANSITION_TIMEOUT { + return Err(MoaError::ProviderError(format!( + "timed out waiting for Daytona sandbox to {operation}; last status was {status:?}" + ))); + } + sleep(LIFECYCLE_POLL_INTERVAL).await; + } + } +} + +fn daytona_hand_status(state: &str) -> HandStatus { + match state.to_ascii_lowercase().as_str() { + "creating" | "pending" | "starting" | "resuming" | "restoring" | "pending_build" + | "building_snapshot" | "pulling_snapshot" | "resizing" | "snapshotting" | "forking" => { + HandStatus::Provisioning + } + "started" | "running" => HandStatus::Running, + "stopping" | "pausing" => HandStatus::Paused, + "paused" | "stopped" => HandStatus::Stopped, + "archived" | "archiving" | "deleted" | "destroying" => HandStatus::Destroyed, + "error" | "failed" | "build_failed" => HandStatus::Failed, + _ => HandStatus::Failed, + } } /// Revision of the Daytona provider's capability declaration. @@ -1096,34 +1217,18 @@ impl HandProvider for DaytonaHandProvider { } async fn execute(&self, handle: &HandHandle, tool: &str, input: &str) -> Result { - let workspace_id = handle.daytona_id()?; - let (account_id, account_generation) = cloud_account(handle, "Daytona")?; - let attempt = self - .attempt(account_id, account_generation, ProviderEndpoint::Toolbox) - .await?; - let payload: Value = serde_json::from_str(input)?; - // Attempt the tool directly rather than resuming on every call. The - // sandbox is only probed and resumed after a failure, and only when it is - // genuinely not running (so the tool never started); this keeps the happy - // path free of a status()+resume() round trip while still recovering a - // sandbox that auto-stopped between calls without risking a double run. - match self - .dispatch_tool(&attempt, workspace_id, tool, input, &payload) + self.execute_bounded(handle, tool, input, ResourceBudget::UNBOUNDED) .await - { - Ok(output) => Ok(output), - Err(error) => match self.status(handle).await { - Ok(HandStatus::Stopped | HandStatus::Paused) => { - self.resume(handle).await?; - let attempt = self - .attempt(account_id, account_generation, ProviderEndpoint::Toolbox) - .await?; - self.dispatch_tool(&attempt, workspace_id, tool, input, &payload) - .await - } - _ => Err(error), - }, - } + } + + async fn execute_within( + &self, + handle: &HandHandle, + tool: &str, + input: &str, + budget: ResourceBudget, + ) -> Result { + self.execute_bounded(handle, tool, input, budget).await } async fn install_files(&self, handle: &HandHandle, files: &[SandboxFile]) -> Result<()> { @@ -1199,16 +1304,8 @@ impl HandProvider for DaytonaHandProvider { let state = value .get("state") .and_then(Value::as_str) - .unwrap_or("started") - .to_ascii_lowercase(); - Ok(match state.as_str() { - "creating" | "pending" | "starting" => HandStatus::Provisioning, - "started" | "running" => HandStatus::Running, - "stopped" => HandStatus::Stopped, - "archived" | "deleted" => HandStatus::Destroyed, - "error" | "failed" => HandStatus::Failed, - _ => HandStatus::Running, - }) + .unwrap_or("unknown"); + Ok(daytona_hand_status(state)) } fn supports_suspend(&self) -> bool { @@ -1224,6 +1321,24 @@ impl HandProvider for DaytonaHandProvider { /// only genuine compute-release-with-filesystem-retention primitive MOA has. async fn suspend(&self, handle: &HandHandle) -> Result<()> { let workspace_id = handle.daytona_id()?; + match self.status(handle).await? { + HandStatus::Stopped => return Ok(()), + HandStatus::Paused => { + return self + .wait_for_status(handle, HandStatus::Stopped, "finish stopping") + .await; + } + HandStatus::Provisioning => { + self.wait_for_status(handle, HandStatus::Running, "become ready before stopping") + .await?; + } + HandStatus::Running => {} + status @ (HandStatus::Destroyed | HandStatus::Failed) => { + return Err(MoaError::ProviderError(format!( + "cannot suspend Daytona sandbox from {status:?}" + ))); + } + } let (account_id, account_generation) = cloud_account(handle, "Daytona")?; let attempt = self .attempt(account_id, account_generation, ProviderEndpoint::Api) @@ -1241,15 +1356,30 @@ impl HandProvider for DaytonaHandProvider { MoaError::ProviderError(format!("failed to stop Daytona sandbox: {error}")) })?; expect_success(response).await?; - Ok(()) + self.wait_for_status(handle, HandStatus::Stopped, "stop") + .await } async fn resume(&self, handle: &HandHandle) -> Result<()> { let workspace_id = handle.daytona_id()?; let status = self.status(handle).await?; - if matches!(status, HandStatus::Running | HandStatus::Provisioning) { + if status == HandStatus::Running { return Ok(()); } + if status == HandStatus::Provisioning { + return self + .wait_for_status(handle, HandStatus::Running, "become ready") + .await; + } + if status == HandStatus::Paused { + self.wait_for_status(handle, HandStatus::Stopped, "finish stopping") + .await?; + } + if matches!(status, HandStatus::Destroyed | HandStatus::Failed) { + return Err(MoaError::ProviderError(format!( + "cannot resume Daytona sandbox from {status:?}" + ))); + } let (account_id, account_generation) = cloud_account(handle, "Daytona")?; let attempt = self .attempt(account_id, account_generation, ProviderEndpoint::Api) @@ -1267,7 +1397,8 @@ impl HandProvider for DaytonaHandProvider { MoaError::ProviderError(format!("failed to start Daytona sandbox: {error}")) })?; expect_success(response).await?; - Ok(()) + self.wait_for_status(handle, HandStatus::Running, "start") + .await } async fn destroy(&self, handle: &HandHandle) -> Result<()> { diff --git a/crates/moa-hands/src/adapters/daytona/tests.rs b/crates/moa-hands/src/adapters/daytona/tests.rs index ef66c1497..d52db159f 100644 --- a/crates/moa-hands/src/adapters/daytona/tests.rs +++ b/crates/moa-hands/src/adapters/daytona/tests.rs @@ -8,6 +8,7 @@ use moa_core::{ traits::{HandProvider, SandboxStorageProvider}, types::hands::{HandHandle, SandboxProfile, SandboxTier}, types::identifiers::{WorkspaceCheckpointId, WorkspaceOperationId}, + types::resource::ResourceBudget, types::sandbox_workspace::{ WorkspaceCheckpointPublishRequest, WorkspaceOperationKind, WorkspaceRevisionRef, WorkspaceStorageOperation, @@ -19,7 +20,7 @@ use tokio::net::{TcpListener, TcpStream}; use super::{ DEFAULT_DAYTONA_IMAGE, DaytonaHandProvider, DaytonaProvisioningIdentity, PROVISIONING_OPERATION_LABEL, PROVISIONING_SPEC_LABEL, ProviderEndpoint, - daytona_auto_stop_minutes, daytona_sandbox_name, volume, + daytona_auto_stop_minutes, daytona_hand_status, daytona_sandbox_name, volume, }; async fn read_request(socket: &mut TcpStream) -> String { @@ -74,6 +75,8 @@ async fn provisions_executes_and_destroys_workspace() { let created_server = created.clone(); let deleted = Arc::new(AtomicBool::new(false)); let deleted_server = deleted.clone(); + let running = Arc::new(AtomicBool::new(false)); + let running_server = running.clone(); let sandbox_name_server = sandbox_name.clone(); let spec_fingerprint_server = spec_fingerprint.clone(); tokio::spawn(async move { @@ -84,6 +87,7 @@ async fn provisions_executes_and_destroys_workspace() { let seen = seen_server.clone(); let created = created_server.clone(); let deleted = deleted_server.clone(); + let running = running_server.clone(); let sandbox_name = sandbox_name_server.clone(); let spec_fingerprint = spec_fingerprint_server.clone(); tokio::spawn(async move { @@ -139,14 +143,30 @@ async fn provisions_executes_and_destroys_workspace() { ( "200 OK", format!( - r#"{{"id":"sbx-123","name":"{sandbox_name}","state":"stopped"}}"# + r#"{{"id":"sbx-123","name":"{sandbox_name}","state":"{}"}}"#, + if running.load(Ordering::SeqCst) { + "started" + } else { + "stopped" + } ), ) } } else if first_line.starts_with("POST /api/sandbox/sbx-123/start ") { + running.store(true, Ordering::SeqCst); + ("200 OK", r#"{"ok":true}"#.to_string()) + } else if first_line.starts_with("POST /api/sandbox/sbx-123/stop ") { + running.store(false, Ordering::SeqCst); ("200 OK", r#"{"ok":true}"#.to_string()) } else if first_line.starts_with("POST /toolbox/sbx-123/process/execute ") { - ("200 OK", r#"{"exitCode":0,"result":"hello\n"}"#.to_string()) + if running.load(Ordering::SeqCst) { + ("200 OK", r#"{"exitCode":0,"result":"hello\n"}"#.to_string()) + } else { + ( + "409 Conflict", + r#"{"error":"sandbox is stopped"}"#.to_string(), + ) + } } else if first_line.starts_with("DELETE /api/sandbox/sbx-123 ") { deleted.store(true, Ordering::SeqCst); ("200 OK", r#"{"ok":true}"#.to_string()) @@ -204,6 +224,23 @@ async fn provisions_executes_and_destroys_workspace() { .unwrap(); assert_eq!(output.process_stdout(), Some("hello\n")); + // Pins: Daytona suspend does not return until the real provider state is + // stopped, and the next dispatch waits for an exact running state. + provider.suspend(&handle).await.unwrap(); + assert_eq!( + provider.status(&handle).await.unwrap(), + moa_core::types::hands::HandStatus::Stopped + ); + let resumed = provider + .execute(&handle, "bash", r#"{"cmd":"echo hello"}"#) + .await + .unwrap(); + assert_eq!(resumed.process_stdout(), Some("hello\n")); + assert_eq!( + provider.status(&handle).await.unwrap(), + moa_core::types::hands::HandStatus::Running + ); + provider.destroy(&handle).await.unwrap(); let seen = seen.lock().await.join("\n"); @@ -221,6 +258,71 @@ async fn provisions_executes_and_destroys_workspace() { ); } +#[test] +fn daytona_transitional_states_never_report_running() { + // Pins: asynchronous stop/start states cannot be mistaken for executable + // or capacity-free terminal states. + assert_eq!( + daytona_hand_status("starting"), + moa_core::types::hands::HandStatus::Provisioning + ); + assert_eq!( + daytona_hand_status("resuming"), + moa_core::types::hands::HandStatus::Provisioning + ); + assert_eq!( + daytona_hand_status("stopping"), + moa_core::types::hands::HandStatus::Paused + ); + assert_eq!( + daytona_hand_status("pausing"), + moa_core::types::hands::HandStatus::Paused + ); + assert_eq!( + daytona_hand_status("stopped"), + moa_core::types::hands::HandStatus::Stopped + ); + assert_eq!( + daytona_hand_status("unexpected"), + moa_core::types::hands::HandStatus::Failed + ); +} + +#[test] +fn daytona_effective_timeout_never_exceeds_default_or_run_budget() { + // Pins: the value declared to the watchdog is also the largest wall-clock + // duration the provider can spend, for bash and non-bash calls alike. + assert_eq!( + crate::tools::bash::effective_synchronous_timeout( + "bash", + r#"{"cmd":"true"}"#, + crate::tools::bash::DEFAULT_BASH_TIMEOUT, + ResourceBudget::UNBOUNDED.time_remaining(chrono::Utc::now()), + ) + .expect("default bash timeout resolves"), + crate::tools::bash::DEFAULT_BASH_TIMEOUT + ); + let bounded = crate::tools::bash::effective_synchronous_timeout( + "file_read", + r#"{"path":"marker.txt"}"#, + crate::tools::bash::DEFAULT_BASH_TIMEOUT, + ResourceBudget::until(chrono::Utc::now() + chrono::Duration::seconds(10)) + .time_remaining(chrono::Utc::now()), + ) + .expect("bounded file timeout resolves"); + assert!(bounded <= std::time::Duration::from_secs(10)); + assert!( + crate::tools::bash::effective_synchronous_timeout( + "file_read", + r#"{"path":"marker.txt"}"#, + crate::tools::bash::DEFAULT_BASH_TIMEOUT, + ResourceBudget::until(chrono::Utc::now() - chrono::Duration::seconds(1)) + .time_remaining(chrono::Utc::now()), + ) + .is_err() + ); +} + #[tokio::test] async fn daytona_rejects_a_mutable_root_outside_the_volume_mount_boundary() { // Pins: volume attachment and checkpoint export share one exact mutable diff --git a/crates/moa-hands/src/adapters/e2b/mod.rs b/crates/moa-hands/src/adapters/e2b/mod.rs index ca707785d..d06302ee9 100644 --- a/crates/moa-hands/src/adapters/e2b/mod.rs +++ b/crates/moa-hands/src/adapters/e2b/mod.rs @@ -7,6 +7,7 @@ mod workspace; #[cfg(test)] mod tests; +use std::borrow::Cow; use std::collections::{HashMap, HashSet}; use std::sync::{Arc, LazyLock}; use std::time::Duration; @@ -43,6 +44,7 @@ use moa_core::{ types::hands::SandboxTierCapabilities, types::hands::validate_sandbox_file_path, types::identifiers::HandProvisioningOperationId, + types::resource::ResourceBudget, types::tools::ToolOutput, }; use reqwest::header::CONTENT_TYPE; @@ -78,7 +80,7 @@ const E2B_SUPPORTED_CAPABILITIES: &[SandboxToolCapability] = &SandboxToolCapabil const DEFAULT_E2B_DOMAIN: &str = "e2b.app"; const DEFAULT_E2B_TEMPLATE: &str = "base"; const DEFAULT_ENVD_PORT: u16 = 49983; -const DEFAULT_COMMAND_TIMEOUT: Duration = Duration::from_secs(300); +const DEFAULT_COMMAND_TIMEOUT: Duration = crate::tools::bash::DEFAULT_BASH_TIMEOUT; const CONNECT_PROTOCOL_VERSION: &str = "1"; const E2B_PROVISIONING_OPERATION_METADATA_KEY: &str = "moa_provisioning_operation_id"; const E2B_PROVISIONING_SPEC_METADATA_KEY: &str = "moa_provisioning_spec_sha256"; @@ -775,6 +777,22 @@ fn sandbox_id(handle: &HandHandle) -> Result<&str> { } } +fn bounded_execution_input<'a>( + tool: &str, + input: &'a str, + timeout: Duration, +) -> Result> { + if tool != "bash" { + return Ok(Cow::Borrowed(input)); + } + let mut payload: Value = serde_json::from_str(input)?; + let object = payload.as_object_mut().ok_or_else(|| { + MoaError::ValidationError("bash tool input must be a JSON object".to_string()) + })?; + object.insert("timeout_secs".to_string(), json!(timeout.as_secs())); + Ok(Cow::Owned(payload.to_string())) +} + fn cloud_account( handle: &HandHandle, ) -> Result<(moa_core::types::identifiers::ProviderAccountId, u64)> { @@ -1178,6 +1196,33 @@ impl HandProvider for E2BHandProvider { } } + async fn execute_within( + &self, + handle: &HandHandle, + tool: &str, + input: &str, + budget: ResourceBudget, + ) -> Result { + let execution_timeout = bash::effective_synchronous_timeout( + tool, + input, + DEFAULT_COMMAND_TIMEOUT, + budget.time_remaining(chrono::Utc::now()), + )?; + let bounded_input = bounded_execution_input(tool, input, execution_timeout)?; + tokio::time::timeout( + execution_timeout, + self.execute(handle, tool, &bounded_input), + ) + .await + .map_err(|_| { + MoaError::ToolError(format!( + "E2B tool execution timed out after {}s", + execution_timeout.as_secs() + )) + })? + } + async fn install_files(&self, handle: &HandHandle, files: &[SandboxFile]) -> Result<()> { let sandbox_id = sandbox_id(handle)?; let (account_id, account_generation) = cloud_account(handle)?; diff --git a/crates/moa-hands/src/adapters/e2b/storage.rs b/crates/moa-hands/src/adapters/e2b/storage.rs index 5adcfdaf4..bcfcc3c26 100644 --- a/crates/moa-hands/src/adapters/e2b/storage.rs +++ b/crates/moa-hands/src/adapters/e2b/storage.rs @@ -7,6 +7,7 @@ use std::path::{Component, Path, PathBuf}; use futures_util::StreamExt as _; use moa_core::error::{MoaError, Result}; use serde::{Deserialize, Deserializer, de::Error as _}; +use sha2::{Digest as _, Sha256}; use tempfile::{Builder, TempDir}; use tokio::io::AsyncWriteExt as _; @@ -80,8 +81,14 @@ where } } -async fn create_operation_temp_dir(purpose: &str) -> Result { - let prefix = format!(".moa-e2b-{purpose}-"); +/// Returns the provider-resource-specific prefix used for operation scratch directories. +pub(super) fn operation_temp_prefix(purpose: &str, sandbox_id: &str) -> String { + let discriminator = format!("{:x}", Sha256::digest(sandbox_id.as_bytes())); + format!(".moa-e2b-{purpose}-{}-", &discriminator[..16]) +} + +async fn create_operation_temp_dir(purpose: &str, sandbox_id: &str) -> Result { + let prefix = operation_temp_prefix(purpose, sandbox_id); let directory = Builder::new().prefix(&prefix).tempdir().map_err(|error| { MoaError::StorageError(format!("create E2B operation temp directory: {error}")) })?; @@ -114,7 +121,7 @@ pub(super) async fn export_data_root( sandbox: &ConnectedSandbox, limits: ArchiveLimits, ) -> Result { - let temporary = create_operation_temp_dir("export").await?; + let temporary = create_operation_temp_dir("export", sandbox_id).await?; let result = export_into_temp(provider, attempt, sandbox_id, sandbox, &temporary, limits).await; let cleanup = cleanup_operation_temp_dir(temporary).await; match (result, cleanup) { @@ -327,7 +334,7 @@ pub(super) async fn restore_checkpoint_data_root( sandbox: &ConnectedSandbox, limits: ArchiveLimits, ) -> Result<()> { - let temporary = create_operation_temp_dir("restore").await?; + let temporary = create_operation_temp_dir("restore", sandbox_id).await?; let restored_root = temporary.path().join("data"); let result = async { store.restore(context, &restored_root).await?; diff --git a/crates/moa-hands/src/adapters/e2b/tests.rs b/crates/moa-hands/src/adapters/e2b/tests.rs index 08cdfa2f3..85a22bc4d 100644 --- a/crates/moa-hands/src/adapters/e2b/tests.rs +++ b/crates/moa-hands/src/adapters/e2b/tests.rs @@ -1,6 +1,7 @@ use std::collections::BTreeSet; use std::path::PathBuf; use std::sync::Arc; +use std::time::Duration; use moa_config::CloudHandProviderKind; use moa_core::{ @@ -9,6 +10,7 @@ use moa_core::{ types::{ hands::{EgressPolicy, HandHandle, HandStatus, SandboxTier}, identifiers::{HandProvisioningOperationId, WorkspaceCheckpointId, WorkspaceOperationId}, + resource::ResourceBudget, sandbox_workspace::{ WorkspaceCheckpointPublishRequest, WorkspaceOperationKind, WorkspaceOperationOutcome, WorkspacePostCommitState, WorkspaceReconcileRequest, WorkspaceRevisionRef, @@ -21,10 +23,11 @@ use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpListener; use tokio::sync::mpsc; -use super::{E2B_PROVISIONING_OPERATION_METADATA_KEY, E2BHandProvider}; +use super::{E2B_PROVISIONING_OPERATION_METADATA_KEY, E2BHandProvider, bounded_execution_input}; struct FixtureE2BApi { addr: std::net::SocketAddr, + sandbox_id: String, create_requests: mpsc::UnboundedReceiver, discovery_requests: mpsc::UnboundedReceiver, } @@ -39,6 +42,8 @@ impl FixtureE2BApi { .expect("fixture E2B API should expose its local address"); let (create_request_tx, create_requests) = mpsc::unbounded_channel(); let (discovery_request_tx, discovery_requests) = mpsc::unbounded_channel(); + let sandbox_id = format!("sbx-{}", uuid::Uuid::new_v4()); + let fixture_sandbox_id = sandbox_id.clone(); tokio::spawn(async move { let mut created_metadata = None; @@ -59,6 +64,7 @@ impl FixtureE2BApi { &mut created_metadata, &create_request_tx, &discovery_request_tx, + &fixture_sandbox_id, ); let headers = format!( "HTTP/1.1 {status}\r\ncontent-type: {content_type}\r\nconnection: close\r\ncontent-length: {}\r\n\r\n", @@ -77,6 +83,7 @@ impl FixtureE2BApi { Self { addr, + sandbox_id, create_requests, discovery_requests, } @@ -107,6 +114,7 @@ fn fixture_response( created_metadata: &mut Option, create_request_tx: &mpsc::UnboundedSender, discovery_request_tx: &mpsc::UnboundedSender, + sandbox_id: &str, ) -> (&'static str, &'static str, String) { if first_line.starts_with("GET /v2/sandboxes?") { let target = first_line @@ -121,7 +129,7 @@ fn fixture_response( || "[]".to_string(), |metadata| { serde_json::json!([{ - "sandboxID": "sbx-123", + "sandboxID": sandbox_id, "metadata": metadata, }]) .to_string() @@ -142,10 +150,16 @@ fn fixture_response( return ( "200 OK", "application/json", - r#"{"sandboxID":"sbx-123","domain":"example.e2b.test","envdAccessToken":"envd-token","envdVersion":"0.1.1"}"#.to_string(), + serde_json::json!({ + "sandboxID": sandbox_id, + "domain": "example.e2b.test", + "envdAccessToken": "envd-token", + "envdVersion": "0.1.1", + }) + .to_string(), ); } - if first_line.starts_with("POST /sandboxes/sbx-123/connect ") { + if first_line.starts_with(&format!("POST /sandboxes/{sandbox_id}/connect ")) { return ( "409 Conflict", "application/json", @@ -193,11 +207,11 @@ fn fixture_response( if first_line.starts_with("GET /files?") { return ("200 OK", "application/octet-stream", "marker".to_string()); } - if first_line.starts_with("DELETE /sandboxes/sbx-123 ") { + if first_line.starts_with(&format!("DELETE /sandboxes/{sandbox_id} ")) { *created_metadata = None; return ("204 No Content", "application/json", String::new()); } - if first_line.starts_with("GET /sandboxes/sbx-123 ") { + if first_line.starts_with(&format!("GET /sandboxes/{sandbox_id} ")) { return created_metadata.as_ref().map_or_else( || { ( @@ -211,7 +225,7 @@ fn fixture_response( "200 OK", "application/json", serde_json::json!({ - "sandboxID": "sbx-123", + "sandboxID": sandbox_id, "state": "running", "metadata": metadata, "domain": "example.e2b.test", @@ -371,6 +385,49 @@ async fn provisions_executes_and_destroys_sandbox() { provider.destroy(&handle).await.unwrap(); } +#[test] +fn e2b_effective_timeout_is_injected_and_never_exceeds_run_budget() { + // Pins: E2B receives the same effective bash bound the watchdog observes, + // while the outer provider deadline also bounds non-bash HTTP operations. + let timeout = crate::tools::bash::effective_synchronous_timeout( + "bash", + r#"{"cmd":"true","timeout_secs":300}"#, + crate::tools::bash::DEFAULT_BASH_TIMEOUT, + ResourceBudget::until(chrono::Utc::now() + chrono::Duration::seconds(10)) + .time_remaining(chrono::Utc::now()), + ) + .expect("bounded bash timeout resolves"); + assert!(timeout <= Duration::from_secs(10)); + let input = bounded_execution_input("bash", r#"{"cmd":"true","timeout_secs":300}"#, timeout) + .expect("bounded bash input rewrites"); + assert_eq!( + serde_json::from_str::(&input) + .expect("rewritten input is JSON") + .get("timeout_secs") + .and_then(Value::as_u64), + Some(timeout.as_secs()) + ); + assert!(matches!( + bounded_execution_input( + "file_read", + r#"{"path":"marker.txt"}"#, + Duration::from_secs(10), + ) + .expect("non-bash input remains valid"), + std::borrow::Cow::Borrowed(_) + )); + assert!( + crate::tools::bash::effective_synchronous_timeout( + "file_read", + r#"{"path":"marker.txt"}"#, + crate::tools::bash::DEFAULT_BASH_TIMEOUT, + ResourceBudget::until(chrono::Utc::now() - chrono::Duration::seconds(1)) + .time_remaining(chrono::Utc::now()), + ) + .is_err() + ); +} + #[tokio::test] async fn e2b_rejects_a_mutable_root_outside_the_checkpoint_boundary() { // Pins: E2B export/import is hard-bound to the standard mutable root, so a @@ -537,7 +594,8 @@ async fn exports_reserved_data_root_through_canonical_archive_and_wipes_temp() { e2b_test_profile(EgressPolicy::DenyAll), ); let binding = spec.workspace.clone(); - let before = e2b_operation_temp_dirs(); + let temp_prefix = super::storage::operation_temp_prefix("export", &fixture.sandbox_id); + let before = e2b_operation_temp_dirs(&temp_prefix); let handle = provider .provision(spec) .await @@ -568,7 +626,7 @@ async fn exports_reserved_data_root_through_canonical_archive_and_wipes_temp() { assert_eq!(archive.manifest.logical_bytes, 6); assert_eq!(archive.manifest.entries.len(), 1); assert_eq!(archive.manifest.entries[0].path, "marker.txt"); - assert_eq!(e2b_operation_temp_dirs(), before); + assert_eq!(e2b_operation_temp_dirs(&temp_prefix), before); provider .destroy(&handle) .await @@ -660,14 +718,14 @@ async fn e2b_commit_reconciliation_retains_the_hand_it_reconciles() { .expect("destroy fixture hand"); } -fn e2b_operation_temp_dirs() -> BTreeSet { +fn e2b_operation_temp_dirs(prefix: &str) -> BTreeSet { std::fs::read_dir(std::env::temp_dir()) .expect("read host temp directory") .filter_map(|entry| entry.ok().map(|entry| entry.path())) .filter(|path| { path.file_name() .and_then(|name| name.to_str()) - .is_some_and(|name| name.starts_with(".moa-e2b-")) + .is_some_and(|name| name.starts_with(prefix)) }) .collect() } diff --git a/crates/moa-hands/src/adapters/local/mod.rs b/crates/moa-hands/src/adapters/local/mod.rs index a5f832b9c..fcea204e5 100644 --- a/crates/moa-hands/src/adapters/local/mod.rs +++ b/crates/moa-hands/src/adapters/local/mod.rs @@ -78,7 +78,7 @@ use crate::tools::{bash, file_outline, file_read, file_search, file_write, grep, const LOCAL_SUPPORTED_CAPABILITIES: &[SandboxToolCapability] = &SandboxToolCapability::ALL; const DEFAULT_DOCKER_IMAGE: &str = "alpine:3.20"; -const DEFAULT_TOOL_TIMEOUT: Duration = Duration::from_secs(300); +const DEFAULT_TOOL_TIMEOUT: Duration = crate::tools::bash::DEFAULT_BASH_TIMEOUT; const DOCKER_DETECTION_TIMEOUT: Duration = Duration::from_secs(2); const DOCKER_TMPFS_OPTIONS: &str = "rw,nosuid,nodev,size=64m"; const HAND_SANDBOX_PREFIX: &str = "hand-"; diff --git a/crates/moa-hands/src/core/dispatch.rs b/crates/moa-hands/src/core/dispatch.rs index 078f08005..735f4fd27 100644 --- a/crates/moa-hands/src/core/dispatch.rs +++ b/crates/moa-hands/src/core/dispatch.rs @@ -13,7 +13,7 @@ use moa_core::{ types::completion::ToolInvocation, types::hands::HandHandle, types::hands::HandStatus, - types::identifiers::{ExecutionRunScopeId, ExecutionTaskScopeId, ToolCallId}, + types::identifiers::{ExecutionRunScopeId, ToolCallId}, types::resource::DeadlineGuard, types::sandbox_workspace::{ExecutionHandReleaseOwner, SandboxWorkspaceScope, WorkspaceEffect}, types::security::ToolCapabilityId, @@ -112,34 +112,6 @@ pub struct JournaledWorkspaceCommit<'a> { pub scope: ToolCallScope<'a>, } -/// One idempotent request to publish an attempt's checkpoint while keeping its compute. -/// -/// This is the continuation-boundary counterpart to -/// [`ExecutionHandReleaseRequest`]. It advances the durable checkpoint head so the -/// next slice can always restore, and deliberately performs no provider teardown: -/// a plain model/tool boundary is not a wait, so destroying and re-provisioning the -/// sandbox between two adjacent slices is pure loss. `retention_deadline_at` is the -/// bound that keeps the retained hand from starving fleet admission. -#[derive(Clone, Copy)] -pub struct ExecutionHandRetentionRequest<'a> { - /// Session whose tenant owns the execution workspace and hand lease. - pub session: &'a SessionMeta, - /// Verified durable execution run. - pub run_id: ExecutionRunScopeId, - /// Verified durable execution task owning the workspace scope. - pub task_id: ExecutionTaskScopeId, - /// Logical task generation the retained compute belongs to. - pub logical_generation: u64, - /// Exact bounded attempt generation publishing this continuation checkpoint. - pub attempt_generation: u64, - /// Absolute instant after which the reaper may destroy the retained sandbox. - /// - /// Never extends the lease's existing idle or hard deadline; it only shortens. - pub retention_deadline_at: chrono::DateTime, - /// Fresh bounded budget for checkpoint publication. - pub scope: ToolCallScope<'a>, -} - /// One idempotent request to release an execution attempt's exact sandbox hand. #[derive(Clone, Copy)] pub struct ExecutionHandReleaseRequest<'a> { diff --git a/crates/moa-hands/src/core/leases.rs b/crates/moa-hands/src/core/leases.rs index 2aa358fe8..67add27af 100644 --- a/crates/moa-hands/src/core/leases.rs +++ b/crates/moa-hands/src/core/leases.rs @@ -23,7 +23,9 @@ use sqlx::{PgPool, Row, types::Json}; use tokio::sync::Mutex; use uuid::Uuid; -use super::sandbox_workspace::capacity::release_active_hand_for_reaper_in_transaction; +use super::sandbox_workspace::capacity::{ + ActiveHandReaperRelease, release_active_hand_for_reaper_in_transaction, +}; /// Maximum wall-clock time the platform allows one provider create dispatch. pub(super) const PROVISIONING_TIMEOUT: Duration = Duration::from_secs(5 * 60); @@ -1200,7 +1202,7 @@ impl HandLeaseStore for PostgresHandLeaseStore { attachment_columns(expected.attachment.as_ref()); let mut conn = self.begin(tenant_id).await?; if expected.attachment.is_some() - && !release_active_hand_for_reaper_in_transaction( + && release_active_hand_for_reaper_in_transaction( conn.as_mut(), tenant_id, expected.provisioning_operation_id, @@ -1208,6 +1210,7 @@ impl HandLeaseStore for PostgresHandLeaseStore { claim_token, ) .await? + != ActiveHandReaperRelease::Released { conn.rollback().await?; return Ok(false); diff --git a/crates/moa-hands/src/core/lifecycle.rs b/crates/moa-hands/src/core/lifecycle.rs index 650d642b5..fb2c39b27 100644 --- a/crates/moa-hands/src/core/lifecycle.rs +++ b/crates/moa-hands/src/core/lifecycle.rs @@ -1066,39 +1066,10 @@ impl ToolRouter { if !lease_expired(&lease) && lease_matches_policy(&lease, policy) => { match self - .resume_durable_lease( - provider, - &lease, - workspace_binding, - &key, - call_scope, - ) + .resume_durable_lease(provider, &lease, &key, call_scope) .await { - // A suspended sandbox has no reserved claim on the slot it - // gave back. Losing that race is terminal for this lease - // rather than something to retry in place: the stopped - // sandbox is handed to the reaper, and admission is refused - // now instead of spinning while the fleet stays full. Safe - // because the continuation boundary published its checkpoint - // before suspending, so a later slice restores the same head - // into fresh compute. - Ok(None) => { - call_scope.admit()?; - let _ = lease_store - .transition_status( - session.tenant_id, - &lease, - HandLeaseStatus::Stale, - ) - .await?; - return Err(MoaError::ValidationError(format!( - "suspended sandbox for session {} provider {provider} lost its \ - active-hands capacity slot to a saturated fleet", - session.id - ))); - } - Ok(Some(handle)) => { + Ok(handle) => { call_scope.admit()?; if lease_store .renew_active(HandLeaseRenewRequest { @@ -1698,21 +1669,14 @@ impl ToolRouter { } } - /// Reattaches one live durable lease, resuming it when the sandbox is stopped. - /// - /// Returns `Ok(None)` when a suspended sandbox could not re-win the - /// active-compute slot it gave back at its continuation boundary. That is a - /// distinct outcome from an error: the lease is not retryable in place and - /// the caller must terminalize it, but nothing is lost, because the boundary - /// published a portable checkpoint before suspending. + /// Reattaches one live durable lease, resuming provider-managed compute when needed. async fn resume_durable_lease( &self, provider: &str, lease: &HandLease, - workspace_binding: &WorkspaceBinding, key: &HandProviderCacheKey, call_scope: ToolCallScope<'_>, - ) -> Result> { + ) -> Result { let lease_handle = lease.handle.as_ref().ok_or_else(|| { MoaError::StorageError(format!( "active hand lease for session {} provider {provider} is missing a handle", @@ -1730,16 +1694,6 @@ impl ToolRouter { match status { HandStatus::Running | HandStatus::Provisioning => {} HandStatus::Paused | HandStatus::Stopped => { - // A continuation boundary that suspends a sandbox releases its - // `ActiveHands` charge so a runnable task can use the slot, which - // makes resuming a fresh admission decision rather than a free - // reattach. Compute must never restart before the slot is re-won. - if !self - .readmit_suspended_hand(lease, workspace_binding) - .await? - { - return Ok(None); - } call_scope.admit()?; provider_impl.resume(&handle).await?; } @@ -1757,26 +1711,7 @@ impl ToolRouter { generation: Some(lease.generation), }, ); - Ok(Some(handle)) - } - - /// Re-wins the active-compute admission slot a suspended hand gave back. - /// - /// Reports `false` for a saturated fleet, which is an ordinary outcome and not - /// an error: the caller drops the warm sandbox and a later slice restores the - /// published checkpoint into fresh compute. That is also the honest semantics - /// — a full fleet does not give a warm slot back for free. Deployments with no - /// capacity repository charge nothing and always re-admit. - async fn readmit_suspended_hand( - &self, - lease: &HandLease, - workspace_binding: &WorkspaceBinding, - ) -> Result { - let Some(capacity) = self.hands.workspace_capacity.as_ref() else { - return Ok(true); - }; - let request = active_hand_capacity_request(workspace_binding, lease)?; - capacity.reacquire_suspended_active_hand(&request).await + Ok(handle) } async fn wait_for_provisioning( diff --git a/crates/moa-hands/src/core/mod.rs b/crates/moa-hands/src/core/mod.rs index 7e45e4902..0d07c551c 100644 --- a/crates/moa-hands/src/core/mod.rs +++ b/crates/moa-hands/src/core/mod.rs @@ -49,7 +49,7 @@ use crate::adapters::local::LocalHandProvider; pub use dispatch::{ AuthorizedToolCall, DeferredWorkspaceToolOutput, ExecutionHandReleaseRequest, - ExecutionHandRetentionRequest, JournaledWorkspaceCommit, PendingConnectorToolOutput, + JournaledWorkspaceCommit, PendingConnectorToolOutput, }; use leases::{HAND_LEASE_SESSION_PAGE_SIZE, HandLeaseStore}; pub use maintenance_provider_inventory::SandboxProviderInventory; @@ -79,7 +79,7 @@ use sandbox_workspace::repository::PostgresWorkspaceRepository; pub use telemetry::truncate_tool_span_text; const DEFAULT_PROVIDER_NAME: &str = "local"; -const DEFAULT_TOOL_TIMEOUT: Duration = Duration::from_secs(300); +const DEFAULT_TOOL_TIMEOUT: Duration = crate::tools::bash::DEFAULT_BASH_TIMEOUT; /// Everything one tool dispatch needs to know about the scope that asked for it. /// diff --git a/crates/moa-hands/src/core/reaper.rs b/crates/moa-hands/src/core/reaper.rs index ddfc1ced8..036639215 100644 --- a/crates/moa-hands/src/core/reaper.rs +++ b/crates/moa-hands/src/core/reaper.rs @@ -37,7 +37,9 @@ use super::leases::{ HandLeaseWorkspaceAttachment, LeaseHandle, PROVISIONING_EMPTY_CONFIRMATION, PROVISIONING_VISIBILITY_GRACE, map_sqlx_error, }; -use super::sandbox_workspace::capacity::release_active_hand_for_reaper_in_transaction; +use super::sandbox_workspace::capacity::{ + ActiveHandReaperRelease, release_active_hand_for_reaper_in_transaction, +}; /// One generation the reaper owns and must destroy. #[derive(Debug, Clone, PartialEq, Eq)] @@ -295,18 +297,22 @@ impl ExpiredHandLeaseClaims for PostgresExpiredHandLeaseClaims { conn.rollback().await?; return Ok(false); } - if claimed.attachment.is_some() - && !release_active_hand_for_reaper_in_transaction( + if claimed.attachment.is_some() { + let release = release_active_hand_for_reaper_in_transaction( conn.as_mut(), claimed.tenant_id, claimed.provisioning_operation_id, claimed.generation, claimed.claim_token, ) - .await? - { - conn.rollback().await?; - return Ok(false); + .await?; + let release_is_safe = matches!(release, ActiveHandReaperRelease::Released) + || (matches!(release, ActiveHandReaperRelease::Missing) + && claimed.handle.is_none()); + if !release_is_safe { + conn.rollback().await?; + return Ok(false); + } } let affected = sqlx::query( r#" diff --git a/crates/moa-hands/src/core/sandbox_workspace/capacity.rs b/crates/moa-hands/src/core/sandbox_workspace/capacity.rs index 197d2c32b..0dbc2fbf9 100644 --- a/crates/moa-hands/src/core/sandbox_workspace/capacity.rs +++ b/crates/moa-hands/src/core/sandbox_workspace/capacity.rs @@ -483,120 +483,6 @@ impl PostgresWorkspaceCapacityRepository { Ok(committed) } - /// Releases active-hand capacity for a hand suspended at a continuation boundary. - /// - /// The lease deliberately stays `active` and keeps its handle: the suspended - /// sandbox still owns its filesystem so the next slice can reattach. Only the - /// compute charge goes back to the fleet, and - /// [`Self::reacquire_suspended_active_hand`] must win it again before the - /// sandbox resumes. Returns `false` when the exact lease or workspace fence - /// no longer holds, in which case the caller must not treat the compute as - /// released. - pub async fn release_suspended_active_hand( - &self, - request: &ActiveHandCapacityRequest, - ) -> Result { - validate_active_hand_request(request)?; - let mut conn = self.begin().await?; - if !active_hand_lease_is_live(conn.as_mut(), request).await? { - conn.rollback().await?; - return Ok(false); - } - let changed = release_active_hand_row(conn.as_mut(), request).await?; - conn.commit().await?; - Ok(changed) - } - - /// Re-admits a suspended hand's active-compute charge before it resumes. - /// - /// Returns `false` when the fleet or tenant has no room. That is an ordinary - /// saturation outcome rather than an error: the caller drops the warm sandbox - /// and lets a later slice restore the published checkpoint into fresh - /// compute. A reservation that is still charged (a replay, or a suspend whose - /// capacity release never committed) is reported as `true` without - /// double-charging. - pub async fn reacquire_suspended_active_hand( - &self, - request: &ActiveHandCapacityRequest, - ) -> Result { - validate_active_hand_request(request)?; - let mut conn = self.begin().await?; - lock_capacity_scope_values( - conn.as_mut(), - request.tenant_id, - request.provider_account_id, - ) - .await?; - if !active_hand_lease_is_live(conn.as_mut(), request).await? { - conn.rollback().await?; - return Err(MoaError::StorageError( - "suspended-hand re-admission lost its exact lease or workspace generation fence" - .to_string(), - )); - } - let state = active_hand_reservation_state(conn.as_mut(), request).await?; - match state.as_deref() { - Some("pending" | "committed" | "reconciling") => { - conn.commit().await?; - return Ok(true); - } - Some("released") => {} - _ => { - conn.rollback().await?; - return Err(MoaError::StorageError( - "suspended hand has no active-hands capacity reservation to re-admit" - .to_string(), - )); - } - } - let quantities = BTreeMap::from([(WorkspaceCapacityDimension::ActiveHands, 1_i64)]); - if let Some(shortfall) = capacity_shortfall( - conn.as_mut(), - request.tenant_id, - request.provider_account_id, - request.provider_account_generation, - &quantities, - ) - .await? - { - conn.rollback().await?; - tracing::info!( - shortfall, - "suspended sandbox lost its active-hands slot to a saturated fleet" - ); - return Ok(false); - } - let readmitted = sqlx::query( - r#" - UPDATE moa.sandbox_capacity_reservations - SET reservation_state = 'committed', expires_at = NULL, updated_at = now() - WHERE tenant_id = $1 AND workspace_id = $2 - AND provider_account_id = $3 AND provider_account_generation = $4 - AND hand_provisioning_operation_id = $5 - AND hand_lease_generation = $6 - AND expected_writer_epoch = $7 - AND expected_instance_generation = $8 - AND resource_dimension = 'active_hands' - AND reservation_state = 'released' - "#, - ) - .bind(request.tenant_id) - .bind(request.workspace_id) - .bind(request.provider_account_id) - .bind(request.provider_account_generation) - .bind(request.provisioning_operation_id) - .bind(request.hand_lease_generation) - .bind(request.expected_writer_epoch) - .bind(request.expected_instance_generation) - .execute(conn.as_mut()) - .await - .map_err(map_sqlx_error)? - .rows_affected() - == 1; - conn.commit().await?; - Ok(readmitted) - } - /// Releases active-hand capacity after exact durable reaper ownership is established. pub async fn release_active_hand_to_reaper( &self, @@ -945,7 +831,7 @@ pub(crate) async fn release_workspace_in_transaction( .bind(tenant_id) .bind(workspace_id) .bind(delete_generation) - .execute(conn) + .execute(&mut *conn) .await .map_err(map_sqlx_error)? .rows_affected() @@ -1284,7 +1170,7 @@ pub async fn commit_active_hand_in_transaction( .bind(request.hand_lease_generation) .bind(request.expected_writer_epoch) .bind(request.expected_instance_generation) - .execute(conn) + .execute(&mut *conn) .await .map_err(map_sqlx_error)? .rows_affected() @@ -1293,18 +1179,28 @@ pub async fn commit_active_hand_in_transaction( /// Releases the active-compute owner held by one exact live durable reaper claim. /// -/// `released` is accepted as an input state and re-asserted as a no-op update: -/// a hand suspended at a continuation boundary already gave its compute charge -/// back while keeping its lease, so requiring a still-charged reservation here -/// would make every suspended hand's eventual destroy roll back forever. +/// The tri-state result distinguishes the recoverable crash window before the +/// reservation insert from a row whose identity matches but generation fences do +/// not. Reapers may accept `Missing` only when provider absence has already been +/// proven and the claimed lease never persisted a handle. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ActiveHandReaperRelease { + /// The exact reservation was present and is now released. + Released, + /// No reservation exists for this provisioning identity. + Missing, + /// A reservation exists for the identity but carries different fences. + Mismatched, +} + pub(crate) async fn release_active_hand_for_reaper_in_transaction( conn: &mut sqlx::PgConnection, tenant_id: TenantId, provisioning_operation_id: HandProvisioningOperationId, hand_lease_generation: i64, claim_token: Uuid, -) -> Result { - Ok(sqlx::query( +) -> Result { + let released = sqlx::query( r#" UPDATE moa.sandbox_capacity_reservations AS reservation SET reservation_state = 'released', updated_at = now() @@ -1334,53 +1230,35 @@ pub(crate) async fn release_active_hand_for_reaper_in_transaction( .bind(provisioning_operation_id) .bind(hand_lease_generation) .bind(claim_token) - .execute(conn) + .execute(&mut *conn) .await .map_err(map_sqlx_error)? .rows_affected() - == 1) -} - -/// Locks and verifies that one exact active lease still owns live attached compute. -/// -/// Used by the suspend/reattach pair, which — unlike provisioning and reaping — -/// moves capacity while the lease stays `active` and keeps its handle. -async fn active_hand_lease_is_live( - conn: &mut sqlx::PgConnection, - request: &ActiveHandCapacityRequest, -) -> Result { - Ok(sqlx::query_scalar::<_, bool>( + == 1; + if released { + return Ok(ActiveHandReaperRelease::Released); + } + let exists = sqlx::query_scalar::<_, bool>( r#" - SELECT TRUE - FROM moa.hand_leases AS lease - JOIN moa.sandbox_workspaces AS workspace - ON workspace.tenant_id = lease.tenant_id - AND workspace.workspace_id = lease.workspace_id - WHERE lease.tenant_id = $1 - AND lease.provisioning_operation_id = $2 - AND lease.generation = $3 - AND lease.status = 'active' - AND lease.handle IS NOT NULL - AND lease.workspace_id = $4 - AND lease.workspace_writer_epoch = $5 - AND lease.workspace_instance_generation = $6 - AND workspace.provider_account_id = $7 - AND workspace.provider_account_generation = $8 - FOR UPDATE OF lease + SELECT EXISTS ( + SELECT 1 + FROM moa.sandbox_capacity_reservations + WHERE tenant_id = $1 + AND hand_provisioning_operation_id = $2 + AND resource_dimension = 'active_hands' + ) "#, ) - .bind(request.tenant_id) - .bind(request.provisioning_operation_id) - .bind(request.hand_lease_generation) - .bind(request.workspace_id) - .bind(request.expected_writer_epoch) - .bind(request.expected_instance_generation) - .bind(request.provider_account_id) - .bind(request.provider_account_generation) - .fetch_optional(conn) + .bind(tenant_id) + .bind(provisioning_operation_id) + .fetch_one(&mut *conn) .await - .map_err(map_sqlx_error)? - .unwrap_or(false)) + .map_err(map_sqlx_error)?; + Ok(if exists { + ActiveHandReaperRelease::Mismatched + } else { + ActiveHandReaperRelease::Missing + }) } async fn release_active_hand_row( @@ -1409,7 +1287,7 @@ async fn release_active_hand_row( .bind(request.hand_lease_generation) .bind(request.expected_writer_epoch) .bind(request.expected_instance_generation) - .execute(conn) + .execute(&mut *conn) .await .map_err(map_sqlx_error)? .rows_affected() diff --git a/crates/moa-hands/src/core/sandbox_workspace/lifecycle.rs b/crates/moa-hands/src/core/sandbox_workspace/lifecycle.rs index 1fe121195..07c034624 100644 --- a/crates/moa-hands/src/core/sandbox_workspace/lifecycle.rs +++ b/crates/moa-hands/src/core/sandbox_workspace/lifecycle.rs @@ -10,13 +10,12 @@ use moa_core::{ WorkspaceOperationId, }, sandbox_workspace::{ - ExecutionHandContinuationDisposition, ExecutionHandReleaseOwner, - ExecutionHandReleaseReceipt, ProviderStorageKind, ProviderStorageRef, - SandboxWorkspaceScope, SandboxWorkspaceState, WorkspaceAttachRequest, WorkspaceBinding, - WorkspaceCheckpointPublishRequest, WorkspaceCheckpointState, - WorkspaceConfirmedDisposition, WorkspaceOperationKind, WorkspaceOperationOutcome, - WorkspacePostCommitState, WorkspaceReconcileRequest, WorkspaceRestoreRequest, - WorkspaceStorageOperation, WorkspaceStoragePrepareRequest, + ExecutionHandReleaseOwner, ExecutionHandReleaseReceipt, ProviderStorageKind, + ProviderStorageRef, SandboxWorkspaceScope, SandboxWorkspaceState, + WorkspaceAttachRequest, WorkspaceBinding, WorkspaceCheckpointPublishRequest, + WorkspaceCheckpointState, WorkspaceConfirmedDisposition, WorkspaceOperationKind, + WorkspaceOperationOutcome, WorkspacePostCommitState, WorkspaceReconcileRequest, + WorkspaceRestoreRequest, WorkspaceStorageOperation, WorkspaceStoragePrepareRequest, }, session::SessionMeta, }, @@ -42,13 +41,12 @@ use moa_observability::{ }; use crate::core::{ - ActiveHand, ExecutionHandReleaseRequest, ExecutionHandRetentionRequest, HandProviderCacheKey, - HandRoute, InstalledManifestMarker, JournaledWorkspaceCommit, ToolCallScope, ToolExecution, - ToolRouter, TrustedSandboxManifest, + ActiveHand, ExecutionHandReleaseRequest, HandProviderCacheKey, HandRoute, + InstalledManifestMarker, JournaledWorkspaceCommit, ToolCallScope, ToolExecution, ToolRouter, + TrustedSandboxManifest, leases::{HandLease, HandLeaseStatus, HandLeaseWorkspaceAttachment}, lifecycle::{ - active_hand_capacity_request, manifest_scope_key, session_provider_key, - workspace_binding_for_hand, workspace_lease_scope, + manifest_scope_key, session_provider_key, workspace_binding_for_hand, workspace_lease_scope, }, telemetry::{ record_workspace_checkpoint, record_workspace_lifecycle, record_workspace_release, @@ -813,10 +811,20 @@ impl ToolRouter { &workspace.workspace_id.0, b"prepare-initial-storage-v1", )); - let deadline_at = call_scope - .budget - .deadline - .unwrap_or_else(|| Utc::now() + ChronoDuration::minutes(5)); + let existing = operations.get(workspace.tenant_id, operation_id).await?; + let deadline_at = existing.as_ref().map_or_else( + || { + call_scope + .budget + .deadline + .unwrap_or_else(|| Utc::now() + ChronoDuration::minutes(5)) + }, + |operation| operation.deadline_at, + ); + let reconcile_not_before = existing.as_ref().map_or_else( + || deadline_at + ChronoDuration::seconds(30), + |operation| operation.reconcile_not_before, + ); let hash_bytes = serde_json::to_vec(&binding)?; let request_hash = format!("sha256:{}", hex::encode(Sha256::digest(hash_bytes))); let intent = WorkspaceOperationIntent { @@ -831,23 +839,42 @@ impl ToolRouter { expected_instance_generation: workspace.instance_generation, expected_checkpoint_generation: workspace.checkpoint_generation, deadline_at, - reconcile_not_before: deadline_at + ChronoDuration::seconds(30), + reconcile_not_before, }; - match operations.get(workspace.tenant_id, operation_id).await? { - Some(existing) - if existing.request_hash == request_hash - && existing.kind == WorkspaceOperationKind::Create => {} - Some(_) => { - return Err(MoaError::StorageError( - "workspace storage preparation replay changed its durable request".to_string(), + let operation = operations.persist_intent(&intent).await?; + match (operation.outcome, operation.confirmed_disposition) { + ( + WorkspaceOperationOutcome::Confirmed, + Some(WorkspaceConfirmedDisposition::ResourcePresent), + ) => return Ok(()), + (WorkspaceOperationOutcome::Confirmed, _) => { + return Err(MoaError::ProviderError( + "workspace storage preparation was durably confirmed absent".to_string(), )); } - None => { - operations.persist_intent(&intent).await?; + (WorkspaceOperationOutcome::Unknown, _) => { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_id.to_string(), + }); + } + (WorkspaceOperationOutcome::NotSent, None) => {} + _ => { + return Err(MoaError::StorageError( + "workspace storage preparation has an inconsistent durable outcome".to_string(), + )); } } failpoints::hit("post_reservation_pre_provider_create").await?; - let result = storage_provider + call_scope.admit()?; + if !operations + .begin_provider_attempt(workspace.tenant_id, operation_id) + .await? + { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_id.to_string(), + }); + } + let result = match storage_provider .prepare_workspace_storage(WorkspaceStoragePrepareRequest { operation: WorkspaceStorageOperation { operation_id, @@ -857,14 +884,42 @@ impl ToolRouter { request_hash, }, }) - .await?; + .await + { + Ok(result) => result, + Err(error) => { + tracing::warn!( + operation_id = %operation_id, + error = %error, + "workspace storage preparation outcome is ambiguous" + ); + operations + .mark_unknown(workspace.tenant_id, operation_id) + .await?; + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_id.to_string(), + }); + } + }; // Every arm records an outcome, so the lifecycle counter carries the real // success/ambiguous ratio rather than only the happy path. match (result.outcome, result.confirmed_disposition) { - (WorkspaceOperationOutcome::Confirmed, Some(disposition)) => { - operations - .confirm_disposition(workspace.tenant_id, operation_id, disposition) - .await?; + ( + WorkspaceOperationOutcome::Confirmed, + Some(WorkspaceConfirmedDisposition::ResourcePresent), + ) => { + if !operations + .confirm_disposition( + workspace.tenant_id, + operation_id, + WorkspaceConfirmedDisposition::ResourcePresent, + ) + .await? + { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_id.to_string(), + }); + } record_workspace_lifecycle( &workspace.provider, SandboxWorkspaceLifecycleOperation::Create, @@ -873,6 +928,32 @@ impl ToolRouter { ); Ok(()) } + ( + WorkspaceOperationOutcome::Confirmed, + Some(WorkspaceConfirmedDisposition::ResourceAbsent), + ) => { + if !operations + .confirm_disposition( + workspace.tenant_id, + operation_id, + WorkspaceConfirmedDisposition::ResourceAbsent, + ) + .await? + { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_id.to_string(), + }); + } + record_workspace_lifecycle( + &workspace.provider, + SandboxWorkspaceLifecycleOperation::Create, + SandboxWorkspaceMetricResult::Failed, + prepare_started_at.elapsed(), + ); + Err(MoaError::ProviderError( + "workspace storage preparation was confirmed absent".to_string(), + )) + } (WorkspaceOperationOutcome::Unknown, None) => { operations .mark_unknown(workspace.tenant_id, operation_id) @@ -975,8 +1056,40 @@ impl ToolRouter { deadline_at: claim.provisioning_deadline_at, reconcile_not_before: claim.provisioning_deadline_at + ChronoDuration::seconds(30), }; - operations.persist_intent(&intent).await?; + let persisted = operations.persist_intent(&intent).await?; + match (persisted.outcome, persisted.confirmed_disposition) { + ( + WorkspaceOperationOutcome::Confirmed, + Some(WorkspaceConfirmedDisposition::ResourcePresent), + ) => return Ok(()), + (WorkspaceOperationOutcome::Confirmed, _) => { + return Err(MoaError::ProviderError(format!( + "workspace {} was durably confirmed absent", + kind.as_str() + ))); + } + (WorkspaceOperationOutcome::Unknown, _) => { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_id.to_string(), + }); + } + (WorkspaceOperationOutcome::NotSent, None) => {} + _ => { + return Err(MoaError::StorageError(format!( + "workspace {} has an inconsistent durable outcome", + kind.as_str() + ))); + } + } call_scope.admit()?; + if !operations + .begin_provider_attempt(binding.tenant_id, operation_id) + .await? + { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_id.to_string(), + }); + } let operation = WorkspaceStorageOperation { operation_id, kind, @@ -1051,7 +1164,7 @@ impl ToolRouter { tracing::warn!( operation_id = %operation_id, error = %error, - "workspace commit provider outcome is ambiguous" + "workspace hydration provider outcome is ambiguous" ); operations .mark_unknown(binding.tenant_id, operation_id) @@ -1062,9 +1175,16 @@ impl ToolRouter { } }; match (result.outcome, result.confirmed_disposition) { - (WorkspaceOperationOutcome::Confirmed, Some(disposition)) => { + ( + WorkspaceOperationOutcome::Confirmed, + Some(WorkspaceConfirmedDisposition::ResourcePresent), + ) => { if !operations - .confirm_disposition(binding.tenant_id, operation_id, disposition) + .confirm_disposition( + binding.tenant_id, + operation_id, + WorkspaceConfirmedDisposition::ResourcePresent, + ) .await? { return Err(MoaError::StorageError( @@ -1086,6 +1206,27 @@ impl ToolRouter { } Ok(()) } + ( + WorkspaceOperationOutcome::Confirmed, + Some(WorkspaceConfirmedDisposition::ResourceAbsent), + ) => { + if !operations + .confirm_disposition( + binding.tenant_id, + operation_id, + WorkspaceConfirmedDisposition::ResourceAbsent, + ) + .await? + { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_id.to_string(), + }); + } + Err(MoaError::ProviderError(format!( + "workspace {} was confirmed absent", + kind.as_str() + ))) + } (WorkspaceOperationOutcome::Unknown, None) => { operations .mark_unknown(binding.tenant_id, operation_id) @@ -1341,263 +1482,6 @@ impl ToolRouter { .await } - /// Publishes one execution-task continuation checkpoint and keeps what it can. - /// - /// A plain model/tool boundary is not a wait: the next slice is enqueued for - /// immediate re-admission, so destroying the sandbox here and restoring it - /// milliseconds later is pure loss — object-store read, decrypt, extract, and a - /// per-file upload round trip on both cloud providers. This publishes the exact - /// same durable checkpoint the release path publishes, so the portable recovery - /// authority advances on every boundary, and then chooses how to keep the - /// sandbox based on what the provider can actually do: - /// - /// * A provider with real compute suspension stops the sandbox **in this - /// call** and hands its `ActiveHands` slot back to the fleet. Release timing - /// is deterministic rather than reaper-lagged, and an idle sandbox stops - /// costing compute and stops competing with runnable work for admission. - /// * A provider without it keeps the hand hot on a deliberately short - /// reaper-owned deadline. That bet only pays off when the next slice arrives - /// fast, so a longer window would only extend the loss. - /// - /// Both paths are safe for the same reason: the checkpoint commits *before* - /// any deadline is armed or any compute is stopped, so losing the warm - /// sandbox is a pure cache miss — the next slice restores from the same head. - pub async fn checkpoint_execution_hand_retaining_compute( - &self, - request: ExecutionHandRetentionRequest<'_>, - ) -> Result { - if request.attempt_generation == 0 || request.logical_generation == 0 { - return Err(MoaError::ValidationError( - "execution task attempt and logical generations must be positive".to_string(), - )); - } - let Some(repository) = self.hands.workspace_repository.as_ref() else { - return Ok(ExecutionHandContinuationDisposition::NoComputeOwned); - }; - let workspace_scope = SandboxWorkspaceScope::ExecutionTask { - run_id: request.run_id, - task_id: request.task_id, - }; - // An attempt that never provisioned a durable workspace or whose lease is no - // longer live has nothing to publish and nothing to keep. Its committed head - // is already the recovery authority, so this is a no-op rather than an error. - let Some(workspace) = repository - .get_by_scope(request.session.tenant_id, &workspace_scope) - .await? - else { - return Ok(ExecutionHandContinuationDisposition::NoComputeOwned); - }; - let lease_scope = workspace_lease_scope(&workspace_scope); - let lease_store = self.hands.hand_leases.as_ref().ok_or_else(|| { - MoaError::StorageError("durable hand lease store missing".to_string()) - })?; - let Some(lease) = lease_store - .get( - request.session.tenant_id, - request.session.id, - &lease_scope, - &workspace.provider, - ) - .await? - else { - return Ok(ExecutionHandContinuationDisposition::NoComputeOwned); - }; - if lease.status != HandLeaseStatus::Active { - return Ok(ExecutionHandContinuationDisposition::NoComputeOwned); - } - let Some(hand) = lease.handle.as_ref().map(|handle| handle.handle.clone()) else { - return Ok(ExecutionHandContinuationDisposition::NoComputeOwned); - }; - - let continuation_key = format!( - "execution-task-continuation-v1:{}:{}:{}", - request.run_id, request.task_id, request.attempt_generation - ); - let tool_call_id = ToolCallId(Uuid::new_v5( - &workspace.workspace_id.0, - continuation_key.as_bytes(), - )); - self.commit_workspace_after_tool(WorkspaceCommitExecution { - session: request.session, - workspace_scope: &workspace_scope, - tool_call_id, - provider_name: &workspace.provider, - hand: &hand, - call_scope: request.scope, - release_compute: false, - }) - .await?; - - let provider_impl = self - .hands - .providers - .get(&workspace.provider) - .ok_or_else(|| { - MoaError::ProviderError(format!( - "hand provider {} is not registered", - workspace.provider - )) - })? - .clone(); - if provider_impl.supports_suspend() { - return self - .suspend_continuation_hand( - &request, - provider_impl.as_ref(), - &workspace, - &lease, - &lease_scope, - &hand, - ) - .await; - } - - // Armed only after the checkpoint commits. Arming it first would let the - // reaper claim and destroy the sandbox in the middle of its own publication. - let started_at = std::time::Instant::now(); - self.bound_retained_hand_lifetime(request, &lease_scope, &workspace.provider) - .await?; - record_workspace_lifecycle( - &workspace.provider, - SandboxWorkspaceLifecycleOperation::Retain, - SandboxWorkspaceMetricResult::Succeeded, - started_at.elapsed(), - ); - Ok(ExecutionHandContinuationDisposition::RetainedHot) - } - - /// Stops a continuation sandbox's compute and returns its admission slot. - /// - /// The provider stop runs before the capacity release on purpose. Releasing - /// first and then failing to stop would under-count a sandbox that is still - /// burning compute; this order can only over-count a sandbox that is already - /// stopped, which the reattach path resolves without double-charging. - async fn suspend_continuation_hand( - &self, - request: &ExecutionHandRetentionRequest<'_>, - provider_impl: &dyn moa_core::traits::HandProvider, - workspace: &SandboxWorkspace, - lease: &HandLease, - lease_scope: &str, - hand: &HandHandle, - ) -> Result { - let started_at = std::time::Instant::now(); - if let Err(error) = self - .run_within_scope(request.scope, provider_impl.suspend(hand)) - .await - { - // Non-fatal by contract: the checkpoint is already published, so the - // caller finishes the ordinary checkpoint-and-destroy path instead of - // leaving a hand hot on a bet that has already lost. - tracing::warn!( - provider = %workspace.provider, - generation = lease.generation, - error = %error, - "continuation sandbox suspension failed; falling back to release" - ); - record_workspace_lifecycle( - &workspace.provider, - SandboxWorkspaceLifecycleOperation::Suspend, - SandboxWorkspaceMetricResult::Failed, - started_at.elapsed(), - ); - return Ok(ExecutionHandContinuationDisposition::SuspendFailed); - } - - // The in-process binding cache hands out an active lease's handle without - // consulting the provider, so a stopped sandbox must be evicted here or the - // next same-process slice would dispatch into compute that is not running. - let cache_key = - session_provider_key(request.session, Some(lease_scope), &workspace.provider); - self.remove_cached_binding_if_matches(&cache_key, hand, Some(lease.generation)) - .await; - - if let Some(capacity) = self.hands.workspace_capacity.as_ref() { - let binding = workspace.binding()?; - let released = capacity - .release_suspended_active_hand(&active_hand_capacity_request(&binding, lease)?) - .await?; - if !released { - // The charge stays held, which is the conservative direction: the - // sandbox really is stopped, so the fleet is only under-admitting. - tracing::warn!( - provider = %workspace.provider, - generation = lease.generation, - "suspended continuation sandbox kept its active-hands charge" - ); - } - } - record_workspace_lifecycle( - &workspace.provider, - SandboxWorkspaceLifecycleOperation::Suspend, - SandboxWorkspaceMetricResult::Succeeded, - started_at.elapsed(), - ); - Ok(ExecutionHandContinuationDisposition::Suspended) - } - - /// Shortens a retained continuation hand's idle deadline to its retention bound. - /// - /// Reuses the ordinary active-lease renewal, which sets the idle deadline under - /// the immutable hard lifetime. The requested bound is additionally clamped to the - /// lease's current idle deadline so retention can only shorten a sandbox's life, - /// never extend it past the policy it was admitted under. - async fn bound_retained_hand_lifetime( - &self, - request: ExecutionHandRetentionRequest<'_>, - lease_scope: &str, - provider: &str, - ) -> Result<()> { - let lease_store = self.hands.hand_leases.as_ref().ok_or_else(|| { - MoaError::StorageError("durable hand lease store missing".to_string()) - })?; - let Some(lease) = lease_store - .get( - request.session.tenant_id, - request.session.id, - lease_scope, - provider, - ) - .await? - else { - return Ok(()); - }; - if lease.status != HandLeaseStatus::Active { - return Ok(()); - } - let Some(attachment) = lease.attachment.clone() else { - return Ok(()); - }; - let retention_deadline_at = lease - .idle_expires_at - .map_or(request.retention_deadline_at, |idle| { - idle.min(request.retention_deadline_at) - }); - if !lease_store - .renew_active(crate::core::leases::HandLeaseRenewRequest { - tenant_id: request.session.tenant_id, - session_id: request.session.id, - worker_id: lease_scope, - provider, - generation: lease.generation, - provisioning_operation_id: lease.provisioning_operation_id, - attachment, - idle_expires_at: retention_deadline_at, - }) - .await? - { - // The lease moved under us, so some other durable owner already governs - // this sandbox's lifetime. The checkpoint is published either way, so the - // worst outcome is that the hand expires on its ordinary idle policy. - tracing::warn!( - provider, - generation = lease.generation, - "retained execution continuation hand kept its ordinary idle deadline" - ); - } - Ok(()) - } - /// Checkpoints one execution-task workspace and releases its exact compute lease. /// /// The returned receipt is the durable proof required before a task may yield to diff --git a/crates/moa-hands/src/core/sandbox_workspace/maintenance/mod.rs b/crates/moa-hands/src/core/sandbox_workspace/maintenance/mod.rs index ec57befbc..1b130d5cb 100644 --- a/crates/moa-hands/src/core/sandbox_workspace/maintenance/mod.rs +++ b/crates/moa-hands/src/core/sandbox_workspace/maintenance/mod.rs @@ -401,19 +401,66 @@ impl WorkspaceMaintenanceCoordinator { Ok(()) } - /// Emits the fleet-wide utilization ratio for every capacity dimension. + /// Emits the highest enforced-scope utilization for every capacity dimension. async fn emit_quota_utilization_metrics(&self) -> Result<()> { let mut conn = maintenance_conn(&self.pool).await?; let rows = sqlx::query( - "SELECT reservation.resource_dimension, \ - sum(reservation.quantity)::BIGINT AS reserved, \ - max(limits.limit_value)::BIGINT AS limit_value \ - FROM moa.sandbox_capacity_reservations AS reservation \ - LEFT JOIN moa.sandbox_tenant_capacity_limits AS limits \ - ON limits.tenant_id = reservation.tenant_id \ - AND limits.resource_dimension = reservation.resource_dimension \ - WHERE reservation.reservation_state <> 'released' \ - GROUP BY reservation.resource_dimension", + r#" + WITH active_reservations AS MATERIALIZED ( + SELECT tenant_id, + provider_account_id, + provider_account_generation, + resource_dimension, + quantity + FROM moa.sandbox_capacity_reservations + WHERE reservation_state IN ('pending', 'committed', 'reconciling') + ), + tenant_usage AS ( + SELECT reservation.tenant_id, + reservation.resource_dimension, + sum(reservation.quantity)::DOUBLE PRECISION AS reserved + FROM active_reservations AS reservation + GROUP BY reservation.tenant_id, reservation.resource_dimension + ), + provider_usage AS ( + SELECT reservation.provider_account_id, + reservation.provider_account_generation, + reservation.resource_dimension, + sum(reservation.quantity)::DOUBLE PRECISION AS reserved + FROM active_reservations AS reservation + GROUP BY reservation.provider_account_id, + reservation.provider_account_generation, + reservation.resource_dimension + ), + scoped_utilization AS ( + SELECT usage.resource_dimension, + usage.reserved, + (limits.configured_limits ->> usage.resource_dimension)::BIGINT AS limit_value + FROM tenant_usage AS usage + JOIN moa.sandbox_tenant_capacity_limits AS limits + ON limits.tenant_id = usage.tenant_id + AND limits.configured_limits ? usage.resource_dimension + UNION ALL + SELECT usage.resource_dimension, + usage.reserved, + (account.configured_limits ->> usage.resource_dimension)::BIGINT AS limit_value + FROM provider_usage AS usage + JOIN moa.sandbox_provider_accounts AS account + ON account.provider_account_id = usage.provider_account_id + AND account.generation = usage.provider_account_generation + AND account.configured_limits ? usage.resource_dimension + ) + SELECT resource_dimension, + max( + CASE + WHEN limit_value > 0 THEN reserved / limit_value::DOUBLE PRECISION + WHEN reserved > 0 THEN 1.0 + ELSE 0.0 + END + )::DOUBLE PRECISION AS utilization + FROM scoped_utilization + GROUP BY resource_dimension + "#, ) .fetch_all(conn.as_mut()) .await @@ -424,18 +471,7 @@ impl WorkspaceMaintenanceCoordinator { let dimension = row .try_get::("resource_dimension") .map_err(map_sqlx)?; - let reserved = row.try_get::("reserved").map_err(map_sqlx)?.max(0); - let limit = row - .try_get::, _>("limit_value") - .map_err(map_sqlx)? - .unwrap_or(0); - // An absent or zero limit means the dimension is unbounded for every tenant - // observed, which is 0.0 pressure rather than a division by zero. - let ratio = if limit > 0 { - reserved as f64 / limit as f64 - } else { - 0.0 - }; + let ratio = row.try_get::("utilization").map_err(map_sqlx)?; ratios.insert(dimension, ratio); } for dimension in all_capacity_dimensions() { diff --git a/crates/moa-hands/src/core/sandbox_workspace/operations.rs b/crates/moa-hands/src/core/sandbox_workspace/operations.rs index 7c0f81991..0faa72a52 100644 --- a/crates/moa-hands/src/core/sandbox_workspace/operations.rs +++ b/crates/moa-hands/src/core/sandbox_workspace/operations.rs @@ -304,33 +304,74 @@ impl PostgresWorkspaceOperationRepository { /// Fences an exact provider attempt as potentially sent without abandoning its commit CAS. /// /// This is the last durable write before provider I/O. It changes only a - /// provably unsent, unclaimed operation to `unknown`; the workspace remains - /// `committing` so a synchronous confirmed result can still atomically - /// publish its checkpoint and head. A subsequent error must call - /// [`Self::mark_unknown`] to move the workspace and reservations into - /// reconciliation. + /// provably unsent, unclaimed operation to `unknown`. Create and hydration + /// operations atomically move the exact workspace fence to `reconciling`; + /// commit keeps `committing` so its synchronous result can still atomically + /// publish the checkpoint and head. A subsequent error must call + /// [`Self::mark_unknown`] to move reservations into reconciliation too. pub async fn begin_provider_attempt( &self, tenant_id: TenantId, operation_id: WorkspaceOperationId, ) -> Result { let mut conn = self.begin(tenant_id).await?; - let affected = sqlx::query( + let row = sqlx::query( r#" UPDATE moa.sandbox_workspace_operations SET outcome_class = 'unknown', confirmed_disposition = NULL, updated_at = now() WHERE tenant_id = $1 AND operation_id = $2 AND outcome_class = 'not_sent' AND claim_token IS NULL + RETURNING workspace_id, operation_kind, expected_writer_epoch, + expected_instance_generation "#, ) .bind(tenant_id) .bind(operation_id) - .execute(conn.as_mut()) + .fetch_optional(conn.as_mut()) .await - .map_err(map_sqlx_error)? - .rows_affected(); + .map_err(map_sqlx_error)?; + if let Some(row) = &row { + let kind: String = row.try_get("operation_kind").map_err(map_sqlx_error)?; + if matches!(kind.as_str(), "create" | "attach" | "restore") { + let workspace_transitioned = sqlx::query( + r#" + UPDATE moa.sandbox_workspaces + SET lifecycle_state = 'reconciling', updated_at = now() + WHERE tenant_id = $1 AND workspace_id = $2 + AND writer_epoch = $3 AND instance_generation = $4 + AND lifecycle_state = CASE + WHEN $5 = 'create' THEN 'creating' + ELSE 'restoring' + END + "#, + ) + .bind(tenant_id) + .bind( + row.try_get::("workspace_id") + .map_err(map_sqlx_error)?, + ) + .bind( + row.try_get::("expected_writer_epoch") + .map_err(map_sqlx_error)?, + ) + .bind( + row.try_get::("expected_instance_generation") + .map_err(map_sqlx_error)?, + ) + .bind(&kind) + .execute(conn.as_mut()) + .await + .map_err(map_sqlx_error)? + .rows_affected() + == 1; + if !workspace_transitioned { + conn.rollback().await?; + return Ok(false); + } + } + } conn.commit().await?; - Ok(affected == 1) + Ok(row.is_some()) } /// Records that a provider request may have been sent and retains reservations. @@ -411,11 +452,10 @@ impl PostgresWorkspaceOperationRepository { /// Confirms one synchronous provider result under the exact operation fence. /// - /// A synchronous absent result is accepted only while the operation remains - /// `not_sent`; once an outcome is ambiguous, absence must go through the - /// claimed two-observation reconciliation path. Present results may resolve - /// either a freshly completed request or an ambiguous request whose exact - /// resource was subsequently verified. + /// The caller invokes this only for the direct result of the provider request + /// whose `not_sent -> unknown` CAS it just won. Delayed recovery never calls + /// this method: it uses the claimed reconciliation methods, where absence + /// requires two separated observations. pub async fn confirm_disposition( &self, tenant_id: TenantId, @@ -430,10 +470,7 @@ impl PostgresWorkspaceOperationRepository { claim_token = NULL, claim_expires_at = NULL, retry_not_before = NULL, updated_at = now() WHERE tenant_id = $1 AND operation_id = $2 - AND ( - outcome_class = 'not_sent' - OR ($3 = 'resource_present' AND outcome_class = 'unknown') - ) + AND outcome_class = 'unknown' AND claim_token IS NULL "#, ) @@ -471,29 +508,17 @@ impl PostgresWorkspaceOperationRepository { .execute(conn.as_mut()) .await .map_err(map_sqlx_error)?; - } - let replay = if affected == 1 { - true - } else { - sqlx::query_scalar::<_, bool>( - r#" - SELECT EXISTS ( - SELECT 1 - FROM moa.sandbox_workspace_operations - WHERE tenant_id = $1 AND operation_id = $2 - AND outcome_class = 'confirmed' AND confirmed_disposition = $3 - ) - "#, + settle_workspace_lifecycle_after_confirmation( + conn.as_mut(), + tenant_id, + operation_id, + disposition, + false, ) - .bind(tenant_id) - .bind(operation_id) - .bind(disposition.as_str()) - .fetch_one(conn.as_mut()) - .await - .map_err(map_sqlx_error)? - }; + .await?; + } conn.commit().await?; - Ok(replay) + Ok(affected == 1) } /// Confirms a present resource only while the exact reaper claim is live. @@ -550,6 +575,14 @@ impl PostgresWorkspaceOperationRepository { .execute(conn.as_mut()) .await .map_err(map_sqlx_error)?; + settle_workspace_lifecycle_after_confirmation( + conn.as_mut(), + claimed.operation.tenant_id, + claimed.operation.operation_id, + WorkspaceConfirmedDisposition::ResourcePresent, + true, + ) + .await?; } conn.commit().await?; Ok(affected == 1) @@ -700,6 +733,14 @@ impl PostgresWorkspaceOperationRepository { .execute(conn.as_mut()) .await .map_err(map_sqlx_error)?; + settle_workspace_lifecycle_after_confirmation( + conn.as_mut(), + claimed.operation.tenant_id, + claimed.operation.operation_id, + WorkspaceConfirmedDisposition::ResourceAbsent, + true, + ) + .await?; } conn.commit().await?; Ok(affected == 1) @@ -860,6 +901,50 @@ impl PostgresWorkspaceOperationRepository { } } +async fn settle_workspace_lifecycle_after_confirmation( + conn: &mut sqlx::PgConnection, + tenant_id: TenantId, + operation_id: WorkspaceOperationId, + disposition: WorkspaceConfirmedDisposition, + reconciled: bool, +) -> Result<()> { + sqlx::query( + r#" + UPDATE moa.sandbox_workspaces AS workspace + SET lifecycle_state = CASE + WHEN operation.operation_kind = 'create' AND $3 = 'resource_present' + THEN CASE WHEN $4 THEN 'ready' ELSE 'creating' END + WHEN operation.operation_kind IN ('attach', 'restore') + AND $3 = 'resource_present' + THEN CASE WHEN $4 THEN 'ready' ELSE 'restoring' END + WHEN operation.operation_kind IN ('create', 'attach', 'restore') + AND $3 = 'resource_absent' THEN 'failed' + ELSE workspace.lifecycle_state + END, + updated_at = now() + FROM moa.sandbox_workspace_operations AS operation + WHERE operation.tenant_id = $1 AND operation.operation_id = $2 + AND operation.outcome_class = 'confirmed' + AND operation.confirmed_disposition = $3 + AND workspace.tenant_id = operation.tenant_id + AND workspace.workspace_id = operation.workspace_id + AND workspace.provider_account_id = operation.provider_account_id + AND workspace.provider_account_generation = operation.provider_account_generation + AND workspace.writer_epoch = operation.expected_writer_epoch + AND workspace.instance_generation = operation.expected_instance_generation + AND workspace.lifecycle_state NOT IN ('deleting', 'deleted') + "#, + ) + .bind(tenant_id) + .bind(operation_id) + .bind(disposition.as_str()) + .bind(reconciled) + .execute(conn) + .await + .map_err(map_sqlx_error)?; + Ok(()) +} + const OPERATION_COLUMNS: &str = "operation_id, tenant_id, workspace_id, provider_account_id, \ provider_account_generation, operation_kind, request_hash, expected_writer_epoch, \ expected_instance_generation, expected_checkpoint_generation, deadline_at, \ diff --git a/crates/moa-hands/src/core/telemetry.rs b/crates/moa-hands/src/core/telemetry.rs index 89f9180b6..88d193dad 100644 --- a/crates/moa-hands/src/core/telemetry.rs +++ b/crates/moa-hands/src/core/telemetry.rs @@ -96,7 +96,7 @@ pub fn record_workspace_quota_decision( record_sandbox_workspace_quota_decision(dimension, decision); } -/// Sets a fleet-wide capacity utilization ratio. +/// Sets the highest enforced tenant or provider-account utilization ratio. pub fn record_workspace_quota_utilization(dimension: WorkspaceCapacityDimension, ratio: f64) { record_sandbox_workspace_quota_utilization(dimension, ratio); } diff --git a/crates/moa-hands/src/lib.rs b/crates/moa-hands/src/lib.rs index 5248246f4..909571dc5 100644 --- a/crates/moa-hands/src/lib.rs +++ b/crates/moa-hands/src/lib.rs @@ -10,10 +10,10 @@ pub use adapters::local::{LOCAL_HAND_CAPABILITIES, LocalHandProvider}; pub use adapters::mcp::{MCPClient, McpDiscoveredTool}; pub use core::{ ActionOrigin, AuthorizedToolCall, CandidateConnector, CatalogDefect, - DeferredWorkspaceToolOutput, ExecutionHandReleaseRequest, ExecutionHandRetentionRequest, - FileProviderCredentialSource, HandLeaseReaper, HandLeaseReaperConfig, HandRoute, - JournaledWorkspaceCommit, MCP_TOOL_REFERENCE_PREFIX, McpCatalogActivation, McpCatalogRefresh, - McpConnectorHealth, PendingConnectorToolOutput, PinnedToolContract, PinnedToolOwner, + DeferredWorkspaceToolOutput, ExecutionHandReleaseRequest, FileProviderCredentialSource, + HandLeaseReaper, HandLeaseReaperConfig, HandRoute, JournaledWorkspaceCommit, + MCP_TOOL_REFERENCE_PREFIX, McpCatalogActivation, McpCatalogRefresh, McpConnectorHealth, + PendingConnectorToolOutput, PinnedToolContract, PinnedToolOwner, PostgresExpiredHandLeaseClaims, PostgresTenantSandboxPolicyStore, PreparedActionInvocation, ProviderCredentialSource, ProviderEndpoint, ProviderHttpAttempt, ProviderSandboxAttempt, SandboxProviderInventory, SessionHandReleasePageOutcome, TenantSandboxPolicyStore, diff --git a/crates/moa-hands/src/tools/bash.rs b/crates/moa-hands/src/tools/bash.rs index 7d23d5412..b03ab0abe 100644 --- a/crates/moa-hands/src/tools/bash.rs +++ b/crates/moa-hands/src/tools/bash.rs @@ -232,6 +232,60 @@ pub async fn execute_docker( /// `timeout_secs: 86400` would otherwise hold a sandbox for a day. pub const MAX_BASH_TIMEOUT_SECS: u64 = 300; +/// Returns the wall-clock bound a tool invocation promises, when it declares one. +/// +/// The attempt watchdog widens its staleness window to this value, because the durable +/// heartbeat is written at step boundaries and never during a step: without the bound, a +/// command that legitimately runs longer than the configured floor reads as a stall. +/// +/// Only bash declares a bound today. Every other tool returns `None` and is held to the +/// configured floor, which is the intended contract rather than an omission: a tool with no +/// declared ceiling has nothing to justify a wider window with. +#[must_use] +pub fn declared_tool_step_bound(tool_name: &str, input: &serde_json::Value) -> Option { + if tool_name != "bash" { + return None; + } + // An unparseable input never reaches a sandbox, so it cannot be running long; the + // floor applies rather than a fabricated bound. + let params = BashToolInput::parse(&input.to_string()).ok()?; + Some(params.timeout(DEFAULT_BASH_TIMEOUT, None, None)) +} + +/// Wall-clock bound applied to a bash call that names no `timeout_secs` of its own. +/// +/// Deliberately well below [`MAX_BASH_TIMEOUT_SECS`]: the ceiling exists so a caller that +/// knows it needs a long command can ask for one, not so every unqualified call holds a +/// sandbox for the maximum. A model that wants five minutes has to say so, which is also +/// what lets the watchdog keep a tight window for everything that does not. +pub const DEFAULT_BASH_TIMEOUT: Duration = Duration::from_secs(120); + +/// Resolves the wall-clock ceiling for one synchronous sandbox tool call. +/// +/// Bash may request its own validated ceiling; every other synchronous tool +/// uses the provider default. In both cases the caller's remaining deadline is +/// authoritative and an allowance below one second is rejected before remote +/// I/O starts. +pub fn effective_synchronous_timeout( + tool_name: &str, + input: &str, + default_timeout: Duration, + run_deadline: Option, +) -> Result { + let timeout = if tool_name == "bash" { + BashToolInput::parse(input)?.timeout(default_timeout, None, run_deadline) + } else { + run_deadline.map_or(default_timeout, |remaining| default_timeout.min(remaining)) + }; + if timeout < Duration::from_secs(1) { + return Err(MoaError::ToolError( + "synchronous tool execution has less than one second remaining before dispatch" + .to_string(), + )); + } + Ok(timeout) +} + /// A caller-supplied bash timeout that has already cleared tool policy. /// /// Validation lives in the type's own deserialization rather than in each diff --git a/crates/moa-hands/tests/daytona_live.rs b/crates/moa-hands/tests/daytona_live.rs index df5a56aa1..e5e6a8b8d 100644 --- a/crates/moa-hands/tests/daytona_live.rs +++ b/crates/moa-hands/tests/daytona_live.rs @@ -745,6 +745,21 @@ async fn daytona_provider_round_trip() { search.to_text() ); + // Pins: the real Daytona stop endpoint reaches exact `stopped` before + // capacity may be released, and execution resumes to exact `running` + // without losing the retained filesystem. + provider.suspend(&handle).await?; + assert_eq!(provider.status(&handle).await?, HandStatus::Stopped); + let resumed_read = provider + .execute( + &handle, + "file_read", + &json!({ "path": file_path }).to_string(), + ) + .await?; + assert_eq!(resumed_read.to_text(), marker); + assert_eq!(provider.status(&handle).await?, HandStatus::Running); + let unsupported_tool = provider .execute( &handle, diff --git a/crates/moa-hands/tests/hands_db/hand_lease_reaper_db.rs b/crates/moa-hands/tests/hands_db/hand_lease_reaper_db.rs index 8d4a612aa..6553d9aa9 100644 --- a/crates/moa-hands/tests/hands_db/hand_lease_reaper_db.rs +++ b/crates/moa-hands/tests/hands_db/hand_lease_reaper_db.rs @@ -343,6 +343,10 @@ async fn cleanup_session_fixture( workspace_id: SandboxWorkspaceId, provider_account_id: ProviderAccountId, ) { + let _ = sqlx::query("DELETE FROM moa.sandbox_capacity_reservations WHERE workspace_id = $1") + .bind(workspace_id) + .execute(pool) + .await; let _ = sqlx::query("DELETE FROM moa.hand_leases WHERE session_id = $1") .bind(session_id) .execute(pool) @@ -430,6 +434,29 @@ async fn seed_expired_active_lease( .await .expect("claim provisioning") .expect("claim is owned"); + sqlx::query( + r#" + INSERT INTO moa.sandbox_capacity_reservations ( + reservation_id, tenant_id, provider_account_id, + provider_account_generation, workspace_id, operation_id, + expected_writer_epoch, expected_instance_generation, + resource_dimension, quantity, hand_provisioning_operation_id, + hand_lease_generation, reservation_state + ) VALUES ( + $1, $2, $3, 1, $4, NULL, 1, 1, + 'active_hands', 1, $5, $6, 'committed' + ) + "#, + ) + .bind(Uuid::now_v7()) + .bind(tenant_id) + .bind(provider_account_id) + .bind(attachment.workspace_id) + .bind(claim.provisioning_operation_id) + .bind(claim.generation) + .execute(pool) + .await + .expect("seed exact active-hand capacity owner"); store .activate(HandLeaseActivateRequest { tenant_id, @@ -586,6 +613,110 @@ async fn competing_replicas_claim_disjoint_generations_without_new_traffic_db() pool.close().await; } +#[tokio::test] +#[ignore = "requires the local compose Postgres via MOA_DATABASE_URL"] +async fn abandoned_pre_capacity_claim_finalizes_only_without_a_mismatched_reservation_db() { + // Pins: a crash after the durable provisioning claim but before active-hand + // capacity reservation is recoverable after provider absence. The same + // recovery must fail closed when that operation identity has a reservation + // carrying different workspace fences. + let pool = pool().await; + let claims = PostgresExpiredHandLeaseClaims::new(pool.clone()); + + for mismatched_reservation in [false, true] { + let tenant_id = TenantId::new(); + let session_id = SessionId::new(); + seed_session(&pool, session_id, tenant_id).await; + let (attachment, provider_account_id) = seed_workspace(&pool, tenant_id, session_id).await; + sqlx::query( + "UPDATE moa.sandbox_workspaces SET lifecycle_state = 'restoring' \ + WHERE tenant_id = $1 AND workspace_id = $2", + ) + .bind(tenant_id) + .bind(attachment.workspace_id) + .execute(&pool) + .await + .expect("seed workspace between writer claim and provider creation"); + let store = PostgresHandLeaseStore::new(pool.clone()); + let policy = lease_policy(seconds(60), seconds(120)); + let provisioning = store + .claim_for_provisioning(HandLeaseProvisionRequest { + session_id, + worker_id: "worker", + tenant_id, + provider: "local", + tier: SandboxTier::Local, + attachment: attachment.clone(), + policy: &policy, + caller_deadline: Some(chrono::Utc::now() - chrono::Duration::minutes(2)), + }) + .await + .expect("persist provisioning claim") + .expect("claim is owned"); + if mismatched_reservation { + sqlx::query( + r#" + INSERT INTO moa.sandbox_capacity_reservations ( + reservation_id, tenant_id, provider_account_id, + provider_account_generation, workspace_id, operation_id, + expected_writer_epoch, expected_instance_generation, + resource_dimension, quantity, hand_provisioning_operation_id, + hand_lease_generation, reservation_state + ) VALUES ( + $1, $2, $3, 1, $4, NULL, $5, $6, + 'active_hands', 1, $7, $8, 'pending' + ) + "#, + ) + .bind(Uuid::now_v7()) + .bind(tenant_id) + .bind(provider_account_id) + .bind(attachment.workspace_id) + .bind(attachment.workspace_writer_epoch) + .bind(attachment.workspace_instance_generation + 1) + .bind(provisioning.provisioning_operation_id) + .bind(provisioning.generation) + .execute(&pool) + .await + .expect("seed same operation with mismatched instance fence"); + } + let claimed = claims + .claim_expired(64, Duration::from_secs(300)) + .await + .expect("claim abandoned provisioning generations") + .into_iter() + .find(|claim| claim.session_id == session_id) + .expect("this abandoned generation is claimable"); + assert!(claimed.handle.is_none()); + + let finalized = claims + .finalize_destroyed(&claimed) + .await + .expect("finalization remains generation fenced"); + assert_eq!( + finalized, !mismatched_reservation, + "absence of a reservation is recoverable, but a mismatched reservation is not" + ); + assert_eq!( + lease_status(&pool, session_id).await, + if mismatched_reservation { + "reaping" + } else { + "destroyed" + } + ); + + cleanup_session_fixture( + &pool, + session_id, + attachment.workspace_id, + provider_account_id, + ) + .await; + } + pool.close().await; +} + #[tokio::test] #[ignore = "requires the local compose Postgres via MOA_DATABASE_URL"] async fn a_failed_destroy_stays_fenced_and_never_returns_to_active_db() { diff --git a/crates/moa-hands/tests/hands_db/sandbox_workspace/dispatch_db.rs b/crates/moa-hands/tests/hands_db/sandbox_workspace/dispatch_db.rs index 9e107dff2..11428d6d0 100644 --- a/crates/moa-hands/tests/hands_db/sandbox_workspace/dispatch_db.rs +++ b/crates/moa-hands/tests/hands_db/sandbox_workspace/dispatch_db.rs @@ -752,6 +752,12 @@ async fn synchronous_absence_and_reconciled_absence_use_distinct_proof_rules_db( .persist_intent(&synchronous) .await .expect("persist synchronous operation"); + assert!( + operations + .begin_provider_attempt(tenant_id, synchronous_id) + .await + .expect("fence the synchronous provider attempt") + ); sqlx::query( r#" INSERT INTO moa.sandbox_capacity_reservations ( diff --git a/crates/moa-hands/tests/hands_db/sandbox_workspace/lifecycle_db.rs b/crates/moa-hands/tests/hands_db/sandbox_workspace/lifecycle_db.rs index f35d8078f..6b5467624 100644 --- a/crates/moa-hands/tests/hands_db/sandbox_workspace/lifecycle_db.rs +++ b/crates/moa-hands/tests/hands_db/sandbox_workspace/lifecycle_db.rs @@ -1,46 +1,27 @@ //! Durable workspace lifecycle, fencing, and reconciliation against Postgres. -use std::{ - collections::HashMap, - path::PathBuf, - sync::{ - Arc, - atomic::{AtomicBool, AtomicUsize, Ordering}, - }, - time::Duration, -}; +use std::{path::PathBuf, time::Duration}; -use async_trait::async_trait; use chrono::{Duration as ChronoDuration, Utc}; -use moa_core::error::{MoaError, Result as MoaResult}; -use moa_core::traits::{HandProvider, SandboxStorageProvider}; +use moa_core::error::MoaError; use moa_core::types::{ - action_policy::{ActionClass, ActionPolicyEffect, CallOrigin, RiskLevel}, + action_policy::CallOrigin, hands::{ - BuiltinPolicyRevision, CpuLimit, DeadlineEnforcement, DiskLimit, EgressMode, EgressPolicy, - HandHandle, HandProviderCapabilities, HandSpec, HandStatus, LifetimeLimit, MemoryLimit, - ResourceSupport, SandboxPolicySnapshot, SandboxProfile, SandboxTier, - SandboxTierCapabilities, + BuiltinPolicyRevision, CpuLimit, DiskLimit, EgressPolicy, HandHandle, LifetimeLimit, + MemoryLimit, SandboxPolicySnapshot, SandboxProfile, SandboxTier, }, identifiers::{ ExecutionCompensationScopeId, ExecutionRunScopeId, ExecutionTaskScopeId, - HandProvisioningOperationId, ModelId, ProviderAccountId, SandboxWorkspaceId, SessionId, - TenantId, WorkspaceCheckpointId, WorkspaceOperationId, + HandProvisioningOperationId, ProviderAccountId, SandboxWorkspaceId, SessionId, TenantId, + WorkspaceCheckpointId, WorkspaceOperationId, }, - resource::ResourceBudget, sandbox_workspace::{ - DurabilityClass, ExecutionHandContinuationDisposition, ExecutionHandReleaseOwner, - ExecutionHandReleaseReceipt, ProviderAccountStorageInventory, ProviderStorageKind, - ProviderStorageRef, SandboxWorkspaceScope, SandboxWorkspaceState, - TenantStoragePurgeRequest, WorkspaceAttachRequest, WorkspaceBinding, - WorkspaceCheckpointPublication, WorkspaceCheckpointPublishRequest, - WorkspaceConfirmedDisposition, WorkspaceOperationKind, WorkspaceOperationOutcome, - WorkspacePostCommitState, WorkspaceReconcileRequest, WorkspaceRestoreRequest, - WorkspaceRevisionRef, WorkspaceStorageDeleteRequest, WorkspaceStorageOperation, - WorkspaceStorageOperationResult, WorkspaceStoragePrepareRequest, + DurabilityClass, ExecutionHandReleaseOwner, ExecutionHandReleaseReceipt, + ProviderStorageKind, ProviderStorageRef, SandboxWorkspaceScope, SandboxWorkspaceState, + WorkspaceBinding, WorkspaceCheckpointPublication, WorkspaceConfirmedDisposition, + WorkspaceOperationKind, WorkspaceOperationOutcome, WorkspacePostCommitState, + WorkspaceRevisionRef, WorkspaceStorageOperation, }, - session::SessionMeta, - tools::{IdempotencyClass, ToolDiffStrategy, ToolInputShape, ToolOutput, ToolPolicySpec}, }; use moa_hands::core::{ leases::{ @@ -61,11 +42,7 @@ use moa_hands::core::{ repository::PostgresWorkspaceRepository, }, }; -use moa_hands::{ - ExecutionHandReleaseRequest, ExecutionHandRetentionRequest, HandRoute, ToolCallScope, - ToolRegistry, ToolRouter, local_development_sandbox_policy, -}; -use sqlx::{Row, postgres::PgPoolOptions}; +use sqlx::postgres::PgPoolOptions; use super::{database_url, seed_session}; @@ -383,872 +360,6 @@ fn create_request( } } -const CONTINUATION_PROVIDER: &str = "continuation-retention"; - -/// Provider double that honours `release_compute` and counts real teardown. -/// -/// It mirrors the shipped adapters on the three behaviours these scenarios depend -/// on: the checkpoint capacity charge is reserved before publication, compute is -/// destroyed inside publication only when the caller asked for a release, and -/// suspension is a declared capability rather than something inferred from a call. -struct ContinuationProvider { - capacity: PostgresWorkspaceCapacityRepository, - destroy_calls: AtomicUsize, - reconcile_calls: AtomicUsize, - suspend_calls: AtomicUsize, - resume_calls: AtomicUsize, - /// Mirrors Daytona (`true`) or local/E2B (`false`) compute-release ability. - suspends: bool, - /// Makes a declared suspension fail the way an unreachable provider would. - suspend_fails: bool, - /// Reports a suspended sandbox as stopped so reattach exercises resume. - suspended: AtomicBool, -} - -const CONTINUATION_CHECKPOINT_BYTES: u64 = 11; - -impl ContinuationProvider { - fn new(pool: &sqlx::PgPool) -> Self { - Self::with_suspension(pool, false, false) - } - - fn with_suspension(pool: &sqlx::PgPool, suspends: bool, suspend_fails: bool) -> Self { - Self { - capacity: PostgresWorkspaceCapacityRepository::new(pool.clone()), - destroy_calls: AtomicUsize::new(0), - reconcile_calls: AtomicUsize::new(0), - suspend_calls: AtomicUsize::new(0), - resume_calls: AtomicUsize::new(0), - suspends, - suspend_fails, - suspended: AtomicBool::new(false), - } - } - - fn confirmed(storage: Option) -> WorkspaceStorageOperationResult { - WorkspaceStorageOperationResult { - outcome: WorkspaceOperationOutcome::Confirmed, - confirmed_disposition: Some(WorkspaceConfirmedDisposition::ResourcePresent), - storage, - checkpoint_publication: None, - post_commit_state: None, - } - } - - fn mutable_storage(binding: &WorkspaceBinding) -> ProviderStorageRef { - ProviderStorageRef { - provider_account_id: binding.provider_account_id, - provider_account_generation: binding.provider_account_generation, - kind: ProviderStorageKind::MutableFilesystem, - resource_id: format!("mutable/{}", binding.workspace_id), - workspace_locator: None, - } - } - - fn published( - operation: &WorkspaceStorageOperation, - post_commit_state: WorkspacePostCommitState, - ) -> WorkspaceStorageOperationResult { - let generation = operation - .binding - .current_revision - .as_ref() - .map_or(1, |parent| parent.generation + 1); - let checkpoint_id = WorkspaceCheckpointId(operation.operation_id.0); - let storage = ProviderStorageRef { - provider_account_id: operation.binding.provider_account_id, - provider_account_generation: operation.binding.provider_account_generation, - kind: ProviderStorageKind::PortableCheckpoint, - resource_id: format!("checkpoint/{checkpoint_id}"), - workspace_locator: None, - }; - WorkspaceStorageOperationResult { - outcome: WorkspaceOperationOutcome::Confirmed, - confirmed_disposition: Some(WorkspaceConfirmedDisposition::ResourcePresent), - storage: Some(storage.clone()), - checkpoint_publication: Some(WorkspaceCheckpointPublication { - revision: WorkspaceRevisionRef { - checkpoint_id, - generation, - format_version: 1, - }, - storage, - manifest_digest: format!("sha256:manifest-{checkpoint_id}"), - logical_bytes: CONTINUATION_CHECKPOINT_BYTES, - }), - post_commit_state: Some(post_commit_state), - } - } -} - -#[async_trait] -impl HandProvider for ContinuationProvider { - fn provider_name(&self) -> &str { - CONTINUATION_PROVIDER - } - - fn capabilities(&self) -> HandProviderCapabilities { - HandProviderCapabilities { - revision: "continuation-retention-hands-v1".to_string(), - tiers: vec![SandboxTierCapabilities { - tier: SandboxTier::Container, - cpu: ResourceSupport::unbounded_only(), - memory: ResourceSupport::unbounded_only(), - ephemeral_disk: ResourceSupport::unbounded_only(), - egress_modes: vec![ - EgressMode::DenyAll, - EgressMode::AllowList, - EgressMode::Unrestricted, - ], - idle_enforcement: DeadlineEnforcement::DurableReaper, - max_lifetime_enforcement: DeadlineEnforcement::DurableReaper, - }], - } - } - - async fn provision(&self, spec: HandSpec) -> MoaResult { - Ok(HandHandle::docker(format!( - "continuation-retention-{}", - spec.provisioning_operation_id - ))) - } - - async fn provisioned_hands( - &self, - _provider_account_id: ProviderAccountId, - _provider_account_generation: u64, - _operation_id: HandProvisioningOperationId, - ) -> MoaResult> { - Ok(Vec::new()) - } - - async fn execute( - &self, - _handle: &HandHandle, - _tool: &str, - _input: &str, - ) -> MoaResult { - Err(MoaError::Unsupported( - "tool execution is outside the continuation-retention scenario".to_string(), - )) - } - - async fn status(&self, _handle: &HandHandle) -> MoaResult { - Ok(if self.suspended.load(Ordering::SeqCst) { - HandStatus::Stopped - } else { - HandStatus::Running - }) - } - - fn supports_suspend(&self) -> bool { - self.suspends - } - - async fn suspend(&self, _handle: &HandHandle) -> MoaResult<()> { - self.suspend_calls.fetch_add(1, Ordering::SeqCst); - if !self.suspends { - return Err(MoaError::Unsupported( - "this continuation provider cannot release compute".to_string(), - )); - } - if self.suspend_fails { - return Err(MoaError::ProviderError( - "continuation provider suspend is unreachable".to_string(), - )); - } - self.suspended.store(true, Ordering::SeqCst); - Ok(()) - } - - async fn resume(&self, _handle: &HandHandle) -> MoaResult<()> { - self.resume_calls.fetch_add(1, Ordering::SeqCst); - self.suspended.store(false, Ordering::SeqCst); - Ok(()) - } - - async fn destroy(&self, _handle: &HandHandle) -> MoaResult<()> { - self.destroy_calls.fetch_add(1, Ordering::SeqCst); - self.suspended.store(false, Ordering::SeqCst); - Ok(()) - } -} - -#[async_trait] -impl SandboxStorageProvider for ContinuationProvider { - fn storage_provider_name(&self) -> &str { - CONTINUATION_PROVIDER - } - - async fn enumerate_account_storage( - &self, - provider_account_id: ProviderAccountId, - provider_account_generation: u64, - ) -> MoaResult { - Ok(ProviderAccountStorageInventory { - provider_account_id, - provider_account_generation, - observed_at: Utc::now(), - resources: Vec::new(), - }) - } - - async fn prepare_workspace_storage( - &self, - request: WorkspaceStoragePrepareRequest, - ) -> MoaResult { - Ok(Self::confirmed(Some(Self::mutable_storage( - &request.operation.binding, - )))) - } - - async fn attach_workspace( - &self, - request: WorkspaceAttachRequest, - ) -> MoaResult { - Ok(Self::confirmed(Some(Self::mutable_storage( - &request.operation.binding, - )))) - } - - async fn publish_workspace_checkpoint( - &self, - request: WorkspaceCheckpointPublishRequest, - ) -> MoaResult { - self.capacity - .reserve_checkpoint_publication(&request.operation, CONTINUATION_CHECKPOINT_BYTES) - .await?; - let post_commit_state = if request.release_compute { - ::destroy(self, &request.hand).await?; - WorkspacePostCommitState::ComputeDestroyed - } else { - WorkspacePostCommitState::AttachmentRetained - }; - Ok(Self::published(&request.operation, post_commit_state)) - } - - async fn restore_workspace( - &self, - _request: WorkspaceRestoreRequest, - ) -> MoaResult { - Ok(Self::confirmed(None)) - } - - async fn delete_workspace_storage( - &self, - _request: WorkspaceStorageDeleteRequest, - ) -> MoaResult { - Err(MoaError::Unsupported( - "delete is outside the continuation-retention scenario".to_string(), - )) - } - - async fn delete_tenant_storage_resource( - &self, - _request: TenantStoragePurgeRequest, - ) -> MoaResult { - Err(MoaError::Unsupported( - "tenant purge is outside the continuation-retention scenario".to_string(), - )) - } - - async fn reconcile_workspace_operation( - &self, - request: WorkspaceReconcileRequest, - ) -> MoaResult { - self.reconcile_calls.fetch_add(1, Ordering::SeqCst); - Ok(Self::published( - request.operation(), - WorkspacePostCommitState::AttachmentRetained, - )) - } - - async fn verify_workspace_storage(&self, _storage: &ProviderStorageRef) -> MoaResult { - Ok(true) - } -} - -fn continuation_router(pool: &sqlx::PgPool, provider: Arc) -> ToolRouter { - let mut registry = ToolRegistry::new(); - registry.register_hand( - "continuation_route_anchor", - "exposes the configured continuation provider route", - serde_json::json!({ "type": "object", "additionalProperties": false }), - ToolPolicySpec { - risk_level: RiskLevel::Low, - default_effect: ActionPolicyEffect::Allow, - action_class: ActionClass::Read, - input_shape: ToolInputShape::Json, - diff_strategy: ToolDiffStrategy::None, - }, - IdempotencyClass::Idempotent, - ); - registry.retarget_hand_tools(vec![HandRoute { - provider: CONTINUATION_PROVIDER.to_string(), - tier: SandboxTier::Container, - policy: SandboxPolicySnapshot::builtin(BuiltinPolicyRevision::RouteUnset), - }]); - let mut hand_providers: HashMap> = HashMap::new(); - hand_providers.insert( - CONTINUATION_PROVIDER.to_string(), - Arc::clone(&provider) as Arc, - ); - ToolRouter::new(registry, hand_providers, local_development_sandbox_policy()) - .with_sandbox_storage_provider(Arc::clone(&provider) as Arc) - .expect("register continuation storage provider") - .with_workspace_repositories(pool.clone()) - .with_hand_lease_store(Arc::new(PostgresHandLeaseStore::new(pool.clone()))) -} - -async fn committed_active_hand_reservations( - pool: &sqlx::PgPool, - tenant_id: TenantId, - workspace_id: SandboxWorkspaceId, -) -> i64 { - sqlx::query( - "SELECT count(*)::BIGINT AS live FROM moa.sandbox_capacity_reservations \ - WHERE tenant_id = $1 AND workspace_id = $2 \ - AND resource_dimension = 'active_hands' \ - AND reservation_state IN ('pending', 'committed')", - ) - .bind(tenant_id) - .bind(workspace_id) - .fetch_one(pool) - .await - .expect("count live active-hand reservations") - .try_get::("live") - .expect("decode live active-hand reservations") -} - -/// One attached execution-task sandbox ready for a continuation-boundary scenario. -struct ContinuationFixture { - _test_db: moa_test_support::postgres::TestDb, - pool: sqlx::PgPool, - tenant_id: TenantId, - session_id: SessionId, - account_id: ProviderAccountId, - workspace_id: SandboxWorkspaceId, - workspace_scope: SandboxWorkspaceScope, - session: SessionMeta, - lease_scope: String, - provider: Arc, - router: ToolRouter, - workspaces: PostgresWorkspaceRepository, - leases: PostgresHandLeaseStore, -} - -/// Attaches one execution-task workspace onto a provider with the given suspend ability. -async fn continuation_fixture(suspends: bool, suspend_fails: bool) -> ContinuationFixture { - let test_db = moa_test_support::postgres::bootstrap_test_db() - .await - .expect("bootstrap isolated current-schema Postgres"); - let pool = test_db.store().pool().clone(); - let tenant_id = TenantId::new(); - let session_id = SessionId::new(); - let account_id = ProviderAccountId::new(); - let workspace_id = SandboxWorkspaceId::new(); - seed_session(&pool, session_id, tenant_id).await; - sqlx::query( - r#" - INSERT INTO moa.sandbox_provider_accounts ( - provider_account_id, generation, provider, isolation_cell, - organization_fingerprint, configured_limits - ) VALUES ($1, 1, $2, $3, $4, '{}'::jsonb) - "#, - ) - .bind(account_id) - .bind(CONTINUATION_PROVIDER) - .bind(format!("continuation-{account_id}")) - .bind(format!("continuation-org-{account_id}")) - .execute(&pool) - .await - .expect("seed continuation provider account"); - let (run_id, _, _) = seed_cancelling_compensation(&pool, tenant_id, session_id).await; - let task_id = seed_cancelling_task(&pool, tenant_id, run_id, "continuation-suspend").await; - let workspace_scope = SandboxWorkspaceScope::ExecutionTask { run_id, task_id }; - let workspaces = PostgresWorkspaceRepository::new(pool.clone()); - workspaces - .create(&CreateWorkspaceRequest { - workspace_id, - tenant_id, - scope: workspace_scope.clone(), - provider: CONTINUATION_PROVIDER.to_string(), - provider_account_id: account_id, - provider_account_generation: 1, - durability_class: DurabilityClass::PortableFilesystem, - retention_deadline_at: None, - }) - .await - .expect("create continuation task workspace"); - - let provider = Arc::new(ContinuationProvider::with_suspension( - &pool, - suspends, - suspend_fails, - )); - let router = continuation_router(&pool, Arc::clone(&provider)); - let session = SessionMeta { - id: session_id, - tenant_id, - model: ModelId::new("continuation-suspend-model"), - ..SessionMeta::default() - }; - router - .attach_managed_workspace(&session, &workspace_scope, workspace_id) - .await - .expect("attach must materialize provider compute and storage"); - let leases = PostgresHandLeaseStore::new(pool.clone()); - ContinuationFixture { - _test_db: test_db, - pool, - tenant_id, - session_id, - account_id, - workspace_id, - workspace_scope, - session, - lease_scope: format!("execution:{run_id}:{task_id}"), - provider, - router, - workspaces, - leases, - } -} - -fn continuation_retention_request<'a>( - fixture: &'a ContinuationFixture, - retention_deadline_at: chrono::DateTime, -) -> ExecutionHandRetentionRequest<'a> { - let SandboxWorkspaceScope::ExecutionTask { run_id, task_id } = fixture.workspace_scope else { - panic!("continuation fixture always owns an execution-task workspace"); - }; - ExecutionHandRetentionRequest { - session: &fixture.session, - run_id, - task_id, - logical_generation: 1, - attempt_generation: 1, - retention_deadline_at, - scope: ToolCallScope::unbounded().with_budget(ResourceBudget::until( - Utc::now() + ChronoDuration::minutes(5), - )), - } -} - -#[tokio::test] -#[ignore = "requires Postgres for an isolated current-schema test database"] -async fn continuation_suspends_compute_and_returns_its_admission_slot_db() { - // Pins: on a provider that can genuinely release compute, a continuation boundary - // stops the sandbox inside the yield rather than leaving it hot for the reaper, - // keeps the lease and handle so the next slice reattaches, and hands the - // `ActiveHands` slot back so a runnable task can be admitted into it. - let fixture = continuation_fixture(true, false).await; - assert_eq!( - committed_active_hand_reservations(&fixture.pool, fixture.tenant_id, fixture.workspace_id) - .await, - 1 - ); - - let disposition = fixture - .router - .checkpoint_execution_hand_retaining_compute(continuation_retention_request( - &fixture, - Utc::now() + ChronoDuration::minutes(2), - )) - .await - .expect("a continuation boundary must publish its checkpoint"); - - assert_eq!( - disposition, - ExecutionHandContinuationDisposition::Suspended, - "a suspend-capable provider must take the suspend path" - ); - assert_eq!(fixture.provider.suspend_calls.load(Ordering::SeqCst), 1); - assert_eq!( - fixture.provider.destroy_calls.load(Ordering::SeqCst), - 0, - "suspension must not destroy the sandbox the next slice will reattach to" - ); - let retained = fixture - .workspaces - .get(fixture.tenant_id, fixture.workspace_id) - .await - .expect("load suspended workspace") - .expect("suspended workspace exists"); - assert_eq!( - retained.checkpoint_generation, 1, - "suspension must still advance the portable recovery head" - ); - let lease = fixture - .leases - .get( - fixture.tenant_id, - fixture.session_id, - &fixture.lease_scope, - CONTINUATION_PROVIDER, - ) - .await - .expect("load suspended continuation lease") - .expect("suspended continuation lease exists"); - assert_eq!( - lease.status, - HandLeaseStatus::Active, - "the lease must survive so the next slice can reattach the same filesystem" - ); - assert!(lease.handle.is_some()); - assert_eq!( - committed_active_hand_reservations(&fixture.pool, fixture.tenant_id, fixture.workspace_id) - .await, - 0, - "a suspended hand must not keep holding fleet admission capacity" - ); -} - -#[tokio::test] -#[ignore = "requires Postgres for an isolated current-schema test database"] -async fn suspended_continuation_hand_reattaches_by_rewinning_capacity_db() { - // Pins: reattaching a suspended sandbox is a fresh admission decision — the charge - // released at the boundary is re-won and the provider is actually resumed, rather - // than the hand being handed back warm while the fleet gauge under-counts it. - let fixture = continuation_fixture(true, false).await; - fixture - .router - .checkpoint_execution_hand_retaining_compute(continuation_retention_request( - &fixture, - Utc::now() + ChronoDuration::minutes(2), - )) - .await - .expect("suspend the continuation sandbox"); - assert_eq!( - committed_active_hand_reservations(&fixture.pool, fixture.tenant_id, fixture.workspace_id) - .await, - 0 - ); - - fixture - .router - .attach_managed_workspace( - &fixture.session, - &fixture.workspace_scope, - fixture.workspace_id, - ) - .await - .expect("the next slice must reattach the suspended sandbox"); - - assert_eq!( - fixture.provider.resume_calls.load(Ordering::SeqCst), - 1, - "a stopped sandbox must be resumed, not dispatched into while stopped" - ); - assert_eq!( - fixture.provider.destroy_calls.load(Ordering::SeqCst), - 0, - "an admitted reattach must reuse the warm sandbox" - ); - assert_eq!( - committed_active_hand_reservations(&fixture.pool, fixture.tenant_id, fixture.workspace_id) - .await, - 1, - "resuming compute must charge the fleet for it again" - ); -} - -#[tokio::test] -#[ignore = "requires Postgres for an isolated current-schema test database"] -async fn saturated_reattach_drops_the_suspended_hand_instead_of_resuming_db() { - // Pins: a suspended sandbox has no reserved claim on its old slot. When the fleet - // filled up while it was stopped, reattach refuses to resume it — which is safe - // only because the boundary published its checkpoint before suspending, making the - // eviction a cache miss rather than lost work. - let fixture = continuation_fixture(true, false).await; - fixture - .router - .checkpoint_execution_hand_retaining_compute(continuation_retention_request( - &fixture, - Utc::now() + ChronoDuration::minutes(2), - )) - .await - .expect("suspend the continuation sandbox"); - - // Every active-hands slot in the provider account is now spoken for. - sqlx::query( - "UPDATE moa.sandbox_provider_accounts \ - SET configured_limits = '{\"active_hands\": 0}'::jsonb \ - WHERE provider_account_id = $1 AND generation = 1", - ) - .bind(fixture.account_id) - .execute(&fixture.pool) - .await - .expect("saturate the provider account"); - - let error = fixture - .router - .attach_managed_workspace( - &fixture.session, - &fixture.workspace_scope, - fixture.workspace_id, - ) - .await - .expect_err("a saturated fleet must not hand back the warm slot for free"); - - assert!( - error.to_string().contains("capacity"), - "reattach must fail on admission, not on some unrelated fault: {error}" - ); - assert_eq!( - fixture.provider.resume_calls.load(Ordering::SeqCst), - 0, - "compute must never restart before its admission slot is re-won" - ); - assert_eq!( - committed_active_hand_reservations(&fixture.pool, fixture.tenant_id, fixture.workspace_id) - .await, - 0, - "a refused reattach must leave the fleet charge released" - ); -} - -#[tokio::test] -#[ignore = "requires Postgres for an isolated current-schema test database"] -async fn failed_suspension_falls_back_to_release_instead_of_staying_hot_db() { - // Pins: a declared-but-failing suspension never leaves a hand hot on a bet that - // already lost. It reports the fallback so the caller finishes the ordinary - // checkpoint-and-destroy path, which returns the capacity. - let fixture = continuation_fixture(true, true).await; - - let disposition = fixture - .router - .checkpoint_execution_hand_retaining_compute(continuation_retention_request( - &fixture, - Utc::now() + ChronoDuration::minutes(2), - )) - .await - .expect("a failed suspension is non-fatal; the checkpoint still publishes"); - - assert_eq!( - disposition, - ExecutionHandContinuationDisposition::SuspendFailed - ); - assert_eq!(fixture.provider.suspend_calls.load(Ordering::SeqCst), 1); - - let SandboxWorkspaceScope::ExecutionTask { run_id, task_id } = fixture.workspace_scope else { - panic!("continuation fixture always owns an execution-task workspace"); - }; - fixture - .router - .checkpoint_and_release_execution_hand(ExecutionHandReleaseRequest { - session: &fixture.session, - run_id, - owner: ExecutionHandReleaseOwner::Task { - task_id, - logical_generation: 1, - }, - attempt_generation: 1, - scope: ToolCallScope::unbounded().with_budget(ResourceBudget::until( - Utc::now() + ChronoDuration::minutes(5), - )), - }) - .await - .expect("the fallback release must complete"); - - assert_eq!(fixture.provider.destroy_calls.load(Ordering::SeqCst), 1); - assert_eq!( - committed_active_hand_reservations(&fixture.pool, fixture.tenant_id, fixture.workspace_id) - .await, - 0, - "the fallback release must return the active-hands slot" - ); -} - -#[tokio::test] -#[ignore = "requires Postgres for an isolated current-schema test database"] -async fn continuation_retains_the_hand_while_a_park_releases_it_db() { - // Pins: on a provider that cannot release compute, a plain model/tool boundary - // publishes its checkpoint and keeps the exact sandbox, its lease, and its admitted - // `ActiveHands` slot — bounded by a shortened idle deadline the reaper owns — while - // the very next genuine park on the same sandbox destroys compute and gives the - // capacity back. - let test_db = moa_test_support::postgres::bootstrap_test_db() - .await - .expect("bootstrap isolated current-schema Postgres"); - let pool = test_db.store().pool().clone(); - let tenant_id = TenantId::new(); - let session_id = SessionId::new(); - let account_id = ProviderAccountId::new(); - let workspace_id = SandboxWorkspaceId::new(); - seed_session(&pool, session_id, tenant_id).await; - sqlx::query( - r#" - INSERT INTO moa.sandbox_provider_accounts ( - provider_account_id, generation, provider, isolation_cell, - organization_fingerprint, configured_limits - ) VALUES ($1, 1, $2, $3, $4, '{}'::jsonb) - "#, - ) - .bind(account_id) - .bind(CONTINUATION_PROVIDER) - .bind(format!("continuation-{account_id}")) - .bind(format!("continuation-org-{account_id}")) - .execute(&pool) - .await - .expect("seed continuation provider account"); - let (run_id, _, _) = seed_cancelling_compensation(&pool, tenant_id, session_id).await; - let task_id = seed_cancelling_task(&pool, tenant_id, run_id, "continuation-retention").await; - let workspace_scope = SandboxWorkspaceScope::ExecutionTask { run_id, task_id }; - let workspaces = PostgresWorkspaceRepository::new(pool.clone()); - workspaces - .create(&CreateWorkspaceRequest { - workspace_id, - tenant_id, - scope: workspace_scope.clone(), - provider: CONTINUATION_PROVIDER.to_string(), - provider_account_id: account_id, - provider_account_generation: 1, - durability_class: DurabilityClass::PortableFilesystem, - retention_deadline_at: None, - }) - .await - .expect("create continuation task workspace"); - - let provider = Arc::new(ContinuationProvider::new(&pool)); - let router = continuation_router(&pool, Arc::clone(&provider)); - let session = SessionMeta { - id: session_id, - tenant_id, - model: ModelId::new("continuation-retention-model"), - ..SessionMeta::default() - }; - router - .attach_managed_workspace(&session, &workspace_scope, workspace_id) - .await - .expect("attach must materialize provider compute and storage"); - let leases = PostgresHandLeaseStore::new(pool.clone()); - let lease_scope = format!("execution:{run_id}:{task_id}"); - let provisioned = leases - .get(tenant_id, session_id, &lease_scope, CONTINUATION_PROVIDER) - .await - .expect("load provisioned continuation lease") - .expect("continuation lease exists"); - assert_eq!(provisioned.status, HandLeaseStatus::Active); - assert_eq!( - committed_active_hand_reservations(&pool, tenant_id, workspace_id).await, - 1 - ); - - let retention_deadline_at = Utc::now() + ChronoDuration::minutes(2); - let disposition = router - .checkpoint_execution_hand_retaining_compute(ExecutionHandRetentionRequest { - session: &session, - run_id, - task_id, - logical_generation: 1, - attempt_generation: 1, - retention_deadline_at, - scope: ToolCallScope::unbounded().with_budget(ResourceBudget::until( - Utc::now() + ChronoDuration::minutes(5), - )), - }) - .await - .expect("a continuation boundary must publish its checkpoint"); - - assert_eq!( - disposition, - ExecutionHandContinuationDisposition::RetainedHot, - "a provider that cannot release compute must fall back to bounded hot retention" - ); - assert_eq!( - provider.suspend_calls.load(Ordering::SeqCst), - 0, - "a provider that declares no suspension must never be asked to suspend" - ); - assert_eq!( - provider.destroy_calls.load(Ordering::SeqCst), - 0, - "a continuation boundary must not destroy the sandbox it is about to resume in" - ); - let retained_workspace = workspaces - .get(tenant_id, workspace_id) - .await - .expect("load retained workspace") - .expect("retained workspace exists"); - assert_eq!(retained_workspace.state, SandboxWorkspaceState::Active); - assert_eq!( - retained_workspace.checkpoint_generation, 1, - "retention must still advance the portable recovery head" - ); - let retained_lease = leases - .get(tenant_id, session_id, &lease_scope, CONTINUATION_PROVIDER) - .await - .expect("load retained continuation lease") - .expect("retained continuation lease exists"); - assert_eq!(retained_lease.status, HandLeaseStatus::Active); - assert!(retained_lease.handle.is_some()); - assert_eq!(retained_lease.generation, provisioned.generation); - let bounded_idle = retained_lease - .idle_expires_at - .expect("a retained hand must carry a reaper-owned retention deadline"); - assert!( - bounded_idle <= retention_deadline_at, - "retention must bound the retained hand at or before its requested deadline" - ); - assert!( - bounded_idle > Utc::now(), - "retention must leave the next slice a window to reattach" - ); - assert_eq!( - committed_active_hand_reservations(&pool, tenant_id, workspace_id).await, - 1, - "a retained hand keeps its admitted active-hands slot" - ); - - // The park path now runs against the exact sandbox the continuation kept alive. - let receipt = router - .checkpoint_and_release_execution_hand(ExecutionHandReleaseRequest { - session: &session, - run_id, - owner: ExecutionHandReleaseOwner::Task { - task_id, - logical_generation: 1, - }, - attempt_generation: 1, - scope: ToolCallScope::unbounded().with_budget(ResourceBudget::until( - Utc::now() + ChronoDuration::minutes(5), - )), - }) - .await - .expect("a genuine park must release the retained sandbox"); - - assert_eq!( - provider.destroy_calls.load(Ordering::SeqCst), - 1, - "a park must destroy exactly the hand the continuation retained" - ); - assert_eq!( - receipt.checkpoint_generation, - Some(2), - "the park publishes the next head on top of the retained continuation checkpoint" - ); - let released_workspace = workspaces - .get(tenant_id, workspace_id) - .await - .expect("load released workspace") - .expect("released workspace exists"); - assert_eq!(released_workspace.state, SandboxWorkspaceState::Ready); - let released_lease = leases - .get(tenant_id, session_id, &lease_scope, CONTINUATION_PROVIDER) - .await - .expect("load released continuation lease") - .expect("released continuation lease exists"); - assert_eq!(released_lease.status, HandLeaseStatus::Destroyed); - assert!(released_lease.handle.is_none()); - assert_eq!( - committed_active_hand_reservations(&pool, tenant_id, workspace_id).await, - 0, - "a park must return the active-hands slot to the fleet" - ); - assert_eq!(provider.reconcile_calls.load(Ordering::SeqCst), 0); -} - #[tokio::test] #[ignore = "requires a fresh V60 compose Postgres via MOA_DATABASE_URL"] async fn cancelling_task_without_owned_compute_gets_exact_absence_receipt_db() { @@ -1982,6 +1093,145 @@ async fn workspace_writer_and_reconciliation_callbacks_are_generation_fenced_db( pool.close().await; } +#[tokio::test] +#[ignore = "requires a fresh V60 compose Postgres via MOA_DATABASE_URL"] +async fn create_operation_replay_never_resends_and_reconciliation_settles_lifecycle_db() { + // Pins: create provider I/O is authorized by one exact not-sent CAS. A crash + // after that CAS leaves the workspace reconciling, replay cannot win the CAS + // again, and only claimed provider evidence returns it to ready or failed. + let pool = PgPoolOptions::new() + .max_connections(4) + .connect(&database_url()) + .await + .expect("test Postgres should be reachable"); + let tenant_id = TenantId::new(); + let account_id = ProviderAccountId::new(); + let workspace_id = SandboxWorkspaceId::new(); + seed_account(&pool, account_id).await; + let workspaces = PostgresWorkspaceRepository::new(pool.clone()); + workspaces + .create(&create_request(tenant_id, workspace_id, account_id)) + .await + .expect("create workspace metadata and lifetime capacity"); + let operations = PostgresWorkspaceOperationRepository::new(pool.clone()); + let operation_id = WorkspaceOperationId::new(); + let now = Utc::now(); + let intent = WorkspaceOperationIntent { + operation_id, + tenant_id, + workspace_id, + provider_account_id: account_id, + provider_account_generation: 1, + kind: WorkspaceOperationKind::Create, + request_hash: format!("sha256:create-replay-{operation_id}"), + expected_writer_epoch: 0, + expected_instance_generation: 0, + expected_checkpoint_generation: 0, + deadline_at: now - ChronoDuration::seconds(2), + reconcile_not_before: now - ChronoDuration::seconds(1), + }; + let persisted = operations + .persist_intent(&intent) + .await + .expect("persist create intent before provider I/O"); + assert_eq!(persisted.outcome, WorkspaceOperationOutcome::NotSent); + assert!( + !operations + .confirm_disposition( + tenant_id, + operation_id, + WorkspaceConfirmedDisposition::ResourcePresent, + ) + .await + .expect("not-sent intent cannot accept a provider disposition") + ); + assert!( + operations + .begin_provider_attempt(tenant_id, operation_id) + .await + .expect("first exact provider-attempt CAS succeeds") + ); + assert!( + !operations + .begin_provider_attempt(tenant_id, operation_id) + .await + .expect("crash replay cannot resend the provider request") + ); + let reconciling = workspaces + .get(tenant_id, workspace_id) + .await + .expect("load workspace after provider-attempt CAS") + .expect("workspace exists"); + assert_eq!(reconciling.state, SandboxWorkspaceState::Reconciling); + + let claimed = operations + .claim_reconciliation(1, Duration::from_secs(30)) + .await + .expect("claim ambiguous create") + .into_iter() + .find(|claim| claim.operation.operation_id == operation_id) + .expect("this create is claimable"); + assert!( + operations + .confirm_present_claimed(&claimed) + .await + .expect("exact live claim confirms provider presence") + ); + let recovered = workspaces + .get(tenant_id, workspace_id) + .await + .expect("load reconciled workspace") + .expect("workspace exists"); + assert_eq!(recovered.state, SandboxWorkspaceState::Ready); + let confirmed = operations + .persist_intent(&intent) + .await + .expect("identical replay loads the durable operation"); + assert_eq!(confirmed.outcome, WorkspaceOperationOutcome::Confirmed); + assert_eq!( + confirmed.confirmed_disposition, + Some(WorkspaceConfirmedDisposition::ResourcePresent) + ); + assert!( + !operations + .confirm_disposition( + tenant_id, + operation_id, + WorkspaceConfirmedDisposition::ResourcePresent, + ) + .await + .expect("confirmed replay cannot re-enter synchronous settlement") + ); + assert!( + !operations + .begin_provider_attempt(tenant_id, operation_id) + .await + .expect("confirmed replay cannot authorize provider I/O") + ); + + sqlx::query("DELETE FROM moa.sandbox_capacity_reservations WHERE workspace_id = $1") + .bind(workspace_id) + .execute(&pool) + .await + .expect("clean capacity"); + sqlx::query("DELETE FROM moa.sandbox_workspace_operations WHERE operation_id = $1") + .bind(operation_id) + .execute(&pool) + .await + .expect("clean operation"); + sqlx::query("DELETE FROM moa.sandbox_workspaces WHERE workspace_id = $1") + .bind(workspace_id) + .execute(&pool) + .await + .expect("clean workspace"); + sqlx::query("DELETE FROM moa.sandbox_provider_accounts WHERE provider_account_id = $1") + .bind(account_id) + .execute(&pool) + .await + .expect("clean provider account"); + pool.close().await; +} + #[tokio::test] #[ignore = "requires a fresh V60 compose Postgres via MOA_DATABASE_URL"] async fn checkpoint_metadata_is_created_before_bytes_and_remains_immutable_db() { diff --git a/crates/moa-hands/tests/hands_db/sandbox_workspace/maintenance_db.rs b/crates/moa-hands/tests/hands_db/sandbox_workspace/maintenance_db.rs index 3d2329814..318cce00f 100644 --- a/crates/moa-hands/tests/hands_db/sandbox_workspace/maintenance_db.rs +++ b/crates/moa-hands/tests/hands_db/sandbox_workspace/maintenance_db.rs @@ -11,6 +11,7 @@ use std::{ use async_trait::async_trait; use chrono::{Duration as ChronoDuration, Utc}; +use metrics_exporter_prometheus::PrometheusBuilder; use moa_core::{ error::Result, types::{ @@ -57,6 +58,9 @@ use moa_hands::core::{ }; use sqlx::{PgPool, Row, postgres::PgPoolOptions}; +use super::sandbox_workspace_retention_db::{ + create_workspace, maintenance_fixture, pools, seed_account as seed_workspace_account, +}; use super::seed_session; fn required_url(name: &str) -> String { @@ -259,6 +263,138 @@ async fn workspace_maintenance_pool_requires_noninheriting_member_login_db() { maintenance.close().await; } +fn quota_ratio(rendered: &str, dimension: &str) -> f64 { + let prefix = + format!("moa_sandbox_workspace_quota_utilization_ratio{{dimension=\"{dimension}\"}} "); + rendered + .lines() + .find_map(|line| line.strip_prefix(&prefix)) + .unwrap_or_else(|| panic!("quota scrape is missing {dimension}:\n{rendered}")) + .parse::() + .unwrap_or_else(|error| panic!("quota ratio for {dimension} is not numeric: {error}")) +} + +#[tokio::test(flavor = "current_thread")] +#[ignore = "requires a fresh V60 database and distinct runtime/workspace-maintenance logins"] +async fn workspace_quota_metrics_use_json_limits_and_highest_enforced_scope_db() { + // Pins: fleet quota telemetry reads the JSONB limit schema used by admission, + // reports the highest tenant/provider-account pressure, distinguishes an + // explicit zero ceiling from an absent unbounded ceiling, and zero-fills it. + let (runtime, maintenance) = pools().await; + let tenant_id = TenantId::new(); + let account_id = ProviderAccountId::new(); + seed_workspace_account(&runtime, account_id).await; + sqlx::query( + "UPDATE moa.sandbox_provider_accounts \ + SET configured_limits = '{\"workspaces\": 10}'::jsonb \ + WHERE provider_account_id = $1 AND generation = 1", + ) + .bind(account_id) + .execute(&runtime) + .await + .expect("seed provider-account workspace ceiling"); + sqlx::query( + "INSERT INTO moa.sandbox_tenant_capacity_limits (tenant_id, configured_limits) \ + VALUES ($1, '{\"workspaces\": 4}'::jsonb)", + ) + .bind(tenant_id) + .execute(&runtime) + .await + .expect("seed tenant workspace ceiling"); + for _ in 0..3 { + create_workspace(&runtime, tenant_id, account_id).await; + } + + let fixture = maintenance_fixture( + &runtime, + &maintenance, + account_id, + moa_config::CheckpointRetentionConfig::default(), + ) + .await; + let recorder = PrometheusBuilder::new().build_recorder(); + let handle = recorder.handle(); + let _recorder_guard = metrics::set_default_local_recorder(&recorder); + + fixture + .coordinator + .emit_fleet_metrics() + .await + .expect("emit tenant-dominated quota snapshot"); + assert_eq!(quota_ratio(&handle.render(), "workspaces"), 0.75); + + sqlx::query( + "UPDATE moa.sandbox_tenant_capacity_limits \ + SET configured_limits = '{\"workspaces\": 30}'::jsonb \ + WHERE tenant_id = $1", + ) + .bind(tenant_id) + .execute(&runtime) + .await + .expect("lower tenant pressure below provider-account pressure"); + fixture + .coordinator + .emit_fleet_metrics() + .await + .expect("emit provider-account-dominated quota snapshot"); + assert_eq!(quota_ratio(&handle.render(), "workspaces"), 0.3); + + sqlx::query( + "UPDATE moa.sandbox_provider_accounts \ + SET configured_limits = '{\"workspaces\": 0}'::jsonb \ + WHERE provider_account_id = $1 AND generation = 1", + ) + .bind(account_id) + .execute(&runtime) + .await + .expect("set explicit zero provider-account ceiling"); + fixture + .coordinator + .emit_fleet_metrics() + .await + .expect("emit over-zero-ceiling quota snapshot"); + assert_eq!(quota_ratio(&handle.render(), "workspaces"), 1.0); + + sqlx::query( + "UPDATE moa.sandbox_provider_accounts \ + SET configured_limits = '{}'::jsonb \ + WHERE provider_account_id = $1 AND generation = 1", + ) + .bind(account_id) + .execute(&runtime) + .await + .expect("remove provider-account ceiling"); + sqlx::query("DELETE FROM moa.sandbox_tenant_capacity_limits WHERE tenant_id = $1") + .bind(tenant_id) + .execute(&runtime) + .await + .expect("remove tenant ceiling"); + fixture + .coordinator + .emit_fleet_metrics() + .await + .expect("emit unbounded quota snapshot"); + assert_eq!(quota_ratio(&handle.render(), "workspaces"), 0.0); + + sqlx::query("DELETE FROM moa.sandbox_capacity_reservations WHERE tenant_id = $1") + .bind(tenant_id) + .execute(&runtime) + .await + .expect("clean isolated capacity reservations"); + sqlx::query("DELETE FROM moa.sandbox_workspaces WHERE tenant_id = $1") + .bind(tenant_id) + .execute(&runtime) + .await + .expect("clean isolated workspaces"); + sqlx::query("DELETE FROM moa.sandbox_provider_accounts WHERE provider_account_id = $1") + .bind(account_id) + .execute(&runtime) + .await + .expect("clean isolated provider account"); + runtime.close().await; + maintenance.close().await; +} + #[tokio::test] #[ignore = "requires distinct runtime and workspace-maintenance Postgres logins"] async fn delayed_checkpoint_reconciliation_atomically_publishes_without_resend_db() { diff --git a/crates/moa-hands/tests/hands_db/sandbox_workspace/purge_db.rs b/crates/moa-hands/tests/hands_db/sandbox_workspace/purge_db.rs index 58c18b339..3f6dc287b 100644 --- a/crates/moa-hands/tests/hands_db/sandbox_workspace/purge_db.rs +++ b/crates/moa-hands/tests/hands_db/sandbox_workspace/purge_db.rs @@ -67,6 +67,12 @@ async fn tenant_purge_fences_access_before_external_delete_and_requires_exact_ab }) .await .expect("persist storage create operation through production repository"); + assert!( + operations + .begin_provider_attempt(tenant_id, operation_id) + .await + .expect("fence the provider create before storage I/O") + ); let storage_resource_id = Uuid::now_v7(); let provider_reference = format!("purge-volume-{storage_resource_id}"); let resources = PostgresWorkspaceStorageResourceRepository::new(runtime.clone()); diff --git a/crates/moa-hands/tests/hands_db/sandbox_workspace/storage_resources_db.rs b/crates/moa-hands/tests/hands_db/sandbox_workspace/storage_resources_db.rs index 0a85a81bd..db4b78c70 100644 --- a/crates/moa-hands/tests/hands_db/sandbox_workspace/storage_resources_db.rs +++ b/crates/moa-hands/tests/hands_db/sandbox_workspace/storage_resources_db.rs @@ -65,6 +65,7 @@ async fn seed_create_operation( ) -> SeededOperation { let workspace_id = SandboxWorkspaceId::new(); let operation_id = WorkspaceOperationId::new(); + let operations = PostgresWorkspaceOperationRepository::new(pool.clone()); PostgresWorkspaceRepository::new(pool.clone()) .create(&CreateWorkspaceRequest { workspace_id, @@ -82,7 +83,7 @@ async fn seed_create_operation( .await .expect("persist logical workspace before provider storage I/O"); let now = Utc::now(); - PostgresWorkspaceOperationRepository::new(pool.clone()) + operations .persist_intent(&WorkspaceOperationIntent { operation_id, tenant_id, @@ -99,6 +100,12 @@ async fn seed_create_operation( }) .await .expect("persist exact create operation before provider storage I/O"); + assert!( + operations + .begin_provider_attempt(tenant_id, operation_id) + .await + .expect("fence the exact create provider attempt") + ); SeededOperation { tenant_id, account_id, diff --git a/crates/moa-migrations/Cargo.toml b/crates/moa-migrations/Cargo.toml index cee252bf2..ce6798fd2 100644 --- a/crates/moa-migrations/Cargo.toml +++ b/crates/moa-migrations/Cargo.toml @@ -11,15 +11,15 @@ refinery = { version = "0.9", features = ["tokio-postgres"] } serde_json.workspace = true sqlx = { workspace = true, features = ["postgres"] } tokio.workspace = true -tokio-postgres = "0.7" +tokio-postgres = { version = "0.7", features = ["with-uuid-1"] } tracing.workspace = true +uuid.workspace = true workspace-hack = { workspace = true } [dev-dependencies] # Migration backfill coverage seeds and reads real UUID-keyed graph rows, so the # test build needs sqlx's UUID mapping on top of the runtime feature set. sqlx = { workspace = true, features = ["postgres", "uuid", "chrono"] } -uuid = { workspace = true } # The lineage acceptance queue derives claim eligibility from its lease pair in a # GENERATED column. Reading those timestamps back is the only way to prove the # derivation is the database's job and not a claimant's. diff --git a/crates/moa-migrations/README.md b/crates/moa-migrations/README.md index 709515c33..70721f8d5 100644 --- a/crates/moa-migrations/README.md +++ b/crates/moa-migrations/README.md @@ -10,7 +10,7 @@ URL; runtime startup validates that the complete history is already present. `migrations/postgres/` is a flat, append-only sequence with exactly one regular file for every version from `V000001` through the current maximum, currently -`V000057`. Filenames +`V000060`. Filenames must match `V__.sql`. `V000001__contiguous_history_epoch.sql` marks the current fresh-install-only @@ -29,8 +29,13 @@ from the migrations that originally created them. V29 remains a no-op marker so the sequence stays contiguous, typed connector origins are V53, the one-way session `paused` to `idle` lifecycle cutover is V54, durable execution-plan compensation is V55, replay-stable ingestion apply outcomes are V56, and -provider-visible hand provisioning operation intents are V57. Any database -that applied an earlier checksum for the rewritten files must be rebuilt; the +provider-visible hand provisioning operation intents are V57. V58 finalizes +durable sandbox-workspace ownership, V59 hard-cuts execution to +bounded long-horizon activations, V60 adds exact sandbox active-compute +capacity. V59 also records each active attempt's step bound, reserves and +settles automatic amendment-planner budget through immutable ledgers, and +normalizes planner-call usage evidence. +Any database that applied an earlier checksum for the rewritten files must be rebuilt; the runner intentionally rejects that divergence before DDL. Do not rewrite a migration after it has shipped in this epoch. Add the next diff --git a/crates/moa-migrations/build.rs b/crates/moa-migrations/build.rs new file mode 100644 index 000000000..614c50b23 --- /dev/null +++ b/crates/moa-migrations/build.rs @@ -0,0 +1,17 @@ +//! Rebuild trigger for the embedded migration set. +//! +//! `refinery::embed_migrations!` reads `migrations/postgres` at compile time, but a macro +//! reading the filesystem is invisible to cargo's change detection: adding or editing a +//! `.sql` file does not touch any `.rs` file, so cargo considers the crate fresh and the +//! binary keeps the migration set it was last compiled with. +//! +//! The failure that produces is expensive to diagnose, because nothing reports a stale +//! embed. Migrations appear to run, the new migration is simply absent, and the first +//! symptom arrives much later as a runtime error from a query naming a column that the +//! migration file on disk plainly creates. +//! +//! Emitting the directory here makes the dependency explicit, so a new or edited migration +//! rebuilds the crate and everything that embeds it. +fn main() { + println!("cargo:rerun-if-changed=migrations/postgres"); +} diff --git a/crates/moa-migrations/migration-ownership.toml b/crates/moa-migrations/migration-ownership.toml index 735ed25ec..dffc5149a 100644 --- a/crates/moa-migrations/migration-ownership.toml +++ b/crates/moa-migrations/migration-ownership.toml @@ -434,6 +434,20 @@ schema = "moa" owner = "moa-execution" readers = ["moa-orchestrator", "moa-analytics"] +[[table]] +name = "execution_amendment_planning_reservation" +schema = "moa" +owner = "moa-execution" +readers = ["moa-orchestrator", "moa-analytics"] +notes = "Immutable budget reservation for one automatic amendment-planner provider call." + +[[table]] +name = "execution_amendment_planning_settlement" +schema = "moa" +owner = "moa-execution" +readers = ["moa-orchestrator", "moa-analytics"] +notes = "Immutable actual usage and cost settlement for one amendment-planning reservation." + [[table]] name = "execution_compensation" schema = "moa" diff --git a/crates/moa-migrations/migrations/postgres/V000059__long_horizon_execution.sql b/crates/moa-migrations/migrations/postgres/V000059__long_horizon_execution.sql index a6df84670..27010a023 100644 --- a/crates/moa-migrations/migrations/postgres/V000059__long_horizon_execution.sql +++ b/crates/moa-migrations/migrations/postgres/V000059__long_horizon_execution.sql @@ -435,6 +435,48 @@ ALTER TABLE moa.execution_run 'completed', 'partial', 'blocked', 'unsupported', 'failed', 'cancelled' )); +-- The expanded long-horizon status vocabulary is also the complete set of +-- states from which a terminal fence may be staged. +ALTER TABLE moa.execution_run + DROP CONSTRAINT execution_run_pending_terminal_check, + ADD CONSTRAINT execution_run_pending_terminal_check CHECK ( + ( + pending_terminal_status IS NULL + AND pending_terminal_reason IS NULL + AND pending_terminal_cause IS NULL + AND pending_terminal_output IS NULL + ) + OR ( + status IN ( + 'awaiting_confirmation', 'queued', 'running', 'waiting_input', + 'waiting_review', 'waiting_signal', 'waiting_timer', 'waiting_external', + 'waiting_replan', 'pause_requested', 'pausing', 'paused', 'compensating' + ) + AND pending_terminal_status IN ( + 'completed','partial','blocked','unsupported','failed','cancelled' + ) + AND pending_terminal_reason IS NOT NULL + AND btrim(pending_terminal_reason) <> '' + AND moa.execution_pending_terminal_payload_is_valid(pending_terminal_cause) + AND moa.execution_terminal_reason_for( + pending_terminal_status, + pending_terminal_cause #> '{terminal_evidence,cause}', + source_kind + ) = pending_terminal_reason + AND ( + ( + pending_terminal_status = 'cancelled' + AND cancellation_reason IS NOT NULL + AND btrim(cancellation_reason) <> '' + ) + OR ( + pending_terminal_status <> 'cancelled' + AND cancellation_reason IS NULL + ) + ) + ) + ); + -- Retire the two terminal-vocabulary values this architecture supersedes. -- -- `scheduler_no_progress` was produced only by the deleted whole-plan scheduler's @@ -651,6 +693,9 @@ ALTER TABLE moa.execution_task ADD COLUMN attempt_started_at TIMESTAMPTZ, ADD COLUMN last_progress_at TIMESTAMPTZ, ADD COLUMN attempt_deadline_at TIMESTAMPTZ, + ADD COLUMN progress_step_bound_seconds INTEGER CHECK ( + progress_step_bound_seconds IS NULL OR progress_step_bound_seconds > 0 + ), ADD COLUMN waiting_since TIMESTAMPTZ, ADD COLUMN ready_at TIMESTAMPTZ, ADD COLUMN active_dispatch_uid UUID, @@ -1078,6 +1123,74 @@ CREATE INDEX execution_amendment_receipt_retention_idx tenant_id, created_at, run_uid, base_plan_revision ); +-- Every automatic amendment-planner call first reserves budget, then records +-- its actual usage in a separate immutable settlement. Together these rows +-- preserve the authorization decision even if mutable run counters are later +-- repaired from the ledger. +CREATE TABLE moa.execution_amendment_planning_reservation ( + reservation_uid UUID PRIMARY KEY, + tenant_id UUID NOT NULL, + contact_id UUID, + contact_scope_id UUID GENERATED ALWAYS AS ( + COALESCE(contact_id, '00000000-0000-0000-0000-000000000000'::UUID) + ) STORED, + run_uid UUID NOT NULL, + base_plan_revision BIGINT NOT NULL CHECK (base_plan_revision >= 1), + call_ordinal SMALLINT NOT NULL CHECK (call_ordinal BETWEEN 0 AND 255), + reserved_cost_microusd BIGINT NOT NULL CHECK (reserved_cost_microusd >= 0), + reserved_tokens BIGINT NOT NULL CHECK (reserved_tokens >= 0), + created_at TIMESTAMPTZ NOT NULL, + CONSTRAINT execution_amendment_planning_reservation_contact_not_nil CHECK ( + contact_id IS NULL + OR contact_id <> '00000000-0000-0000-0000-000000000000'::UUID + ), + CONSTRAINT execution_amendment_planning_reservation_run_fkey + FOREIGN KEY (run_uid, tenant_id, contact_scope_id) + REFERENCES moa.execution_run (run_uid, tenant_id, contact_scope_id), + CONSTRAINT execution_amendment_planning_reservation_logical_key + UNIQUE (run_uid, base_plan_revision, call_ordinal), + CONSTRAINT execution_amendment_planning_reservation_scope_key + UNIQUE (reservation_uid, tenant_id, contact_scope_id, run_uid) +); + +CREATE TABLE moa.execution_amendment_planning_settlement ( + settlement_uid UUID PRIMARY KEY, + reservation_uid UUID NOT NULL UNIQUE, + tenant_id UUID NOT NULL, + contact_id UUID, + contact_scope_id UUID GENERATED ALWAYS AS ( + COALESCE(contact_id, '00000000-0000-0000-0000-000000000000'::UUID) + ) STORED, + run_uid UUID NOT NULL, + actual_cost_microusd BIGINT NOT NULL CHECK (actual_cost_microusd >= 0), + actual_tokens BIGINT NOT NULL CHECK (actual_tokens >= 0), + budget_overrun BOOLEAN NOT NULL, + settled_at TIMESTAMPTZ NOT NULL, + CONSTRAINT execution_amendment_planning_settlement_contact_not_nil CHECK ( + contact_id IS NULL + OR contact_id <> '00000000-0000-0000-0000-000000000000'::UUID + ), + CONSTRAINT execution_amendment_planning_settlement_reservation_fkey + FOREIGN KEY (reservation_uid, tenant_id, contact_scope_id, run_uid) + REFERENCES moa.execution_amendment_planning_reservation ( + reservation_uid, tenant_id, contact_scope_id, run_uid + ) +); + +-- Planner audits carry exact normalized usage even when later compilation or +-- amendment application fails. +ALTER TABLE moa.execution_planner_call_audit + ADD COLUMN input_tokens_uncached BIGINT NOT NULL DEFAULT 0 + CHECK (input_tokens_uncached >= 0), + ADD COLUMN input_tokens_cache_write BIGINT NOT NULL DEFAULT 0 + CHECK (input_tokens_cache_write >= 0), + ADD COLUMN input_tokens_cache_read BIGINT NOT NULL DEFAULT 0 + CHECK (input_tokens_cache_read >= 0), + ADD COLUMN output_tokens BIGINT NOT NULL DEFAULT 0 + CHECK (output_tokens >= 0), + ADD COLUMN cost_microusd BIGINT NOT NULL DEFAULT 0 + CHECK (cost_microusd >= 0); + -- Replan-stop evaluation persists one bounded controller handoff. The exact -- compensation fence consumes this row atomically; no controller activation -- rescans task history to reconstruct the decision. @@ -2814,6 +2927,8 @@ BEGIN 'execution_node_state', 'execution_completion_scan', 'execution_amendment_receipt', + 'execution_amendment_planning_reservation', + 'execution_amendment_planning_settlement', 'execution_replan_stop_intent', 'execution_external_job', 'execution_trigger', @@ -2960,6 +3075,12 @@ FOR EACH ROW EXECUTE FUNCTION moa.reject_tenant_id_change(); CREATE TRIGGER execution_amendment_receipt_immutable_guard BEFORE UPDATE OR DELETE ON moa.execution_amendment_receipt FOR EACH ROW EXECUTE FUNCTION moa.reject_execution_immutable_payload(); +CREATE TRIGGER execution_amendment_planning_reservation_immutable_guard +BEFORE UPDATE OR DELETE ON moa.execution_amendment_planning_reservation +FOR EACH ROW EXECUTE FUNCTION moa.reject_execution_immutable_payload(); +CREATE TRIGGER execution_amendment_planning_settlement_immutable_guard +BEFORE UPDATE OR DELETE ON moa.execution_amendment_planning_settlement +FOR EACH ROW EXECUTE FUNCTION moa.reject_execution_immutable_payload(); CREATE TRIGGER execution_replan_stop_intent_immutable_guard BEFORE UPDATE OR DELETE ON moa.execution_replan_stop_intent FOR EACH ROW EXECUTE FUNCTION moa.reject_execution_replan_stop_intent_mutation(); @@ -3000,6 +3121,8 @@ FOR EACH ROW EXECUTE FUNCTION moa.reject_tenant_id_change(); SELECT moa.apply_tenant_rls('moa.execution_node_state'); SELECT moa.apply_tenant_rls('moa.execution_completion_scan'); SELECT moa.apply_tenant_rls('moa.execution_amendment_receipt'); +SELECT moa.apply_contact_rls('moa.execution_amendment_planning_reservation'::REGCLASS); +SELECT moa.apply_contact_rls('moa.execution_amendment_planning_settlement'::REGCLASS); SELECT moa.apply_tenant_rls('moa.execution_replan_stop_intent'); SELECT moa.apply_tenant_rls('moa.execution_trigger'); SELECT moa.apply_tenant_rls('moa.execution_dispatch_outbox'); @@ -3052,6 +3175,8 @@ BEGIN 'execution_node_state', 'execution_completion_scan', 'execution_amendment_receipt', + 'execution_amendment_planning_reservation', + 'execution_amendment_planning_settlement', 'execution_replan_stop_intent', 'execution_trigger', 'execution_dispatch_outbox', @@ -3085,30 +3210,34 @@ BEGIN END $execution_long_horizon_purge_fences$; --- Delete children before execution_compensation/task/run. Shift through a +-- Delete execution-owned children before planner audits, tasks, and runs. Shift through a -- remote range to preserve the catalog's unique stage ordering. UPDATE moa.tenant_purge_catalog SET stage_order = stage_order + 1000 WHERE stage_order >= ( SELECT stage_order FROM moa.tenant_purge_catalog - WHERE stage_name = 'moa.execution_compensation' + WHERE stage_name = 'moa.execution_planner_call_audit' ); UPDATE moa.tenant_purge_catalog -SET stage_order = stage_order - 985 +SET stage_order = stage_order - 983 WHERE stage_order >= 1000; INSERT INTO moa.tenant_purge_catalog ( stage_order, stage_name, table_schema, table_name, scope_mode, action_mode ) -SELECT compensation.stage_order - execution_stage.stage_offset, +SELECT planner_audit.stage_order - execution_stage.stage_offset, execution_stage.stage_name, 'moa', execution_stage.table_name, 'tenant_id', 'delete' -FROM moa.tenant_purge_catalog AS compensation +FROM moa.tenant_purge_catalog AS planner_audit CROSS JOIN (VALUES + (17, 'moa.execution_amendment_planning_settlement', + 'execution_amendment_planning_settlement'), + (16, 'moa.execution_amendment_planning_reservation', + 'execution_amendment_planning_reservation'), (15, 'moa.execution_replan_stop_intent', 'execution_replan_stop_intent'), (14, 'moa.execution_amendment_receipt', 'execution_amendment_receipt'), (13, 'moa.execution_task_checkpoint', 'execution_task_checkpoint'), @@ -3127,10 +3256,10 @@ CROSS JOIN (VALUES 'execution_terminal_archive_segment'), (1, 'moa.execution_terminal_archive', 'execution_terminal_archive') ) AS execution_stage(stage_offset, stage_name, table_name) -WHERE compensation.stage_name = 'moa.execution_compensation'; +WHERE planner_audit.stage_name = 'moa.execution_planner_call_audit'; COMMENT ON TABLE moa.tenant_purge_catalog IS - 'Closed 157-table tenant-offboarding residue surface. Fleet capacity-bucket rows, sandbox provider accounts, and inventory findings are global maintenance authority; the two nullable-scope simulator certification authority tables are also intentionally global and absent.'; + 'Closed 159-table tenant-offboarding residue surface. Fleet capacity-bucket rows, sandbox provider accounts, and inventory findings are global maintenance authority; the two nullable-scope simulator certification authority tables are also intentionally global and absent.'; DO $execution_long_horizon_purge_function$ DECLARE @@ -3144,8 +3273,8 @@ BEGIN RAISE EXCEPTION 'unexpected V58 tenant purge function definition' USING ERRCODE = '55000'; END IF; - replacement := replace(predecessor, 'catalog_count <> 142', 'catalog_count <> 157'); - replacement := replace(replacement, 'exactly 142 tables', 'exactly 157 tables'); + replacement := replace(predecessor, 'catalog_count <> 142', 'catalog_count <> 159'); + replacement := replace(replacement, 'exactly 142 tables', 'exactly 159 tables'); EXECUTE replacement; END $execution_long_horizon_purge_function$; diff --git a/crates/moa-migrations/migrations/postgres/V000060__sandbox_active_compute_capacity.sql b/crates/moa-migrations/migrations/postgres/V000060__sandbox_active_compute_capacity.sql index 64d17458b..4834add06 100644 --- a/crates/moa-migrations/migrations/postgres/V000060__sandbox_active_compute_capacity.sql +++ b/crates/moa-migrations/migrations/postgres/V000060__sandbox_active_compute_capacity.sql @@ -326,7 +326,7 @@ FROM moa.tenant_purge_catalog AS checkpoint_stage WHERE checkpoint_stage.stage_name = 'moa.sandbox_workspace_checkpoints'; COMMENT ON TABLE moa.tenant_purge_catalog IS - 'Closed 158-table tenant-offboarding residue surface. Fleet capacity-bucket rows, sandbox provider accounts, sandbox provider inventory claims, and inventory findings are global maintenance authority; the two nullable-scope simulator certification authority tables are also intentionally global and absent.'; + 'Closed 160-table tenant-offboarding residue surface. Fleet capacity-bucket rows, sandbox provider accounts, sandbox provider inventory claims, and inventory findings are global maintenance authority; the two nullable-scope simulator certification authority tables are also intentionally global and absent.'; DO $sandbox_hand_release_receipt_purge_function$ DECLARE @@ -335,13 +335,13 @@ DECLARE BEGIN SELECT pg_get_functiondef('moa.run_tenant_purge_batch(uuid,text)'::REGPROCEDURE) INTO predecessor; - IF predecessor NOT LIKE '%catalog_count <> 157%' - OR predecessor NOT LIKE '%exactly 157 tables%' THEN + IF predecessor NOT LIKE '%catalog_count <> 159%' + OR predecessor NOT LIKE '%exactly 159 tables%' THEN RAISE EXCEPTION 'unexpected V59 tenant purge function definition' USING ERRCODE = '55000'; END IF; - replacement := replace(predecessor, 'catalog_count <> 157', 'catalog_count <> 158'); - replacement := replace(replacement, 'exactly 157 tables', 'exactly 158 tables'); + replacement := replace(predecessor, 'catalog_count <> 159', 'catalog_count <> 160'); + replacement := replace(replacement, 'exactly 159 tables', 'exactly 160 tables'); EXECUTE replacement; END $sandbox_hand_release_receipt_purge_function$; diff --git a/crates/moa-migrations/src/lib.rs b/crates/moa-migrations/src/lib.rs index 526514682..bf34b0db1 100644 --- a/crates/moa-migrations/src/lib.rs +++ b/crates/moa-migrations/src/lib.rs @@ -310,7 +310,7 @@ async fn rewrite_archived_session_statuses(client: &mut Client) -> Result { .context("begin archived session status rewrite")?; let rows = tx .query( - "SELECT session_id::TEXT AS session_id, payload, content_digest \ + "SELECT session_id, payload, content_digest \ FROM public.session_event_archives \ ORDER BY session_id FOR UPDATE", &[], @@ -320,7 +320,7 @@ async fn rewrite_archived_session_statuses(client: &mut Client) -> Result { let mut rewrites = Vec::new(); for row in rows { - let session_id: String = row.get("session_id"); + let session_id: uuid::Uuid = row.get("session_id"); let payload: Vec = row.get("payload"); let stored_digest: Vec = row.get("content_digest"); let actual_digest = blake3::hash(&payload); @@ -361,7 +361,7 @@ async fn rewrite_archived_session_statuses(client: &mut Client) -> Result { .execute( "UPDATE public.session_event_archives \ SET payload = $2, content_digest = $3 \ - WHERE session_id = $1::UUID", + WHERE session_id = $1", &[session_id, payload, digest], ) .await @@ -382,14 +382,14 @@ async fn rewrite_archived_session_statuses(client: &mut Client) -> Result { let verification_rows = tx .query( - "SELECT session_id::TEXT AS session_id, payload, content_digest \ + "SELECT session_id, payload, content_digest \ FROM public.session_event_archives ORDER BY session_id", &[], ) .await .context("verify rewritten session event archives")?; for row in verification_rows { - let session_id: String = row.get("session_id"); + let session_id: uuid::Uuid = row.get("session_id"); let payload: Vec = row.get("payload"); let stored_digest: Vec = row.get("content_digest"); if stored_digest.as_slice() != blake3::hash(&payload).as_bytes() { diff --git a/crates/moa-migrations/tests/run_idempotency_db/connectors.rs b/crates/moa-migrations/tests/run_idempotency_db/connectors.rs index 0acd1c338..6e20dd023 100644 --- a/crates/moa-migrations/tests/run_idempotency_db/connectors.rs +++ b/crates/moa-migrations/tests/run_idempotency_db/connectors.rs @@ -1067,6 +1067,10 @@ async fn tenant_connector_use_grants_enforce_same_tenant_rls_and_restrict_deleti ) .fetch_all(&target) .await?; + let purge_catalog_count: i64 = + sqlx::query_scalar("SELECT count(*) FROM moa.tenant_purge_catalog") + .fetch_one(&target) + .await?; let purge_definition: String = sqlx::query_scalar( "SELECT pg_get_functiondef('moa.run_tenant_purge_batch(uuid,text)'::REGPROCEDURE)", ) @@ -1185,6 +1189,7 @@ async fn tenant_connector_use_grants_enforce_same_tenant_rls_and_restrict_deleti policy_names, foreign_keys, purge_stages, + purge_catalog_count, purge_definition, visible_count, neighbour_visible, @@ -1215,6 +1220,7 @@ async fn tenant_connector_use_grants_enforce_same_tenant_rls_and_restrict_deleti policy_names, foreign_keys, purge_stages, + purge_catalog_count, purge_definition, visible_count, neighbour_visible, @@ -1342,19 +1348,23 @@ async fn tenant_connector_use_grants_enforce_same_tenant_rls_and_restrict_deleti "every registry parent must use NO ACTION so inverse rows cannot disappear by cascade" ); assert_eq!( - purge_stages, + purge_stages + .iter() + .map(|(_, stage_name)| stage_name.as_str()) + .collect::>(), vec![ - (26, "moa.connector_action_invocations".to_string()), - (27, "moa.connector_action_bindings".to_string()), - (28, "moa.connector_connection_use_grants".to_string()), - (29, "moa.connector_connections".to_string()), - (57, "public.contacts".to_string()), - (66, "public.agents".to_string()), - (69, "public.users".to_string()), - ] + "moa.connector_action_invocations", + "moa.connector_action_bindings", + "moa.connector_connection_use_grants", + "moa.connector_connections", + "public.contacts", + "public.agents", + "public.users", + ], + "connector children and inverse grants must purge before every referenced parent" ); - assert!(purge_definition.contains("catalog_count <> 131")); - assert!(purge_definition.contains("exactly 131 tables")); + assert!(purge_definition.contains(&format!("catalog_count <> {purge_catalog_count}"))); + assert!(purge_definition.contains(&format!("exactly {purge_catalog_count} tables"))); assert_eq!(visible_count, 3); assert_eq!(neighbour_visible, 0); assert_eq!(cross_rls_fact.0.as_deref(), Some("42501")); @@ -1425,11 +1435,12 @@ async fn knowledge_connection_parent_constraint_and_replay_ledgers_are_strict_db .bind(tenant_id) .execute(&target) .await?; - let unknown_error = + let unknown_error = format!( + "{:#}", apply_through_migration(&target_url, "knowledge_connection_parent_constraint") .await .expect_err("unknown providers must fail the closed V52 catch-up") - .to_string(); + ); sqlx::query("DELETE FROM moa.knowledge_connections WHERE connection_uid = $1") .bind(unknown_connection) .execute(&target) @@ -1468,11 +1479,12 @@ async fn knowledge_connection_parent_constraint_and_replay_ledgers_are_strict_db .bind(tenant_id) .execute(&target) .await?; - let incompatible_error = + let incompatible_error = format!( + "{:#}", apply_through_migration(&target_url, "knowledge_connection_parent_constraint") .await .expect_err("an incompatible pre-existing parent must fail closed") - .to_string(); + ); sqlx::query( "UPDATE moa.connector_connections SET built_in_key = 'knowledge:merge' \ WHERE connection_uid = $1", @@ -2033,11 +2045,22 @@ async fn knowledge_connection_parent_constraint_and_replay_ledgers_are_strict_db missing_scope_visible, ) = outcome.expect("V52 migration assertions should complete"); - assert!(unknown_error.contains("no closed connector parent mapping")); - assert!(incompatible_error.contains("incompatible connector parent")); + assert!( + unknown_error.contains("no closed connector parent mapping"), + "migration error chain lost the closed-mapping cause: {unknown_error}" + ); + assert!( + incompatible_error.contains("incompatible connector parent"), + "migration error chain lost the incompatible-parent cause: {incompatible_error}" + ); assert_eq!( first, - expected_migration_labels_from("knowledge_connection_parent_constraint") + vec![ + expected_migration_labels_from("knowledge_connection_parent_constraint") + .into_iter() + .next() + .expect("V52 label must be embedded") + ] ); assert!(second.is_empty(), "V52 replay must be a no-op: {second:?}"); assert_eq!( @@ -2204,7 +2227,7 @@ async fn knowledge_connection_parent_constraint_and_replay_ledgers_are_strict_db true, true, true, - false, + true, false, ), ( @@ -2251,9 +2274,9 @@ async fn knowledge_connection_parent_constraint_and_replay_ledgers_are_strict_db (31, "moa.connector_connections".to_string()), ] ); - assert_eq!(purge_count, 134); - assert!(purge_definition.contains("catalog_count <> 142")); - assert!(purge_definition.contains("exactly 142 tables")); + assert_eq!(purge_count, 133); + assert!(purge_definition.contains(&format!("catalog_count <> {purge_count}"))); + assert!(purge_definition.contains(&format!("exactly {purge_count} tables"))); assert_eq!(visible, (1, 5)); assert_eq!(neighbour_visible, 0); assert_eq!(cross_tenant_fact.0.as_deref(), Some("42501")); diff --git a/crates/moa-migrations/tests/run_idempotency_db/execution_and_security_catalog.rs b/crates/moa-migrations/tests/run_idempotency_db/execution_and_security_catalog.rs index 8b9e0cda6..2b5cbb506 100644 --- a/crates/moa-migrations/tests/run_idempotency_db/execution_and_security_catalog.rs +++ b/crates/moa-migrations/tests/run_idempotency_db/execution_and_security_catalog.rs @@ -1706,7 +1706,7 @@ async fn long_horizon_execution_cutover_rejects_live_runs_and_installs_fenced_ca let catalog_shape: (bool, bool, bool, bool, bool, bool) = sqlx::query_as( r#" SELECT - (SELECT count(*) = 143 + (SELECT count(*) = 169 FROM information_schema.columns WHERE table_schema = 'moa' AND ( @@ -1727,9 +1727,27 @@ async fn long_horizon_execution_cutover_rejects_live_runs_and_installs_fenced_ca 'terminal_archive_hash', 'terminal_details_archived_at' )) OR + (table_name = 'execution_planner_call_audit' AND column_name IN ( + 'input_tokens_uncached', 'input_tokens_cache_write', + 'input_tokens_cache_read', 'output_tokens', 'cost_microusd' + )) + OR + (table_name = 'execution_amendment_planning_reservation' AND column_name IN ( + 'reservation_uid', 'tenant_id', 'contact_id', 'contact_scope_id', + 'run_uid', 'base_plan_revision', 'call_ordinal', + 'reserved_cost_microusd', 'reserved_tokens', 'created_at' + )) + OR + (table_name = 'execution_amendment_planning_settlement' AND column_name IN ( + 'settlement_uid', 'reservation_uid', 'tenant_id', 'contact_id', + 'contact_scope_id', 'run_uid', 'actual_cost_microusd', + 'actual_tokens', 'budget_overrun', 'settled_at' + )) + OR (table_name = 'execution_task' AND column_name IN ( 'attempt_generation', 'attempt_state', 'attempt_started_at', - 'last_progress_at', 'attempt_deadline_at', 'waiting_since', + 'last_progress_at', 'progress_step_bound_seconds', + 'attempt_deadline_at', 'waiting_since', 'ready_at', 'active_dispatch_uid', 'dispatch_sequence', 'external_job_uid', 'failure_fingerprint' )) @@ -1811,8 +1829,28 @@ async fn long_horizon_execution_cutover_rejects_live_runs_and_installs_fenced_ca 'next_run_at', 'scheduled_generation', 'claim_owner', 'claimed_generation', 'claim_expires_at' )) + ) + AND EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conrelid = 'moa.execution_task'::REGCLASS + AND contype = 'c' + AND pg_get_constraintdef(oid) + LIKE '%progress_step_bound_seconds IS NULL%progress_step_bound_seconds > 0%' + ) + AND EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conrelid = 'moa.execution_run'::REGCLASS + AND conname = 'execution_run_pending_terminal_check' + AND convalidated + AND pg_get_constraintdef(oid) LIKE '%waiting_signal%' + AND pg_get_constraintdef(oid) LIKE '%waiting_timer%' + AND pg_get_constraintdef(oid) LIKE '%waiting_external%' + AND pg_get_constraintdef(oid) LIKE '%pause_requested%' + AND pg_get_constraintdef(oid) LIKE '%compensating%' )), - (SELECT count(*) = 15 + (SELECT count(*) = 17 FROM pg_class relation JOIN pg_namespace namespace ON namespace.oid = relation.relnamespace WHERE namespace.nspname = 'moa' @@ -1825,11 +1863,13 @@ async fn long_horizon_execution_cutover_rejects_live_runs_and_installs_fenced_ca 'execution_external_job_callback_receipt', 'execution_completion_scan', 'execution_amendment_receipt', + 'execution_amendment_planning_reservation', + 'execution_amendment_planning_settlement', 'execution_replan_stop_intent', 'execution_task_checkpoint', 'execution_terminal_archive', 'execution_terminal_archive_segment' )), - (SELECT count(*) = 15 AND bool_and(relrowsecurity AND relforcerowsecurity) + (SELECT count(*) = 17 AND bool_and(relrowsecurity AND relforcerowsecurity) FROM pg_class relation JOIN pg_namespace namespace ON namespace.oid = relation.relnamespace WHERE namespace.nspname = 'moa' @@ -1841,11 +1881,13 @@ async fn long_horizon_execution_cutover_rejects_live_runs_and_installs_fenced_ca 'execution_external_job_callback_receipt', 'execution_completion_scan', 'execution_amendment_receipt', + 'execution_amendment_planning_reservation', + 'execution_amendment_planning_settlement', 'execution_replan_stop_intent', 'execution_task_checkpoint', 'execution_terminal_archive', 'execution_terminal_archive_segment' )), - (SELECT count(*) = 44 + (SELECT count(*) = 49 FROM pg_indexes WHERE schemaname = 'moa' AND indexname IN ( @@ -1893,6 +1935,11 @@ async fn long_horizon_execution_cutover_rejects_live_runs_and_installs_fenced_ca ,'execution_run_overdue_deadline_idx' ,'execution_task_active_attempt_started_idx' ,'execution_compensation_active_attempt_started_idx' + ,'execution_amendment_planning_reservation_pkey' + ,'execution_amendment_planning_reservation_logical_key' + ,'execution_amendment_planning_reservation_scope_key' + ,'execution_amendment_planning_settlement_pkey' + ,'execution_amendment_planning_settlement_reservation_uid_key' )), moa.execution_admitted_identity_is_valid(admitted_identity, tenant_id) AND activation_state = 'terminal' diff --git a/crates/moa-migrations/tests/run_idempotency_db/execution_compensation.rs b/crates/moa-migrations/tests/run_idempotency_db/execution_compensation.rs index 8a1504e9c..b9afc43b6 100644 --- a/crates/moa-migrations/tests/run_idempotency_db/execution_compensation.rs +++ b/crates/moa-migrations/tests/run_idempotency_db/execution_compensation.rs @@ -58,10 +58,7 @@ async fn execution_compensation_schema_and_transitions_are_strict_db() { .await?; let version_neutral_shapes: (bool, bool, bool) = sqlx::query_as( "SELECT moa.execution_plan_definition_is_valid(initial_plan -> 'definition'), \ - moa.execution_plan_definition_is_valid( \ - (initial_plan -> 'definition') \ - || '{\"schema_version\":2}'::JSONB \ - ), \ + moa.execution_plan_definition_is_current(initial_plan -> 'definition'), \ NOT capability_catalog ? 'schema_version' \ FROM moa.execution_run WHERE run_uid = $1", ) @@ -125,7 +122,7 @@ async fn execution_compensation_schema_and_transitions_are_strict_db() { ); assert!(compensation_schema); assert!(action_review_owner_schema); - assert_eq!(version_neutral_shapes, (true, false, true)); + assert_eq!(version_neutral_shapes, (false, true, true)); assert_eq!(compensation_reason.as_deref(), Some("compensation_failed")); } diff --git a/crates/moa-migrations/tests/run_idempotency_db/protocol.rs b/crates/moa-migrations/tests/run_idempotency_db/protocol.rs index c6dcccf6a..106c39e22 100644 --- a/crates/moa-migrations/tests/run_idempotency_db/protocol.rs +++ b/crates/moa-migrations/tests/run_idempotency_db/protocol.rs @@ -115,7 +115,7 @@ async fn final_schema_omits_retired_relations_columns_and_indexes_db() { #[tokio::test] #[ignore = "requires a superuser-capable local Postgres via MOA_DATABASE_URL"] async fn migration_protocol_pristine_apply_is_exact_and_idempotent_db() { - // Pins: a pristine database applies the exact contiguous V1..V58 epoch, + // Pins: a pristine database applies the exact contiguous current epoch, // validates as complete, and reports no work on a second public-runner call. let admin_url = test_database_url(); let db_name = unique_db_name(); @@ -166,10 +166,12 @@ async fn migration_protocol_pristine_apply_is_exact_and_idempotent_db() { let (first, second, history, removed_token_vault_tables_absent) = outcome.expect("central migration runs should complete on a fresh database"); let expected_labels = expected_migration_labels(); + let current_version = current_migration_version(); + assert_eq!(current_version, 60, "the current epoch must end at V000060"); assert_eq!( - expected_labels.len(), - 58, - "the epoch must contain exactly 58 migrations" + i32::try_from(expected_labels.len()).expect("migration count must fit i32"), + current_version, + "the epoch must contain exactly one migration for every version" ); assert_eq!( first, expected_labels, @@ -180,8 +182,8 @@ async fn migration_protocol_pristine_apply_is_exact_and_idempotent_db() { .iter() .map(|(version, _)| *version) .collect::>(), - (1..=58).collect::>(), - "refinery history must be exactly contiguous from V1 through V58" + (1..=current_version).collect::>(), + "refinery history must be exactly contiguous through the current maximum" ); assert!( second.is_empty(), @@ -416,7 +418,7 @@ async fn migration_protocol_parallel_fresh_databases_retry_shared_role_catalog_r #[ignore = "requires a superuser-capable local Postgres via MOA_DATABASE_URL"] async fn migration_protocol_exact_prefix_resumes_db() { // Pins: a database with an exact new-epoch prefix resumes at the next - // semantic migration and becomes a complete V1..V58 history. + // semantic migration and becomes a complete current history. let admin_url = test_database_url(); let db_name = unique_db_name(); let admin = PgPoolOptions::new() @@ -472,6 +474,7 @@ async fn migration_protocol_exact_prefix_resumes_db() { let (prefix, resumed, second, partial_error, versions) = outcome.expect("exact contiguous prefix should resume successfully"); let expected = expected_migration_labels(); + let current_version = current_migration_version(); let prefix_len = usize::try_from( migration_version("execution_analytics").expect("execution analytics must be embedded"), ) @@ -483,10 +486,12 @@ async fn migration_protocol_exact_prefix_resumes_db() { "completed history must not reapply: {second:?}" ); assert!( - partial_error.contains("incomplete: found 28 of 57 expected rows"), + partial_error.contains(&format!( + "incomplete: found {prefix_len} of {current_version} expected rows" + )), "complete-history validation must distinguish a valid prefix: {partial_error}" ); - assert_eq!(versions, (1..=58).collect::>()); + assert_eq!(versions, (1..=current_version).collect::>()); } #[tokio::test] diff --git a/crates/moa-migrations/tests/run_idempotency_db/session_status.rs b/crates/moa-migrations/tests/run_idempotency_db/session_status.rs index 1dbf912d1..0a0edf242 100644 --- a/crates/moa-migrations/tests/run_idempotency_db/session_status.rs +++ b/crates/moa-migrations/tests/run_idempotency_db/session_status.rs @@ -35,6 +35,8 @@ async fn session_status_idle_rewrites_live_and_archived_state_idempotently_db() let live_session_id = uuid::Uuid::new_v4(); let archived_session_id = uuid::Uuid::new_v4(); for (session_id, archived) in [(live_session_id, false), (archived_session_id, true)] { + let label = format!("session-status-{session_id}"); + let mut tx = target.begin().await?; sqlx::query( "INSERT INTO sessions \ (id, tenant_id, storage_partition_id, user_id, status, model, \ @@ -45,8 +47,23 @@ async fn session_status_idle_rewrites_live_and_archived_state_idempotently_db() .bind(session_id) .bind(tenant_id) .bind(archived) - .execute(&target) + .execute(&mut *tx) .await?; + sqlx::query( + "INSERT INTO session_agent_context \ + (session_id, tenant_id, storage_partition_id, user_id, \ + agent_definition_ref, agent_revision_uid, policy_hash, display_name, \ + policy_snapshot) \ + VALUES ($1, $2, $2::TEXT, $3, 'agent://system-default', \ + '00000000-0000-4000-8000-000000000a02', \ + 'session-status-test-policy', 'Session Status Test Agent', '{}'::JSONB)", + ) + .bind(session_id) + .bind(tenant_id) + .bind(&label) + .execute(&mut *tx) + .await?; + tx.commit().await?; } for (sequence_num, from, to) in [(0_i64, "running", "paused"), (1, "paused", "running")] { @@ -210,15 +227,7 @@ async fn session_status_idle_rewrites_live_and_archived_state_idempotently_db() old_live_values, cutover_receipts, ) = outcome.expect("session status migration should complete"); - assert_eq!( - first, - vec![ - expected_migration_labels() - .last() - .expect("V54 label must exist") - .clone() - ] - ); + assert_eq!(first, expected_migration_labels_from("session_status_idle")); assert!(second.is_empty(), "second migration run must apply no SQL"); assert_eq!(second_archive, (first_bytes, first_digest)); assert_eq!(old_live_values, 0); diff --git a/crates/moa-migrations/tests/run_idempotency_db/support.rs b/crates/moa-migrations/tests/run_idempotency_db/support.rs index c670ff499..0c95a0d24 100644 --- a/crates/moa-migrations/tests/run_idempotency_db/support.rs +++ b/crates/moa-migrations/tests/run_idempotency_db/support.rs @@ -1,5 +1,7 @@ //! Shared fixtures and catalog helpers for the migration database test lane. +use anyhow::Context; + pub(super) use sqlx::postgres::PgPoolOptions; pub(super) use sqlx::{Executor, PgPool}; @@ -352,6 +354,16 @@ pub(super) fn expected_migration_labels() -> Vec { migrations.into_iter().map(|(_, label)| label).collect() } +/// Returns the current maximum embedded migration version. +pub(super) fn current_migration_version() -> i32 { + embedded_for_cutover_proof::migrations::runner() + .get_migrations() + .iter() + .map(refinery::Migration::version) + .max() + .expect("the central migration epoch must not be empty") +} + /// Returns the embedded migration labels from one semantic migration onward. /// /// A scenario that applies through the preceding migration and then runs the @@ -389,7 +401,7 @@ pub(super) fn migration_version(migration_name: &str) -> Result Result, Box> { +) -> anyhow::Result> { let version = migration_version(migration_name)?; let mut migrations = embedded_for_cutover_proof::migrations::runner() .get_migrations() @@ -410,7 +422,8 @@ pub(super) async fn apply_through_migration( let result = runner .set_target(refinery::Target::Version(version)) .run_async(&mut client) - .await; + .await + .with_context(|| format!("apply migrations through {migration_name}")); drop(client); connection_task.await??; let unlock_result = sqlx::query("SELECT pg_advisory_unlock($1)") diff --git a/crates/moa-migrations/tests/run_idempotency_db/tenant_purge.rs b/crates/moa-migrations/tests/run_idempotency_db/tenant_purge.rs index e72dc5f43..e238451ea 100644 --- a/crates/moa-migrations/tests/run_idempotency_db/tenant_purge.rs +++ b/crates/moa-migrations/tests/run_idempotency_db/tenant_purge.rs @@ -709,7 +709,7 @@ async fn seed_tenant_purge_activated_release_chain( #[tokio::test] #[ignore = "requires a superuser-capable local Postgres via MOA_DATABASE_URL"] async fn bounded_tenant_purge_final_schema_executes_bounded_batches_db() { - // Pins: a pristine final schema persists exactly 158 purge stages, installs + // Pins: a pristine final schema persists exactly 160 purge stages, installs // statement fences, and advances a real purge in fixed-size batches. let admin_url = test_database_url(); let db_name = unique_db_name(); @@ -833,7 +833,7 @@ async fn bounded_tenant_purge_final_schema_executes_bounded_batches_db() { ) .fetch_one(&target) .await?; - let legacy_release_cleanup: (bool, i64, i64, bool, String) = sqlx::query_as( + let legacy_release_cleanup: (bool, i64, bool, String) = sqlx::query_as( r#" WITH legacy_tables(table_name) AS ( VALUES @@ -860,21 +860,7 @@ async fn bounded_tenant_purge_final_schema_executes_bounded_batches_db() { 'artifact_release_partition_purge' ) ), - ( - SELECT count(*) - FROM legacy_tables AS legacy - WHERE has_table_privilege( - 'moa_artifact_releaser', - format('moa.%I', legacy.table_name), - 'SELECT' - ) - OR has_table_privilege( - 'moa_artifact_releaser', - format('moa.%I', legacy.table_name), - 'DELETE' - ) - ), - has_schema_privilege('moa_artifact_releaser', 'moa', 'USAGE'), + to_regrole('moa_artifact_releaser') IS NULL, pg_get_functiondef('moa.artifact_activation_audit_guard()'::REGPROCEDURE) "#, ) @@ -1308,7 +1294,7 @@ async fn bounded_tenant_purge_final_schema_executes_bounded_batches_db() { true, ) ); - assert_eq!(catalog_count, 158); + assert_eq!(catalog_count, 160); assert_eq!( trigger_kinds, vec![ @@ -1341,24 +1327,20 @@ async fn bounded_tenant_purge_final_schema_executes_bounded_batches_db() { legacy_release_cleanup.1, 0, "all legacy release read/delete policies must be absent" ); - assert_eq!( - legacy_release_cleanup.2, 0, - "the inert releaser role must retain no release-table privileges" - ); assert!( - !legacy_release_cleanup.3, - "the inert releaser role must retain no moa schema usage" + legacy_release_cleanup.2, + "the retired artifact releaser role must be absent" ); assert!( legacy_release_cleanup - .4 + .3 .contains("moa.tenant_purge_bypass_valid") - && !legacy_release_cleanup.4.contains("moa_artifact_releaser") + && !legacy_release_cleanup.3.contains("moa_artifact_releaser") && !legacy_release_cleanup - .4 + .3 .contains("artifact_release_purge_partition"), "the audit guard must admit deletion only through the validated bounded purge: {}", - legacy_release_cleanup.4 + legacy_release_cleanup.3 ); assert_eq!( audit_guard_contract, @@ -1395,7 +1377,7 @@ async fn bounded_tenant_purge_final_schema_executes_bounded_batches_db() { ); assert_eq!( bounded_facts, - (0, 0, 1, 1001, 2012, 0, 1), + (0, 0, 1, 1001, 2014, 0, 1), "the target release-policy set must be gone while the neighboring policy survives" ); assert_eq!( @@ -1419,8 +1401,8 @@ async fn bounded_tenant_purge_final_schema_executes_bounded_batches_db() { #[tokio::test] #[ignore = "requires a superuser-capable local Postgres via MOA_DATABASE_URL"] async fn sandbox_workspace_purge_catalog_db() { - // Pins: a full clean apply lands a 158-stage catalog (V58 took it to 142, V59 - // to 157, V60 to 158) with all workspace rows fenced and checkpoint + // Pins: a full clean apply lands a 160-stage catalog (V58 took it to 142, V59 + // to 159, and V60 to 160) with all workspace rows fenced and checkpoint // head/parent ordering encoded in the bounded owner-only purge function. let admin_url = test_database_url(); let db_name = unique_db_name(); @@ -1463,6 +1445,18 @@ async fn sandbox_workspace_purge_catalog_db() { ]) .fetch_all(&target) .await?; + let planning_stages: Vec = sqlx::query_scalar( + "SELECT stage_name FROM moa.tenant_purge_catalog \ + WHERE stage_name = ANY($1) ORDER BY stage_order", + ) + .bind(vec![ + "moa.execution_amendment_planning_settlement", + "moa.execution_amendment_planning_reservation", + "moa.execution_planner_call_audit", + "moa.execution_run", + ]) + .fetch_all(&target) + .await?; let fence_count: i64 = sqlx::query_scalar( "SELECT count(*) FROM information_schema.triggers \ WHERE event_object_schema = 'moa' \ @@ -1480,6 +1474,8 @@ async fn sandbox_workspace_purge_catalog_db() { "sandbox_workspace_grants", "sandbox_storage_resources", "sandbox_capacity_reservations", + "execution_amendment_planning_reservation", + "execution_amendment_planning_settlement", ]) .fetch_one(&target) .await?; @@ -1535,6 +1531,7 @@ async fn sandbox_workspace_purge_catalog_db() { second, catalog_count, workspace_stages, + planning_stages, fence_count, global_catalog_count, purge_definition, @@ -1551,6 +1548,7 @@ async fn sandbox_workspace_purge_catalog_db() { second, catalog_count, workspace_stages, + planning_stages, fence_count, global_catalog_count, purge_definition, @@ -1559,7 +1557,17 @@ async fn sandbox_workspace_purge_catalog_db() { ) = outcome.expect("sandbox workspace purge schema assertions should complete"); assert_eq!(first, expected_migration_labels()); assert!(second.is_empty(), "V58 must not reapply: {second:?}"); - assert_eq!(catalog_count, 158); + assert_eq!(catalog_count, 160); + assert_eq!( + planning_stages, + vec![ + "moa.execution_amendment_planning_settlement", + "moa.execution_amendment_planning_reservation", + "moa.execution_planner_call_audit", + "moa.execution_run", + ], + "settlements must purge before reservations and both before their audit/run parents" + ); assert_eq!( workspace_stages, vec![ @@ -1594,10 +1602,10 @@ async fn sandbox_workspace_purge_catalog_db() { ), ] ); - assert_eq!(fence_count, 14); + assert_eq!(fence_count, 18); assert_eq!(global_catalog_count, 0); - assert!(purge_definition.contains("catalog_count <> 158")); - assert!(purge_definition.contains("exactly 158 tables")); + assert!(purge_definition.contains("catalog_count <> 160")); + assert!(purge_definition.contains("exactly 160 tables")); assert!(purge_definition.contains("SET current_checkpoint_id = NULL")); assert!(purge_definition.contains("ORDER BY target.generation DESC")); assert_eq!(checkpoint_columns.len(), 7); diff --git a/crates/moa-observability/src/runtime_metrics.rs b/crates/moa-observability/src/runtime_metrics.rs index 35e54b567..fea81f82e 100644 --- a/crates/moa-observability/src/runtime_metrics.rs +++ b/crates/moa-observability/src/runtime_metrics.rs @@ -1346,11 +1346,12 @@ pub fn record_sandbox_workspace_quota_decision( .increment(1); } -/// Sets fleet quota utilization for one capacity dimension. +/// Sets the highest enforced-scope quota utilization for one capacity dimension. /// -/// Callers must aggregate across tenants before recording. Per-tenant values are -/// intentionally not accepted because a shared unlabeled gauge would otherwise -/// expose only the last tenant observed while tenant labels would be unbounded. +/// Callers must aggregate the tenant and provider-account scopes before recording. +/// Per-scope values are intentionally not accepted because a shared gauge would +/// otherwise expose only the last scope observed while identity labels would be +/// unbounded. pub fn record_sandbox_workspace_quota_utilization( dimension: WorkspaceCapacityDimension, ratio: f64, diff --git a/crates/moa-orchestrator/src/services/execution_amendment_planner.rs b/crates/moa-orchestrator/src/services/execution_amendment_planner.rs index 67c58f4f8..4280aaeb8 100644 --- a/crates/moa-orchestrator/src/services/execution_amendment_planner.rs +++ b/crates/moa-orchestrator/src/services/execution_amendment_planner.rs @@ -34,6 +34,7 @@ use moa_core::{ execution_planning::ExecutionPlanningAuditEnvelope, identifiers::ModelId, model::ModelCapabilities, + resource::{ResourceAmounts, ResourceBudget}, }, }; use moa_execution::{ @@ -41,6 +42,11 @@ use moa_execution::{ capability::amendment_hash, repository::{ ExecutionRepository, ExecutionRunRecord, ExecutionScope, + planning_budget::{ + AmendmentPlanningCallReconcileOutcome, AmendmentPlanningCallReconcileRequest, + AmendmentPlanningCallReservation, AmendmentPlanningCallReservationOutcome, + AmendmentPlanningCallReservationRequest, PlanningUsage, + }, replan_stop::{NewExecutionReplanStopIntent, ReplanStopIntentWriteOutcome}, }, state::ExecutionRunStatus, @@ -53,7 +59,9 @@ use serde_json::json; use crate::services::{ execution::ExecutionClient, - llm_gateway::{LLMCompletionAction, LLMGatewayClient, completion_idempotency_key}, + llm_gateway::{ + BoundedCompletionRequest, LLMCompletionAction, LLMGatewayClient, completion_idempotency_key, + }, }; pub use preparation::{ @@ -160,6 +168,10 @@ impl ExecutionAmendmentPlanner for ExecutionAmendmentPlannerImpl { let provider = RestateAmendmentPlannerProvider { ctx: &ctx, + repository: self.repository.clone(), + scope, + config: &self.config, + deadline_at: prepared.remaining_budget.deadline_at, run_uid: target.run_uid, plan_revision: target.base_plan_revision, next_attempt: AtomicUsize::new(0), @@ -222,6 +234,18 @@ impl ExecutionAmendmentPlanner for ExecutionAmendmentPlannerImpl { .await .map(Json::from); } + ExecutionAmendmentPlanningResultKind::BudgetExhausted { message } => { + return self + .record_planner_stop( + &ctx, + scope, + prepared.origin, + ReplanStopReason::BudgetExhausted, + message, + ) + .await + .map(Json::from); + } }; let response = submit_amendment(&ctx, &target, &prepared.admitted_identity, amendment) @@ -500,6 +524,10 @@ pub(crate) async fn dispatch_parked_replan_planning( /// Restate-journaled planner provider that routes every model call through the gateway. struct RestateAmendmentPlannerProvider<'a, 'ctx> { ctx: &'a Context<'ctx>, + repository: ExecutionRepository, + scope: ExecutionScope, + config: &'a ExecutionConfig, + deadline_at: Option>, run_uid: uuid::Uuid, plan_revision: u64, next_attempt: AtomicUsize, @@ -519,15 +547,77 @@ impl LLMProvider for RestateAmendmentPlannerProvider<'_, '_> { &self, request: SharedCompletionRequest, ) -> moa_core::error::Result { - // Restate's JSON transport requires the owned durable DTO. This is the - // explicit serialization boundary after in-process shared routing. let request = CompletionRequest::from_view(&request); - // Amendment generation and its bounded repair are sequential planner calls. let attempt = self.next_attempt.fetch_add(1, Ordering::Relaxed); + let ordinal = u8::try_from(attempt).map_err(|_| { + moa_core::error::MoaError::ValidationError( + "amendment planner call ordinal exceeds u8".to_string(), + ) + })?; + let estimate = amendment_planning_call_estimate(&request, self.config)?; + let repository = self.repository.clone(); + let scope = self.scope; + let run_uid = self.run_uid; + let plan_revision = self.plan_revision; + let reservation = self + .ctx + .run(|| { + let repository = repository.clone(); + async move { + let outcome = repository + .reserve_amendment_planning_call( + scope, + AmendmentPlanningCallReservationRequest { + run_uid, + base_plan_revision: plan_revision, + call_ordinal: ordinal, + reservation: estimate, + now: chrono::Utc::now(), + }, + ) + .await + .map_err(crate::workflows::errors::execution_error_to_handler_error)?; + Ok::<_, HandlerError>(Json::from(JournaledPlanningReservation::from(&outcome))) + } + }) + .name(format!( + "execution_amendment_reserve_{}_{}_{}", + self.run_uid, self.plan_revision, attempt + )) + .await + .map_err(|error| moa_core::error::MoaError::ProviderError(error.to_string()))? + .into_inner(); + match reservation { + JournaledPlanningReservation::Proceed => {} + JournaledPlanningReservation::Denied => { + return Err(moa_core::error::MoaError::BudgetExhausted( + "automatic amendment planning exhausted the approved run budget".to_string(), + )); + } + JournaledPlanningReservation::Conflict | JournaledPlanningReservation::NotFound => { + return Err(moa_core::error::MoaError::ProviderError( + "automatic amendment planning reservation conflicted with durable state" + .to_string(), + )); + } + } + let response = crate::restate_identity::replay_safe_request( self.ctx .service_client::() - .complete(Json::from(request)) + .complete_bounded(Json::from(BoundedCompletionRequest { + request, + budget: ResourceBudget::new( + self.deadline_at, + Some(ResourceAmounts { + cost_micro_usd: estimate.cost_microusd, + tokens: estimate.tokens, + turns: 0, + model_calls: 1, + tool_calls: 0, + }), + ), + })) .idempotency_key(completion_idempotency_key( self.ctx.invocation_id(), LLMCompletionAction::ExecutionAmendment { @@ -538,13 +628,185 @@ impl LLMProvider for RestateAmendmentPlannerProvider<'_, '_> { )), ) .call() - .await - .map_err(|error| moa_core::error::MoaError::ProviderError(error.to_string()))? - .into_inner(); + .await; + let response = match response { + Ok(response) => response.into_inner(), + Err(error) => { + self.reconcile_call(ordinal, attempt, PlanningUsage::default()) + .await?; + return Err(moa_core::error::MoaError::ProviderError(error.to_string())); + } + }; + let actual = amendment_planning_call_usage(&response)?; + self.reconcile_call(ordinal, attempt, actual).await?; Ok(CompletionStream::from_response(response)) } } +impl RestateAmendmentPlannerProvider<'_, '_> { + async fn reconcile_call( + &self, + ordinal: u8, + attempt: usize, + actual: PlanningUsage, + ) -> moa_core::error::Result<()> { + let repository = self.repository.clone(); + let scope = self.scope; + let run_uid = self.run_uid; + let plan_revision = self.plan_revision; + let reconcile = self + .ctx + .run(|| { + let repository = repository.clone(); + async move { + let outcome = repository + .reconcile_amendment_planning_call( + scope, + AmendmentPlanningCallReconcileRequest { + run_uid, + base_plan_revision: plan_revision, + call_ordinal: ordinal, + actual, + settled_at: chrono::Utc::now(), + }, + ) + .await + .map_err(crate::workflows::errors::execution_error_to_handler_error)?; + Ok::<_, HandlerError>(Json::from(JournaledPlanningReconcile::from(&outcome))) + } + }) + .name(format!( + "execution_amendment_reconcile_{}_{}_{}", + self.run_uid, self.plan_revision, attempt + )) + .await + .map_err(|error| moa_core::error::MoaError::ProviderError(error.to_string()))? + .into_inner(); + if reconcile != JournaledPlanningReconcile::Settled { + return Err(moa_core::error::MoaError::ProviderError( + "automatic amendment planning reconciliation conflicted with durable state" + .to_string(), + )); + } + Ok(()) + } +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +enum JournaledPlanningReservation { + Proceed, + Denied, + Conflict, + NotFound, +} + +impl From<&AmendmentPlanningCallReservationOutcome> for JournaledPlanningReservation { + fn from(value: &AmendmentPlanningCallReservationOutcome) -> Self { + match value { + AmendmentPlanningCallReservationOutcome::Granted(_) + | AmendmentPlanningCallReservationOutcome::ReplayedOpen(_) + | AmendmentPlanningCallReservationOutcome::AlreadySettled(_) => Self::Proceed, + AmendmentPlanningCallReservationOutcome::Denied(_) => Self::Denied, + AmendmentPlanningCallReservationOutcome::Conflict => Self::Conflict, + AmendmentPlanningCallReservationOutcome::NotFound => Self::NotFound, + } + } +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +enum JournaledPlanningReconcile { + Settled, + Conflict, + NotFound, +} + +impl From<&AmendmentPlanningCallReconcileOutcome> for JournaledPlanningReconcile { + fn from(value: &AmendmentPlanningCallReconcileOutcome) -> Self { + match value { + AmendmentPlanningCallReconcileOutcome::Applied(_) + | AmendmentPlanningCallReconcileOutcome::Replayed(_) => Self::Settled, + AmendmentPlanningCallReconcileOutcome::Conflict => Self::Conflict, + AmendmentPlanningCallReconcileOutcome::NotFound => Self::NotFound, + } + } +} + +fn amendment_planning_call_estimate( + request: &CompletionRequest, + config: &ExecutionConfig, +) -> moa_core::error::Result { + // The shared estimator sizes the provider-visible messages and tools. A twofold headroom keeps + // this authorization conservative across model tokenizers without charging DTO serialization + // syntax that the provider never sees. + let input_token_count = moa_core::types::context::sum_message_tokens(&request.messages) + .saturating_add( + request + .tools + .iter() + .map(|tool| moa_core::types::context::estimate_text_tokens(&tool.to_string())) + .sum(), + ) + .saturating_mul(2); + let input_tokens = u64::try_from(input_token_count).map_err(|_| { + moa_core::error::MoaError::ValidationError( + "amendment planner request token estimate exceeds u64".to_string(), + ) + })?; + let output_tokens = + u64::try_from(request.max_output_tokens.unwrap_or_default()).map_err(|_| { + moa_core::error::MoaError::ValidationError( + "amendment planner output limit exceeds u64".to_string(), + ) + })?; + let tokens = input_tokens.checked_add(output_tokens).ok_or_else(|| { + moa_core::error::MoaError::ValidationError( + "amendment planner token reservation overflow".to_string(), + ) + })?; + let priced_cost = request + .model + .as_ref() + .and_then(|model| moa_providers::pricing_for_model(model.as_str())) + .map(|pricing| { + let input_rate = pricing + .input_per_mtok + .max(pricing.cached_input_per_mtok.unwrap_or_default()) + .max(pricing.cache_write_per_mtok()); + let dollars = input_tokens as f64 / 1_000_000.0 * input_rate + + output_tokens as f64 / 1_000_000.0 * pricing.output_per_mtok; + moa_core::types::resource::ResourceAmounts::cost_micro_usd_from_dollars(dollars) + .unwrap_or(u64::MAX) + }) + .unwrap_or_default(); + Ok(AmendmentPlanningCallReservation { + cost_microusd: priced_cost.max(config.agent_turn_cost_microusd), + tokens, + }) +} + +fn amendment_planning_call_usage( + response: &moa_core::types::completion::CompletionResponse, +) -> moa_core::error::Result { + let input = u64::try_from(response.usage.total_input_tokens()).map_err(|_| { + moa_core::error::MoaError::ValidationError( + "amendment planner input usage exceeds u64".to_string(), + ) + })?; + let output = u64::try_from(response.usage.output_tokens).map_err(|_| { + moa_core::error::MoaError::ValidationError( + "amendment planner output usage exceeds u64".to_string(), + ) + })?; + Ok(PlanningUsage { + cost_microusd: moa_providers::pricing_for_model(response.model.as_str()) + .map(|pricing| pricing.cost_micros(&response.usage)) + .unwrap_or_default(), + tokens: input.saturating_add(output), + }) +} + #[cfg(feature = "integration")] fn automatic_amendment_planner_paused() -> bool { std::env::var("MOA_EXECUTION_TEST_PAUSE_AMENDMENT_PLANNER").as_deref() == Ok("true") diff --git a/crates/moa-orchestrator/src/services/execution_amendment_planner/tests.rs b/crates/moa-orchestrator/src/services/execution_amendment_planner/tests.rs index 99755e2ab..8560873da 100644 --- a/crates/moa-orchestrator/src/services/execution_amendment_planner/tests.rs +++ b/crates/moa-orchestrator/src/services/execution_amendment_planner/tests.rs @@ -4,8 +4,9 @@ use super::preparation::{ bounded_failure_evidence, narrow_amendment_context, narrow_authorized_capability_refs, }; use super::{ - AmendmentPlanningOrigin, ReplanStopReason, amendment_planning_identity, - parked_run_needs_amendment, planner_stop_amendment, + AmendmentPlanningOrigin, ReplanStopReason, amendment_planning_call_estimate, + amendment_planning_call_usage, amendment_planning_identity, parked_run_needs_amendment, + planner_stop_amendment, }; use std::collections::BTreeSet; @@ -32,6 +33,44 @@ fn unbounded_budget() -> ExecutionBudgetLimit { } } +#[test] +fn amendment_planner_call_estimate_and_usage_are_cost_and_token_only_offline() { + // Pins: every automatic amendment model call reserves a conservative request-plus-output token + // bound, then reconciles the provider's exact normalized tokens and model-priced cost. + let mut request = moa_core::types::completion::CompletionRequest::new("repair the plan"); + request.model = Some(moa_core::types::identifiers::ModelId::new( + "claude-sonnet-4-6", + )); + request.max_output_tokens = Some(32_768); + let estimate = + amendment_planning_call_estimate(&request, &moa_config::ExecutionConfig::default()) + .expect("bounded planner request should estimate"); + let expected_input = + u64::try_from(moa_core::types::context::sum_message_tokens(&request.messages) * 2) + .expect("fixture estimate should fit u64"); + assert_eq!(estimate.tokens, 32_768 + expected_input); + assert!(estimate.cost_microusd >= 100_000); + + let response = moa_core::types::completion::CompletionResponse { + text: "{}".to_string(), + content: Vec::new(), + stop_reason: moa_core::types::completion::StopReason::EndTurn, + model: moa_core::types::identifiers::ModelId::new("claude-sonnet-4-6"), + usage: moa_core::types::completion::TokenUsage { + input_tokens_uncached: 1_000, + input_tokens_cache_write: 200, + input_tokens_cache_read: 300, + output_tokens: 400, + }, + duration_ms: 1, + thought_signature: None, + }; + let actual = amendment_planning_call_usage(&response) + .expect("authoritative provider usage should reconcile"); + assert_eq!(actual.tokens, 1_900); + assert!(actual.cost_microusd > 0); +} + fn reference(name: &str) -> CapabilityReference { CapabilityReference { name: name.to_string(), diff --git a/crates/moa-orchestrator/src/services/tool_executor.rs b/crates/moa-orchestrator/src/services/tool_executor.rs index 511fa4b86..1a5241b1a 100644 --- a/crates/moa-orchestrator/src/services/tool_executor.rs +++ b/crates/moa-orchestrator/src/services/tool_executor.rs @@ -25,7 +25,6 @@ use moa_core::{ ExecutionCompensationScopeId, ExecutionRunScopeId, ExecutionTaskScopeId, SessionId, TenantId, ToolCallId, }, - types::sandbox_workspace::ExecutionHandContinuationDisposition, types::sandbox_workspace::ExecutionHandReleaseOwner, types::sandbox_workspace::ExecutionHandReleaseReceipt, types::sandbox_workspace::SandboxWorkspaceScope, @@ -68,9 +67,9 @@ use moa_execution::wire::{ ExecutionToolDispatchRejection, }; use moa_hands::{ - DeferredWorkspaceToolOutput, ExecutionHandReleaseRequest, ExecutionHandRetentionRequest, - JournaledWorkspaceCommit, PendingConnectorToolOutput, SessionHandReleasePageOutcome, - ToolCallScope, ToolCatalogPin, ToolCatalogSnapshot, ToolExecution, ToolRouter, + DeferredWorkspaceToolOutput, ExecutionHandReleaseRequest, JournaledWorkspaceCommit, + PendingConnectorToolOutput, SessionHandReleasePageOutcome, ToolCallScope, ToolCatalogPin, + ToolCatalogSnapshot, ToolExecution, ToolRouter, }; use moa_security::{ OutputClassification, ToolInputCanaryScreening, classify_tool_output, @@ -551,11 +550,6 @@ pub trait ToolExecutor { request: Json, ) -> Result, HandlerError>; - /// Publishes one continuation checkpoint and keeps its sandbox for the next slice. - async fn checkpoint_execution_hands_retaining_compute( - request: Json, - ) -> Result, HandlerError>; - /// Releases the generation-independent hand scope owned by one compensation. async fn release_execution_compensation_hands( request: Json, @@ -687,28 +681,6 @@ pub struct CheckpointAndReleaseExecutionHandsRequest { pub release_deadline_at: chrono::DateTime, } -/// Exact bounded execution sandbox to checkpoint and keep across a continuation. -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -#[serde(deny_unknown_fields)] -pub struct CheckpointExecutionHandsRetainingComputeRequest { - /// Verified tenant that owns the execution run and lease. - pub tenant_id: TenantId, - /// Authoritative parent session loaded from the run. - pub session_id: SessionId, - /// Owning execution run. - pub run_uid: uuid::Uuid, - /// Task whose durable workspace scope owns the retained sandbox. - pub task_id: ExecutionTaskScopeId, - /// Logical task generation the retained compute belongs to. - pub logical_generation: u64, - /// Exact active-attempt generation publishing this continuation checkpoint. - pub attempt_generation: u64, - /// Fresh absolute bound for checkpoint publication. - pub publish_deadline_at: chrono::DateTime, - /// Absolute instant after which the reaper may destroy the retained sandbox. - pub retention_deadline_at: chrono::DateTime, -} - /// Request to release one settled compensation's scoped hands. #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] #[serde(deny_unknown_fields)] @@ -2535,71 +2507,6 @@ impl ToolExecutor for ToolExecutorImpl { .await?) } - #[tracing::instrument(skip(self, ctx, request))] - // SAFETY: internal bounded-attempt continuation; the authoritative Session is loaded and its - // exact tenant/run/task generation is fenced again by the durable workspace repository. - async fn checkpoint_execution_hands_retaining_compute( - &self, - ctx: Context<'_>, - request: Json, - ) -> Result, HandlerError> { - crate::ctx::adopt_incoming_trace_parent(&ctx); - annotate_restate_handler_span( - "ToolExecutor", - "checkpoint_execution_hands_retaining_compute", - ); - let request = request.into_inner(); - let session_store = self.session_access.sessions.clone(); - let session_id = request.session_id; - let session = ctx - .run(|| async move { - session_store - .get_session(session_id) - .await - .map(Json::from) - .map_err(moa_error_to_handler_error) - }) - .name(format!("load_task_continuation_session:{session_id}")) - .await? - .into_inner(); - if session.tenant_id != request.tenant_id { - return Err( - TerminalError::new("execution continuation session tenant mismatch").into(), - ); - } - let router = self.router.clone(); - let run_id = ExecutionRunScopeId(request.run_uid); - let task_id = request.task_id; - let logical_generation = request.logical_generation; - let attempt_generation = request.attempt_generation; - let publish_deadline_at = request.publish_deadline_at; - let retention_deadline_at = request.retention_deadline_at; - Ok(ctx - .run(|| async move { - router - .checkpoint_execution_hand_retaining_compute(ExecutionHandRetentionRequest { - session: &session, - run_id, - task_id, - logical_generation, - attempt_generation, - retention_deadline_at, - scope: ToolCallScope::unbounded().with_budget( - moa_core::types::resource::ResourceBudget::until(publish_deadline_at), - ), - }) - .await - .map(Json::from) - .map_err(moa_error_to_handler_error) - }) - .name(format!( - "checkpoint_retaining_execution_hand:{}:{}:{}", - request.run_uid, request.task_id, request.attempt_generation - )) - .retry_policy(RunRetryPolicy::new().max_attempts(1)) - .await?) - } - #[tracing::instrument(skip(self, ctx, request))] // SAFETY: internal terminal-task teardown reclaims only the typed run/task hand scope and returns no caller-owned data. async fn release_execution_task_hands( diff --git a/crates/moa-orchestrator/src/workflows/execution_task_attempt/active.rs b/crates/moa-orchestrator/src/workflows/execution_task_attempt/active.rs index 5910168ca..62e5ca163 100644 --- a/crates/moa-orchestrator/src/workflows/execution_task_attempt/active.rs +++ b/crates/moa-orchestrator/src/workflows/execution_task_attempt/active.rs @@ -2,6 +2,8 @@ use std::collections::{BTreeMap, BTreeSet}; +use chrono::{DateTime, Utc}; + use moa_artifacts::execution_plan::{ CapabilityReference, ExecutionFailureClass, ExecutionTaskOutcome, ExecutionTaskResult, ExecutionUsage, @@ -24,7 +26,7 @@ use moa_execution::{ capability::{CapabilitySource, ExecutionCapability}, repository::task::{ NewTaskAttemptCheckpoint, TaskAttemptCheckpointKind, TaskAttemptCheckpointRecord, - TaskAttemptCheckpointWriteOutcome, TaskAttemptRecord, + TaskAttemptCheckpointWriteOutcome, TaskAttemptProgressOutcome, TaskAttemptRecord, }, schema::validate_instance, state::{LogicalTaskKind, completed_task_outcome, failed_task_outcome}, @@ -344,8 +346,13 @@ pub(super) async fn execute_task_attempt( /// Durable step boundary at which an active attempt reports progress. #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum AttemptHeartbeat { + /// One model completion is about to start. The bounded gateway budget cannot outlive the + /// attempt deadline, so the persisted stall window covers that exact call. + ModelTurnStart, /// One model turn returned, so the following tool dispatch starts its own stall window. ModelTurn, + /// One governed tool invocation is about to start with its declared stall bound. + ToolCallStart { bound: Option }, /// One governed tool invocation returned, so sandbox release and continuation persistence /// start their own stall window. ToolCall, @@ -355,7 +362,9 @@ impl AttemptHeartbeat { /// Deterministic journal step name for this boundary. const fn observation_step(self) -> &'static str { match self { + Self::ModelTurnStart => "task_attempt_model_turn_start_progress_at", Self::ModelTurn => "task_attempt_model_turn_progress_at", + Self::ToolCallStart { .. } => "task_attempt_tool_call_start_progress_at", Self::ToolCall => "task_attempt_tool_call_progress_at", } } @@ -363,38 +372,146 @@ impl AttemptHeartbeat { /// Deterministic journal step name for the persisted heartbeat. const fn write_step(self) -> &'static str { match self { + Self::ModelTurnStart => "record_task_attempt_model_turn_start_progress", Self::ModelTurn => "record_task_attempt_model_turn_progress", + Self::ToolCallStart { .. } => "record_task_attempt_tool_call_start_progress", Self::ToolCall => "record_task_attempt_tool_call_progress", } } + + /// Upper bound of the step this boundary opens, when that step declares one. + /// + /// Post-return boundaries clear the bound back to the configured heartbeat floor. + fn step_bound_seconds( + self, + request: &ExecutionTaskAttemptRequest, + observed_at: DateTime, + ) -> Option { + match self { + Self::ModelTurnStart => Some(AttemptStepBound::UntilAttemptDeadline) + .and_then(|bound| bound.seconds(request, observed_at)), + Self::ToolCallStart { bound } => { + bound.and_then(|bound| bound.seconds(request, observed_at)) + } + Self::ModelTurn | Self::ToolCall => None, + } + } +} + +/// Returns the bound to record for the capability step this attempt is about to dispatch. +/// +/// An external provider start remains bounded by the attempt deadline so its recovery trigger, +/// rather than the task watchdog, retains authority over an ambiguous start. +fn capability_step_bound( + requires_sandbox: bool, + async_mode: &ToolAsyncMode, + tool_call: &ToolCallContent, +) -> Option { + if requires_sandbox || matches!(async_mode, ToolAsyncMode::MayReturnExternalJob { .. }) { + return Some(AttemptStepBound::UntilAttemptDeadline); + } + moa_hands::tools::bash::declared_tool_step_bound( + &tool_call.invocation.name, + &tool_call.invocation.input, + ) + .and_then(|bound| u32::try_from(bound.as_secs()).ok()) + .map(AttemptStepBound::Declared) +} + +/// Upper bound a dispatching step declares for itself. +/// +/// `UntilAttemptDeadline` is resolved against the journaled heartbeat instant rather than a +/// fresh clock read, because a workflow that reads the wall clock outside the journal +/// produces a different value on replay. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum AttemptStepBound { + /// The step named its own ceiling in seconds. + Declared(u32), + /// The step runs until the attempt deadline and must not be cut short before it. + UntilAttemptDeadline, +} + +impl AttemptStepBound { + /// Resolves this bound to seconds against one journaled observation instant. + fn seconds( + self, + request: &ExecutionTaskAttemptRequest, + observed_at: DateTime, + ) -> Option { + match self { + Self::Declared(seconds) => Some(seconds), + // Rounded up so a sub-second remainder still outlasts the deadline it covers. + Self::UntilAttemptDeadline => u32::try_from( + request + .attempt_deadline_at + .signed_duration_since(observed_at) + .num_seconds() + .saturating_add(1), + ) + .ok() + .filter(|seconds| *seconds > 0), + } + } } -/// Advances the active attempt's durable progress clock at one completed step boundary. +/// Advances the active attempt's durable progress clock and returns whether it still owns it. /// -/// Only an attempt that currently owns active capacity is heartbeated; the repository call is -/// fenced on the exact dispatch and rejects a parked, superseded, or already-settled attempt, -/// so a waiting task can never appear to make progress. The observation timestamp is journaled -/// through `durable_utc_now` so replay reuses the recorded instant instead of a fresh clock -/// read, and the repository write itself is monotonic. +/// The exact dispatch fence prevents a parked, superseded, or settled attempt from regaining +/// dispatch authority. The observation time is journaled for replay stability. async fn record_attempt_heartbeat( workflow: &ExecutionTaskAttemptImpl, ctx: &WorkflowContext<'_>, request: &ExecutionTaskAttemptRequest, boundary: AttemptHeartbeat, -) -> Result<(), HandlerError> { +) -> Result { let observed_at = durable_utc_now(ctx, boundary.observation_step()).await?; let repository = workflow.repository.clone(); let fence = task_attempt_fence(request); - ctx.run(|| async move { - repository - .record_task_attempt_progress(fence, observed_at) - .await - .map(Json::from) - .map_err(crate::workflows::errors::execution_error_to_handler_error) - }) - .name(boundary.write_step()) - .await?; - Ok(()) + Ok(ctx + .run(|| async move { + repository + .record_task_attempt_progress( + fence, + observed_at, + boundary.step_bound_seconds(request, observed_at), + ) + .await + .map(|outcome| Json::from(attempt_progress_retains_ownership(outcome))) + .map_err(crate::workflows::errors::execution_error_to_handler_error) + }) + .name(boundary.write_step()) + .await? + .into_inner()) +} + +const fn attempt_progress_retains_ownership(outcome: TaskAttemptProgressOutcome) -> bool { + matches!( + outcome, + TaskAttemptProgressOutcome::Applied | TaskAttemptProgressOutcome::Replayed + ) +} + +/// Records the exact capability bound and confirms ownership before provider dispatch. +async fn begin_capability_dispatch( + workflow: &ExecutionTaskAttemptImpl, + ctx: &WorkflowContext<'_>, + request: &ExecutionTaskAttemptRequest, + capability: &ExecutionCapability, + tool_call: &ToolCallContent, +) -> Result { + record_attempt_heartbeat( + workflow, + ctx, + request, + AttemptHeartbeat::ToolCallStart { + bound: capability_step_bound( + capability.requires_sandbox, + &capability.async_mode, + tool_call, + ), + }, + ) + .await } async fn persist_external_start_checkpoint( @@ -533,6 +650,9 @@ async fn execute_direct_capability( return Ok(ActiveTaskAttemptExit::OwnershipLost); } } + if !begin_capability_dispatch(workflow, ctx, request, capability, &tool_call).await? { + return Ok(ActiveTaskAttemptExit::OwnershipLost); + } let governed = invoke_governed_tool( ctx, GovernedInvocationRequest { @@ -560,7 +680,9 @@ async fn execute_direct_capability( workflow.channel_adapters.as_ref(), ) .await?; - record_attempt_heartbeat(workflow, ctx, request, AttemptHeartbeat::ToolCall).await?; + if !record_attempt_heartbeat(workflow, ctx, request, AttemptHeartbeat::ToolCall).await? { + return Ok(ActiveTaskAttemptExit::OwnershipLost); + } usage.tool_calls = usage.tool_calls.saturating_add(1); classify_capability_outcome(capability, governed, usage) } @@ -1104,6 +1226,11 @@ async fn execute_agent_turn( }; let owner = LLMCompletionOwner::execution_task_attempt(request.dispatch_uid); attach_completion_owner(&mut completion, &owner); + if !record_attempt_heartbeat(workflow, ctx, request, AttemptHeartbeat::ModelTurnStart) + .await? + { + return Ok(ActiveTaskAttemptExit::OwnershipLost); + } let response = crate::restate_identity::replay_safe_request( ctx.service_client::() .complete_bounded(Json::from(BoundedCompletionRequest { @@ -1121,7 +1248,9 @@ async fn execute_agent_turn( .call() .await? .into_inner(); - record_attempt_heartbeat(workflow, ctx, request, AttemptHeartbeat::ModelTurn).await?; + if !record_attempt_heartbeat(workflow, ctx, request, AttemptHeartbeat::ModelTurn).await? { + return Ok(ActiveTaskAttemptExit::OwnershipLost); + } usage.tokens = usage .tokens .saturating_add(response.usage.total_input_tokens() as u64) @@ -1179,6 +1308,20 @@ async fn execute_agent_turn( } pending_tool_calls = tool_calls; next_turn = next_turn.saturating_add(1); + return Ok(ActiveTaskAttemptExit::Continue { + continuation: agent_continuation( + messages, + next_turn, + usage, + security_circuit, + disabled_capabilities, + AgentPending { + review: None, + tool_calls: pending_tool_calls, + external: pending_external, + }, + ), + }); } let invocation = pending_tool_calls.remove(0); @@ -1270,6 +1413,9 @@ async fn execute_agent_turn( return Ok(ActiveTaskAttemptExit::OwnershipLost); } } + if !begin_capability_dispatch(workflow, ctx, request, capability, &tool_call).await? { + return Ok(ActiveTaskAttemptExit::OwnershipLost); + } let governed = invoke_governed_tool( ctx, GovernedInvocationRequest { @@ -1297,7 +1443,9 @@ async fn execute_agent_turn( workflow.channel_adapters.as_ref(), ) .await?; - record_attempt_heartbeat(workflow, ctx, request, AttemptHeartbeat::ToolCall).await?; + if !record_attempt_heartbeat(workflow, ctx, request, AttemptHeartbeat::ToolCall).await? { + return Ok(ActiveTaskAttemptExit::OwnershipLost); + } usage.tool_calls = usage.tool_calls.saturating_add(1); match governed { GovernedInvocationOutcome::Completed(result) @@ -1623,12 +1771,112 @@ fn serialized_len(value: &T) -> u64 { #[cfg(test)] mod tests { - use chrono::{TimeZone, Utc}; + use chrono::{Duration, TimeZone, Utc}; use moa_artifacts::execution_plan::ExecutionUsage; - use moa_core::types::{completion::ToolInvocation, context::ContextMessage}; + use moa_core::types::{ + completion::ToolInvocation, context::ContextMessage, identifiers::TenantId, + }; + use moa_execution::state::ExecutionTaskId; use super::*; + // Pins: every pre-provider heartbeat is an ownership check, not telemetry. A stale, + // absent, or non-running attempt must stop before model or tool dispatch, while exact + // replay of an already-journaled heartbeat retains authority. + #[test] + fn heartbeat_verdict_stops_dispatch_after_ownership_loss_offline() { + assert!(attempt_progress_retains_ownership( + TaskAttemptProgressOutcome::Applied + )); + assert!(attempt_progress_retains_ownership( + TaskAttemptProgressOutcome::Replayed + )); + for lost in [ + TaskAttemptProgressOutcome::NotFound, + TaskAttemptProgressOutcome::Stale, + TaskAttemptProgressOutcome::InvalidState, + ] { + assert!(!attempt_progress_retains_ownership(lost)); + } + } + + // Pins: a healthy model or tool call whose declared duration exceeds the configured + // heartbeat floor remains live for that exact step, and the first post-return heartbeat + // clears the widened bound back to the ordinary orchestration floor. + #[test] + fn model_and_tool_steps_are_bounded_before_dispatch_and_cleared_after_return_offline() { + let observed_at = Utc + .with_ymd_and_hms(2026, 8, 13, 12, 0, 0) + .single() + .expect("fixture timestamp is valid"); + let request = ExecutionTaskAttemptRequest { + dispatch_uid: Uuid::from_u128(1), + capacity_reservation_uid: Uuid::from_u128(2), + watchdog_trigger_uid: Uuid::from_u128(3), + watchdog_dispatch_uid: Uuid::from_u128(4), + run_uid: Uuid::from_u128(5), + task_id: ExecutionTaskId::from_uuid(Uuid::from_u128(6)), + controller_generation: 7, + attempt_generation: 8, + attempt_deadline_at: observed_at + Duration::seconds(121), + tenant_id: TenantId(Uuid::from_u128(9)), + }; + + assert_eq!( + AttemptHeartbeat::ModelTurnStart.step_bound_seconds(&request, observed_at), + Some(122), + ); + assert_eq!( + AttemptHeartbeat::ToolCallStart { + bound: Some(AttemptStepBound::Declared(90)), + } + .step_bound_seconds(&request, observed_at), + Some(90), + ); + assert_eq!( + AttemptHeartbeat::ModelTurn.step_bound_seconds(&request, observed_at), + None, + ); + assert_eq!( + AttemptHeartbeat::ToolCall.step_bound_seconds(&request, observed_at), + None, + ); + } + + // Pins: sandbox lifecycle work shares the active attempt deadline because provisioning, + // restore, install, execution, and commit can outlive the command timeout alone. A + // non-sandbox synchronous call keeps its narrower declared execution bound. + #[test] + fn sandbox_capability_uses_attempt_bound_while_non_sandbox_keeps_tool_bound_offline() { + let tool_call = ToolCallContent { + invocation: ToolInvocation { + id: Some("bounded-bash".to_string()), + name: "bash".to_string(), + input: json!({"cmd": "sleep 1", "timeout_secs": 90}), + }, + provider_metadata: None, + }; + + assert_eq!( + capability_step_bound(true, &ToolAsyncMode::SynchronousOnly, &tool_call), + Some(AttemptStepBound::UntilAttemptDeadline), + ); + assert_eq!( + capability_step_bound(false, &ToolAsyncMode::SynchronousOnly, &tool_call), + Some(AttemptStepBound::Declared(90)), + ); + assert_eq!( + capability_step_bound( + false, + &ToolAsyncMode::MayReturnExternalJob { + provider: "fixture".to_string(), + }, + &tool_call, + ), + Some(AttemptStepBound::UntilAttemptDeadline), + ); + } + // Pins: a continuation that cannot fit in the bounded durable payload is rejected // before persistence so callers must decompose or request a replan. #[test] diff --git a/crates/moa-orchestrator/src/workflows/execution_task_attempt/watchdog.rs b/crates/moa-orchestrator/src/workflows/execution_task_attempt/watchdog.rs index a47d7d954..3731ca9fe 100644 --- a/crates/moa-orchestrator/src/workflows/execution_task_attempt/watchdog.rs +++ b/crates/moa-orchestrator/src/workflows/execution_task_attempt/watchdog.rs @@ -6,12 +6,11 @@ use moa_artifacts::execution_plan::{ }; use moa_core::types::tools::IdempotencyClass; use moa_execution::{ - capability::CapabilitySource, repository::{ ExecutionAttemptState, ExecutionScope, task::{ - ActiveAttemptLiveness, TaskAttemptFence, TaskAttemptRecord, - TaskAttemptSettlementOutcome, UnstartedTaskAttemptDisposition, + ActiveAttemptLiveness, TaskAttemptCheckpointRecord, TaskAttemptFence, + TaskAttemptRecord, TaskAttemptSettlementOutcome, UnstartedTaskAttemptDisposition, classify_active_attempt_liveness, }, }, @@ -30,7 +29,9 @@ use crate::{ attempt_slice::durable_utc_now_shared, errors::execution_error_to_handler_error, execution_task_attempt::{ - ExecutionTaskAttemptImpl, task_attempt_fence, + ExecutionTaskAttemptImpl, + active::{TaskAttemptContinuation, TaskAttemptContinuationState}, + task_attempt_fence, yielding::{begin_release_shared, checkpoint_task_hands_shared}, }, }, @@ -109,18 +110,19 @@ pub(super) async fn handle_task_attempt_watchdog( )); } let now = durable_utc_now_shared(ctx, "task_watchdog_observed_at").await?; - let liveness = - classify_active_attempt_liveness(&workflow.config, deadline, task.last_progress_at, now); + let liveness = classify_active_attempt_liveness( + &workflow.config, + deadline, + task.last_progress_at, + task.progress_step_bound_seconds + .and_then(|seconds| chrono::TimeDelta::try_seconds(i64::from(seconds))), + now, + ); if !liveness.is_expired() { return Ok(watchdog_result( ExecutionAttemptWatchdogResponseOutcome::RetryDelivery, )); } - cancel_completion_owner( - ctx, - LLMCompletionOwner::execution_task_attempt(request.dispatch_uid), - ) - .await?; if task.status == ExecutionTaskStatus::Dispatching && task.attempt_state == ExecutionAttemptState::Dispatching { @@ -180,7 +182,48 @@ pub(super) async fn handle_task_attempt_watchdog( attempt_deadline_at: deadline, tenant_id: request.tenant_id, }; + let repository = workflow.repository.clone(); + let fence = task_attempt_fence(&attempt_request); + let external_start_recovery_owns_attempt = ctx + .run(|| async move { + repository + .load_current_task_external_start_recovery(fence) + .await + .map(|recovery| Json::from(recovery.is_some())) + .map_err(execution_error_to_handler_error) + }) + .name("load_task_external_start_recovery_for_watchdog") + .await? + .into_inner(); + if external_start_recovery_owns_attempt { + // The provider start may already have committed. Its exact recovery trigger is the + // only authority allowed to bind or prove-not-sent; watchdog teardown would supersede + // that trigger and could orphan provider work. + return Ok(watchdog_result( + ExecutionAttemptWatchdogResponseOutcome::RetryDelivery, + )); + } + let repository = workflow.repository.clone(); + let checkpoint = ctx + .run(|| async move { + repository + .load_task_attempt_checkpoint(scope, request.run_uid, request.task_id) + .await + .map(Json::from) + .map_err(execution_error_to_handler_error) + }) + .name("load_task_attempt_checkpoint_for_watchdog") + .await? + .into_inner(); let started = TaskAttemptRecord { run, task }; + let disposition = classify_stale_attempt( + task_effect_idempotency(&started, checkpoint.as_ref()).map_err(TerminalError::new)?, + ); + cancel_completion_owner( + ctx, + LLMCompletionOwner::execution_task_attempt(request.dispatch_uid), + ) + .await?; let Some(started) = begin_release_shared( workflow, ctx, @@ -195,7 +238,6 @@ pub(super) async fn handle_task_attempt_watchdog( )); }; let receipt = checkpoint_task_hands_shared(ctx, &attempt_request, &started).await?; - let disposition = classify_stale_attempt(task_effect_idempotency(&started)); let outcome = expired_attempt_outcome(disposition, liveness, started.task.actual.clone()); let outcome = exhaust_retry_outcome(started.task.attempt, &started.task.retry, outcome); let retry_at = matches!( @@ -277,33 +319,97 @@ fn watchdog_result(outcome: ExecutionAttemptWatchdogResponseOutcome) -> TaskAtte TaskAttemptWatchdogResult { outcome } } -fn task_effect_idempotency(started: &TaskAttemptRecord) -> IdempotencyClass { - let references: Vec<_> = match &started.task.kind { - LogicalTaskKind::Capability { reference } => vec![reference], - LogicalTaskKind::Agent { - capability_refs, .. - } => capability_refs.iter().collect(), - LogicalTaskKind::Output { .. } - | LogicalTaskKind::Review { .. } - | LogicalTaskKind::WaitSignal { .. } - | LogicalTaskKind::WaitUntil { .. } - | LogicalTaskKind::CompletionVerifier { .. } => Vec::new(), - }; - if references.into_iter().any(|reference| { +fn task_effect_idempotency( + started: &TaskAttemptRecord, + checkpoint: Option<&TaskAttemptCheckpointRecord>, +) -> Result { + let direct_capability = |reference: &moa_artifacts::execution_plan::CapabilityReference| { started .run .catalog .capabilities .iter() .find(|capability| capability.reference == *reference) - .is_none_or(|capability| { - capability.idempotency_class == IdempotencyClass::NonIdempotent - || matches!(capability.source, CapabilitySource::Model) + .map(|capability| capability.idempotency_class) + .ok_or_else(|| { + "active attempt capability is absent from its pinned catalog".to_string() }) - }) { - IdempotencyClass::NonIdempotent - } else { - IdempotencyClass::Idempotent + }; + match &started.task.kind { + LogicalTaskKind::Capability { reference } => direct_capability(reference), + LogicalTaskKind::Agent { .. } | LogicalTaskKind::CompletionVerifier { .. } => { + let Some(checkpoint) = checkpoint else { + // No model response has crossed a durable boundary yet. The in-flight gateway + // completion is cancellation-fenced and idempotency-keyed. + return Ok(IdempotencyClass::Idempotent); + }; + let continuation = + serde_json::from_value::(checkpoint.payload.clone()) + .map_err(|error| format!("decode watchdog task continuation: {error}"))?; + match persisted_in_flight_effect(continuation) { + PersistedInFlightEffect::Model => Ok(IdempotencyClass::Idempotent), + PersistedInFlightEffect::Classified(idempotency) => Ok(idempotency), + PersistedInFlightEffect::Capability(tool_name) => started + .run + .catalog + .capabilities + .iter() + .find(|capability| { + capability.source.model_visible_tool_name() == Some(tool_name.as_str()) + }) + .map(|capability| capability.idempotency_class) + .ok_or_else(|| { + format!( + "pending watchdog capability `{}` is absent from the pinned catalog", + tool_name + ) + }), + PersistedInFlightEffect::UnboundExternalStart => { + Err("external-start checkpoint has no matching recovery authority".to_string()) + } + } + } + LogicalTaskKind::Output { .. } + | LogicalTaskKind::Review { .. } + | LogicalTaskKind::WaitSignal { .. } + | LogicalTaskKind::WaitUntil { .. } => Ok(IdempotencyClass::Idempotent), + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +enum PersistedInFlightEffect { + Model, + Capability(String), + Classified(IdempotencyClass), + UnboundExternalStart, +} + +fn persisted_in_flight_effect(continuation: TaskAttemptContinuation) -> PersistedInFlightEffect { + match continuation.state { + TaskAttemptContinuationState::Agent { + pending_review, + pending_tool_calls, + pending_external, + .. + } => { + if let Some(pending) = pending_external { + return PersistedInFlightEffect::Classified(pending.effect_idempotency); + } + if let Some(pending) = pending_review { + return PersistedInFlightEffect::Classified(pending.effect_idempotency); + } + pending_tool_calls + .first() + .map_or(PersistedInFlightEffect::Model, |invocation| { + PersistedInFlightEffect::Capability(invocation.name.clone()) + }) + } + TaskAttemptContinuationState::CapabilityReview { pending_review, .. } => { + PersistedInFlightEffect::Classified(pending_review.effect_idempotency) + } + TaskAttemptContinuationState::CapabilityExternalStart { .. } => { + PersistedInFlightEffect::UnboundExternalStart + } } } @@ -327,8 +433,59 @@ fn task_watchdog_settlement_response( #[cfg(test)] mod tests { + use std::collections::BTreeMap; + + use moa_artifacts::execution_plan::ExecutionUsage; + use moa_core::types::{ + completion::ToolInvocation, context::ContextMessage, security::SecurityCircuitState, + }; + use super::*; + // Pins: after a model boundary the watchdog classifies only the first exact pending + // invocation. An unrelated non-idempotent capability elsewhere in the agent catalog must + // not turn an idempotent in-flight effect into UnknownOutcome. + #[test] + fn watchdog_classifies_the_exact_persisted_agent_phase_offline() { + let continuation = TaskAttemptContinuation { + schema_version: 1, + state: TaskAttemptContinuationState::Agent { + messages: vec![ContextMessage::user("run the exact queued tools")], + next_turn: 1, + usage: ExecutionUsage { + cost_microusd: 0, + tokens: 0, + tool_calls: 0, + retrieved_bytes: 0, + }, + security_circuit: SecurityCircuitState::default(), + disabled_capabilities: BTreeMap::new(), + pending_review: None, + pending_tool_calls: vec![ + ToolInvocation { + id: Some("safe-first".to_string()), + name: "read_exact".to_string(), + input: serde_json::json!({}), + }, + ToolInvocation { + id: Some("ambiguous-later".to_string()), + name: "write_later".to_string(), + input: serde_json::json!({}), + }, + ], + pending_external: None, + }, + review_resolution: None, + external_job_resolution: None, + workspace_release_receipt_id: None, + }; + + assert_eq!( + persisted_in_flight_effect(continuation), + PersistedInFlightEffect::Capability("read_exact".to_string()) + ); + } + // Pins: a watchdog must never automatically resend an ambiguous // non-idempotent effect, while an idempotent effect remains recoverable. #[test] diff --git a/crates/moa-orchestrator/src/workflows/execution_task_attempt/yielding.rs b/crates/moa-orchestrator/src/workflows/execution_task_attempt/yielding.rs index 420d70c6d..41585048f 100644 --- a/crates/moa-orchestrator/src/workflows/execution_task_attempt/yielding.rs +++ b/crates/moa-orchestrator/src/workflows/execution_task_attempt/yielding.rs @@ -3,9 +3,7 @@ use chrono::{Duration, Utc}; use moa_artifacts::execution_plan::{ExecutionTaskOutcome, ExecutionTaskResult}; use moa_core::types::action_policy::{ActionReviewOwner, ExecutionTaskOrigin}; -use moa_core::types::sandbox_workspace::{ - ExecutionHandContinuationDisposition, ExecutionHandReleaseReceipt, -}; +use moa_core::types::sandbox_workspace::ExecutionHandReleaseReceipt; use moa_execution::{ repository::{ ExecutionAttemptState, ExecutionScope, @@ -30,10 +28,7 @@ use crate::{ services::{ action_reviews::{AcknowledgeExecutionActionReviewRequest, ActionReviewsClient}, llm_gateway::{LLMCompletionOwner, cancel_completion_owner}, - tool_executor::{ - CheckpointAndReleaseExecutionHandsRequest, - CheckpointExecutionHandsRetainingComputeRequest, ToolExecutorClient, - }, + tool_executor::{CheckpointAndReleaseExecutionHandsRequest, ToolExecutorClient}, }, workflows::{ attempt_slice::durable_utc_now_shared, @@ -171,24 +166,10 @@ pub(super) async fn yield_continuation( else { return Ok(()); }; - // A continuation is not a wait: the yield below marks the task ready and enqueues - // its run activation immediately, so the next slice is admitted in seconds. The - // durable checkpoint head still advances here, and the sandbox is kept — suspended - // where the provider can genuinely release compute, briefly hot where it cannot — - // instead of destroyed and re-provisioned across a zero-length yield. Every genuine - // park — review, input, external job, cancel, pause — still releases unconditionally. - let disposition = continue_task_hands_workflow(ctx, request, &started).await?; - // A failed suspension leaves the hand running with no owner willing to bet on it, - // so it finishes the ordinary checkpoint-and-destroy path and fences the resulting - // receipt into this checkpoint exactly as a genuine park would. - let workspace_release_receipt = match disposition { - ExecutionHandContinuationDisposition::SuspendFailed => { - checkpoint_task_hands_workflow(ctx, request, &started).await? - } - ExecutionHandContinuationDisposition::NoComputeOwned - | ExecutionHandContinuationDisposition::Suspended - | ExecutionHandContinuationDisposition::RetainedHot => None, - }; + // Every bounded activation yield is storage-only. Publish the portable checkpoint, + // destroy provider compute, and release the exact hand lease before redispatching the + // continuation; the next slice restores into freshly admitted compute. + let workspace_release_receipt = checkpoint_task_hands_workflow(ctx, request, &started).await?; let payload = continuation.to_bounded_json().map_err(TerminalError::new)?; let repository = workflow.repository.clone(); let checkpoint = NewTaskAttemptCheckpoint { @@ -525,44 +506,6 @@ fn release_claim(outcome: TaskAttemptReleaseClaimOutcome) -> Option, - request: &ExecutionTaskAttemptRequest, - started: &TaskAttemptRecord, -) -> Result { - let publish_started_at = durable_utc_now(ctx, "task_hand_continuation_started_at").await?; - Ok(crate::restate_identity::replay_safe_request( - ctx.service_client::() - .checkpoint_execution_hands_retaining_compute(Json::from( - CheckpointExecutionHandsRetainingComputeRequest { - tenant_id: request.tenant_id, - session_id: started.run.session_id, - run_uid: request.run_uid, - task_id: moa_core::types::identifiers::ExecutionTaskScopeId( - request.task_id.as_uuid(), - ), - logical_generation: started.task.generation, - attempt_generation: request.attempt_generation, - publish_deadline_at: task_hand_release_deadline(publish_started_at), - retention_deadline_at: task_hand_retention_deadline( - publish_started_at, - request.attempt_deadline_at, - ), - }, - )), - ) - .call() - .await? - .into_inner()) -} - /// Obtains provider-verified checkpoint and release proof for the attempt's hands. pub(super) async fn checkpoint_task_hands_workflow( ctx: &WorkflowContext<'_>, @@ -627,44 +570,12 @@ fn task_hand_release_deadline(release_started_at: chrono::DateTime) -> chro release_started_at + Duration::minutes(5) } -/// How long a continuation boundary may keep an unsuspendable sandbox hot. -/// -/// This bound only applies to providers that cannot actually release compute, where -/// the sandbox stays fully billed and keeps its `ActiveHands` admission slot for the -/// whole window. That is a bet that the next slice arrives before the window closes, -/// and it only pays off when it arrives *fast*: a longer window does not raise the -/// odds, it just extends the loss when the bet fails — hot idle compute plus a slot -/// withheld from runnable work plus, in the end, the full restore anyway. -/// -/// One reaper interval is therefore the whole budget. It matches -/// `HandLeaseReaperConfig::interval` (30s by default, -/// `crates/moa-hands/src/core/reaper.rs`), which is also the granularity at which the -/// deadline can actually be enforced — a shorter value would not be observed sooner, -/// and a longer one buys nothing the fast path needs. Providers with real suspension -/// never reach this constant: they release compute in the yield path instead. -const TASK_CONTINUATION_HAND_RETENTION: Duration = Duration::seconds(30); - -/// Bounds retention by both the retention window and the attempt's own deadline. -/// -/// The sandbox was admitted under this attempt's compute deadline, so retention never -/// carries it past that instant even when the window would allow it. -fn task_hand_retention_deadline( - published_at: chrono::DateTime, - attempt_deadline_at: chrono::DateTime, -) -> chrono::DateTime { - (published_at + TASK_CONTINUATION_HAND_RETENTION).min(attempt_deadline_at) -} - #[cfg(test)] mod tests { use chrono::{Duration, TimeZone, Utc}; use moa_execution::wire::ExecutionAttemptCancelReason; - use moa_hands::core::reaper::HandLeaseReaperConfig; - use super::{ - TASK_CONTINUATION_HAND_RETENTION, TaskCancelSettlement, task_cancel_settlement, - task_hand_release_deadline, task_hand_retention_deadline, - }; + use super::{TaskCancelSettlement, task_cancel_settlement, task_hand_release_deadline}; #[test] fn pause_cancel_uses_nonterminal_release_finalizer() { @@ -691,44 +602,6 @@ mod tests { } } - #[test] - fn continuation_retention_never_outlives_the_attempt_that_was_admitted_for_it() { - // Pins: a retained continuation hand is bounded by the retention window and, when - // the attempt ends sooner, by the attempt deadline it was admitted under. Losing - // either bound would let one task hold a fleet active-hands slot indefinitely. - let published_at = Utc - .with_ymd_and_hms(2026, 8, 12, 9, 0, 0) - .single() - .expect("fixture timestamp is valid"); - - let roomy_attempt_deadline = published_at + Duration::hours(1); - assert_eq!( - task_hand_retention_deadline(published_at, roomy_attempt_deadline), - published_at + TASK_CONTINUATION_HAND_RETENTION, - ); - - let expiring_attempt_deadline = published_at + Duration::seconds(10); - assert_eq!( - task_hand_retention_deadline(published_at, expiring_attempt_deadline), - expiring_attempt_deadline, - ); - assert!(TASK_CONTINUATION_HAND_RETENTION < Duration::minutes(10)); - } - - #[test] - fn hot_retention_never_outlives_one_reaper_sweep() { - // Pins: hot retention on an unsuspendable provider is one reaper interval and no - // more. The window is fully billed compute that also withholds an admission slot - // from runnable work, so a longer bet does not improve the odds of the next slice - // arriving — it only enlarges the loss when the bet fails. The bound is the - // 30s `HandLeaseReaperConfig::interval` default that actually enforces it. - assert_eq!( - TASK_CONTINUATION_HAND_RETENTION, - Duration::from_std(HandLeaseReaperConfig::default().interval) - .expect("the reaper interval fits a chrono duration"), - ); - } - #[test] fn overdue_watchdog_gets_a_fresh_bounded_sandbox_release_deadline() { // Pins: an expired compute deadline still permits one bounded checkpoint/destroy cycle; diff --git a/crates/moa-orchestrator/tests/long_horizon_execution_canary_live.rs b/crates/moa-orchestrator/tests/long_horizon_execution_canary_live.rs index 3d506f4ba..251de86b8 100644 --- a/crates/moa-orchestrator/tests/long_horizon_execution_canary_live.rs +++ b/crates/moa-orchestrator/tests/long_horizon_execution_canary_live.rs @@ -17,12 +17,7 @@ const RESTATE_KEY_BATCH_SIZE: usize = 250; async fn deployed_long_horizon_invariants_hold_for_24_hours_live() -> Result<()> { // Pins: an explicitly selected external deployment is sampled for a full // 24 hours; an instantaneous healthy sample cannot satisfy this canary. - if !canary_selected("24h")? { - eprintln!( - "SKIPPED long-horizon canary 24h: MOA_LONG_HORIZON_CANARY_WINDOW selects the other window" - ); - return Ok(()); - } + require_canary_selection("24h")?; run_canary(Duration::from_secs(24 * 60 * 60)).await } @@ -31,33 +26,39 @@ async fn deployed_long_horizon_invariants_hold_for_24_hours_live() -> Result<()> async fn deployed_long_horizon_invariants_hold_for_seven_days_live() -> Result<()> { // Pins: the seven-day deployment soak continuously rejects overdue runs, // parked compute ownership, and still-live attempt invocations. - if !canary_selected("7d")? { - eprintln!( - "SKIPPED long-horizon canary 7d: MOA_LONG_HORIZON_CANARY_WINDOW selects the other window" - ); - return Ok(()); - } + require_canary_selection("7d")?; run_canary(Duration::from_secs(7 * 24 * 60 * 60)).await } -fn canary_selected(expected: &str) -> Result { - if std::env::var("MOA_RUN_LONG_HORIZON_CANARY").as_deref() != Ok("1") { - // Both cases are `#[ignore]`d, so reaching this point means the binary was - // explicitly selected with `--run-ignored`. Returning `Ok(false)` here used to - // report a green 24h/7d soak that sampled nothing, making an unauthorized sweep - // indistinguishable from a real deployment canary in CI logs. +fn require_canary_selection(expected: &str) -> Result<()> { + let enabled = std::env::var("MOA_RUN_LONG_HORIZON_CANARY").as_deref() == Ok("1"); + let selected = std::env::var("MOA_LONG_HORIZON_CANARY_WINDOW").ok(); + validate_canary_selection(enabled, selected.as_deref(), expected) +} + +fn validate_canary_selection(enabled: bool, selected: Option<&str>, expected: &str) -> Result<()> { + if !enabled { + // Both canaries are ignored, so reaching this point means this exact case + // was explicitly selected. A successful skip would make a run that sampled + // nothing indistinguishable from a completed deployment soak. bail!( "long-horizon canary was explicitly selected without MOA_RUN_LONG_HORIZON_CANARY=1; \ refusing to report a passing soak that sampled nothing" ); } - let selected = std::env::var("MOA_LONG_HORIZON_CANARY_WINDOW").context( + let selected = selected.context( "MOA_RUN_LONG_HORIZON_CANARY=1 requires MOA_LONG_HORIZON_CANARY_WINDOW=24h or 7d", )?; if selected != "24h" && selected != "7d" { bail!("MOA_LONG_HORIZON_CANARY_WINDOW must be exactly 24h or 7d"); } - Ok(selected == expected) + if selected != expected { + bail!( + "selected long-horizon canary window {selected}, but invoked the {expected} canary; \ + select exactly one matching canary test" + ); + } + Ok(()) } async fn run_canary(window: Duration) -> Result<()> { @@ -278,3 +279,48 @@ async fn assert_no_live_restate_invocations( } Ok(()) } + +#[cfg(test)] +mod selection_tests { + use super::validate_canary_selection; + + #[test] + fn canary_selection_accepts_only_the_matching_opted_in_window_offline() { + // Pins: selecting the exact opted-in canary is the only successful + // preflight; the other named canary cannot false-pass as a skip. + validate_canary_selection(true, Some("24h"), "24h") + .expect("the matching opted-in canary window should be accepted"); + let error = validate_canary_selection(true, Some("7d"), "24h") + .expect_err("a mismatched named canary must fail closed"); + assert_eq!( + error.to_string(), + "selected long-horizon canary window 7d, but invoked the 24h canary; select exactly one matching canary test" + ); + } + + #[test] + fn canary_selection_rejects_missing_gate_or_invalid_window_offline() { + // Pins: directly running an ignored canary without the explicit gate or + // with a mistyped window cannot report a successful soak. + let missing_gate = validate_canary_selection(false, Some("24h"), "24h") + .expect_err("the explicit canary gate is required"); + assert_eq!( + missing_gate.to_string(), + "long-horizon canary was explicitly selected without MOA_RUN_LONG_HORIZON_CANARY=1; refusing to report a passing soak that sampled nothing" + ); + + let invalid_window = validate_canary_selection(true, Some("week"), "7d") + .expect_err("only the documented canary windows are valid"); + assert_eq!( + invalid_window.to_string(), + "MOA_LONG_HORIZON_CANARY_WINDOW must be exactly 24h or 7d" + ); + + let missing_window = validate_canary_selection(true, None, "7d") + .expect_err("the selected canary window is required"); + assert_eq!( + missing_window.to_string(), + "MOA_RUN_LONG_HORIZON_CANARY=1 requires MOA_LONG_HORIZON_CANARY_WINDOW=24h or 7d" + ); + } +} diff --git a/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/burst_admission.rs b/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/burst_admission.rs index a1c2067cb..9480a79b1 100644 --- a/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/burst_admission.rs +++ b/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/burst_admission.rs @@ -141,6 +141,18 @@ async fn one_thousand_common_wakes_bound_capacity_invocations_and_oldest_ready_a let controller = fixture .fixture_capability() .context("thousand-wake fixture omitted capability controller")?; + // `PHASE_TIMEOUT` bounds the first drain wave, not the wait for the timer that + // releases it. The runs are parked on an absolute wake that is still in the future + // here — by the assertion above, at least `PRE_WAKE_MARGIN_SECONDS` of it remains, + // and after fast admission far more than that. Starting a 90s observation now + // measures the pre-wake window instead of the drain and fails before the wake can + // fire, and it fails *sooner* the faster admission was. Wait out the remaining + // pre-wake window first, then bound the wave itself. + let until_wake = common_wake + .signed_duration_since(Utc::now()) + .to_std() + .unwrap_or(Duration::ZERO); + tokio::time::sleep(until_wake).await; controller.wait_for_calls(FLEET_CAP, PHASE_TIMEOUT).await?; tokio::time::sleep(Duration::from_millis(500)).await; assert_eq!(controller.calls().len(), FLEET_CAP); @@ -229,7 +241,12 @@ async fn one_thousand_common_wakes_bound_capacity_invocations_and_oldest_ready_a maximum_oldest_ready_seconds <= 60.0, "oldest ready task exceeded bounded age: {maximum_oldest_ready_seconds}s" ); - await_tenant_run_count(&pool, tenant_id, "completed", RUN_COUNT, PHASE_TIMEOUT).await?; + // The terminal settle is the tail of the same fleet-capped drain the loop above + // budgets `DRAIN_BUDGET` for: every one of `RUN_COUNT` runs still has to finish + // through a `FLEET_CAP` slot. `PHASE_TIMEOUT` bounds a single observation, so + // applying it here measures one phase against work that is `RUN_COUNT / FLEET_CAP` + // waves deep and fails partway through steady progress rather than on a stall. + await_tenant_run_count(&pool, tenant_id, "completed", RUN_COUNT, DRAIN_BUDGET).await?; let first = status(&test, &runs[0]).await?; let last = status(&test, &runs[RUN_COUNT - 1]).await?; assert_eq!(first.output, Some(json!({"completed": true}))); diff --git a/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/deadline_and_waits.rs b/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/deadline_and_waits.rs index 1fc06d8fd..bce1cb275 100644 --- a/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/deadline_and_waits.rs +++ b/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/deadline_and_waits.rs @@ -240,6 +240,13 @@ async fn watchdog_retries_idempotent_but_never_resends_ambiguous_effect_service_ "MOA_EXECUTION_ACTIVE_ATTEMPT_TIMEOUT_SECONDS".to_string(), "2".to_string(), ), + // Staleness must stay strictly below the attempt timeout, so a scenario that + // shortens the timeout has to shorten this with it or the orchestrator refuses + // to boot. + ( + "MOA_EXECUTION_ATTEMPT_HEARTBEAT_STALENESS_SECONDS".to_string(), + "1".to_string(), + ), ( "MOA_EXECUTION_TRIGGER_RECONCILIATION_CADENCE_SECONDS".to_string(), "1".to_string(), diff --git a/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/disaster_recovery.rs b/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/disaster_recovery.rs index bc911de9e..3f23831c1 100644 --- a/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/disaster_recovery.rs +++ b/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/disaster_recovery.rs @@ -14,7 +14,16 @@ async fn unbound_external_start_recovers_provider_job_without_replaying_attempt_ let fixture = external_job_execution_fixture(vec![ ( "MOA_EXECUTION_ACTIVE_ATTEMPT_TIMEOUT_SECONDS".to_string(), - "5".to_string(), + "15".to_string(), + ), + // Two constraints bracket this value. It must stay strictly below the attempt + // timeout or the orchestrator refuses to boot, and it must outlast the provider + // start this scenario performs: the start declares no bound of its own, so the + // floor is its whole window, and a floor under it kills the attempt before the + // start commits and leaves no provider job to recover. + ( + "MOA_EXECUTION_ATTEMPT_HEARTBEAT_STALENESS_SECONDS".to_string(), + "10".to_string(), ), ( "MOA_EXECUTION_TRIGGER_RECONCILIATION_CADENCE_SECONDS".to_string(), @@ -272,7 +281,14 @@ async fn running_ambiguous_attempt_is_not_redriven_after_total_restate_loss_serv vec![ ( "MOA_EXECUTION_ACTIVE_ATTEMPT_TIMEOUT_SECONDS".to_string(), - "5".to_string(), + "15".to_string(), + ), + // Staleness must stay strictly below the attempt timeout, so a scenario that + // shortens the timeout has to shorten this with it or the orchestrator refuses + // to boot. + ( + "MOA_EXECUTION_ATTEMPT_HEARTBEAT_STALENESS_SECONDS".to_string(), + "10".to_string(), ), ( "MOA_EXECUTION_TRIGGER_RECONCILIATION_CADENCE_SECONDS".to_string(), diff --git a/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/pause_and_external.rs b/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/pause_and_external.rs index 9f9f39a00..cf3f926b3 100644 --- a/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/pause_and_external.rs +++ b/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/pause_and_external.rs @@ -805,7 +805,7 @@ async fn agent_action_review_parks_then_resumes_persisted_continuation_service_e u64::try_from(checkpoint.try_get::("task_generation")?)?, waiting.generation ); - assert_eq!(checkpoint.try_get::("attempt_generation")?, 1); + assert_eq!(checkpoint.try_get::("attempt_generation")?, 2); assert_eq!(checkpoint.try_get::("released_capacity")?, 1); assert_eq!(checkpoint.try_get::("controller_activations")?, 1); let release_receipt: Value = checkpoint @@ -903,15 +903,15 @@ async fn agent_action_review_parks_then_resumes_persisted_continuation_service_e .bind(task_id(&completed_task).as_uuid()) .fetch_one(&pool) .await?; - assert_eq!(redispatch.try_get::("dispatch_count")?, 2); - assert_eq!(redispatch.try_get::("distinct_dispatches")?, 2); + assert_eq!(redispatch.try_get::("dispatch_count")?, 3); + assert_eq!(redispatch.try_get::("distinct_dispatches")?, 3); assert_eq!( redispatch.try_get::, _>("first_generation")?, Some(1) ); assert_eq!( redispatch.try_get::, _>("last_generation")?, - Some(2) + Some(3) ); let terminal = await_run_status(&test, &run, ExecutionRunStatus::Completed).await?; assert_eq!(terminal.output, Some(json!({"review": "complete"}))); diff --git a/crates/xtask/src/execution_trace_manifest.rs b/crates/xtask/src/execution_trace_manifest.rs index 37f230837..cef1fbb58 100644 --- a/crates/xtask/src/execution_trace_manifest.rs +++ b/crates/xtask/src/execution_trace_manifest.rs @@ -1070,13 +1070,6 @@ const SENDERS: &[SenderManifestEntry] = &[ "ToolExecutorClient", "checkpoint_and_release_execution_hands" ), - sender!( - "crates/moa-orchestrator/src/workflows/execution_task_attempt/yielding.rs", - "continue_task_hands_workflow", - TRACE_HELPER, - "ToolExecutorClient", - "checkpoint_execution_hands_retaining_compute" - ), sender!( "crates/moa-orchestrator/src/workflows/execution_task_attempt/yielding.rs", "park_review", diff --git a/docker-compose.yml b/docker-compose.yml index 6a0764481..9240ce9a8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -277,7 +277,12 @@ services: - | /usr/local/bin/moa-orchestrator wait-status-cutover \ --database-url "$${MOA_DATABASE_URL}" - exec /usr/local/bin/moa-orchestrator + # `--port` is required, not cosmetic: the binary defaults to 10020, while + # restate-bootstrap registers this deployment at `http://moa-orchestrator:9080` + # and the session-status-migrator serves 9080 explicitly. Without it the + # registration fails with META0003 connection refused and no handler is ever + # reachable, even though the container itself looks healthy. + exec /usr/local/bin/moa-orchestrator --port 9080 environment: HOME: /var/lib/moa MOA_DATABASE_URL: postgres://moa_owner:dev@postgres:5432/moa diff --git a/docs/01-architecture-overview.md b/docs/01-architecture-overview.md index 8d1a16564..b17e34c17 100644 --- a/docs/01-architecture-overview.md +++ b/docs/01-architecture-overview.md @@ -173,7 +173,7 @@ its operation enum has exactly eight variants: 8. `Output { value }` resolves and validates the terminal output. Every wait expiry uses `ExecutionWaitPolicy { expiry, on_expiry }`, where -`on_expiry` is `FailTask`, `FailTask`, or `ContinueWith { output }`. +`on_expiry` is `FailTask` or `ContinueWith { output }`. `ExecutionTemporalTarget::At { at }` is an exact UTC instant and is allowed in one-off compiled plans. `After { delay_seconds }` is nonzero and resolves from the instant the task actually enters its wait, not from planning or run @@ -247,6 +247,12 @@ consume one logical task. `ExecutionConfig` provides one planner repair attempt. It either regenerates from the frozen context after a strict response-schema rejection or repairs one parsed compiler-rejected candidate; malformed raw provider output is never replayed. +Every automatic amendment-planner provider call, including that sole repair, +atomically reserves its conservative cost and token ceiling from the live run +before gateway dispatch, reconciles authoritative response usage exactly once, +and persists the run revision and call ordinal that incurred the spend. A denied +reservation issues no gateway call. Restate replay reuses the same reservation, +gateway idempotency key, settlement, and planner-call audit attribution. `repeated_failure_limit = 3`, and tenant-independent defaults of `max_tasks = 10_000`, `max_tokens = 10_000_000`, `max_tool_calls = 100_000`, `max_retrieved_bytes = 10_000_000_000`, and @@ -685,7 +691,7 @@ policy. | Security events | Postgres | Signed OCSF v1.3 events in `security_events` | The central PostgreSQL migration inventory is a fresh-install-only chain of -exactly 57 files, `V000001..V000057`. The ownership manifest contains one entry +exactly 60 files, `V000001..V000060`. The ownership manifest contains one entry for every logical table family. `xtask check-migrations` rejects gaps, extra files, and missing or stale ownership entries. The 2026-08-03 hard-reset epoch removes the retired per-user token-vault tables from their original catalog diff --git a/docs/02-brain-orchestration.md b/docs/02-brain-orchestration.md index 458a4706c..bf2d69b14 100644 --- a/docs/02-brain-orchestration.md +++ b/docs/02-brain-orchestration.md @@ -515,6 +515,10 @@ instruction-only skills and capabilities with bounded turns and budgets. They cannot mutate the graph. Unexpected conditions return typed `NeedsInput` or `NeedsReplan`; every amendment is compiled, authorization-narrowing, budgeted, persisted in `plan_history`, and applied only to pending or downstream work. +Each automatic amendment generation or repair call first reserves cost and +tokens from the run ledger; unavailable capacity stops before gateway dispatch, +while completed calls reconcile exact provider usage and retain per-call audit +attribution under replay. Repeated hashes, recurring failure fingerprints, no progress, deadline, or resource exhaustion terminate with exact partial/blocked coverage instead of an infinite loop. diff --git a/docs/10-technology-stack.md b/docs/10-technology-stack.md index 814b20fb0..7867de8f8 100644 --- a/docs/10-technology-stack.md +++ b/docs/10-technology-stack.md @@ -74,7 +74,10 @@ provider-visible hand operation identities, absolute create deadlines, delayed reconciliation, and a database generation-rotation guard that rejects pre-V57 writers before provider I/O. V59 installs bounded execution activations, triggers, schedules, external jobs, admission, and the -dispatch outbox; V60 installs exact active-compute capacity reservations. +dispatch outbox, persists the current attempt step bound so watchdog staleness +respects the work actually in flight, and durably reserves and settles +automatic amendment-planner provider-call budget. V60 installs exact +active-compute capacity reservations. ## External Services @@ -267,7 +270,7 @@ and deployment setup. Key groups: | `MOA_SKILL_BUDGET_*` | skill manifest budget controls | | `MOA_EXECUTION_*` | planner repair, task/token/tool/retrieval/cost defaults, unattended confirmation threshold, deadlines, and the positive per-run live-task window | | `MOA_CLOUD_*` | remote hand provider settings | -| `MOA_RESTATE_*` and `MOA_ORCHESTRATOR_*` | Normal-runtime Restate ingress and optional health URL; bootstrap Admin access is an explicit command argument | +| `MOA_RESTATE_*` and `MOA_ORCHESTRATOR_*` | Normal-runtime Restate ingress and optional health URL; bootstrap receives explicit Admin access, while only the singleton maintenance observer may use `MOA_RESTATE_ADMIN_URL` for read-only deployment-drain queries | | `MOA_AUTH_*`, `MOA_AUTHZ_*`, `MOA_ASYNC_AUTHZ_*`, `MOA_AUDIT_SECURITY_*` | identity, authorization, builtin async authorization challenges, and OCSF security-event audit | | `MOA_SESSION_BLOB_*` | claim-check blob backend, threshold, and explicit local path when filesystem blobs are used | | `MOA_SESSION_ATTACHMENT_*` | session upload object storage backend, bucket, prefix, endpoint, and cloud credentials | diff --git a/docs/12-restate-architecture.md b/docs/12-restate-architecture.md index 97fd87201..c9cd034f2 100644 --- a/docs/12-restate-architecture.md +++ b/docs/12-restate-architecture.md @@ -365,9 +365,10 @@ that identity; no activation re-derives authority from ambient request state. Pending and waiting rows are storage-only. `ExecutionRunController/advance` is keyed by `run_uid`. One activation claims a -persisted wake epoch, applies at most `maximum_activation_steps`, dispatches at -most `dispatch_batch_size` stable ready rows, records aggregate progress, and -returns. `ExecutionTaskAttempt/run` executes one task generation within the +persisted wake epoch, charges bounded scheduler inspection and settlement work +against `maximum_activation_steps`, independently dispatches at most +`dispatch_batch_size` stable ready rows, records aggregate progress, and returns. +`ExecutionTaskAttempt/run` executes one task generation within the active-attempt timeout. Compensation uses one bounded `ExecutionCompensationAttempt` slice per immutable dispatch identity. No activation sleeps until a product event or retains an attached child for the @@ -391,7 +392,9 @@ one-off plan. Nonzero `After { delay_seconds }` is resolved from the instant the task enters the wait. Reusable templates reject `At` and use `After`, so earlier dependency duration cannot make a template timer stale. Entering any wait persists `due_at`, releases attempt and hand capacity, and schedules an immutable -trigger; expiry follows `FailTask`, `FailTask`, or `ContinueWith { output }`. +trigger. Node-owned waits expire through `FailTask` or `ContinueWith { output }`; +the plan-level runtime-input policy accepts only `FailTask` because it has no +single node output schema against which to validate a continuation value. Before dispatch, the repository atomically reserves worst-case microusd, tokens, tasks, tool calls, retrieved bytes, deadline allowance, and tenant/fleet @@ -595,10 +598,12 @@ to one recovery replica and then zero when its invocation count reaches zero. Deployment requirements: - Postgres/Neon for product data. -- Restate ingress URL for runtime invocation. Normal replicas have no Admin API - configuration or network grant; Operator owns registration and version - retention, while the revisioned bootstrap Job receives its Admin URL as an - explicit command argument. +- Restate ingress URL for runtime invocation. Versioned serving replicas and the + singleton maintenance owner receive ingress grants for durable delivery and + reconciliation, but serving replicas have no Admin API configuration or network + grant. Operator owns registration and version retention, the revisioned bootstrap + Job receives its Admin URL as an explicit command argument, and maintenance has a + narrow Admin grant only for read-only deployment-drain observation. - Redis-compatible Valkey for Session turn admission, pacing, and shared runtime cache coordination. Orchestrator startup fails if this backend is absent or resolves to process-local memory. diff --git a/docs/17-observability.md b/docs/17-observability.md index 87c1bb78a..ffaef53f1 100644 --- a/docs/17-observability.md +++ b/docs/17-observability.md @@ -102,19 +102,19 @@ bounded Restate activation state, compact session events, and trace attributes. Pending and every waiting phase are storage-only; only admitted attempts may own active capacity or hands. -Fleet health uses bounded labels only, and every exported series backs an alert: -oldest-ready age, overdue deadlines, trigger/outbox lag and dead letters, oldest -active-attempt age, admission utilization by resource and fleet/tenant-peak scope, +Fleet health uses bounded labels only. Alert-driving series cover oldest-ready age, +overdue deadlines, trigger/outbox lag and dead letters, oldest active-attempt and +external-job age, admission utilization by resource and fleet/tenant-peak scope, durable reconciliation and retention last-success ages, and parked tasks retaining -hands. None carry a tenant, run, task, deployment, or provider account identifier. -IDs belong in traces and Postgres drilldown. +hands. Bounded aggregate diagnostics such as the per-phase run census and tenant +maximum share remain available to operational dashboards even when they do not +have a dedicated alert. None carry a tenant, run, task, deployment, or provider +account identifier. IDs belong in traces and Postgres drilldown. -A metric that no alert consumes is not exported. Per-phase run census, tenant -maximum share, oldest external-job age, and old Restate deployment age/replica-hours -were removed rather than left as series nothing reads. `k8s/scripts/validate-observability.sh` enforces the invariant directly: every `pub fn record_*` in `runtime_metrics.rs` must have a caller outside that file and -outside `tests/`, so a recorder can never again be declared without being wired. +outside `tests/`, so a recorder can never again be declared without a production +producer. Alert and dashboard inventories separately pin the operational consumers. Reconciliation and retention expose separate durable health receipts. Trigger/outbox repair drives `moa_execution_maintenance_*`; terminal-evidence retention drives diff --git a/docs/22-load-and-chaos-testing.md b/docs/22-load-and-chaos-testing.md index 7156c3814..bbc814399 100644 --- a/docs/22-load-and-chaos-testing.md +++ b/docs/22-load-and-chaos-testing.md @@ -193,9 +193,15 @@ env -u MOA_ANTHROPIC_API_KEY -u MOA_OPENAI_API_KEY \ MOA_LONG_HORIZON_CANARY_WINDOW=24h \ cargo nextest run -p moa-orchestrator --locked \ --test long_horizon_execution_canary_live \ + -E 'test(/^deployed_long_horizon_invariants_hold_for_24_hours_live$/)' \ --run-ignored ignored-only --no-tests fail ``` +Use the corresponding exact seven-day test name when +`MOA_LONG_HORIZON_CANARY_WINDOW=7d`. A named canary fails closed when its test +name and configured window differ, so a wrong selection cannot report a +successful soak that sampled nothing. + T3 certifies the 10k+ QPS claim as arithmetic validated by measurement: `replicas_needed = ceil(10_000 / per_replica_rate)` must be ≤ HPA max, and a scale-out run at the computed replica count must sustain the target rate diff --git a/docs/23-environment-variables.md b/docs/23-environment-variables.md index 28eb82bf0..e404a6f02 100644 --- a/docs/23-environment-variables.md +++ b/docs/23-environment-variables.md @@ -84,7 +84,7 @@ Grouped by top-level config section. `_unset_`/`_none_` means the field is | `MOA_EXECUTION_AGENT_TURN_RETRIEVED_BYTES` | `execution.agent_turn_retrieved_bytes` | 10000000 | Worst-case retrieved-byte estimate for one agent turn | | `MOA_EXECUTION_AGENT_TURN_TOKENS` | `execution.agent_turn_tokens` | 8000 | Worst-case token estimate for one agent turn | | `MOA_EXECUTION_AGENT_TURN_TOOL_CALLS` | `execution.agent_turn_tool_calls` | 8 | Worst-case governed tool-call estimate for one agent turn | -| `MOA_EXECUTION_ATTEMPT_HEARTBEAT_STALENESS_SECONDS` | `execution.attempt_heartbeat_staleness_seconds` | 120 | Interval without durable attempt progress after which an active attempt is classified stalled; must be less than the active attempt timeout | +| `MOA_EXECUTION_ATTEMPT_HEARTBEAT_STALENESS_SECONDS` | `execution.attempt_heartbeat_staleness_seconds` | 120 | Minimum no-progress window used between steps or when the in-flight step declares no bound; a declared step bound widens only that attempt's window to at least the bound plus safety margin. This configured floor must be less than the active attempt timeout | | `MOA_EXECUTION_DISPATCH_BATCH_SIZE` | `execution.dispatch_batch_size` | 64 | Maximum ready task attempts dispatched by one controller activation | | `MOA_EXECUTION_MAX_COST_MICROUSD` | `execution.max_cost_microusd` | 100000000 | Default run cost limit in integer micro-USD | | `MOA_EXECUTION_MAX_FLEET_ACTIVE_RUNS` | `execution.max_fleet_active_runs` | 1000 | Fleet ceiling for admitted non-parked execution runs | @@ -698,6 +698,7 @@ not trip the unknown-variable audit. They do not affect application config. | `MOA_LINEAGE_SINK` | Lineage sink selection: unset/`null`, `otel`, or `postgres`; ClickHouse is analytics-only | | `MOA_PERSIST_TURN_METRICS` | Persist per-turn metrics rows | | `MOA_PROVIDERS_OVERRIDE` | Provider-catalog override (tests/tools) | +| `MOA_RESTATE_ADMIN_URL` | Optional Restate Admin endpoint override used only by the singleton maintenance deployment-drain observer; serving replicas must not receive it | | `MOA_SCIM_BASE_URL` | SCIM base URL | | `MOA_TOXIPROXY_URL` | Toxiproxy control URL (chaos tests) | | `MOA_TURBOPUFFER_LIVE_NEWS_FACTS` | Live Turbopuffer news-facts eval fixture | diff --git a/docs/examples/artifacts/patterns/custom-logic.skill.yaml b/docs/examples/artifacts/patterns/custom-logic.skill.yaml index 83d3fe916..6309b5d80 100644 --- a/docs/examples/artifacts/patterns/custom-logic.skill.yaml +++ b/docs/examples/artifacts/patterns/custom-logic.skill.yaml @@ -14,9 +14,11 @@ definition: path: SKILL.md inputs: type: object + required: [priority, retry] properties: priority: type: string + enum: [high, standard] retry: type: boolean execution_plan: @@ -44,10 +46,11 @@ definition: kind: fail_task input_schema: type: object - required: [priority] + required: [priority, retry] properties: priority: type: string + enum: [high, standard] retry: type: boolean output_schema: diff --git a/docs/operations/restate-operations.md b/docs/operations/restate-operations.md index d3cf1d0e0..0f0913adf 100644 --- a/docs/operations/restate-operations.md +++ b/docs/operations/restate-operations.md @@ -364,7 +364,8 @@ scripts/cutover-long-horizon-execution.sh \ --restate-ingress-url https://RESTATE_INGRESS_HOST \ --old-deployment-id dp_EXACT_OLD_ID \ --new-deployment-uri https://IMMUTABLE_NEW_HANDLER_URI \ - --archive-dir /explicit/precreated/empty/archive/directory + --archive-dir /explicit/precreated/empty/archive/directory \ + --session-events-schema public ``` That first pass is read-only and must print zero nonterminal Postgres runs and diff --git a/k8s/base/10-restate-cluster.yaml b/k8s/base/10-restate-cluster.yaml index 20cd0c559..1df37cbbd 100644 --- a/k8s/base/10-restate-cluster.yaml +++ b/k8s/base/10-restate-cluster.yaml @@ -69,6 +69,14 @@ spec: podSelector: matchLabels: app.kubernetes.io/name: moa-restate-bootstrap + # The singleton maintenance owner invokes durable reconciliation + # handlers through ingress but does not expose a Restate handler port. + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: moa-system + podSelector: + matchLabels: + app.kubernetes.io/name: moa-maintenance node: # Alloy discovers and scrapes each Restate pod directly on the node # port. Keep this path inside the cluster without exposing it through a @@ -86,6 +94,14 @@ spec: podSelector: matchLabels: app.kubernetes.io/name: moa-restate-bootstrap + # The singleton maintenance owner performs read-only deployment-drain + # queries. Serving replicas remain excluded from the Admin surface. + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: moa-system + podSelector: + matchLabels: + app.kubernetes.io/name: moa-maintenance config: | auto-provision = false shutdown-timeout = "10m" diff --git a/k8s/scripts/validate-observability.sh b/k8s/scripts/validate-observability.sh index b567a9dbe..5cff0b713 100755 --- a/k8s/scripts/validate-observability.sh +++ b/k8s/scripts/validate-observability.sh @@ -486,6 +486,47 @@ import yaml root = pathlib.Path(sys.argv[1]) +restate_cluster_path = root / "k8s/base/10-restate-cluster.yaml" +restate_cluster = yaml.safe_load(restate_cluster_path.read_text(encoding="utf-8")) +network_peers = ( + restate_cluster.get("spec", {}) + .get("security", {}) + .get("networkPeers", {}) +) +def moa_system_peer(name): + return { + "namespaceSelector": { + "matchLabels": {"kubernetes.io/metadata.name": "moa-system"} + }, + "podSelector": { + "matchLabels": {"app.kubernetes.io/name": name} + }, + } + +ingress_peers = network_peers.get("ingress", []) +expected_ingress_peers = [ + moa_system_peer("moa-edge"), + moa_system_peer("moa-orchestrator"), + moa_system_peer("moa-restate-bootstrap"), + moa_system_peer("moa-maintenance"), +] +if ingress_peers != expected_ingress_peers: + raise SystemExit( + "Restate ingress peers must be exactly edge, serving orchestrator, " + "bootstrap, and the singleton maintenance reconciliation owner" + ) + +admin_peers = network_peers.get("admin", []) +expected_admin_peers = [ + moa_system_peer("moa-restate-bootstrap"), + moa_system_peer("moa-maintenance"), +] +if admin_peers != expected_admin_peers: + raise SystemExit( + "Restate Admin peers must be exactly bootstrap plus the singleton " + "maintenance drain observer; serving replicas must remain excluded" + ) + restate_deployment = yaml.safe_load( (root / "k8s/base/20-orchestrator-deployment.yaml").read_text(encoding="utf-8") ) diff --git a/ops/prometheus/alerts/sandbox-workspaces.yaml b/ops/prometheus/alerts/sandbox-workspaces.yaml index 9c954376c..c430bc8c7 100644 --- a/ops/prometheus/alerts/sandbox-workspaces.yaml +++ b/ops/prometheus/alerts/sandbox-workspaces.yaml @@ -59,8 +59,8 @@ spec: labels: severity: warning annotations: - summary: Sandbox workspace fleet capacity is nearly exhausted - description: "Fleet utilization for {{ $labels.dimension }} is {{ $value | humanizePercentage }} (threshold 90%). Inspect provider inventory and durable reservations before increasing limits." + summary: A sandbox workspace capacity scope is nearly exhausted + description: "The highest enforced tenant or provider-account utilization for {{ $labels.dimension }} is {{ $value | humanizePercentage }} (threshold 90%). Inspect provider inventory and durable reservations before increasing limits." runbook_url: https://github.com/hwuiwon/moa/blob/main/docs/19-data-operations.md#workspace-capacity-pressure - alert: MOASandboxWorkspaceCheckpointFailures diff --git a/scripts/cutover-long-horizon-execution.sh b/scripts/cutover-long-horizon-execution.sh index ce2df4543..64163302b 100755 --- a/scripts/cutover-long-horizon-execution.sh +++ b/scripts/cutover-long-horizon-execution.sh @@ -161,7 +161,7 @@ require_value --new-deployment-uri "${NEW_DEPLOYMENT_URI}" require_value --archive-dir "${ARCHIVE_DIR}" require_value --session-events-schema "${SESSION_EVENTS_SCHEMA}" -for command in cargo curl find grep jq pg_dump psql restate seq tee tr wc; do +for command in cargo curl find grep head jq pg_dump psql restate seq tee tr wc; do require_cmd "${command}" done @@ -205,13 +205,22 @@ restate_cli() { } # The exact central-migration identities this cutover applies. Comparing names -# as well as versions is what lets a resumed invocation treat V000060 as "the -# migration stage already ran" instead of "some other chain reached 60". +# as well as versions is what lets a resumed invocation treat the current +# migration stage as complete instead of accepting some other chain at the same +# version. The V59 prefix remains resumable if V60 was not recorded. readonly APPLIED_MIGRATIONS_EXPECTED=$'59|long_horizon_execution\n60|sandbox_active_compute_capacity' +readonly CUTOVER_MIGRATION_MAX=60 applied_migration_identities() { psql -X "${DATABASE_ADMIN_URL}" --set=ON_ERROR_STOP=1 --tuples-only --no-align \ - --command "SELECT version, name FROM public.refinery_schema_history WHERE version IN (59, 60) ORDER BY version;" + --command "SELECT version, name FROM public.refinery_schema_history WHERE version BETWEEN 59 AND ${CUTOVER_MIGRATION_MAX} ORDER BY version;" +} + +expected_migration_prefix() { + local migration_position="$1" + local prefix_length=$((migration_position - 58)) + [[ "${prefix_length}" -gt 0 ]] || return 0 + head -n "${prefix_length}" <<<"${APPLIED_MIGRATIONS_EXPECTED}" } restate_query() { @@ -317,7 +326,9 @@ MIGRATION_POSITION="$( printf 'latest central migration: V%06d\n' "${MIGRATION_POSITION}" case "${MIGRATION_POSITION}" in - 58) + 58|59) + [[ "$(applied_migration_identities)" == "$(expected_migration_prefix "${MIGRATION_POSITION}")" ]] \ + || die "the database's applied V59-V${MIGRATION_POSITION} identities do not match this cutover" MIGRATION_STAGE="pending" ;; 60) @@ -326,7 +337,7 @@ case "${MIGRATION_POSITION}" in MIGRATION_STAGE="complete" ;; *) - die "the database must be at exactly V000058 before, or exactly V000060 after, the V59/V60 hard cut" + die "the database must be at an exact V000058-V000060 prefix of the V59/V60 hard cut" ;; esac diff --git a/scripts/run-clean-e2e.sh b/scripts/run-clean-e2e.sh index 5fc93d4be..c6b124bce 100755 --- a/scripts/run-clean-e2e.sh +++ b/scripts/run-clean-e2e.sh @@ -808,9 +808,12 @@ if [[ "${LIVE}" -eq 1 ]]; then if [[ "${RUN_LONG_HORIZON}" -eq 1 ]]; then # The suite owns its disposable Restate/Postgres/Valkey stack. Strip both - # external-stack discovery and provider keys so the lane remains hermetic - # and cannot accidentally consume billed provider credit. + # external-stack discovery, ambient runtime-cache selection, and provider + # keys so the lane remains hermetic and cannot accidentally consume billed + # provider credit. run_without_external_orchestrator env \ + -u MOA_RUNTIME_CACHE_BACKEND \ + -u MOA_RUNTIME_CACHE_REDIS_URL \ -u MOA_ANTHROPIC_API_KEY \ -u MOA_OPENAI_API_KEY \ -u MOA_GOOGLE_API_KEY \ From ac18f7426207e98d5ba7e36c0f2ec0c1b57d949f Mon Sep 17 00:00:00 2001 From: Hwuiwon Kim Date: Thu, 13 Aug 2026 16:42:33 -0400 Subject: [PATCH 05/21] modularize repository workflows and fix certification gaps --- .agents/subsystems.toml | 333 +++ AGENTS.md | 17 + Cargo.lock | 2 + crates/moa-auth/AGENTS.md | 12 + crates/moa-connectors/AGENTS.md | 12 + crates/moa-core/src/lib.rs | 1 - crates/moa-eval/core/src/lib.rs | 1 + .../core}/src/transcript.rs | 5 +- .../scenarios/long_conversation/RECORDING.md | 2 +- .../long_conversation/provider_recorded.rs | 2 +- .../long_conversation/transcript_runner.rs | 2 +- .../long_conversation_foundation_eval.rs | 2 +- .../tests/long_conversation_smoke_eval.rs | 2 +- crates/moa-execution/AGENTS.md | 12 + .../moa-execution/src/repository/capacity.rs | 54 +- .../src/repository/compensation.rs | 2122 +------------- .../compensation/pending_terminal.rs | 1964 +++++++++++++ .../execution_db/completion_projection_db.rs | 14 + .../execution_db/execution_capacity_db.rs | 67 + crates/moa-hands/AGENTS.md | 12 + .../src/core/sandbox_workspace/lifecycle.rs | 2496 +---------------- .../sandbox_workspace/lifecycle/commit.rs | 498 ++++ .../lifecycle/execution_release.rs | 749 +++++ .../sandbox_workspace/lifecycle/management.rs | 615 ++++ .../lifecycle/materialization.rs | 649 +++++ .../repository/checkpoints.rs | 13 +- .../hands_db/sandbox_workspace/dispatch_db.rs | 37 +- crates/moa-memory/AGENTS.md | 11 + .../postgres/V000058__sandbox_workspaces.sql | 12 +- .../tests/run_idempotency_db/hand_leases.rs | 28 + crates/moa-orchestrator/AGENTS.md | 13 + .../src/workflows/execution_task_attempt.rs | 1 + .../execution_task_attempt/active.rs | 1905 +------------ .../execution_task_attempt/active/agent.rs | 1018 +++++++ .../active/capability.rs | 456 +++ .../active/heartbeat.rs | 293 ++ .../execution_task_attempt/continuation.rs | 235 ++ .../execution_task_attempt/external.rs | 2 +- .../execution_task_attempt/watchdog.rs | 2 +- .../execution_task_attempt/yielding.rs | 2 +- .../burst_admission.rs | 35 +- crates/moa-providers/Cargo.toml | 1 + .../tests/openai_responses_envelope.rs | 2 +- crates/moa-test-support/Cargo.toml | 3 + crates/moa-test-support/README.md | 4 +- .../tests/fixtures_round_trip.rs | 2 +- crates/xtask/README.md | 15 + crates/xtask/src/execution_trace_manifest.rs | 54 +- crates/xtask/src/main.rs | 5 +- crates/xtask/src/subsystem_map.rs | 1249 +++++++++ ...13-codex-efficient-repository-structure.md | 156 ++ .../repository-structure-inventory.md | 62 + 52 files changed, 8767 insertions(+), 6494 deletions(-) create mode 100644 .agents/subsystems.toml create mode 100644 crates/moa-auth/AGENTS.md create mode 100644 crates/moa-connectors/AGENTS.md rename crates/{moa-core => moa-eval/core}/src/transcript.rs (98%) create mode 100644 crates/moa-execution/AGENTS.md create mode 100644 crates/moa-execution/src/repository/compensation/pending_terminal.rs create mode 100644 crates/moa-hands/AGENTS.md create mode 100644 crates/moa-hands/src/core/sandbox_workspace/lifecycle/commit.rs create mode 100644 crates/moa-hands/src/core/sandbox_workspace/lifecycle/execution_release.rs create mode 100644 crates/moa-hands/src/core/sandbox_workspace/lifecycle/management.rs create mode 100644 crates/moa-hands/src/core/sandbox_workspace/lifecycle/materialization.rs create mode 100644 crates/moa-memory/AGENTS.md create mode 100644 crates/moa-orchestrator/AGENTS.md create mode 100644 crates/moa-orchestrator/src/workflows/execution_task_attempt/active/agent.rs create mode 100644 crates/moa-orchestrator/src/workflows/execution_task_attempt/active/capability.rs create mode 100644 crates/moa-orchestrator/src/workflows/execution_task_attempt/active/heartbeat.rs create mode 100644 crates/moa-orchestrator/src/workflows/execution_task_attempt/continuation.rs create mode 100644 crates/xtask/src/subsystem_map.rs create mode 100644 docs/engineering-discipline/plans/2026-08-13-codex-efficient-repository-structure.md create mode 100644 docs/engineering-discipline/repository-structure-inventory.md diff --git a/.agents/subsystems.toml b/.agents/subsystems.toml new file mode 100644 index 000000000..338459247 --- /dev/null +++ b/.agents/subsystems.toml @@ -0,0 +1,333 @@ +version = 1 + +[audit] +max_agents = 4 +report_word_limit = 1200 +artifact_root = "target/agent-audits" + +[[subsystem]] +id = "platform-core-configuration" +owner = "moa-core" +path_prefixes = [ + "crates/moa-core/", + "crates/moa-config/", + "crates/moa-crypto/", + "crates/moa-db/", + "crates/moa-kms/", + "crates/moa-runtime-store/", + "crates/moa-wire/", +] +docs = [ + "docs/01-architecture-overview.md", + "docs/08-security.md", + "docs/10-technology-stack.md", + "docs/15-architecture-policy.md", + "docs/23-environment-variables.md", +] +agent_files = ["AGENTS.md"] +test_profiles = ["fast-pr", "db-session"] +make_targets = ["test-fast", "test-db-session"] + +[[subsystem]] +id = "execution-domain-artifacts" +owner = "moa-execution" +path_prefixes = ["crates/moa-execution/", "crates/moa-artifacts/"] +docs = [ + "docs/01-architecture-overview.md", + "docs/02-brain-orchestration.md", + "docs/09-skills-and-learning.md", + "docs/15-architecture-policy.md", +] +agent_files = ["AGENTS.md"] +local_agents = [ + { path_prefix = "crates/moa-execution/", file = "crates/moa-execution/AGENTS.md" }, +] +test_profiles = ["fast-pr", "db-session", "execution-eval-pr"] +make_targets = ["test-fast", "test-db-session", "e2e-clean-live"] + +[[subsystem.live_gates]] +id = "execution-service-e2e" +make_target = "e2e-clean-live" +authorization_env = ["MOA_RUN_LIVE_E2E"] +services = ["docker", "postgres", "restate", "openfga", "valkey"] +billed = false + +[[subsystem]] +id = "orchestration-edge-sessions-messaging" +owner = "moa-orchestrator" +path_prefixes = [ + "crates/moa-orchestrator/", + "crates/moa-edge/", + "crates/moa-session/", + "crates/moa-messaging/", +] +docs = [ + "docs/02-brain-orchestration.md", + "docs/03-communication-layer.md", + "docs/05-session-event-log.md", + "docs/12-restate-architecture.md", +] +agent_files = ["AGENTS.md"] +local_agents = [ + { path_prefix = "crates/moa-orchestrator/", file = "crates/moa-orchestrator/AGENTS.md" }, +] +test_profiles = [ + "fast-pr", + "db-session", + "db-memory", + "restate-service-e2e", + "orchestrator-service-e2e", + "restate-recovery-pr", +] +make_targets = ["test-fast", "test-db-session", "test-db-memory", "e2e-clean-live"] + +[[subsystem.live_gates]] +id = "orchestrator-service-e2e" +make_target = "e2e-clean-live" +authorization_env = ["MOA_RUN_LIVE_E2E"] +services = ["docker", "postgres", "restate", "openfga", "valkey", "pii"] +billed = false + +[[subsystem]] +id = "hands-sandbox-workspaces" +owner = "moa-hands" +path_prefixes = ["crates/moa-hands/"] +docs = [ + "docs/06-hands-and-mcp.md", + "docs/08-security.md", + "docs/25-sandbox-workspaces.md", +] +agent_files = ["AGENTS.md"] +local_agents = [ + { path_prefix = "crates/moa-hands/", file = "crates/moa-hands/AGENTS.md" }, +] +test_profiles = ["fast-pr", "db-session", "db-memory", "restate-recovery-pr"] +make_targets = ["test-fast", "test-db-session", "test-db-memory", "e2e-clean-live"] + +[[subsystem.live_gates]] +id = "sandbox-workspace-service-e2e" +make_target = "e2e-clean-live" +authorization_env = ["MOA_RUN_LIVE_E2E"] +services = ["docker", "postgres", "restate", "openfga", "valkey"] +billed = false + +[[subsystem]] +id = "connectors-knowledge-outbound-security" +owner = "moa-connectors" +path_prefixes = [ + "crates/moa-connectors/", + "crates/moa-knowledge/", + "crates/moa-security/", +] +docs = [ + "docs/08-security.md", + "docs/21-tenant-knowledge-base.md", + "docs/24-connectors-and-connections.md", +] +agent_files = ["AGENTS.md"] +local_agents = [ + { path_prefix = "crates/moa-connectors/", file = "crates/moa-connectors/AGENTS.md" }, +] +test_profiles = ["fast-pr", "db-session", "db-memory", "restate-service-e2e"] +make_targets = ["test-fast", "test-db-session", "test-db-memory", "e2e-clean-live"] + +[[subsystem.live_gates]] +id = "connector-service-e2e" +make_target = "e2e-clean-live" +authorization_env = ["MOA_RUN_LIVE_E2E"] +services = ["docker", "postgres", "restate", "openfga"] +billed = false + +[[subsystem]] +id = "providers-model-governance" +owner = "moa-providers" +path_prefixes = ["crates/moa-providers/"] +docs = [ + "docs/08-security.md", + "docs/10-technology-stack.md", + "docs/23-environment-variables.md", +] +agent_files = ["AGENTS.md"] +test_profiles = ["fast-pr", "provider-e2e"] +make_targets = ["test-fast", "test-provider-e2e"] + +[[subsystem.live_gates]] +id = "provider-e2e" +make_target = "test-provider-e2e" +authorization_env = ["MOA_RUN_LIVE_E2E", "MOA_RUN_LIVE_PROVIDER_TESTS"] +credentials_any_of = ["MOA_ANTHROPIC_API_KEY", "MOA_OPENAI_API_KEY", "MOA_GOOGLE_API_KEY"] +services = ["docker", "postgres", "restate", "openfga", "valkey"] +billed = true + +[[subsystem]] +id = "memory-retrieval-brain-context" +owner = "moa-brain" +path_prefixes = ["crates/moa-memory/", "crates/moa-retrieval/", "crates/moa-brain/"] +docs = [ + "docs/04-memory-architecture.md", + "docs/07-context-pipeline.md", + "docs/16-evaluation.md", + "docs/prompt-caching-architecture.md", +] +agent_files = ["AGENTS.md"] +local_agents = [ + { path_prefix = "crates/moa-memory/", file = "crates/moa-memory/AGENTS.md" }, +] +test_profiles = ["fast-pr", "db-memory", "eval-recorded", "provider-e2e"] +make_targets = ["test-fast", "test-db-memory", "test-provider-e2e"] + +[[subsystem.live_gates]] +id = "brain-provider-e2e" +make_target = "test-provider-e2e" +authorization_env = ["MOA_RUN_LIVE_E2E", "MOA_RUN_LIVE_PROVIDER_TESTS"] +credentials_any_of = ["MOA_ANTHROPIC_API_KEY", "MOA_OPENAI_API_KEY", "MOA_GOOGLE_API_KEY"] +services = ["docker", "postgres", "restate", "openfga", "valkey"] +billed = true + +[[subsystem]] +id = "auth-principals-contacts" +owner = "moa-authz" +path_prefixes = ["crates/moa-auth/", "crates/moa-agents/", "crates/moa-contacts/"] +docs = [ + "docs/03-communication-layer.md", + "docs/08-security.md", + "docs/14-multi-tenancy-and-learning.md", + "docs/15-architecture-policy.md", +] +agent_files = ["AGENTS.md"] +local_agents = [ + { path_prefix = "crates/moa-auth/", file = "crates/moa-auth/AGENTS.md" }, +] +test_profiles = ["fast-pr", "db-session", "authz-pentest"] +make_targets = ["test-fast", "test-db-session", "test-authz-pentest"] + +[[subsystem]] +id = "lineage-observability-analytics" +owner = "moa-lineage-core" +path_prefixes = [ + "crates/moa-lineage/", + "crates/moa-analytics/", + "crates/moa-analytics-export/", + "crates/moa-observability/", + "crates/moa-ocsf/", +] +docs = ["docs/17-observability.md", "docs/18-performance.md", "docs/analytics.md"] +agent_files = ["AGENTS.md"] +test_profiles = ["fast-pr", "db-session", "clickhouse-docker"] +make_targets = ["test-fast", "test-db-session", "test-clickhouse"] + +[[subsystem.live_gates]] +id = "clickhouse-docker" +make_target = "test-clickhouse" +authorization_env = ["MOA_RUN_CLICKHOUSE_DOCKER_TESTS"] +services = ["docker", "clickhouse"] +billed = false + +[[subsystem]] +id = "skills-experiments-eval-test-support" +owner = "moa-eval" +path_prefixes = [ + "crates/moa-skills/", + "crates/moa-experiments/", + "crates/moa-eval/", + "crates/moa-loadtest/", + "crates/moa-test-support/", +] +docs = [ + "docs/09-skills-and-learning.md", + "docs/16-evaluation.md", + "docs/20-testing.md", + "docs/22-load-and-chaos-testing.md", +] +agent_files = ["AGENTS.md"] +test_profiles = [ + "fast-pr", + "db-session", + "db-memory", + "eval-recorded", + "behavior-lab-service-e2e", + "execution-eval-pr", + "loadtest-service-e2e", +] +make_targets = [ + "test-fast", + "test-db-session", + "test-db-memory", + "test-behavior-lab-live", +] + +[[subsystem.live_gates]] +id = "behavior-lab-live" +make_target = "test-behavior-lab-live" +authorization_env = ["MOA_RUN_LIVE_E2E", "MOA_RUN_LIVE_PROVIDER_TESTS"] +credentials_any_of = ["MOA_ANTHROPIC_API_KEY", "MOA_OPENAI_API_KEY", "MOA_GOOGLE_API_KEY"] +budget_env = ["MOA_BEHAVIOR_LAB_BUDGET_USD"] +services = ["docker", "postgres", "restate", "openfga", "valkey"] +billed = true + +[[subsystem]] +id = "migrations-database-operations" +owner = "moa-migrations" +path_prefixes = ["crates/moa-migrations/"] +docs = [ + "docs/15-architecture-policy.md", + "docs/19-data-operations.md", + "docs/20-testing.md", +] +agent_files = ["AGENTS.md"] +test_profiles = ["fast-pr", "db-session"] +make_targets = ["test-fast", "test-db-session"] + +[[subsystem]] +id = "repository-tooling-deployment-docs" +owner = "xtask" +path_prefixes = [ + ".agents/", + ".cargo/", + ".claude/", + ".codex/", + ".config/", + ".githooks/", + ".github/", + ".vscode/", + "crates/xtask/", + "crates/workspace-hack/", + "dashboards/", + "docker/", + "docs/", + "k8s/", + "live/", + "ops/", + "paper/", + "scripts/", + "services/", + ".dockerignore", + ".env.example", + ".envrc", + ".gitignore", + ".mcp.json", + "AGENTS.md", + "ARCHITECTURE.md", + "CHANGELOG.md", + "CLAUDE.md", + "CONTRIBUTING.md", + "Cargo.lock", + "Cargo.toml", + "Dockerfile", + "LICENSE", + "Makefile", + "README.md", + "SEQUENCE-DIAGRAMS.md", + "docker-compose.chaos.yml", + "docker-compose.edge.yml", + "docker-compose.yml", + "rust-toolchain.toml", +] +docs = [ + "docs/10-technology-stack.md", + "docs/15-architecture-policy.md", + "docs/20-testing.md", +] +agent_files = ["AGENTS.md"] +test_profiles = ["fast-pr", "ci"] +make_targets = ["test-fast", "test-ci", "test-affected"] diff --git a/AGENTS.md b/AGENTS.md index e04aed1cc..c866b226f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -123,3 +123,20 @@ For codebase questions, first use `codegraph_explore` when a `.codegraph/` index exists. If MCP tools are not available, use `./scripts/codegraph explore ""`. For focused local checks, use `./scripts/codegraph node`, `query`, `callers`, `callees`, or `impact`. + +## Bounded Repository Workflow + +Before broad repository work, resolve changed or named paths through +`.agents/subsystems.toml`; use `cargo xtask plan-subsystem-audit` when a durable +context packet is useful. Discovery is read-only first. Use at most four +discovery agents by default, give each only its routed paths, canonical docs, +and applicable `AGENTS.md` files, and reconcile their exact-path evidence before +editing. + +Implementation write sets must be disjoint. One integration owner runs broad +Cargo, Docker, or E2E validation after workers stop editing; workers run only +focused checks for their owned surface. Bound terminal output, keep durable +reports under `target/agent-audits/`, and update that run's `checkpoint.md` after +each completed phase so later work can resume without replaying discovery. +Live, billed, credentialed, 24-hour, and 7-day gates require explicit user +authorization; a registry entry describes prerequisites but never grants it. diff --git a/Cargo.lock b/Cargo.lock index 9dc0c1b90..b94da8b05 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4414,6 +4414,7 @@ dependencies = [ "metrics", "moa-config", "moa-core", + "moa-eval-core", "moa-memory-graph", "moa-memory-pii", "moa-observability", @@ -4587,6 +4588,7 @@ dependencies = [ "moa-config", "moa-core", "moa-crypto", + "moa-eval-core", "moa-execution", "moa-kms", "moa-migrations", diff --git a/crates/moa-auth/AGENTS.md b/crates/moa-auth/AGENTS.md new file mode 100644 index 000000000..f0d705168 --- /dev/null +++ b/crates/moa-auth/AGENTS.md @@ -0,0 +1,12 @@ +# Auth Instructions + +Read `docs/03-communication-layer.md`, `docs/08-security.md`, and the auth +placement rules in `docs/15-architecture-policy.md`. Keep schema constants, +authorization checks/outbox, provider identity, credential-vault behavior, and +bootstrap responsibilities in their documented child crates. Protected reads +must follow authz, delegated writes must retain delegation, and deletes must +enqueue inverse tuples transactionally. + +Use `fast-pr`, `db-session`, and the serialized `authz-pentest` profile. Local +OpenFGA/Postgres prerequisites are infrastructure checks, not permission to run +external identity-provider or credentialed tests. diff --git a/crates/moa-connectors/AGENTS.md b/crates/moa-connectors/AGENTS.md new file mode 100644 index 000000000..80c74e70a --- /dev/null +++ b/crates/moa-connectors/AGENTS.md @@ -0,0 +1,12 @@ +# Connector Instructions + +Read `docs/08-security.md`, `docs/21-tenant-knowledge-base.md`, and +`docs/24-connectors-and-connections.md`. `moa-connectors` owns generic tenant +connection lifecycle, bindings, constrained HTTP execution, and invocation +ledgers; immutable definitions stay in `moa-artifacts`, knowledge projections in +`moa-knowledge`, and destination admission in `moa-security`. Do not add the +forbidden dependencies recorded in `docs/15-architecture-policy.md`. + +Use `fast-pr`, `db-session`, and `db-memory` locally. Service/provider checks +need the named clean-E2E flags, services, credentials, and separate live +authorization recorded in the subsystem registry. diff --git a/crates/moa-core/src/lib.rs b/crates/moa-core/src/lib.rs index 5b4b8faf1..8e5ecba76 100644 --- a/crates/moa-core/src/lib.rs +++ b/crates/moa-core/src/lib.rs @@ -14,7 +14,6 @@ pub mod session_engine; pub mod session_replay; pub mod shell; pub mod traits; -pub mod transcript; pub mod truncation; pub mod types; pub mod workspace; diff --git a/crates/moa-eval/core/src/lib.rs b/crates/moa-eval/core/src/lib.rs index 125de6c8f..e88885f3b 100644 --- a/crates/moa-eval/core/src/lib.rs +++ b/crates/moa-eval/core/src/lib.rs @@ -16,6 +16,7 @@ pub mod reliability; pub mod replay; pub mod resource_report; pub mod results; +pub mod transcript; pub mod types; pub use admission::{ diff --git a/crates/moa-core/src/transcript.rs b/crates/moa-eval/core/src/transcript.rs similarity index 98% rename from crates/moa-core/src/transcript.rs rename to crates/moa-eval/core/src/transcript.rs index 6b78840a4..74a4afb61 100644 --- a/crates/moa-core/src/transcript.rs +++ b/crates/moa-eval/core/src/transcript.rs @@ -7,10 +7,7 @@ use std::path::Path; use serde::{Deserialize, Serialize}; use thiserror::Error; -use crate::{ - types::completion::StopReason, types::completion::TokenUsage, - types::completion::ToolCallContent, -}; +use moa_core::types::completion::{StopReason, TokenUsage, ToolCallContent}; /// A recorded scenario transcript. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] diff --git a/crates/moa-eval/scenarios/long_conversation/RECORDING.md b/crates/moa-eval/scenarios/long_conversation/RECORDING.md index 0ecb94c0b..214b50d8c 100644 --- a/crates/moa-eval/scenarios/long_conversation/RECORDING.md +++ b/crates/moa-eval/scenarios/long_conversation/RECORDING.md @@ -13,7 +13,7 @@ To re-record a scenario: 2. Export the relevant live provider key locally. Do not commit secrets or shell history containing secrets. 3. Run the scenario through a dedicated recorder or fixture-generation workflow. 4. Capture the provider stream with the recording wrapper and save it as `/transcript.jsonl`. -5. Validate it with `moa_core::transcript::Transcript::read_jsonl`. +5. Validate it with `moa_eval_core::transcript::Transcript::read_jsonl`. 6. Run `cargo test -p moa-eval --test long_conversation_smoke_eval --locked -- --ignored`. The current in-repo long-conversation runner replays recorded transcripts; it diff --git a/crates/moa-eval/src/long_conversation/provider_recorded.rs b/crates/moa-eval/src/long_conversation/provider_recorded.rs index b02508188..321bc155d 100644 --- a/crates/moa-eval/src/long_conversation/provider_recorded.rs +++ b/crates/moa-eval/src/long_conversation/provider_recorded.rs @@ -3,7 +3,6 @@ use std::sync::{Arc, Mutex}; use async_trait::async_trait; -use moa_core::transcript::{ProviderEvent, Transcript}; use moa_core::{ error::MoaError, error::Result as MoaResult, traits::LLMProvider, types::completion::CompletionContent, types::completion::CompletionRequestView, @@ -12,6 +11,7 @@ use moa_core::{ types::completion::TokenUsage, types::context::MessageRole, types::identifiers::ModelId, types::model::ModelCapabilities, types::model::TokenPricing, types::model::ToolCallFormat, }; +use moa_eval_core::transcript::{ProviderEvent, Transcript}; const COMPACTION_MAX_OUTPUT_TOKENS: usize = 700; const RECORDED_COMPACTION_SUMMARY: &str = diff --git a/crates/moa-eval/src/long_conversation/transcript_runner.rs b/crates/moa-eval/src/long_conversation/transcript_runner.rs index ec1bf32bb..85fb598e1 100644 --- a/crates/moa-eval/src/long_conversation/transcript_runner.rs +++ b/crates/moa-eval/src/long_conversation/transcript_runner.rs @@ -16,7 +16,6 @@ use moa_brain::{ runtime_events::RuntimeEvent, }; use moa_config::MoaConfig; -use moa_core::transcript::Transcript; use moa_core::{ error::MoaError, events::Event, traits::LLMProvider, traits::SessionStore, types::completion::CompletionRequestView, types::completion::CompletionStream, @@ -31,6 +30,7 @@ use moa_core::{ types::segments::TaskSegment, types::segments::deterministic_segment_id, types::session::SessionMeta, }; +use moa_eval_core::transcript::Transcript; use moa_eval_core::{ AgentConfig, ConversationCost, EngineOptions, Error, EvalResult, EvalScore, EvalScoreValue, EvalStatus, LongConversationMode, LongSessionInterleaving, LongTestCase, Result, TestCase, diff --git a/crates/moa-eval/tests/long_conversation_foundation_eval.rs b/crates/moa-eval/tests/long_conversation_foundation_eval.rs index 70d6586d6..72e040217 100644 --- a/crates/moa-eval/tests/long_conversation_foundation_eval.rs +++ b/crates/moa-eval/tests/long_conversation_foundation_eval.rs @@ -2,7 +2,6 @@ use std::sync::Arc; use moa_config::MoaConfig; -use moa_core::transcript::{ProviderEvent, Transcript, Turn, UserUtterance}; use moa_core::{ types::completion::CompletionRequest, types::completion::StopReason, types::completion::TokenUsage, types::identifiers::SessionId, @@ -15,6 +14,7 @@ use moa_eval::long_conversation::{ SafetyScores, ScoreCard, ToolScores, TurnUsage, compute_input_cached_ratio, compute_prefix_stability, }; +use moa_eval_core::transcript::{ProviderEvent, Transcript, Turn, UserUtterance}; use moa_eval_core::{ AgentConfig, EngineOptions, EvalStatus, LongConversationMode, LongTestCase, TestCase, TestCaseKind, TestSuite, diff --git a/crates/moa-eval/tests/long_conversation_smoke_eval.rs b/crates/moa-eval/tests/long_conversation_smoke_eval.rs index 2cb5b6a7a..9b2fc0be9 100644 --- a/crates/moa-eval/tests/long_conversation_smoke_eval.rs +++ b/crates/moa-eval/tests/long_conversation_smoke_eval.rs @@ -15,7 +15,6 @@ use async_trait::async_trait; use moa_artifacts::document::ArtifactStatus; use moa_artifacts::registry::{ArtifactRegistry, NewArtifactDraft, NewArtifactFile}; use moa_core::shell::{has_action_policy_unsafe_shell_syntax, split_shell_chain}; -use moa_core::transcript::{ProviderEvent, Transcript, Turn, UserUtterance}; use moa_core::{ error::MoaError, events::Event, traits::LLMProvider, types::action_policy::ActionRuleScope, types::completion::CompletionRequestView, types::completion::CompletionResponse, @@ -26,6 +25,7 @@ use moa_core::{ }; use moa_eval::fixture_ids::tenant_id_from_storage_partition; use moa_eval::long_conversation::{Budgets, RecordedScriptedProvider, run_scenario_with_provider}; +use moa_eval_core::transcript::{ProviderEvent, Transcript, Turn, UserUtterance}; use moa_eval_core::{ ActionPolicyOverride, ActionPolicyRuleOverride, AgentConfig, EngineOptions, LongConversationMode, LongSessionInterleaving, LongTestCase, SecondaryLongSession, TestCase, diff --git a/crates/moa-execution/AGENTS.md b/crates/moa-execution/AGENTS.md new file mode 100644 index 000000000..099b7ca72 --- /dev/null +++ b/crates/moa-execution/AGENTS.md @@ -0,0 +1,12 @@ +# Execution Instructions + +Read the execution ownership section of `docs/01-architecture-overview.md` and +`docs/15-architecture-policy.md`. Keep compiler, interpreter, and scheduler +transitions pure; repositories own SQL and fencing, while Restate remains in +`moa-orchestrator`. Preserve SQL lock/order, idempotency keys, generation +fences, accounting, and public `ExecutionRepository` paths. + +Use `fast-pr` for pure logic and `db-session` for repository behavior. The +deterministic `execution-eval-pr` service lane runs through the clean E2E +harness without live authorization; set live flags only for explicitly live or +provider-backed targets. diff --git a/crates/moa-execution/src/repository/capacity.rs b/crates/moa-execution/src/repository/capacity.rs index e05ae934d..bf8f54e8b 100644 --- a/crates/moa-execution/src/repository/capacity.rs +++ b/crates/moa-execution/src/repository/capacity.rs @@ -359,6 +359,11 @@ impl ExecutionRepository { let mut admitted = Vec::with_capacity(bounded_limit); let mut saturated_tenants = Vec::::new(); let mut exhausted_runs = Vec::::new(); + // The transaction retains each tenant row lock until commit. Cache its remaining + // capacity so a same-tenant batch does not re-issue the identical INSERT and FOR UPDATE + // for every admitted task while still decrementing only after all durable state for that + // task has been written below. + let mut tenant_available_by_id = BTreeMap::::new(); while admitted.len() < bounded_limit { let Some(tenant_id) = select_fair_ready_tenant( &mut conn, @@ -371,21 +376,28 @@ impl ExecutionRepository { else { break; }; - ensure_tenant_bucket( - conn.as_mut(), - TenantId::from(tenant_id), - "active_tasks", - i64::from(config.max_tenant_active_tasks), - ) - .await?; - let tenant_available = lock_capacity_bucket( - conn.as_mut(), - "tenant", - Some(tenant_id), - "active_tasks", - i64::from(config.max_tenant_active_tasks), - ) - .await?; + let tenant_available = match tenant_available_by_id.get(&tenant_id) { + Some(available) => *available, + None => { + ensure_tenant_bucket( + conn.as_mut(), + TenantId::from(tenant_id), + "active_tasks", + i64::from(config.max_tenant_active_tasks), + ) + .await?; + let available = lock_capacity_bucket( + conn.as_mut(), + "tenant", + Some(tenant_id), + "active_tasks", + i64::from(config.max_tenant_active_tasks), + ) + .await?; + tenant_available_by_id.insert(tenant_id, available); + available + } + }; if tenant_available == 0 { saturated_tenants.push(tenant_id); continue; @@ -565,6 +577,18 @@ impl ExecutionRepository { ) .await?; advance_tenant_fairness(&mut conn, tenant_id, now).await?; + let tenant_available = tenant_available_by_id.get_mut(&tenant_id).ok_or_else(|| { + Error::InvalidRepositoryData { + message: "admitted task has no locked tenant capacity bucket".to_string(), + } + })?; + *tenant_available = + tenant_available + .checked_sub(1) + .ok_or_else(|| Error::InvalidRepositoryData { + message: "locked tenant active_tasks capacity was over-admitted" + .to_string(), + })?; admitted.push(ExecutionAdmissionItem { dispatch_uid, capacity_reservation_uid, diff --git a/crates/moa-execution/src/repository/compensation.rs b/crates/moa-execution/src/repository/compensation.rs index f7262cf79..dcd6dfa84 100644 --- a/crates/moa-execution/src/repository/compensation.rs +++ b/crates/moa-execution/src/repository/compensation.rs @@ -1,5 +1,7 @@ //! Compensation registration, fencing, reverse-order claims, and finalization. +mod pending_terminal; + use super::*; use super::{ capacity::{ @@ -51,10 +53,6 @@ use moa_artifacts::execution_plan::ExecutionCancelPolicy; use moa_config::ExecutionConfig; use moa_core::types::sandbox_workspace::{ExecutionHandReleaseOwner, ExecutionHandReleaseReceipt}; -const PENDING_TERMINAL_CANCEL_NAMESPACE: Uuid = - Uuid::from_u128(0xd3d4_9744_5c24_58cc_8be8_4806_faba_1837); -const MAX_PENDING_TERMINAL_PAGE_SIZE: u32 = 1_000; - /// Durable lifecycle of one bounded compensation-attempt slice. #[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] pub enum CompensationAttemptState { @@ -331,415 +329,7 @@ pub enum CompensationExternalJobSettlementOutcome { NotFound, } -async fn load_replan_stop_task( - conn: &mut ScopedConn<'_>, - run_uid: Uuid, - task_id: ExecutionTaskId, -) -> Result> { - sqlx::query(LOAD_TASK_FOR_UPDATE_SQL) - .bind(run_uid) - .bind(task_id.as_uuid()) - .fetch_optional(conn.as_mut()) - .await - .map_err(sqlx_error)? - .map(|row| task_from_row(&row)) - .transpose() -} - -fn replan_stop_receipt_audit(receipt: &ReplanStopReceipt, recorded_at: DateTime) -> Value { - json!({ - "kind": "replan_stop_fenced", - "accepted": true, - "task_id": receipt.task_id, - "task_generation": receipt.task_generation, - "base_plan_revision": receipt.base_plan_revision, - "amendment_hash": receipt.amendment_hash, - "recorded_at": recorded_at, - }) -} - impl ExecutionRepository { - /// Fences one due approved deadline and advances one bounded terminal-drain page. - #[allow(clippy::too_many_arguments)] - pub async fn fence_deadline_and_enqueue_settlement( - &self, - config: &ExecutionConfig, - scope: ExecutionScope, - run_uid: Uuid, - controller_generation: u64, - expected_wake_epoch: u64, - now: DateTime, - page_limit: u32, - ) -> Result { - validate_pending_terminal_page_limit(page_limit)?; - let mut conn = scope.begin(&self.pool).await?; - let Some(run) = load_and_lock_pending_terminal_run(&mut conn, config, run_uid).await? - else { - conn.commit().await.map_err(storage_error)?; - return Ok(PendingTerminalAdvanceOutcome::NotFound); - }; - if run.controller_generation != controller_generation - || run.wake_epoch != expected_wake_epoch - { - conn.commit().await.map_err(storage_error)?; - return Ok(PendingTerminalAdvanceOutcome::Conflict); - } - if expected_wake_epoch <= run.processed_wake_epoch { - let commit = replayed_pending_terminal_commit(&mut conn, config, run).await?; - conn.commit().await.map_err(storage_error)?; - return Ok(PendingTerminalAdvanceOutcome::Replayed(Box::new(commit))); - } - let Some(deadline_at) = run.approved_budget.deadline_at else { - return Err(Error::InvalidRepositoryData { - message: "durable execution run is missing its approved deadline".to_string(), - }); - }; - if deadline_at > now || run.status.is_terminal() { - conn.commit().await.map_err(storage_error)?; - return Ok(PendingTerminalAdvanceOutcome::Conflict); - } - let requirement_count = u64::try_from(run.goal.requirements.len()).map_err(|_| { - Error::InvalidRepositoryData { - message: "execution requirement count exceeds u64".to_string(), - } - })?; - let pending = PendingExecutionTerminal { - status: ExecutionRunStatus::Failed, - reason: ExecutionTerminalReason::DeadlineExceeded, - terminal_evidence: ExecutionTerminalEvidence { - cause: ExecutionTerminalCause::LimitStop { - reason: ExecutionLimitStop::DeadlineExceeded, - }, - satisfied_requirement_count: 0, - requirement_count, - }, - completion_check_results: Vec::new(), - terminal_gaps: vec!["approved execution deadline elapsed".to_string()], - output: run.output.clone(), - cancellation_reason: None, - }; - pending.validate()?; - let new_pending = run.pending_terminal.is_none().then_some(pending); - advance_pending_terminal_page_in_conn( - conn, - config, - run, - controller_generation, - expected_wake_epoch, - new_pending, - now, - page_limit, - ) - .await - } - - /// Persists one completion-derived terminal intent and advances its first bounded drain page. - #[allow(clippy::too_many_arguments)] - pub async fn fence_completion_terminal_and_enqueue_settlement( - &self, - config: &ExecutionConfig, - scope: ExecutionScope, - run_uid: Uuid, - controller_generation: u64, - expected_wake_epoch: u64, - pending: PendingExecutionTerminal, - now: DateTime, - page_limit: u32, - ) -> Result { - validate_pending_terminal_page_limit(page_limit)?; - pending.validate()?; - let mut conn = scope.begin(&self.pool).await?; - let Some(run) = load_and_lock_pending_terminal_run(&mut conn, config, run_uid).await? - else { - conn.commit().await.map_err(storage_error)?; - return Ok(PendingTerminalAdvanceOutcome::NotFound); - }; - if run.controller_generation != controller_generation - || run.wake_epoch != expected_wake_epoch - { - conn.commit().await.map_err(storage_error)?; - return Ok(PendingTerminalAdvanceOutcome::Conflict); - } - if expected_wake_epoch <= run.processed_wake_epoch { - let commit = replayed_pending_terminal_commit(&mut conn, config, run).await?; - conn.commit().await.map_err(storage_error)?; - return Ok(PendingTerminalAdvanceOutcome::Replayed(Box::new(commit))); - } - if run.status.is_terminal() - || run - .pending_terminal - .as_ref() - .is_some_and(|current| current != &pending) - { - conn.commit().await.map_err(storage_error)?; - return Ok(PendingTerminalAdvanceOutcome::Conflict); - } - advance_pending_terminal_page_in_conn( - conn, - config, - run, - controller_generation, - expected_wake_epoch, - Some(pending), - now, - page_limit, - ) - .await - } - - /// Persists an exact replan-stop receipt and advances its first bounded terminal-drain page. - #[allow(clippy::too_many_arguments)] - pub async fn fence_replan_stop_and_enqueue_settlement( - &self, - config: &ExecutionConfig, - scope: ExecutionScope, - run_uid: Uuid, - controller_generation: u64, - expected_revision: u64, - expected_wake_epoch: u64, - pending: PendingExecutionTerminal, - receipt: ReplanStopReceipt, - now: DateTime, - page_limit: u32, - ) -> Result { - validate_pending_terminal_page_limit(page_limit)?; - pending.validate()?; - if receipt.base_plan_revision != expected_revision - || !matches!( - pending.terminal_evidence.cause, - ExecutionTerminalCause::ReplanStop { .. } - ) - { - return Err(Error::InvalidRepositoryInput { - message: "replan-stop receipt must match the fenced revision and terminal cause" - .to_string(), - }); - } - let mut conn = scope.begin(&self.pool).await?; - let Some(run) = load_and_lock_pending_terminal_run(&mut conn, config, run_uid).await? - else { - conn.commit().await.map_err(storage_error)?; - return Ok(PendingTerminalAdvanceOutcome::NotFound); - }; - if run.controller_generation != controller_generation - || run.plan_revision != expected_revision - || run.wake_epoch != expected_wake_epoch - || run.status.is_terminal() - || run.status == ExecutionRunStatus::Compensating - || run - .pending_terminal - .as_ref() - .is_some_and(|current| current != &pending) - { - conn.commit().await.map_err(storage_error)?; - return Ok(PendingTerminalAdvanceOutcome::Conflict); - } - let Some(task) = load_replan_stop_task(&mut conn, run_uid, receipt.task_id).await? else { - conn.commit().await.map_err(storage_error)?; - return Ok(PendingTerminalAdvanceOutcome::NotFound); - }; - let receipt_exists: bool = sqlx::query_scalar( - "SELECT EXISTS (SELECT 1 FROM moa.execution_amendment_receipt \ - WHERE tenant_id=$1 AND run_uid=$2 AND base_plan_revision=$3 \ - AND amendment_hash=$4 AND receipt_kind='replan_stop' \ - AND superseded_task_id=$5 AND task_generation=$6 \ - AND cardinality(task_ids_to_release)=0)", - ) - .bind(run.tenant_id.0) - .bind(run.run_uid) - .bind(to_i64( - receipt.base_plan_revision, - "replan-stop plan revision", - )?) - .bind(receipt.amendment_hash.to_string()) - .bind(receipt.task_id.as_uuid()) - .bind(to_i64( - receipt.task_generation, - "replan-stop task generation", - )?) - .fetch_one(conn.as_mut()) - .await - .map_err(sqlx_error)?; - let intent = sqlx::query( - "SELECT tenant_id,controller_generation,wake_epoch,origin_task_id,task_generation, \ - base_plan_revision,stop_reason,amendment_hash \ - FROM moa.execution_replan_stop_intent WHERE run_uid=$1 FOR UPDATE", - ) - .bind(run.run_uid) - .fetch_optional(conn.as_mut()) - .await - .map_err(sqlx_error)?; - if run.pending_terminal.is_some() { - if !receipt_exists || intent.is_some() { - conn.commit().await.map_err(storage_error)?; - return Ok(PendingTerminalAdvanceOutcome::Conflict); - } - if expected_wake_epoch <= run.processed_wake_epoch { - let commit = replayed_pending_terminal_commit(&mut conn, config, run).await?; - conn.commit().await.map_err(storage_error)?; - return Ok(PendingTerminalAdvanceOutcome::Replayed(Box::new(commit))); - } - return advance_pending_terminal_page_in_conn( - conn, - config, - run, - controller_generation, - expected_wake_epoch, - None, - now, - page_limit, - ) - .await; - } - let Some(intent) = intent else { - conn.commit().await.map_err(storage_error)?; - return Ok(PendingTerminalAdvanceOutcome::Conflict); - }; - let ExecutionTerminalCause::ReplanStop { - reason: expected_stop_reason, - } = &pending.terminal_evidence.cause - else { - return Err(Error::InvalidRepositoryData { - message: "replan-stop fence lost its validated terminal cause".to_string(), - }); - }; - let expected_stop_reason = expected_stop_reason.as_str(); - let intent_exact = intent.try_get::("tenant_id").map_err(row_error)? - == run.tenant_id.0 - && required_u64(&intent, "controller_generation")? == controller_generation - && required_u64(&intent, "wake_epoch")? == expected_wake_epoch - && intent - .try_get::("origin_task_id") - .map_err(row_error)? - == receipt.task_id.as_uuid() - && required_u64(&intent, "task_generation")? == receipt.task_generation - && required_u64(&intent, "base_plan_revision")? == receipt.base_plan_revision - && intent - .try_get::("stop_reason") - .map_err(row_error)? - == expected_stop_reason - && intent - .try_get::("amendment_hash") - .map_err(row_error)? - == receipt.amendment_hash.to_string(); - if receipt_exists - || !intent_exact - || task.plan_revision != receipt.base_plan_revision - || task.generation != receipt.task_generation - || task.status != ExecutionTaskStatus::WaitingReplan - || !matches!( - task.current_outcome.as_ref().map(|outcome| &outcome.result), - Some(ExecutionTaskResult::NeedsReplan { .. }) - ) - { - conn.commit().await.map_err(storage_error)?; - return Ok(PendingTerminalAdvanceOutcome::Conflict); - } - sqlx::query( - "INSERT INTO moa.execution_amendment_receipt \ - (tenant_id,run_uid,base_plan_revision,amendment_hash,receipt_kind, \ - superseded_task_id,task_generation,task_ids_to_release,created_at) \ - VALUES ($1,$2,$3,$4,'replan_stop',$5,$6,'{}'::UUID[],$7)", - ) - .bind(run.tenant_id.0) - .bind(run.run_uid) - .bind(to_i64( - receipt.base_plan_revision, - "replan-stop plan revision", - )?) - .bind(receipt.amendment_hash.to_string()) - .bind(receipt.task_id.as_uuid()) - .bind(to_i64( - receipt.task_generation, - "replan-stop task generation", - )?) - .bind(now) - .execute(conn.as_mut()) - .await - .map_err(sqlx_error)?; - let deleted = sqlx::query( - "DELETE FROM moa.execution_replan_stop_intent WHERE tenant_id=$1 AND run_uid=$2 \ - AND controller_generation=$3 AND wake_epoch=$4", - ) - .bind(run.tenant_id.0) - .bind(run.run_uid) - .bind(to_i64(controller_generation, "controller generation")?) - .bind(to_i64(expected_wake_epoch, "expected wake epoch")?) - .execute(conn.as_mut()) - .await - .map_err(sqlx_error)?; - if deleted.rows_affected() != 1 { - return Err(Error::InvalidRepositoryData { - message: "replan-stop fence lost its exact durable intent".to_string(), - }); - } - sqlx::query(APPEND_TASK_OUTCOME_AUDIT_SQL) - .bind(run_uid) - .bind(task.task_id.as_uuid()) - .bind(replan_stop_receipt_audit(&receipt, now)) - .fetch_one(conn.as_mut()) - .await - .map_err(sqlx_error)?; - advance_pending_terminal_page_in_conn( - conn, - config, - run, - controller_generation, - expected_wake_epoch, - Some(pending), - now, - page_limit, - ) - .await - } - - /// Advances one bounded page of an already-fenced pending-terminal drain. - #[allow(clippy::too_many_arguments)] - pub async fn advance_pending_terminal_settlement( - &self, - config: &ExecutionConfig, - scope: ExecutionScope, - run_uid: Uuid, - controller_generation: u64, - expected_wake_epoch: u64, - now: DateTime, - page_limit: u32, - ) -> Result { - validate_pending_terminal_page_limit(page_limit)?; - let mut conn = scope.begin(&self.pool).await?; - let Some(run) = load_and_lock_pending_terminal_run(&mut conn, config, run_uid).await? - else { - conn.commit().await.map_err(storage_error)?; - return Ok(PendingTerminalAdvanceOutcome::NotFound); - }; - if run.controller_generation != controller_generation - || run.wake_epoch != expected_wake_epoch - { - conn.commit().await.map_err(storage_error)?; - return Ok(PendingTerminalAdvanceOutcome::Conflict); - } - if expected_wake_epoch <= run.processed_wake_epoch { - let commit = replayed_pending_terminal_commit(&mut conn, config, run).await?; - conn.commit().await.map_err(storage_error)?; - return Ok(PendingTerminalAdvanceOutcome::Replayed(Box::new(commit))); - } - if run.pending_terminal.is_none() || run.status.is_terminal() { - conn.commit().await.map_err(storage_error)?; - return Ok(PendingTerminalAdvanceOutcome::Conflict); - } - advance_pending_terminal_page_in_conn( - conn, - config, - run, - controller_generation, - expected_wake_epoch, - None, - now, - page_limit, - ) - .await - } - /// Admits the highest unsettled compensation into one bounded durable slice. pub async fn admit_next_compensation_attempt( &self, @@ -2486,279 +2076,43 @@ async fn load_existing_compensation_admission( }) } -enum PendingCompensationDrive { - Admitted(Box), - Replayed(Box), - CapacityUnavailable { retry_at: DateTime }, - ExternalCancellation(ExecutionDispatchRecord), - Parked, - Complete, - ManualRepair(CompensationRegistrationProjection), +fn compensation_attempt_state_from_row(row: &PgRow) -> Result { + row.try_get::("attempt_state") + .map_err(row_error)? + .parse() } -async fn drive_pending_terminal_compensation_in_conn( - conn: &mut ScopedConn<'_>, - config: &ExecutionConfig, - run: &ExecutionRunRecord, - now: DateTime, -) -> Result { - let nonterminal_forward_exists: bool = sqlx::query_scalar( - "SELECT EXISTS (SELECT 1 FROM moa.execution_task WHERE run_uid=$1 \ - AND status NOT IN ('completed','skipped','failed','cancelled','unknown_outcome'))", - ) - .bind(run.run_uid) - .fetch_one(conn.as_mut()) - .await - .map_err(sqlx_error)?; - // `manual_repair_required` is deliberately NOT rejected here. Settling a compensation - // attempt with a non-retryable failure sets that flag on the run, so rejecting it made - // the very next controller activation return a terminal repository error and the run - // sat in `compensating` forever instead of terminalizing `Failed`/`CompensationFailed`. - // The flag means "stop driving automatically and hand this to an operator", which is a - // `ManualRepair` outcome, not an invalid state — see the check below the registration - // load, which needs the row to report which compensation is stuck. - if run.status != ExecutionRunStatus::Compensating - || run.pending_terminal.is_none() - || nonterminal_forward_exists - { - return Err(Error::InvalidRepositoryData { - message: "bounded compensation driver entered from an invalid run state".to_string(), - }); - } - let Some(row) = sqlx::query( - "SELECT * FROM moa.execution_compensation WHERE run_uid=$1 \ - AND status <> 'completed' ORDER BY registered_sequence DESC \ - LIMIT 1 FOR UPDATE", - ) - .bind(run.run_uid) - .fetch_optional(conn.as_mut()) - .await - .map_err(sqlx_error)? - else { - return Ok(PendingCompensationDrive::Complete); - }; - let registration = compensation_from_row(&row)?; - let attempt_state = compensation_attempt_state_from_row(&row)?; - if run.manual_repair_required - || matches!( - registration.status, - CompensationStatus::Failed | CompensationStatus::UnknownOutcome - ) - { - return Ok(PendingCompensationDrive::ManualRepair(registration)); - } - if attempt_state == CompensationAttemptState::Dispatching { - return Ok(PendingCompensationDrive::Replayed(Box::new( - load_existing_compensation_admission(conn, config, run, &row, ®istration).await?, - ))); - } - if attempt_state == CompensationAttemptState::WaitingExternal { - let external_job_uid = row - .try_get::, _>("external_job_uid") - .map_err(row_error)? - .ok_or_else(|| Error::InvalidRepositoryData { - message: "waiting-external compensation lost its exact external job UID" - .to_string(), - })?; - let owner = ExecutionExternalJobOwner::Compensation { - compensation_id: registration.compensation_id.as_uuid(), - compensation_generation: registration.generation, - compensation_attempt_generation: required_u64(&row, "attempt_generation")?, - }; - return match request_external_job_cancellation_in_conn( - conn, - config, - external_job_uid, - owner, - now, - ) - .await? - { - ExecutionExternalJobCancellationRequestOutcome::Applied(dispatch) - | ExecutionExternalJobCancellationRequestOutcome::Replayed(dispatch) => { - Ok(PendingCompensationDrive::ExternalCancellation(dispatch)) - } - ExecutionExternalJobCancellationRequestOutcome::UnboundPendingRecovery => { - Ok(PendingCompensationDrive::Parked) - } - ExecutionExternalJobCancellationRequestOutcome::AlreadyTerminal => { - let job = load_external_job_for_update_in_conn(conn.as_mut(), external_job_uid) - .await? - .ok_or_else(|| Error::InvalidRepositoryData { - message: "terminal compensation external job disappeared".to_string(), - })?; - settle_external_job_terminal_in_conn(conn, &job, now).await?; - Ok(PendingCompensationDrive::Parked) - } - ExecutionExternalJobCancellationRequestOutcome::NotFound - | ExecutionExternalJobCancellationRequestOutcome::Stale => { - Err(Error::InvalidRepositoryData { - message: "waiting-external compensation has a stale external job owner" - .to_string(), - }) - } - }; - } - if matches!( - attempt_state, - CompensationAttemptState::Running - | CompensationAttemptState::Cancelling - | CompensationAttemptState::WaitingReview - ) { - return Ok(PendingCompensationDrive::Parked); - } - if attempt_state != CompensationAttemptState::Idle - || !matches!( - registration.status, - CompensationStatus::Pending | CompensationStatus::Running - ) - { - return Err(Error::InvalidRepositoryData { - message: "highest reverse-order compensation is not dispatchable or settled" - .to_string(), - }); - } - let retry_at = checked_retry_at(config, now)?; - if !compensation_capacity_available(conn, run.tenant_id).await? { - return Ok(PendingCompensationDrive::CapacityUnavailable { retry_at }); - } - if registration.status == CompensationStatus::Pending && registration.outcome.is_none() { - let forward_task = - load_forward_task(conn, run.run_uid, registration.forward_task_id).await?; - let reservation = - compensation_reservation(run, ®istration, forward_task.retry.max_attempts)?; - let mut ledger = budget_ledger(run); - if ledger.try_reserve(reservation).is_err() { - let failed = - terminalize_compensation_budget_rejection(conn, run, ®istration, reservation) - .await?; - return Ok(PendingCompensationDrive::ManualRepair(failed)); - } - persist_run_budget(conn, run.run_uid, &ledger, false).await?; - } - let deadline = checked_attempt_deadline(config, now)?; - let attempt_generation = required_u64(&row, "attempt_generation")?; - let dispatch_uid = Uuid::now_v7(); - let watchdog_uid = Uuid::now_v7(); - let reservation_uid = reserve_compensation_attempt_capacity( - conn, - config, - run, - ®istration, - attempt_generation, - deadline, - now, - ) - .await?; - let watchdog = create_trigger_with_dispatch_in_conn( - conn.as_mut(), - config, - &compensation_trigger( - run, - ®istration, - attempt_generation, - watchdog_uid, - ExecutionTriggerKind::CompensationWatchdog, - deadline, - json!({}), - ), - ) - .await?; - let attempt_request = ExecutionCompensationAttemptRequest { - dispatch_uid, - capacity_reservation_uid: reservation_uid, - watchdog_trigger_uid: watchdog.trigger.trigger_uid, - watchdog_dispatch_uid: watchdog.dispatch.dispatch_uid, - run_uid: run.run_uid, - compensation_id: registration.compensation_id, - compensation_generation: registration.generation, - compensation_attempt_generation: attempt_generation, - controller_generation: run.controller_generation, - attempt_deadline_at: deadline, - tenant_id: run.tenant_id, - }; - let dispatch = enqueue_dispatch_in_conn( - conn.as_mut(), - &compensation_dispatch(run, &attempt_request, now)?, - ) - .await?; - let updated = sqlx::query( - "UPDATE moa.execution_compensation SET status='running', \ - attempt_state='dispatching', attempt_started_at=$6, \ - last_progress_at=GREATEST(last_progress_at,$6), \ - attempt_deadline_at=$7, waiting_since=NULL, active_dispatch_uid=$8, \ - dispatch_sequence=dispatch_sequence+1, started_at=COALESCE(started_at,$6), \ - updated_at=NOW() WHERE run_uid=$1 AND compensation_id=$2 AND generation=$3 \ - AND attempt_generation=$4 AND attempt_state='idle' \ - AND status IN ('pending','running') AND EXISTS ( \ - SELECT 1 FROM moa.execution_run AS current_run \ - WHERE current_run.run_uid=$1 AND current_run.controller_generation=$5) \ - RETURNING *", - ) - .bind(run.run_uid) - .bind(registration.compensation_id.as_uuid()) - .bind(to_i64(registration.generation, "compensation generation")?) - .bind(to_i64( - attempt_generation, - "compensation attempt generation", - )?) - .bind(to_i64(run.controller_generation, "controller generation")?) - .bind(now) - .bind(deadline) - .bind(dispatch_uid) - .fetch_optional(conn.as_mut()) - .await - .map_err(sqlx_error)? - .ok_or_else(|| Error::InvalidRepositoryData { - message: "bounded compensation admission lost its exact row lock".to_string(), - })?; - Ok(PendingCompensationDrive::Admitted(Box::new( - CompensationAttemptAdmission { - attempt: compensation_attempt_from_row(&updated, run)?, - capacity_reservation_uid: reservation_uid, - dispatch, - watchdog, - }, - ))) -} - -fn compensation_attempt_state_from_row(row: &PgRow) -> Result { - row.try_get::("attempt_state") - .map_err(row_error)? - .parse() -} - -fn compensation_release_intent_from_row( - row: &PgRow, -) -> Result> { - row.try_get::, _>("release_intent") - .map_err(row_error)? - .map(|label| match label.as_str() { - "outcome" => Ok(ExecutionCompensationReleaseIntent::Outcome), - "retry" => Ok(ExecutionCompensationReleaseIntent::Retry), - "review" => Ok(ExecutionCompensationReleaseIntent::Review), - "external_job" => Ok(ExecutionCompensationReleaseIntent::ExternalJob), - "pause" => Ok(ExecutionCompensationReleaseIntent::Pause), - "watchdog" => Ok(ExecutionCompensationReleaseIntent::Watchdog), - "deadline" => Ok(ExecutionCompensationReleaseIntent::Deadline), - "run_terminal" => Ok(ExecutionCompensationReleaseIntent::RunTerminal), - _ => Err(Error::InvalidRepositoryData { - message: format!("unknown compensation release intent `{label}`"), - }), - }) - .transpose() -} - -fn compensation_cancel_request_fence( - request: &ExecutionCompensationAttemptCancelRequest, -) -> CompensationAttemptFence { - CompensationAttemptFence { - run_uid: request.run_uid, - compensation_id: request.compensation_id, - controller_generation: request.controller_generation, - compensation_generation: request.compensation_generation, - attempt_generation: request.compensation_attempt_generation, - dispatch_uid: request.active_dispatch_uid, +fn compensation_release_intent_from_row( + row: &PgRow, +) -> Result> { + row.try_get::, _>("release_intent") + .map_err(row_error)? + .map(|label| match label.as_str() { + "outcome" => Ok(ExecutionCompensationReleaseIntent::Outcome), + "retry" => Ok(ExecutionCompensationReleaseIntent::Retry), + "review" => Ok(ExecutionCompensationReleaseIntent::Review), + "external_job" => Ok(ExecutionCompensationReleaseIntent::ExternalJob), + "pause" => Ok(ExecutionCompensationReleaseIntent::Pause), + "watchdog" => Ok(ExecutionCompensationReleaseIntent::Watchdog), + "deadline" => Ok(ExecutionCompensationReleaseIntent::Deadline), + "run_terminal" => Ok(ExecutionCompensationReleaseIntent::RunTerminal), + _ => Err(Error::InvalidRepositoryData { + message: format!("unknown compensation release intent `{label}`"), + }), + }) + .transpose() +} + +fn compensation_cancel_request_fence( + request: &ExecutionCompensationAttemptCancelRequest, +) -> CompensationAttemptFence { + CompensationAttemptFence { + run_uid: request.run_uid, + compensation_id: request.compensation_id, + controller_generation: request.controller_generation, + compensation_generation: request.compensation_generation, + attempt_generation: request.compensation_attempt_generation, + dispatch_uid: request.active_dispatch_uid, } } @@ -3838,17 +3192,6 @@ fn force_terminal_failure_if_exhausted( } } -fn validate_pending_terminal_page_limit(page_limit: u32) -> Result<()> { - if page_limit == 0 || page_limit > MAX_PENDING_TERMINAL_PAGE_SIZE { - return Err(Error::InvalidRepositoryInput { - message: format!( - "pending-terminal page limit must be between 1 and {MAX_PENDING_TERMINAL_PAGE_SIZE}" - ), - }); - } - Ok(()) -} - /// Settles one terminal provider job into its exact waiting compensation attempt. pub(super) async fn settle_external_job_terminal_in_conn( conn: &mut ScopedConn<'_>, @@ -4045,1069 +3388,60 @@ pub(super) async fn settle_external_job_terminal_in_conn( )) } -async fn load_and_lock_pending_terminal_run( - conn: &mut ScopedConn<'_>, - config: &ExecutionConfig, - run_uid: Uuid, -) -> Result> { - let Some(visible_row) = sqlx::query(LOAD_RUN_SQL) - .bind(run_uid) - .fetch_optional(conn.as_mut()) - .await - .map_err(sqlx_error)? - else { - return Ok(None); - }; - let visible = run_from_row(&visible_row)?; - prelock_capacity_dimensions_in_tx( - conn.as_mut(), - config, - visible.tenant_id, - &[ - ExecutionCapacityDimension::ActiveRuns, - ExecutionCapacityDimension::ActiveTasks, - ExecutionCapacityDimension::ParkedRuns, - ExecutionCapacityDimension::ScheduledTriggers, - ExecutionCapacityDimension::ExternalJobs, - ], - ) - .await?; - let row = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) - .bind(run_uid) - .fetch_one(conn.as_mut()) - .await - .map_err(sqlx_error)?; - let run = run_from_row(&row)?; - if run.tenant_id != visible.tenant_id { - return Err(Error::InvalidRepositoryData { - message: "execution run tenant changed while acquiring compensation capacity locks" - .to_string(), - }); +fn compensation_release_intent_label(intent: ExecutionCompensationReleaseIntent) -> &'static str { + match intent { + ExecutionCompensationReleaseIntent::Outcome => "outcome", + ExecutionCompensationReleaseIntent::Retry => "retry", + ExecutionCompensationReleaseIntent::Review => "review", + ExecutionCompensationReleaseIntent::ExternalJob => "external_job", + ExecutionCompensationReleaseIntent::Pause => "pause", + ExecutionCompensationReleaseIntent::Watchdog => "watchdog", + ExecutionCompensationReleaseIntent::Deadline => "deadline", + ExecutionCompensationReleaseIntent::RunTerminal => "run_terminal", } - Ok(Some(run)) } -async fn replayed_pending_terminal_commit( - conn: &mut ScopedConn<'_>, - config: &ExecutionConfig, - run: ExecutionRunRecord, -) -> Result { - let work_remaining: bool = sqlx::query_scalar( - "SELECT EXISTS (SELECT 1 FROM moa.execution_task WHERE run_uid=$1 \ - AND status NOT IN ('completed','skipped','failed','cancelled','unknown_outcome')) \ - OR EXISTS (SELECT 1 FROM moa.execution_compensation WHERE run_uid=$1 \ - AND status <> 'completed') \ - OR EXISTS (SELECT 1 FROM moa.execution_capacity_reservation WHERE run_uid=$1 \ - AND resource_dimension IN ('active_tasks','scheduled_triggers','external_jobs') \ - AND state IN ('reserved','reconciling'))", - ) - .bind(run.run_uid) - .fetch_one(conn.as_mut()) - .await - .map_err(sqlx_error)?; - let compensation_admission = if run.status == ExecutionRunStatus::Compensating { - let row = sqlx::query( - "SELECT * FROM moa.execution_compensation WHERE run_uid=$1 \ - AND status <> 'completed' ORDER BY registered_sequence DESC LIMIT 1 FOR UPDATE", - ) - .bind(run.run_uid) - .fetch_optional(conn.as_mut()) - .await - .map_err(sqlx_error)?; - if let Some(row) = row { - if compensation_attempt_state_from_row(&row)? == CompensationAttemptState::Dispatching { - let registration = compensation_from_row(&row)?; - Some(Box::new( - load_existing_compensation_admission(conn, config, &run, &row, ®istration) - .await?, - )) - } else { - None - } - } else { - None +fn validate_compensation_settlement_intent( + intent: ExecutionCompensationReleaseIntent, + outcome: &ExecutionCompensationOutcome, +) -> Result<()> { + let retryable_failure = matches!( + outcome, + ExecutionCompensationOutcome::Failed { + retryable: true, + .. } - } else { - None + ); + let valid = match intent { + ExecutionCompensationReleaseIntent::Outcome => !retryable_failure, + ExecutionCompensationReleaseIntent::Retry + | ExecutionCompensationReleaseIntent::Watchdog => retryable_failure, + ExecutionCompensationReleaseIntent::Deadline + | ExecutionCompensationReleaseIntent::RunTerminal => !retryable_failure, + ExecutionCompensationReleaseIntent::Review + | ExecutionCompensationReleaseIntent::ExternalJob + | ExecutionCompensationReleaseIntent::Pause => false, }; - let continuation = load_pending_terminal_continuation(conn, &run).await?; - let stage = if run.status.is_terminal() { - if run.manual_repair_required { - PendingTerminalAdvanceStage::ManualRepairRequired - } else { - PendingTerminalAdvanceStage::Finalized - } - } else if compensation_admission.is_some() { - PendingTerminalAdvanceStage::CompensationQueued - } else if work_remaining { - PendingTerminalAdvanceStage::Draining + if valid { + Ok(()) } else { - PendingTerminalAdvanceStage::EnqueuedPage - }; - Ok(PendingTerminalAdvanceCommit { - run, - stage, - settled_task_count: 0, - drained_trigger_count: 0, - cancellation_dispatches: Vec::new(), - compensation_admission, - continuation: continuation.map(Box::new), - work_remaining, - }) -} - -async fn load_pending_terminal_continuation( - conn: &mut ScopedConn<'_>, - run: &ExecutionRunRecord, -) -> Result> { - let row = sqlx::query( - "SELECT dispatch_uid, not_before_at, payload, wake_epoch \ - FROM moa.execution_dispatch_outbox WHERE run_uid=$1 \ - AND dispatch_kind='run_activation' AND controller_generation=$2 \ - AND payload->>'source_wake_epoch'=$3 ORDER BY created_at DESC LIMIT 1", - ) - .bind(run.run_uid) - .bind(to_i64(run.controller_generation, "controller generation")?) - .bind(run.processed_wake_epoch.to_string()) - .fetch_optional(conn.as_mut()) - .await - .map_err(sqlx_error)?; - let Some(row) = row else { - return Ok(None); - }; - let wake_epoch = required_u64(&row, "wake_epoch")?; - let request = NewExecutionDispatch { - dispatch_uid: row.try_get("dispatch_uid").map_err(row_error)?, - tenant_id: run.tenant_id, - run_uid: Some(run.run_uid), - task_id: None, - compensation_id: None, - trigger_uid: None, - external_job_uid: None, - kind: ExecutionDispatchKind::RunActivation, - controller_generation: Some(run.controller_generation), - wake_epoch: Some(wake_epoch), - attempt_generation: None, - compensation_generation: None, - compensation_attempt_generation: None, - not_before_at: row.try_get("not_before_at").map_err(row_error)?, - payload: row.try_get("payload").map_err(row_error)?, - }; - enqueue_dispatch_in_conn(conn.as_mut(), &request) - .await - .map(Some) + Err(Error::InvalidRepositoryInput { + message: format!( + "compensation release intent `{}` does not match its settlement path", + compensation_release_intent_label(intent) + ), + }) + } } -#[allow(clippy::too_many_arguments)] -async fn advance_pending_terminal_page_in_conn( - mut conn: ScopedConn<'_>, - config: &ExecutionConfig, - mut run: ExecutionRunRecord, - controller_generation: u64, - expected_wake_epoch: u64, - new_pending: Option, - now: DateTime, - page_limit: u32, -) -> Result { - if let Some(pending) = new_pending { - if let Some(current) = &run.pending_terminal { - if current != &pending { - conn.commit().await.map_err(storage_error)?; - return Ok(PendingTerminalAdvanceOutcome::Conflict); - } - } else { - let row = sqlx::query( - "UPDATE moa.execution_run SET pending_terminal_status=$4, \ - pending_terminal_reason=$5, pending_terminal_cause=$6, \ - pending_terminal_output=$7, cancellation_reason=$8, \ - next_wake_at=NULL, \ - updated_at=$9 WHERE run_uid=$1 AND controller_generation=$2 \ - AND wake_epoch=$3 AND pending_terminal_status IS NULL \ - AND status NOT IN ('completed','partial','blocked','unsupported', \ - 'failed','cancelled','compensating') RETURNING *", - ) - .bind(run.run_uid) - .bind(to_i64(controller_generation, "controller generation")?) - .bind(to_i64(expected_wake_epoch, "expected wake epoch")?) - .bind(pending.status.as_str()) - .bind(pending.reason.as_str()) - .bind(serde_json::to_value(PendingTerminalEvidencePayload { - terminal_evidence: pending.terminal_evidence.clone(), - completion_check_results: pending.completion_check_results.clone(), - terminal_gaps: pending.terminal_gaps.clone(), - })?) - .bind(&pending.output) - .bind(&pending.cancellation_reason) - .bind(now) - .fetch_optional(conn.as_mut()) - .await - .map_err(sqlx_error)?; - let Some(row) = row else { - conn.rollback().await.map_err(storage_error)?; - return Ok(PendingTerminalAdvanceOutcome::Conflict); - }; - run = run_from_row(&row)?; - } - } - let pending = run - .pending_terminal - .clone() - .ok_or_else(|| Error::InvalidRepositoryData { - message: "terminal drain lost its pending terminal intent".to_string(), - })?; - let cancel_reason = if pending.reason == ExecutionTerminalReason::DeadlineExceeded { - ExecutionAttemptCancelReason::DeadlineExceeded - } else { - ExecutionAttemptCancelReason::RunTerminal - }; - let task_rows = sqlx::query( - "SELECT task.* FROM moa.execution_task AS task WHERE task.run_uid=$1 \ - AND task.status NOT IN ('completed','skipped','failed','cancelled','unknown_outcome') \ - AND task.attempt_state <> 'cancelling' \ - ORDER BY CASE WHEN task.attempt_state IN ('dispatching','running') THEN 0 ELSE 1 END, \ - task.task_id LIMIT $2 FOR UPDATE", - ) - .bind(run.run_uid) - .bind(i64::from(page_limit)) - .fetch_all(conn.as_mut()) - .await - .map_err(sqlx_error)?; - let tasks = task_rows - .iter() - .map(task_from_row) - .collect::>>()?; - let processed_task_count = - u32::try_from(tasks.len()).map_err(|_| Error::InvalidRepositoryData { - message: "terminal drain task page exceeds u32".to_string(), - })?; - let storage_task_ids = tasks - .iter() - .filter(|task| { - task.status != ExecutionTaskStatus::WaitingExternal - && !matches!( - task.attempt_state, - ExecutionAttemptState::Dispatching | ExecutionAttemptState::Running - ) - }) - .map(|task| task.task_id.as_uuid()) - .collect::>(); - supersede_storage_task_waits(&mut conn, run.run_uid, run.tenant_id.0, &storage_task_ids) - .await?; - let mut settled_task_count = 0_u64; - let mut cancellation_dispatches = Vec::with_capacity(tasks.len()); - for task in tasks { - if task.status == ExecutionTaskStatus::WaitingExternal { - let external_job_uid = - task.external_job_uid - .ok_or_else(|| Error::InvalidRepositoryData { - message: "waiting-external task lost its exact external job UID" - .to_string(), - })?; - let owner = ExecutionExternalJobOwner::Task { - task_id: task.task_id.as_uuid(), - attempt_generation: task.attempt_generation, - }; - match request_external_job_cancellation_in_conn( - &mut conn, - config, - external_job_uid, - owner, - now, - ) - .await? - { - ExecutionExternalJobCancellationRequestOutcome::Applied(dispatch) - | ExecutionExternalJobCancellationRequestOutcome::Replayed(dispatch) => { - cancellation_dispatches.push(dispatch); - } - ExecutionExternalJobCancellationRequestOutcome::UnboundPendingRecovery => {} - ExecutionExternalJobCancellationRequestOutcome::AlreadyTerminal => { - let job = load_external_job_for_update_in_conn(conn.as_mut(), external_job_uid) - .await? - .ok_or_else(|| Error::InvalidRepositoryData { - message: "terminal external job disappeared under its owner fence" - .to_string(), - })?; - settle_task_external_job_terminal_in_conn(&mut conn, &job, now).await?; - } - ExecutionExternalJobCancellationRequestOutcome::NotFound - | ExecutionExternalJobCancellationRequestOutcome::Stale => { - return Err(Error::InvalidRepositoryData { - message: "waiting-external task has a stale external job owner".to_string(), - }); - } - } - continue; - } - if matches!( - task.attempt_state, - ExecutionAttemptState::Dispatching | ExecutionAttemptState::Running - ) { - cancellation_dispatches.push( - enqueue_pending_terminal_task_cancellation( - &mut conn, - &run, - &task, - cancel_reason, - pending.reason, - now, - ) - .await?, - ); - continue; - } - let original_status = task.status; - match record_task_outcome_in_conn( - &mut conn, - run.run_uid, - task.task_id, - task.generation, - cancelled_task_outcome( - format!("run terminal fence: {}", pending.reason.as_str()), - task.actual.clone(), - ), - ) - .await? - { - TaskOutcomeWrite::Applied { task, .. } | TaskOutcomeWrite::Replayed { task, .. } => { - transition_node_counters_in_tx( - &mut conn, - run.run_uid, - &task.node_id, - &task.item_key, - original_status, - ExecutionTaskStatus::Cancelled, - ) - .await?; - } - TaskOutcomeWrite::Rejected { reason, .. } => { - return Err(Error::InvalidRepositoryData { - message: format!("terminal drain task settlement was rejected: {reason:?}"), - }); - } - TaskOutcomeWrite::NotFound => { - return Err(Error::InvalidRepositoryData { - message: "terminal drain lost a row-locked task".to_string(), - }); - } - } - settled_task_count = - settled_task_count - .checked_add(1) - .ok_or_else(|| Error::InvalidRepositoryData { - message: "terminal drain settled-task count overflow".to_string(), - })?; - } - - let task_dispatch_count = cancellation_dispatches.len(); - let remaining_slots = page_limit.saturating_sub(processed_task_count); - if remaining_slots > 0 && run.status != ExecutionRunStatus::Compensating { - let compensation_rows = sqlx::query( - "SELECT compensation.* FROM moa.execution_compensation AS compensation \ - WHERE compensation.run_uid=$1 \ - AND compensation.attempt_state IN ('dispatching','running') \ - ORDER BY compensation.registered_sequence DESC LIMIT $2 FOR UPDATE", - ) - .bind(run.run_uid) - .bind(i64::from(remaining_slots)) - .fetch_all(conn.as_mut()) - .await - .map_err(sqlx_error)?; - for row in compensation_rows { - cancellation_dispatches.push( - enqueue_pending_terminal_compensation_cancellation( - &mut conn, - &run, - &row, - cancel_reason, - pending.reason, - now, - ) - .await?, - ); - } - } - let compensation_cancellation_count = cancellation_dispatches - .len() - .checked_sub(task_dispatch_count) - .and_then(|count| u32::try_from(count).ok()) - .ok_or_else(|| Error::InvalidRepositoryData { - message: "terminal drain compensation cancellation count overflow".to_string(), - })?; - let charged_after_cancellations = processed_task_count - .checked_add(compensation_cancellation_count) - .ok_or_else(|| Error::InvalidRepositoryData { - message: "terminal drain page accounting overflow after cancellation".to_string(), - })?; - let trigger_slots = page_limit.saturating_sub(charged_after_cancellations); - - let nonterminal_forward_count: i64 = sqlx::query_scalar( - "SELECT COUNT(*) FROM moa.execution_task WHERE run_uid=$1 \ - AND status NOT IN ('completed','skipped','failed','cancelled','unknown_outcome')", - ) - .bind(run.run_uid) - .fetch_one(conn.as_mut()) - .await - .map_err(sqlx_error)?; - let actionable_forward_exists: bool = sqlx::query_scalar( - "SELECT EXISTS (SELECT 1 FROM moa.execution_task WHERE run_uid=$1 \ - AND status NOT IN ('completed','skipped','failed','cancelled','unknown_outcome') \ - AND attempt_state <> 'cancelling' AND status <> 'waiting_external')", - ) - .bind(run.run_uid) - .fetch_one(conn.as_mut()) - .await - .map_err(sqlx_error)?; - let active_count = active_attempt_capacity_count(&mut conn, run.run_uid).await?; - let has_registrations: bool = sqlx::query_scalar( - "SELECT EXISTS (SELECT 1 FROM moa.execution_compensation WHERE run_uid=$1)", - ) - .bind(run.run_uid) - .fetch_one(conn.as_mut()) - .await - .map_err(sqlx_error)?; - let retain_cancelled_effects = pending.status == ExecutionRunStatus::Cancelled - && run.active_plan.definition.cancel_policy == ExecutionCancelPolicy::RetainEffects; - let should_compensate = has_registrations && !retain_cancelled_effects; - let active_trigger_exists: bool = sqlx::query_scalar( - "SELECT EXISTS (SELECT 1 FROM moa.execution_trigger WHERE run_uid=$1 \ - AND state = 'pending')", - ) - .bind(run.run_uid) - .fetch_one(conn.as_mut()) - .await - .map_err(sqlx_error)?; - let cleanup_triggers_now = nonterminal_forward_count == 0 - && active_count == 0 - && !should_compensate - && run.status != ExecutionRunStatus::Compensating; - let (mut drained_trigger_count, mut trigger_work_remaining) = - if cleanup_triggers_now && trigger_slots > 0 { - let page = drain_run_triggers_page_in_conn(&mut conn, &run, trigger_slots).await?; - (page.drained_trigger_count, page.work_remaining) - } else if cleanup_triggers_now { - (0, active_trigger_exists) - } else { - (0, false) - }; - let ready_count: i64 = sqlx::query_scalar( - "SELECT COUNT(*) FROM moa.execution_task WHERE run_uid=$1 AND status='ready'", - ) - .bind(run.run_uid) - .fetch_one(conn.as_mut()) - .await - .map_err(sqlx_error)?; - - let mut stage = PendingTerminalAdvanceStage::Draining; - let mut work_remaining = nonterminal_forward_count > 0 || active_count > 0; - let mut continuation_payload = None; - let mut continuation_not_before = now; - let mut checkpoint_status = run.status; - let mut checkpoint_active_count = active_count; - let mut compensation_admission = None; - if actionable_forward_exists || trigger_work_remaining { - stage = PendingTerminalAdvanceStage::EnqueuedPage; - work_remaining = true; - continuation_payload = Some(json!({ - "reason":"pending_terminal_page", - "source_wake_epoch": expected_wake_epoch, - })); - } else if nonterminal_forward_count == 0 && active_count == 0 { - if should_compensate && run.status != ExecutionRunStatus::Compensating { - stage = PendingTerminalAdvanceStage::EnqueuedPage; - checkpoint_status = ExecutionRunStatus::Compensating; - work_remaining = true; - continuation_payload = Some(json!({ - "reason":"pending_terminal_compensation", - "source_wake_epoch": expected_wake_epoch, - })); - } else if run.status == ExecutionRunStatus::Compensating { - match drive_pending_terminal_compensation_in_conn(&mut conn, config, &run, now).await? { - PendingCompensationDrive::Admitted(admission) - | PendingCompensationDrive::Replayed(admission) => { - stage = PendingTerminalAdvanceStage::CompensationQueued; - work_remaining = true; - compensation_admission = Some(admission); - checkpoint_active_count = - active_attempt_capacity_count(&mut conn, run.run_uid).await?; - } - PendingCompensationDrive::CapacityUnavailable { retry_at } => { - stage = PendingTerminalAdvanceStage::EnqueuedPage; - work_remaining = true; - continuation_not_before = retry_at; - continuation_payload = Some(json!({ - "reason":"pending_terminal_compensation_capacity", - "source_wake_epoch": expected_wake_epoch, - })); - } - PendingCompensationDrive::ExternalCancellation(dispatch) => { - cancellation_dispatches.push(dispatch); - work_remaining = true; - } - PendingCompensationDrive::Parked => { - work_remaining = true; - } - PendingCompensationDrive::ManualRepair(registration) => { - if trigger_slots > 0 { - let page = - drain_run_triggers_page_in_conn(&mut conn, &run, trigger_slots).await?; - drained_trigger_count = page.drained_trigger_count; - trigger_work_remaining = page.work_remaining; - } else { - trigger_work_remaining = active_trigger_exists; - } - if trigger_work_remaining { - stage = PendingTerminalAdvanceStage::EnqueuedPage; - work_remaining = true; - continuation_payload = Some(json!({ - "reason":"pending_terminal_manual_repair_cleanup", - "source_wake_epoch": expected_wake_epoch, - })); - } else { - let non_lifetime_capacity_exists: bool = sqlx::query_scalar( - "SELECT EXISTS (SELECT 1 FROM moa.execution_capacity_reservation \ - WHERE run_uid=$1 AND resource_dimension IN \ - ('active_tasks','scheduled_triggers','external_jobs') \ - AND state IN ('reserved','reconciling'))", - ) - .bind(run.run_uid) - .fetch_one(conn.as_mut()) - .await - .map_err(sqlx_error)?; - if non_lifetime_capacity_exists { - return Err(Error::InvalidRepositoryData { - message: "failed compensation retained non-lifetime capacity" - .to_string(), - }); - } - let failure = compensation_failure_pending(&pending, ®istration)?; - replace_pending_terminal_exact( - &mut conn, - &run, - &pending, - &failure, - controller_generation, - expected_wake_epoch, - now, - ) - .await?; - let finalized = finalize_pending_terminal_exact( - &mut conn, - &run, - &failure, - controller_generation, - expected_wake_epoch, - now, - ) - .await?; - conn.commit().await.map_err(storage_error)?; - return Ok(PendingTerminalAdvanceOutcome::Applied(Box::new( - PendingTerminalAdvanceCommit { - run: finalized, - stage: PendingTerminalAdvanceStage::ManualRepairRequired, - settled_task_count, - drained_trigger_count, - cancellation_dispatches, - compensation_admission: None, - continuation: None, - work_remaining: false, - }, - ))); - } - } - PendingCompensationDrive::Complete => { - if trigger_slots > 0 { - let page = - drain_run_triggers_page_in_conn(&mut conn, &run, trigger_slots).await?; - drained_trigger_count = page.drained_trigger_count; - trigger_work_remaining = page.work_remaining; - } else { - trigger_work_remaining = active_trigger_exists; - } - if trigger_work_remaining { - stage = PendingTerminalAdvanceStage::EnqueuedPage; - work_remaining = true; - continuation_payload = Some(json!({ - "reason":"pending_terminal_trigger_cleanup", - "source_wake_epoch": expected_wake_epoch, - })); - } else { - let non_lifetime_capacity_exists: bool = sqlx::query_scalar( - "SELECT EXISTS (SELECT 1 FROM moa.execution_capacity_reservation \ - WHERE run_uid=$1 AND resource_dimension IN \ - ('active_tasks','scheduled_triggers','external_jobs') \ - AND state IN ('reserved','reconciling'))", - ) - .bind(run.run_uid) - .fetch_one(conn.as_mut()) - .await - .map_err(sqlx_error)?; - if non_lifetime_capacity_exists { - return Err(Error::InvalidRepositoryData { - message: "completed compensation retained non-lifetime capacity" - .to_string(), - }); - } - let finalized = finalize_pending_terminal_exact( - &mut conn, - &run, - &pending, - controller_generation, - expected_wake_epoch, - now, - ) - .await?; - conn.commit().await.map_err(storage_error)?; - return Ok(PendingTerminalAdvanceOutcome::Applied(Box::new( - PendingTerminalAdvanceCommit { - run: finalized, - stage: PendingTerminalAdvanceStage::Finalized, - settled_task_count, - drained_trigger_count, - cancellation_dispatches, - compensation_admission: None, - continuation: None, - work_remaining: false, - }, - ))); - } - } - } - } else { - let non_lifetime_capacity_exists: bool = sqlx::query_scalar( - "SELECT EXISTS (SELECT 1 FROM moa.execution_capacity_reservation WHERE run_uid=$1 \ - AND resource_dimension IN ('active_tasks','scheduled_triggers','external_jobs') \ - AND state IN ('reserved','reconciling'))", - ) - .bind(run.run_uid) - .fetch_one(conn.as_mut()) - .await - .map_err(sqlx_error)?; - if non_lifetime_capacity_exists { - work_remaining = true; - } else { - let finalized = finalize_pending_terminal_exact( - &mut conn, - &run, - &pending, - controller_generation, - expected_wake_epoch, - now, - ) - .await?; - stage = if finalized.manual_repair_required { - PendingTerminalAdvanceStage::ManualRepairRequired - } else { - PendingTerminalAdvanceStage::Finalized - }; - conn.commit().await.map_err(storage_error)?; - return Ok(PendingTerminalAdvanceOutcome::Applied(Box::new( - PendingTerminalAdvanceCommit { - run: finalized, - stage, - settled_task_count, - drained_trigger_count, - cancellation_dispatches, - compensation_admission: None, - continuation: None, - work_remaining: false, - }, - ))); - } - } - } - - let checkpointed = checkpoint_pending_terminal_wake( - &mut conn, - run.run_uid, - controller_generation, - expected_wake_epoch, - checkpoint_status, - u64::try_from(ready_count).map_err(|_| Error::InvalidRepositoryData { - message: "terminal drain ready-task count is negative".to_string(), - })?, - checkpoint_active_count, - now, - ) - .await?; - let continuation = if let Some(payload) = continuation_payload { - Some(Box::new( - enqueue_run_activation_in_conn( - conn.as_mut(), - checkpointed.tenant_id, - checkpointed.run_uid, - checkpointed.controller_generation, - continuation_not_before, - payload, - ) - .await?, - )) - } else { - None - }; - let row = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) - .bind(run.run_uid) - .fetch_one(conn.as_mut()) - .await - .map_err(sqlx_error)?; - run = run_from_row(&row)?; - conn.commit().await.map_err(storage_error)?; - Ok(PendingTerminalAdvanceOutcome::Applied(Box::new( - PendingTerminalAdvanceCommit { - run, - stage, - settled_task_count, - drained_trigger_count, - cancellation_dispatches, - compensation_admission, - continuation, - work_remaining, - }, - ))) -} - -async fn enqueue_pending_terminal_task_cancellation( - conn: &mut ScopedConn<'_>, - run: &ExecutionRunRecord, - task: &ExecutionTaskRecord, - reason: ExecutionAttemptCancelReason, - terminal_reason: ExecutionTerminalReason, - now: DateTime, -) -> Result { - let row = sqlx::query( - "SELECT reservation.reservation_uid, trigger.trigger_uid \ - FROM moa.execution_capacity_reservation AS reservation \ - JOIN moa.execution_trigger AS trigger ON trigger.run_uid=reservation.run_uid \ - AND trigger.task_id=reservation.task_id \ - AND trigger.controller_generation=reservation.controller_generation \ - AND trigger.attempt_generation=reservation.attempt_generation \ - AND trigger.trigger_kind='task_watchdog' \ - AND trigger.state = 'pending' \ - WHERE reservation.run_uid=$1 AND reservation.task_id=$2 \ - AND reservation.controller_generation=$3 AND reservation.attempt_generation=$4 \ - AND reservation.resource_dimension='active_tasks' \ - AND reservation.state IN ('reserved','reconciling') FOR UPDATE OF reservation, trigger", - ) - .bind(run.run_uid) - .bind(task.task_id.as_uuid()) - .bind(to_i64(run.controller_generation, "controller generation")?) - .bind(to_i64(task.attempt_generation, "task attempt generation")?) - .fetch_optional(conn.as_mut()) - .await - .map_err(sqlx_error)? - .ok_or_else(|| Error::InvalidRepositoryData { - message: format!( - "active task {} is missing its exact capacity or watchdog receipt", - task.task_id - ), - })?; - let active_dispatch_uid = - task.active_dispatch_uid - .ok_or_else(|| Error::InvalidRepositoryData { - message: format!( - "active task {} is missing its dispatch identity", - task.task_id - ), - })?; - let capacity_reservation_uid: Uuid = row.try_get("reservation_uid").map_err(row_error)?; - let watchdog_trigger_uid: Uuid = row.try_get("trigger_uid").map_err(row_error)?; - let cancellation_dispatch_uid = pending_terminal_cancel_dispatch_uid( - active_dispatch_uid, - run.controller_generation, - terminal_reason, - ); - let cancelling = sqlx::query( - "UPDATE moa.execution_task SET attempt_state='cancelling', \ - last_progress_at=GREATEST(last_progress_at,$6), updated_at=NOW() \ - WHERE run_uid=$1 AND task_id=$2 \ - AND generation=$3 AND attempt_generation=$4 AND active_dispatch_uid=$5 \ - AND attempt_state IN ('dispatching','running')", - ) - .bind(run.run_uid) - .bind(task.task_id.as_uuid()) - .bind(to_i64(task.generation, "task generation")?) - .bind(to_i64(task.attempt_generation, "task attempt generation")?) - .bind(active_dispatch_uid) - .bind(now) - .execute(conn.as_mut()) - .await - .map_err(sqlx_error)?; - if cancelling.rows_affected() != 1 { - return Err(Error::InvalidRepositoryData { - message: format!("task {} lost its terminal cancellation fence", task.task_id), - }); - } - let reconciling = sqlx::query( - "UPDATE moa.execution_capacity_reservation SET state='reconciling', updated_at=$2 \ - WHERE reservation_uid=$1 AND state IN ('reserved','reconciling')", - ) - .bind(capacity_reservation_uid) - .bind(now) - .execute(conn.as_mut()) - .await - .map_err(sqlx_error)?; - if reconciling.rows_affected() != 1 { - return Err(Error::InvalidRepositoryData { - message: format!("task {} lost its active-capacity receipt", task.task_id), - }); - } - let payload = serde_json::to_value(ExecutionTaskAttemptCancelRequest { - cancellation_dispatch_uid, - tenant_id: run.tenant_id, - run_uid: run.run_uid, - task_id: task.task_id, - controller_generation: run.controller_generation, - attempt_controller_generation: run.controller_generation, - task_generation: task.generation, - attempt_generation: task.attempt_generation, - active_dispatch_uid, - capacity_reservation_uid, - watchdog_trigger_uid, - reason, - })?; - enqueue_dispatch_in_conn( - conn.as_mut(), - &NewExecutionDispatch { - dispatch_uid: cancellation_dispatch_uid, - tenant_id: run.tenant_id, - run_uid: Some(run.run_uid), - task_id: Some(task.task_id.as_uuid()), - compensation_id: None, - trigger_uid: None, - external_job_uid: task.external_job_uid, - kind: ExecutionDispatchKind::TaskAttemptCancel, - controller_generation: Some(run.controller_generation), - wake_epoch: None, - attempt_generation: Some(task.attempt_generation), - compensation_generation: None, - compensation_attempt_generation: None, - not_before_at: now, - payload, - }, - ) - .await -} - -async fn enqueue_pending_terminal_compensation_cancellation( - conn: &mut ScopedConn<'_>, - run: &ExecutionRunRecord, - compensation_row: &PgRow, - reason: ExecutionAttemptCancelReason, - terminal_reason: ExecutionTerminalReason, - now: DateTime, -) -> Result { - let registration = compensation_from_row(compensation_row)?; - let attempt_generation = required_u64(compensation_row, "attempt_generation")?; - let active_dispatch_uid: Uuid = compensation_row - .try_get("active_dispatch_uid") - .map_err(row_error)?; - let receipt = sqlx::query( - "SELECT reservation.reservation_uid, trigger.trigger_uid \ - FROM moa.execution_capacity_reservation AS reservation \ - JOIN moa.execution_trigger AS trigger ON trigger.run_uid=reservation.run_uid \ - AND trigger.compensation_id=reservation.compensation_id \ - AND trigger.controller_generation=reservation.controller_generation \ - AND trigger.compensation_generation=reservation.compensation_generation \ - AND trigger.compensation_attempt_generation=reservation.compensation_attempt_generation \ - AND trigger.trigger_kind='compensation_watchdog' \ - AND trigger.state = 'pending' \ - WHERE reservation.run_uid=$1 AND reservation.compensation_id=$2 \ - AND reservation.controller_generation=$3 AND reservation.compensation_generation=$4 \ - AND reservation.compensation_attempt_generation=$5 \ - AND reservation.resource_dimension='active_tasks' \ - AND reservation.state IN ('reserved','reconciling') FOR UPDATE OF reservation, trigger", - ) - .bind(run.run_uid) - .bind(registration.compensation_id.as_uuid()) - .bind(to_i64(run.controller_generation, "controller generation")?) - .bind(to_i64(registration.generation, "compensation generation")?) - .bind(to_i64( - attempt_generation, - "compensation attempt generation", - )?) - .fetch_optional(conn.as_mut()) - .await - .map_err(sqlx_error)? - .ok_or_else(|| Error::InvalidRepositoryData { - message: format!( - "active compensation {} is missing its exact capacity or watchdog receipt", - registration.compensation_id - ), - })?; - let capacity_reservation_uid: Uuid = receipt.try_get("reservation_uid").map_err(row_error)?; - let watchdog_trigger_uid: Uuid = receipt.try_get("trigger_uid").map_err(row_error)?; - let cancellation_dispatch_uid = pending_terminal_cancel_dispatch_uid( - active_dispatch_uid, - run.controller_generation, - terminal_reason, - ); - let intent = compensation_release_intent(reason); - let cancelling = sqlx::query( - "UPDATE moa.execution_compensation SET attempt_state='cancelling', \ - release_intent=$7, last_progress_at=GREATEST(last_progress_at,$6), \ - updated_at=NOW() \ - WHERE run_uid=$1 AND compensation_id=$2 \ - AND generation=$3 AND attempt_generation=$4 AND active_dispatch_uid=$5 \ - AND attempt_state IN ('dispatching','running')", - ) - .bind(run.run_uid) - .bind(registration.compensation_id.as_uuid()) - .bind(to_i64(registration.generation, "compensation generation")?) - .bind(to_i64( - attempt_generation, - "compensation attempt generation", - )?) - .bind(active_dispatch_uid) - .bind(now) - .bind(compensation_release_intent_label(intent)) - .execute(conn.as_mut()) - .await - .map_err(sqlx_error)?; - if cancelling.rows_affected() != 1 { - return Err(Error::InvalidRepositoryData { - message: format!( - "compensation {} lost its terminal cancellation fence", - registration.compensation_id - ), - }); - } - let reconciling = sqlx::query( - "UPDATE moa.execution_capacity_reservation SET state='reconciling', updated_at=$2 \ - WHERE reservation_uid=$1 AND state IN ('reserved','reconciling')", - ) - .bind(capacity_reservation_uid) - .bind(now) - .execute(conn.as_mut()) - .await - .map_err(sqlx_error)?; - if reconciling.rows_affected() != 1 { - return Err(Error::InvalidRepositoryData { - message: format!( - "compensation {} lost its active-capacity receipt", - registration.compensation_id - ), - }); - } - let payload = serde_json::to_value(ExecutionCompensationAttemptCancelRequest { - cancellation_dispatch_uid, - tenant_id: run.tenant_id, - run_uid: run.run_uid, - compensation_id: registration.compensation_id, - controller_generation: run.controller_generation, - attempt_controller_generation: run.controller_generation, - compensation_generation: registration.generation, - compensation_attempt_generation: attempt_generation, - active_dispatch_uid, - capacity_reservation_uid, - watchdog_trigger_uid, - intent, - })?; - enqueue_dispatch_in_conn( - conn.as_mut(), - &NewExecutionDispatch { - dispatch_uid: cancellation_dispatch_uid, - tenant_id: run.tenant_id, - run_uid: Some(run.run_uid), - task_id: None, - compensation_id: Some(registration.compensation_id.as_uuid()), - trigger_uid: None, - external_job_uid: None, - kind: ExecutionDispatchKind::CompensationAttemptCancel, - controller_generation: Some(run.controller_generation), - wake_epoch: None, - attempt_generation: None, - compensation_generation: Some(registration.generation), - compensation_attempt_generation: Some(attempt_generation), - not_before_at: now, - payload, - }, - ) - .await -} - -fn pending_terminal_cancel_dispatch_uid( - active_dispatch_uid: Uuid, - controller_generation: u64, - terminal_reason: ExecutionTerminalReason, -) -> Uuid { - let name = format!( - "{active_dispatch_uid}:{controller_generation}:{}", - terminal_reason.as_str() - ); - Uuid::new_v5(&PENDING_TERMINAL_CANCEL_NAMESPACE, name.as_bytes()) -} - -fn compensation_release_intent( - reason: ExecutionAttemptCancelReason, -) -> ExecutionCompensationReleaseIntent { - match reason { - ExecutionAttemptCancelReason::DeadlineExceeded => { - ExecutionCompensationReleaseIntent::Deadline - } - ExecutionAttemptCancelReason::RunTerminal => { - ExecutionCompensationReleaseIntent::RunTerminal - } - ExecutionAttemptCancelReason::PauseRequested => ExecutionCompensationReleaseIntent::Pause, - ExecutionAttemptCancelReason::ExternalJobStarted => { - ExecutionCompensationReleaseIntent::ExternalJob - } - } -} - -fn compensation_release_intent_label(intent: ExecutionCompensationReleaseIntent) -> &'static str { - match intent { - ExecutionCompensationReleaseIntent::Outcome => "outcome", - ExecutionCompensationReleaseIntent::Retry => "retry", - ExecutionCompensationReleaseIntent::Review => "review", - ExecutionCompensationReleaseIntent::ExternalJob => "external_job", - ExecutionCompensationReleaseIntent::Pause => "pause", - ExecutionCompensationReleaseIntent::Watchdog => "watchdog", - ExecutionCompensationReleaseIntent::Deadline => "deadline", - ExecutionCompensationReleaseIntent::RunTerminal => "run_terminal", - } -} - -fn validate_compensation_settlement_intent( - intent: ExecutionCompensationReleaseIntent, - outcome: &ExecutionCompensationOutcome, -) -> Result<()> { - let retryable_failure = matches!( - outcome, - ExecutionCompensationOutcome::Failed { - retryable: true, - .. - } - ); - let valid = match intent { - ExecutionCompensationReleaseIntent::Outcome => !retryable_failure, - ExecutionCompensationReleaseIntent::Retry - | ExecutionCompensationReleaseIntent::Watchdog => retryable_failure, - ExecutionCompensationReleaseIntent::Deadline - | ExecutionCompensationReleaseIntent::RunTerminal => !retryable_failure, - ExecutionCompensationReleaseIntent::Review - | ExecutionCompensationReleaseIntent::ExternalJob - | ExecutionCompensationReleaseIntent::Pause => false, - }; - if valid { - Ok(()) - } else { - Err(Error::InvalidRepositoryInput { - message: format!( - "compensation release intent `{}` does not match its settlement path", - compensation_release_intent_label(intent) - ), - }) - } -} - -fn compensation_outcome_from_review_resolution( - resolution: &ExecutionActionReviewResolution, -) -> Result { - Ok(match resolution { - ExecutionActionReviewResolution::Completed { tool_output } => { - ExecutionCompensationOutcome::Completed { - output: tool_output.clone(), - usage: zero_usage(), +fn compensation_outcome_from_review_resolution( + resolution: &ExecutionActionReviewResolution, +) -> Result { + Ok(match resolution { + ExecutionActionReviewResolution::Completed { tool_output } => { + ExecutionCompensationOutcome::Completed { + output: tool_output.clone(), + usage: zero_usage(), } } ExecutionActionReviewResolution::UnknownOutcome { message } => { @@ -5147,107 +3481,6 @@ fn compensation_outcome_from_review_resolution( }) } -async fn supersede_storage_task_waits( - conn: &mut ScopedConn<'_>, - run_uid: Uuid, - tenant_id: Uuid, - task_ids: &[Uuid], -) -> Result<()> { - if task_ids.is_empty() { - return Ok(()); - } - let trigger_uids = sqlx::query_scalar::<_, Uuid>( - "UPDATE moa.execution_trigger SET state='superseded', updated_at=NOW() \ - WHERE run_uid=$1 AND task_id = ANY($2::UUID[]) \ - AND trigger_kind <> 'task_watchdog' AND state = 'pending' \ - RETURNING trigger_uid", - ) - .bind(run_uid) - .bind(task_ids) - .fetch_all(conn.as_mut()) - .await - .map_err(sqlx_error)?; - if trigger_uids.is_empty() { - return Ok(()); - } - sqlx::query( - "UPDATE moa.execution_dispatch_outbox \ - SET state='cancelled', claim_owner=NULL, claimed_at=NULL, claim_expires_at=NULL, \ - updated_at=NOW() \ - WHERE trigger_uid = ANY($1::UUID[]) AND state IN ('pending','dispatching')", - ) - .bind(&trigger_uids) - .execute(conn.as_mut()) - .await - .map_err(sqlx_error)?; - let receipt_count: i64 = sqlx::query_scalar( - "SELECT COUNT(*) FROM moa.execution_capacity_reservation AS reservation \ - JOIN moa.execution_trigger AS trigger \ - ON trigger.trigger_uid=reservation.trigger_uid \ - AND trigger.tenant_id=reservation.tenant_id \ - AND trigger.run_uid IS NOT DISTINCT FROM reservation.run_uid \ - AND trigger.controller_generation IS NOT DISTINCT FROM reservation.controller_generation \ - WHERE reservation.trigger_uid = ANY($1::UUID[]) \ - AND reservation.tenant_id=$2 AND reservation.run_uid=$3 \ - AND reservation.resource_dimension='scheduled_triggers'", - ) - .bind(&trigger_uids) - .bind(tenant_id) - .bind(run_uid) - .fetch_one(conn.as_mut()) - .await - .map_err(sqlx_error)?; - if usize::try_from(receipt_count).ok() != Some(trigger_uids.len()) { - return Err(Error::InvalidRepositoryData { - message: "storage-wait trigger capacity receipts do not match their exact owners" - .to_string(), - }); - } - let released_quantities = sqlx::query_scalar::<_, i64>( - "UPDATE moa.execution_capacity_reservation \ - SET state='released', released_at=NOW(), updated_at=NOW() \ - WHERE trigger_uid = ANY($1::UUID[]) AND tenant_id=$2 AND run_uid=$3 \ - AND resource_dimension='scheduled_triggers' \ - AND state IN ('reserved','reconciling') AND released_at IS NULL \ - RETURNING quantity", - ) - .bind(&trigger_uids) - .bind(tenant_id) - .bind(run_uid) - .fetch_all(conn.as_mut()) - .await - .map_err(sqlx_error)?; - let released_quantity = released_quantities - .into_iter() - .try_fold(0_i64, i64::checked_add) - .ok_or_else(|| Error::InvalidRepositoryData { - message: "storage-wait trigger capacity quantity overflowed PostgreSQL BIGINT" - .to_string(), - })?; - if released_quantity == 0 { - return Ok(()); - } - let buckets = sqlx::query( - "UPDATE moa.execution_capacity_bucket \ - SET reserved_quantity=reserved_quantity-$2, version=version+1, updated_at=NOW() \ - WHERE resource_dimension='scheduled_triggers' AND reserved_quantity >= $2 \ - AND ((scope_kind='fleet' AND tenant_id IS NULL) \ - OR (scope_kind='tenant' AND tenant_id=$1))", - ) - .bind(tenant_id) - .bind(released_quantity) - .execute(conn.as_mut()) - .await - .map_err(sqlx_error)?; - if buckets.rows_affected() != 2 { - return Err(Error::InvalidRepositoryData { - message: "storage-wait trigger release did not decrement both capacity buckets" - .to_string(), - }); - } - Ok(()) -} - async fn active_attempt_capacity_count(conn: &mut ScopedConn<'_>, run_uid: Uuid) -> Result { let count: i64 = sqlx::query_scalar( "SELECT COUNT(*) FROM moa.execution_capacity_reservation WHERE run_uid=$1 \ @@ -5293,195 +3526,6 @@ async fn reconcile_run_after_compensation_capacity_release( } #[allow(clippy::too_many_arguments)] -async fn checkpoint_pending_terminal_wake( - conn: &mut ScopedConn<'_>, - run_uid: Uuid, - controller_generation: u64, - expected_wake_epoch: u64, - status: ExecutionRunStatus, - ready_task_count: u64, - active_task_count: u64, - now: DateTime, -) -> Result { - let row = sqlx::query( - "UPDATE moa.execution_run SET status=$4, activation_state='idle', \ - next_wake_at=NULL, waiting_since=NULL, ready_task_count=$5, \ - active_task_count=$6, processed_wake_epoch=$3, activation_failure_count=0, \ - last_progress_at=GREATEST(last_progress_at,$7), updated_at=NOW() \ - WHERE run_uid=$1 AND controller_generation=$2 AND wake_epoch >= $3 \ - AND processed_wake_epoch < $3 \ - AND activation_state IN ('queued','advancing','paused') RETURNING *", - ) - .bind(run_uid) - .bind(to_i64(controller_generation, "controller generation")?) - .bind(to_i64(expected_wake_epoch, "expected wake epoch")?) - .bind(status.as_str()) - .bind(to_i64(ready_task_count, "ready task count")?) - .bind(to_i64(active_task_count, "active task count")?) - .bind(now) - .fetch_optional(conn.as_mut()) - .await - .map_err(sqlx_error)? - .ok_or_else(|| Error::InvalidRepositoryData { - message: "terminal drain lost its controller wake checkpoint fence".to_string(), - })?; - run_from_row(&row) -} - -fn compensation_failure_pending( - original: &PendingExecutionTerminal, - registration: &CompensationRegistrationProjection, -) -> Result { - let outcome = registration - .outcome - .clone() - .ok_or_else(|| Error::InvalidRepositoryData { - message: "failed compensation is missing its terminal outcome".to_string(), - })?; - let pending = PendingExecutionTerminal { - status: ExecutionRunStatus::Failed, - reason: ExecutionTerminalReason::CompensationFailed, - terminal_evidence: ExecutionTerminalEvidence { - cause: ExecutionTerminalCause::CompensationFailure { - original_status: original.status, - original_reason: original.reason, - original_cause: Box::new(original.terminal_evidence.cause.clone()), - compensation_id: registration.compensation_id, - outcome, - }, - satisfied_requirement_count: original.terminal_evidence.satisfied_requirement_count, - requirement_count: original.terminal_evidence.requirement_count, - }, - completion_check_results: original.completion_check_results.clone(), - terminal_gaps: original.terminal_gaps.clone(), - output: original.output.clone(), - cancellation_reason: None, - }; - pending.validate()?; - Ok(pending) -} - -async fn replace_pending_terminal_exact( - conn: &mut ScopedConn<'_>, - run: &ExecutionRunRecord, - expected: &PendingExecutionTerminal, - replacement: &PendingExecutionTerminal, - controller_generation: u64, - expected_wake_epoch: u64, - now: DateTime, -) -> Result<()> { - let expected_payload = serde_json::to_value(PendingTerminalEvidencePayload { - terminal_evidence: expected.terminal_evidence.clone(), - completion_check_results: expected.completion_check_results.clone(), - terminal_gaps: expected.terminal_gaps.clone(), - })?; - let replacement_payload = serde_json::to_value(PendingTerminalEvidencePayload { - terminal_evidence: replacement.terminal_evidence.clone(), - completion_check_results: replacement.completion_check_results.clone(), - terminal_gaps: replacement.terminal_gaps.clone(), - })?; - let updated = sqlx::query( - "UPDATE moa.execution_run SET pending_terminal_status=$5, \ - pending_terminal_reason=$6, pending_terminal_cause=$7, pending_terminal_output=$8, \ - cancellation_reason=NULL, manual_repair_required=TRUE, updated_at=$9 \ - WHERE run_uid=$1 AND controller_generation=$2 AND wake_epoch >= $3 \ - AND pending_terminal_cause=$4 AND status='compensating'", - ) - .bind(run.run_uid) - .bind(to_i64(controller_generation, "controller generation")?) - .bind(to_i64(expected_wake_epoch, "expected wake epoch")?) - .bind(expected_payload) - .bind(replacement.status.as_str()) - .bind(replacement.reason.as_str()) - .bind(replacement_payload) - .bind(&replacement.output) - .bind(now) - .execute(conn.as_mut()) - .await - .map_err(sqlx_error)?; - if updated.rows_affected() != 1 { - return Err(Error::InvalidRepositoryData { - message: "compensation failure lost its exact pending-terminal replacement fence" - .to_string(), - }); - } - Ok(()) -} - -async fn finalize_pending_terminal_exact( - conn: &mut ScopedConn<'_>, - run: &ExecutionRunRecord, - pending: &PendingExecutionTerminal, - controller_generation: u64, - expected_wake_epoch: u64, - now: DateTime, -) -> Result { - release_owned_run_capacity_in_tx( - conn.as_mut(), - run.tenant_id, - run.run_uid, - run.controller_generation, - ) - .await?; - let evidence_payload = serde_json::to_value(PendingTerminalEvidencePayload { - terminal_evidence: pending.terminal_evidence.clone(), - completion_check_results: pending.completion_check_results.clone(), - terminal_gaps: pending.terminal_gaps.clone(), - })?; - let row = sqlx::query( - "UPDATE moa.execution_run SET status=$4, terminal_reason=$5, terminal_cause=$6, \ - terminal_satisfied_requirement_count=$7, terminal_requirement_count=$8, \ - completion_check_results=$9, terminal_gaps=$10, output=$11, \ - pending_terminal_status=NULL, pending_terminal_reason=NULL, \ - pending_terminal_cause=NULL, pending_terminal_output=NULL, \ - reserved_cost_microusd=0, reserved_tokens=0, reserved_tasks=0, \ - reserved_tool_calls=0, reserved_retrieved_bytes=0, \ - activation_state='terminal', waiting_reasons='[]'::JSONB, next_wake_at=NULL, \ - waiting_task_count=0, waiting_input_task_count=0, waiting_review_task_count=0, \ - waiting_signal_task_count=0, waiting_timer_task_count=0, \ - waiting_external_task_count=0, waiting_replan_task_count=0, \ - waiting_input_user_task_count=0, waiting_input_tenant_admin_task_count=0, \ - waiting_input_external_task_count=0, waiting_reasons_truncated=FALSE, \ - waiting_since=NULL, ready_task_count=0, active_task_count=0, \ - processed_wake_epoch=$3, activation_failure_count=0, completed_at=$12, \ - last_progress_at=GREATEST(last_progress_at,$12), updated_at=NOW() \ - WHERE run_uid=$1 AND controller_generation=$2 AND wake_epoch >= $3 \ - AND processed_wake_epoch < $3 AND pending_terminal_cause=$13 \ - AND NOT EXISTS (SELECT 1 FROM moa.execution_task WHERE run_uid=$1 \ - AND status NOT IN ('completed','skipped','failed','cancelled','unknown_outcome')) \ - AND NOT EXISTS (SELECT 1 FROM moa.execution_capacity_reservation WHERE run_uid=$1 \ - AND resource_dimension IN ('active_tasks','scheduled_triggers','external_jobs') \ - AND state IN ('reserved','reconciling')) \ - RETURNING *", - ) - .bind(run.run_uid) - .bind(to_i64(controller_generation, "controller generation")?) - .bind(to_i64(expected_wake_epoch, "expected wake epoch")?) - .bind(pending.status.as_str()) - .bind(pending.reason.as_str()) - .bind(serde_json::to_value(&pending.terminal_evidence.cause)?) - .bind(to_i64( - pending.terminal_evidence.satisfied_requirement_count, - "terminal satisfied requirement count", - )?) - .bind(to_i64( - pending.terminal_evidence.requirement_count, - "terminal requirement count", - )?) - .bind(serde_json::to_value(&pending.completion_check_results)?) - .bind(serde_json::to_value(&pending.terminal_gaps)?) - .bind(&pending.output) - .bind(now) - .bind(evidence_payload) - .fetch_optional(conn.as_mut()) - .await - .map_err(sqlx_error)? - .ok_or_else(|| Error::InvalidRepositoryData { - message: "terminal drain lost its final exact fence".to_string(), - })?; - run_from_row(&row) -} - async fn nonterminal_task_exists(conn: &mut ScopedConn<'_>, run_uid: Uuid) -> Result { sqlx::query_scalar( "SELECT EXISTS (SELECT 1 FROM moa.execution_task WHERE run_uid=$1 \ diff --git a/crates/moa-execution/src/repository/compensation/pending_terminal.rs b/crates/moa-execution/src/repository/compensation/pending_terminal.rs new file mode 100644 index 000000000..3a76085c1 --- /dev/null +++ b/crates/moa-execution/src/repository/compensation/pending_terminal.rs @@ -0,0 +1,1964 @@ +//! Pending-terminal fencing, bounded drain, compensation coordination, and finalization. + +use super::*; + +const PENDING_TERMINAL_CANCEL_NAMESPACE: Uuid = + Uuid::from_u128(0xd3d4_9744_5c24_58cc_8be8_4806_faba_1837); +const MAX_PENDING_TERMINAL_PAGE_SIZE: u32 = 1_000; + +async fn load_replan_stop_task( + conn: &mut ScopedConn<'_>, + run_uid: Uuid, + task_id: ExecutionTaskId, +) -> Result> { + sqlx::query(LOAD_TASK_FOR_UPDATE_SQL) + .bind(run_uid) + .bind(task_id.as_uuid()) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + .map(|row| task_from_row(&row)) + .transpose() +} + +fn replan_stop_receipt_audit(receipt: &ReplanStopReceipt, recorded_at: DateTime) -> Value { + json!({ + "kind": "replan_stop_fenced", + "accepted": true, + "task_id": receipt.task_id, + "task_generation": receipt.task_generation, + "base_plan_revision": receipt.base_plan_revision, + "amendment_hash": receipt.amendment_hash, + "recorded_at": recorded_at, + }) +} + +impl ExecutionRepository { + /// Fences one due approved deadline and advances one bounded terminal-drain page. + #[allow(clippy::too_many_arguments)] + pub async fn fence_deadline_and_enqueue_settlement( + &self, + config: &ExecutionConfig, + scope: ExecutionScope, + run_uid: Uuid, + controller_generation: u64, + expected_wake_epoch: u64, + now: DateTime, + page_limit: u32, + ) -> Result { + validate_pending_terminal_page_limit(page_limit)?; + let mut conn = scope.begin(&self.pool).await?; + let Some(run) = load_and_lock_pending_terminal_run(&mut conn, config, run_uid).await? + else { + conn.commit().await.map_err(storage_error)?; + return Ok(PendingTerminalAdvanceOutcome::NotFound); + }; + if run.controller_generation != controller_generation + || run.wake_epoch != expected_wake_epoch + { + conn.commit().await.map_err(storage_error)?; + return Ok(PendingTerminalAdvanceOutcome::Conflict); + } + if expected_wake_epoch <= run.processed_wake_epoch { + let commit = replayed_pending_terminal_commit(&mut conn, config, run).await?; + conn.commit().await.map_err(storage_error)?; + return Ok(PendingTerminalAdvanceOutcome::Replayed(Box::new(commit))); + } + let Some(deadline_at) = run.approved_budget.deadline_at else { + return Err(Error::InvalidRepositoryData { + message: "durable execution run is missing its approved deadline".to_string(), + }); + }; + if deadline_at > now || run.status.is_terminal() { + conn.commit().await.map_err(storage_error)?; + return Ok(PendingTerminalAdvanceOutcome::Conflict); + } + let requirement_count = u64::try_from(run.goal.requirements.len()).map_err(|_| { + Error::InvalidRepositoryData { + message: "execution requirement count exceeds u64".to_string(), + } + })?; + let pending = PendingExecutionTerminal { + status: ExecutionRunStatus::Failed, + reason: ExecutionTerminalReason::DeadlineExceeded, + terminal_evidence: ExecutionTerminalEvidence { + cause: ExecutionTerminalCause::LimitStop { + reason: ExecutionLimitStop::DeadlineExceeded, + }, + satisfied_requirement_count: 0, + requirement_count, + }, + completion_check_results: Vec::new(), + terminal_gaps: vec!["approved execution deadline elapsed".to_string()], + output: run.output.clone(), + cancellation_reason: None, + }; + pending.validate()?; + let new_pending = run.pending_terminal.is_none().then_some(pending); + advance_pending_terminal_page_in_conn( + conn, + config, + run, + controller_generation, + expected_wake_epoch, + new_pending, + now, + page_limit, + ) + .await + } + + /// Persists one completion-derived terminal intent and advances its first bounded drain page. + #[allow(clippy::too_many_arguments)] + pub async fn fence_completion_terminal_and_enqueue_settlement( + &self, + config: &ExecutionConfig, + scope: ExecutionScope, + run_uid: Uuid, + controller_generation: u64, + expected_wake_epoch: u64, + pending: PendingExecutionTerminal, + now: DateTime, + page_limit: u32, + ) -> Result { + validate_pending_terminal_page_limit(page_limit)?; + pending.validate()?; + let mut conn = scope.begin(&self.pool).await?; + let Some(run) = load_and_lock_pending_terminal_run(&mut conn, config, run_uid).await? + else { + conn.commit().await.map_err(storage_error)?; + return Ok(PendingTerminalAdvanceOutcome::NotFound); + }; + if run.controller_generation != controller_generation + || run.wake_epoch != expected_wake_epoch + { + conn.commit().await.map_err(storage_error)?; + return Ok(PendingTerminalAdvanceOutcome::Conflict); + } + if expected_wake_epoch <= run.processed_wake_epoch { + let commit = replayed_pending_terminal_commit(&mut conn, config, run).await?; + conn.commit().await.map_err(storage_error)?; + return Ok(PendingTerminalAdvanceOutcome::Replayed(Box::new(commit))); + } + if run.status.is_terminal() + || run + .pending_terminal + .as_ref() + .is_some_and(|current| current != &pending) + { + conn.commit().await.map_err(storage_error)?; + return Ok(PendingTerminalAdvanceOutcome::Conflict); + } + advance_pending_terminal_page_in_conn( + conn, + config, + run, + controller_generation, + expected_wake_epoch, + Some(pending), + now, + page_limit, + ) + .await + } + + /// Persists an exact replan-stop receipt and advances its first bounded terminal-drain page. + #[allow(clippy::too_many_arguments)] + pub async fn fence_replan_stop_and_enqueue_settlement( + &self, + config: &ExecutionConfig, + scope: ExecutionScope, + run_uid: Uuid, + controller_generation: u64, + expected_revision: u64, + expected_wake_epoch: u64, + pending: PendingExecutionTerminal, + receipt: ReplanStopReceipt, + now: DateTime, + page_limit: u32, + ) -> Result { + validate_pending_terminal_page_limit(page_limit)?; + pending.validate()?; + if receipt.base_plan_revision != expected_revision + || !matches!( + pending.terminal_evidence.cause, + ExecutionTerminalCause::ReplanStop { .. } + ) + { + return Err(Error::InvalidRepositoryInput { + message: "replan-stop receipt must match the fenced revision and terminal cause" + .to_string(), + }); + } + let mut conn = scope.begin(&self.pool).await?; + let Some(run) = load_and_lock_pending_terminal_run(&mut conn, config, run_uid).await? + else { + conn.commit().await.map_err(storage_error)?; + return Ok(PendingTerminalAdvanceOutcome::NotFound); + }; + if run.controller_generation != controller_generation + || run.plan_revision != expected_revision + || run.wake_epoch != expected_wake_epoch + || run.status.is_terminal() + || run.status == ExecutionRunStatus::Compensating + || run + .pending_terminal + .as_ref() + .is_some_and(|current| current != &pending) + { + conn.commit().await.map_err(storage_error)?; + return Ok(PendingTerminalAdvanceOutcome::Conflict); + } + let Some(task) = load_replan_stop_task(&mut conn, run_uid, receipt.task_id).await? else { + conn.commit().await.map_err(storage_error)?; + return Ok(PendingTerminalAdvanceOutcome::NotFound); + }; + let receipt_exists: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM moa.execution_amendment_receipt \ + WHERE tenant_id=$1 AND run_uid=$2 AND base_plan_revision=$3 \ + AND amendment_hash=$4 AND receipt_kind='replan_stop' \ + AND superseded_task_id=$5 AND task_generation=$6 \ + AND cardinality(task_ids_to_release)=0)", + ) + .bind(run.tenant_id.0) + .bind(run.run_uid) + .bind(to_i64( + receipt.base_plan_revision, + "replan-stop plan revision", + )?) + .bind(receipt.amendment_hash.to_string()) + .bind(receipt.task_id.as_uuid()) + .bind(to_i64( + receipt.task_generation, + "replan-stop task generation", + )?) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let intent = sqlx::query( + "SELECT tenant_id,controller_generation,wake_epoch,origin_task_id,task_generation, \ + base_plan_revision,stop_reason,amendment_hash \ + FROM moa.execution_replan_stop_intent WHERE run_uid=$1 FOR UPDATE", + ) + .bind(run.run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if run.pending_terminal.is_some() { + if !receipt_exists || intent.is_some() { + conn.commit().await.map_err(storage_error)?; + return Ok(PendingTerminalAdvanceOutcome::Conflict); + } + if expected_wake_epoch <= run.processed_wake_epoch { + let commit = replayed_pending_terminal_commit(&mut conn, config, run).await?; + conn.commit().await.map_err(storage_error)?; + return Ok(PendingTerminalAdvanceOutcome::Replayed(Box::new(commit))); + } + return advance_pending_terminal_page_in_conn( + conn, + config, + run, + controller_generation, + expected_wake_epoch, + None, + now, + page_limit, + ) + .await; + } + let Some(intent) = intent else { + conn.commit().await.map_err(storage_error)?; + return Ok(PendingTerminalAdvanceOutcome::Conflict); + }; + let ExecutionTerminalCause::ReplanStop { + reason: expected_stop_reason, + } = &pending.terminal_evidence.cause + else { + return Err(Error::InvalidRepositoryData { + message: "replan-stop fence lost its validated terminal cause".to_string(), + }); + }; + let expected_stop_reason = expected_stop_reason.as_str(); + let intent_exact = intent.try_get::("tenant_id").map_err(row_error)? + == run.tenant_id.0 + && required_u64(&intent, "controller_generation")? == controller_generation + && required_u64(&intent, "wake_epoch")? == expected_wake_epoch + && intent + .try_get::("origin_task_id") + .map_err(row_error)? + == receipt.task_id.as_uuid() + && required_u64(&intent, "task_generation")? == receipt.task_generation + && required_u64(&intent, "base_plan_revision")? == receipt.base_plan_revision + && intent + .try_get::("stop_reason") + .map_err(row_error)? + == expected_stop_reason + && intent + .try_get::("amendment_hash") + .map_err(row_error)? + == receipt.amendment_hash.to_string(); + if receipt_exists + || !intent_exact + || task.plan_revision != receipt.base_plan_revision + || task.generation != receipt.task_generation + || task.status != ExecutionTaskStatus::WaitingReplan + || !matches!( + task.current_outcome.as_ref().map(|outcome| &outcome.result), + Some(ExecutionTaskResult::NeedsReplan { .. }) + ) + { + conn.commit().await.map_err(storage_error)?; + return Ok(PendingTerminalAdvanceOutcome::Conflict); + } + sqlx::query( + "INSERT INTO moa.execution_amendment_receipt \ + (tenant_id,run_uid,base_plan_revision,amendment_hash,receipt_kind, \ + superseded_task_id,task_generation,task_ids_to_release,created_at) \ + VALUES ($1,$2,$3,$4,'replan_stop',$5,$6,'{}'::UUID[],$7)", + ) + .bind(run.tenant_id.0) + .bind(run.run_uid) + .bind(to_i64( + receipt.base_plan_revision, + "replan-stop plan revision", + )?) + .bind(receipt.amendment_hash.to_string()) + .bind(receipt.task_id.as_uuid()) + .bind(to_i64( + receipt.task_generation, + "replan-stop task generation", + )?) + .bind(now) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let deleted = sqlx::query( + "DELETE FROM moa.execution_replan_stop_intent WHERE tenant_id=$1 AND run_uid=$2 \ + AND controller_generation=$3 AND wake_epoch=$4", + ) + .bind(run.tenant_id.0) + .bind(run.run_uid) + .bind(to_i64(controller_generation, "controller generation")?) + .bind(to_i64(expected_wake_epoch, "expected wake epoch")?) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if deleted.rows_affected() != 1 { + return Err(Error::InvalidRepositoryData { + message: "replan-stop fence lost its exact durable intent".to_string(), + }); + } + sqlx::query(APPEND_TASK_OUTCOME_AUDIT_SQL) + .bind(run_uid) + .bind(task.task_id.as_uuid()) + .bind(replan_stop_receipt_audit(&receipt, now)) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + advance_pending_terminal_page_in_conn( + conn, + config, + run, + controller_generation, + expected_wake_epoch, + Some(pending), + now, + page_limit, + ) + .await + } + + /// Advances one bounded page of an already-fenced pending-terminal drain. + #[allow(clippy::too_many_arguments)] + pub async fn advance_pending_terminal_settlement( + &self, + config: &ExecutionConfig, + scope: ExecutionScope, + run_uid: Uuid, + controller_generation: u64, + expected_wake_epoch: u64, + now: DateTime, + page_limit: u32, + ) -> Result { + validate_pending_terminal_page_limit(page_limit)?; + let mut conn = scope.begin(&self.pool).await?; + let Some(run) = load_and_lock_pending_terminal_run(&mut conn, config, run_uid).await? + else { + conn.commit().await.map_err(storage_error)?; + return Ok(PendingTerminalAdvanceOutcome::NotFound); + }; + if run.controller_generation != controller_generation + || run.wake_epoch != expected_wake_epoch + { + conn.commit().await.map_err(storage_error)?; + return Ok(PendingTerminalAdvanceOutcome::Conflict); + } + if expected_wake_epoch <= run.processed_wake_epoch { + let commit = replayed_pending_terminal_commit(&mut conn, config, run).await?; + conn.commit().await.map_err(storage_error)?; + return Ok(PendingTerminalAdvanceOutcome::Replayed(Box::new(commit))); + } + if run.pending_terminal.is_none() || run.status.is_terminal() { + conn.commit().await.map_err(storage_error)?; + return Ok(PendingTerminalAdvanceOutcome::Conflict); + } + advance_pending_terminal_page_in_conn( + conn, + config, + run, + controller_generation, + expected_wake_epoch, + None, + now, + page_limit, + ) + .await + } +} + +enum PendingCompensationDrive { + Admitted(Box), + Replayed(Box), + CapacityUnavailable { retry_at: DateTime }, + ExternalCancellation(ExecutionDispatchRecord), + Parked, + Complete, + ManualRepair(CompensationRegistrationProjection), +} + +async fn drive_pending_terminal_compensation_in_conn( + conn: &mut ScopedConn<'_>, + config: &ExecutionConfig, + run: &ExecutionRunRecord, + now: DateTime, +) -> Result { + let nonterminal_forward_exists: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM moa.execution_task WHERE run_uid=$1 \ + AND status NOT IN ('completed','skipped','failed','cancelled','unknown_outcome'))", + ) + .bind(run.run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + // `manual_repair_required` is deliberately NOT rejected here. Settling a compensation + // attempt with a non-retryable failure sets that flag on the run, so rejecting it made + // the very next controller activation return a terminal repository error and the run + // sat in `compensating` forever instead of terminalizing `Failed`/`CompensationFailed`. + // The flag means "stop driving automatically and hand this to an operator", which is a + // `ManualRepair` outcome, not an invalid state — see the check below the registration + // load, which needs the row to report which compensation is stuck. + if run.status != ExecutionRunStatus::Compensating + || run.pending_terminal.is_none() + || nonterminal_forward_exists + { + return Err(Error::InvalidRepositoryData { + message: "bounded compensation driver entered from an invalid run state".to_string(), + }); + } + let Some(row) = sqlx::query( + "SELECT * FROM moa.execution_compensation WHERE run_uid=$1 \ + AND status <> 'completed' ORDER BY registered_sequence DESC \ + LIMIT 1 FOR UPDATE", + ) + .bind(run.run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + return Ok(PendingCompensationDrive::Complete); + }; + let registration = compensation_from_row(&row)?; + let attempt_state = compensation_attempt_state_from_row(&row)?; + if run.manual_repair_required + || matches!( + registration.status, + CompensationStatus::Failed | CompensationStatus::UnknownOutcome + ) + { + return Ok(PendingCompensationDrive::ManualRepair(registration)); + } + if attempt_state == CompensationAttemptState::Dispatching { + return Ok(PendingCompensationDrive::Replayed(Box::new( + load_existing_compensation_admission(conn, config, run, &row, ®istration).await?, + ))); + } + if attempt_state == CompensationAttemptState::WaitingExternal { + let external_job_uid = row + .try_get::, _>("external_job_uid") + .map_err(row_error)? + .ok_or_else(|| Error::InvalidRepositoryData { + message: "waiting-external compensation lost its exact external job UID" + .to_string(), + })?; + let owner = ExecutionExternalJobOwner::Compensation { + compensation_id: registration.compensation_id.as_uuid(), + compensation_generation: registration.generation, + compensation_attempt_generation: required_u64(&row, "attempt_generation")?, + }; + return match request_external_job_cancellation_in_conn( + conn, + config, + external_job_uid, + owner, + now, + ) + .await? + { + ExecutionExternalJobCancellationRequestOutcome::Applied(dispatch) + | ExecutionExternalJobCancellationRequestOutcome::Replayed(dispatch) => { + Ok(PendingCompensationDrive::ExternalCancellation(dispatch)) + } + ExecutionExternalJobCancellationRequestOutcome::UnboundPendingRecovery => { + Ok(PendingCompensationDrive::Parked) + } + ExecutionExternalJobCancellationRequestOutcome::AlreadyTerminal => { + let job = load_external_job_for_update_in_conn(conn.as_mut(), external_job_uid) + .await? + .ok_or_else(|| Error::InvalidRepositoryData { + message: "terminal compensation external job disappeared".to_string(), + })?; + settle_external_job_terminal_in_conn(conn, &job, now).await?; + Ok(PendingCompensationDrive::Parked) + } + ExecutionExternalJobCancellationRequestOutcome::NotFound + | ExecutionExternalJobCancellationRequestOutcome::Stale => { + Err(Error::InvalidRepositoryData { + message: "waiting-external compensation has a stale external job owner" + .to_string(), + }) + } + }; + } + if matches!( + attempt_state, + CompensationAttemptState::Running + | CompensationAttemptState::Cancelling + | CompensationAttemptState::WaitingReview + ) { + return Ok(PendingCompensationDrive::Parked); + } + if attempt_state != CompensationAttemptState::Idle + || !matches!( + registration.status, + CompensationStatus::Pending | CompensationStatus::Running + ) + { + return Err(Error::InvalidRepositoryData { + message: "highest reverse-order compensation is not dispatchable or settled" + .to_string(), + }); + } + let retry_at = checked_retry_at(config, now)?; + if !compensation_capacity_available(conn, run.tenant_id).await? { + return Ok(PendingCompensationDrive::CapacityUnavailable { retry_at }); + } + if registration.status == CompensationStatus::Pending && registration.outcome.is_none() { + let forward_task = + load_forward_task(conn, run.run_uid, registration.forward_task_id).await?; + let reservation = + compensation_reservation(run, ®istration, forward_task.retry.max_attempts)?; + let mut ledger = budget_ledger(run); + if ledger.try_reserve(reservation).is_err() { + let failed = + terminalize_compensation_budget_rejection(conn, run, ®istration, reservation) + .await?; + return Ok(PendingCompensationDrive::ManualRepair(failed)); + } + persist_run_budget(conn, run.run_uid, &ledger, false).await?; + } + let deadline = checked_attempt_deadline(config, now)?; + let attempt_generation = required_u64(&row, "attempt_generation")?; + let dispatch_uid = Uuid::now_v7(); + let watchdog_uid = Uuid::now_v7(); + let reservation_uid = reserve_compensation_attempt_capacity( + conn, + config, + run, + ®istration, + attempt_generation, + deadline, + now, + ) + .await?; + let watchdog = create_trigger_with_dispatch_in_conn( + conn.as_mut(), + config, + &compensation_trigger( + run, + ®istration, + attempt_generation, + watchdog_uid, + ExecutionTriggerKind::CompensationWatchdog, + deadline, + json!({}), + ), + ) + .await?; + let attempt_request = ExecutionCompensationAttemptRequest { + dispatch_uid, + capacity_reservation_uid: reservation_uid, + watchdog_trigger_uid: watchdog.trigger.trigger_uid, + watchdog_dispatch_uid: watchdog.dispatch.dispatch_uid, + run_uid: run.run_uid, + compensation_id: registration.compensation_id, + compensation_generation: registration.generation, + compensation_attempt_generation: attempt_generation, + controller_generation: run.controller_generation, + attempt_deadline_at: deadline, + tenant_id: run.tenant_id, + }; + let dispatch = enqueue_dispatch_in_conn( + conn.as_mut(), + &compensation_dispatch(run, &attempt_request, now)?, + ) + .await?; + let updated = sqlx::query( + "UPDATE moa.execution_compensation SET status='running', \ + attempt_state='dispatching', attempt_started_at=$6, \ + last_progress_at=GREATEST(last_progress_at,$6), \ + attempt_deadline_at=$7, waiting_since=NULL, active_dispatch_uid=$8, \ + dispatch_sequence=dispatch_sequence+1, started_at=COALESCE(started_at,$6), \ + updated_at=NOW() WHERE run_uid=$1 AND compensation_id=$2 AND generation=$3 \ + AND attempt_generation=$4 AND attempt_state='idle' \ + AND status IN ('pending','running') AND EXISTS ( \ + SELECT 1 FROM moa.execution_run AS current_run \ + WHERE current_run.run_uid=$1 AND current_run.controller_generation=$5) \ + RETURNING *", + ) + .bind(run.run_uid) + .bind(registration.compensation_id.as_uuid()) + .bind(to_i64(registration.generation, "compensation generation")?) + .bind(to_i64( + attempt_generation, + "compensation attempt generation", + )?) + .bind(to_i64(run.controller_generation, "controller generation")?) + .bind(now) + .bind(deadline) + .bind(dispatch_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + .ok_or_else(|| Error::InvalidRepositoryData { + message: "bounded compensation admission lost its exact row lock".to_string(), + })?; + Ok(PendingCompensationDrive::Admitted(Box::new( + CompensationAttemptAdmission { + attempt: compensation_attempt_from_row(&updated, run)?, + capacity_reservation_uid: reservation_uid, + dispatch, + watchdog, + }, + ))) +} + +fn validate_pending_terminal_page_limit(page_limit: u32) -> Result<()> { + if page_limit == 0 || page_limit > MAX_PENDING_TERMINAL_PAGE_SIZE { + return Err(Error::InvalidRepositoryInput { + message: format!( + "pending-terminal page limit must be between 1 and {MAX_PENDING_TERMINAL_PAGE_SIZE}" + ), + }); + } + Ok(()) +} + +async fn load_and_lock_pending_terminal_run( + conn: &mut ScopedConn<'_>, + config: &ExecutionConfig, + run_uid: Uuid, +) -> Result> { + let Some(visible_row) = sqlx::query(LOAD_RUN_SQL) + .bind(run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + else { + return Ok(None); + }; + let visible = run_from_row(&visible_row)?; + prelock_capacity_dimensions_in_tx( + conn.as_mut(), + config, + visible.tenant_id, + &[ + ExecutionCapacityDimension::ActiveRuns, + ExecutionCapacityDimension::ActiveTasks, + ExecutionCapacityDimension::ParkedRuns, + ExecutionCapacityDimension::ScheduledTriggers, + ExecutionCapacityDimension::ExternalJobs, + ], + ) + .await?; + let row = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) + .bind(run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let run = run_from_row(&row)?; + if run.tenant_id != visible.tenant_id { + return Err(Error::InvalidRepositoryData { + message: "execution run tenant changed while acquiring compensation capacity locks" + .to_string(), + }); + } + Ok(Some(run)) +} + +async fn replayed_pending_terminal_commit( + conn: &mut ScopedConn<'_>, + config: &ExecutionConfig, + run: ExecutionRunRecord, +) -> Result { + let work_remaining: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM moa.execution_task WHERE run_uid=$1 \ + AND status NOT IN ('completed','skipped','failed','cancelled','unknown_outcome')) \ + OR EXISTS (SELECT 1 FROM moa.execution_compensation WHERE run_uid=$1 \ + AND status <> 'completed') \ + OR EXISTS (SELECT 1 FROM moa.execution_capacity_reservation WHERE run_uid=$1 \ + AND resource_dimension IN ('active_tasks','scheduled_triggers','external_jobs') \ + AND state IN ('reserved','reconciling'))", + ) + .bind(run.run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let compensation_admission = if run.status == ExecutionRunStatus::Compensating { + let row = sqlx::query( + "SELECT * FROM moa.execution_compensation WHERE run_uid=$1 \ + AND status <> 'completed' ORDER BY registered_sequence DESC LIMIT 1 FOR UPDATE", + ) + .bind(run.run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if let Some(row) = row { + if compensation_attempt_state_from_row(&row)? == CompensationAttemptState::Dispatching { + let registration = compensation_from_row(&row)?; + Some(Box::new( + load_existing_compensation_admission(conn, config, &run, &row, ®istration) + .await?, + )) + } else { + None + } + } else { + None + } + } else { + None + }; + let continuation = load_pending_terminal_continuation(conn, &run).await?; + let stage = if run.status.is_terminal() { + if run.manual_repair_required { + PendingTerminalAdvanceStage::ManualRepairRequired + } else { + PendingTerminalAdvanceStage::Finalized + } + } else if compensation_admission.is_some() { + PendingTerminalAdvanceStage::CompensationQueued + } else if work_remaining { + PendingTerminalAdvanceStage::Draining + } else { + PendingTerminalAdvanceStage::EnqueuedPage + }; + Ok(PendingTerminalAdvanceCommit { + run, + stage, + settled_task_count: 0, + drained_trigger_count: 0, + cancellation_dispatches: Vec::new(), + compensation_admission, + continuation: continuation.map(Box::new), + work_remaining, + }) +} + +async fn load_pending_terminal_continuation( + conn: &mut ScopedConn<'_>, + run: &ExecutionRunRecord, +) -> Result> { + let row = sqlx::query( + "SELECT dispatch_uid, not_before_at, payload, wake_epoch \ + FROM moa.execution_dispatch_outbox WHERE run_uid=$1 \ + AND dispatch_kind='run_activation' AND controller_generation=$2 \ + AND payload->>'source_wake_epoch'=$3 ORDER BY created_at DESC LIMIT 1", + ) + .bind(run.run_uid) + .bind(to_i64(run.controller_generation, "controller generation")?) + .bind(run.processed_wake_epoch.to_string()) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(row) = row else { + return Ok(None); + }; + let wake_epoch = required_u64(&row, "wake_epoch")?; + let request = NewExecutionDispatch { + dispatch_uid: row.try_get("dispatch_uid").map_err(row_error)?, + tenant_id: run.tenant_id, + run_uid: Some(run.run_uid), + task_id: None, + compensation_id: None, + trigger_uid: None, + external_job_uid: None, + kind: ExecutionDispatchKind::RunActivation, + controller_generation: Some(run.controller_generation), + wake_epoch: Some(wake_epoch), + attempt_generation: None, + compensation_generation: None, + compensation_attempt_generation: None, + not_before_at: row.try_get("not_before_at").map_err(row_error)?, + payload: row.try_get("payload").map_err(row_error)?, + }; + enqueue_dispatch_in_conn(conn.as_mut(), &request) + .await + .map(Some) +} + +#[allow(clippy::too_many_arguments)] +async fn advance_pending_terminal_page_in_conn( + mut conn: ScopedConn<'_>, + config: &ExecutionConfig, + mut run: ExecutionRunRecord, + controller_generation: u64, + expected_wake_epoch: u64, + new_pending: Option, + now: DateTime, + page_limit: u32, +) -> Result { + if let Some(pending) = new_pending { + if let Some(current) = &run.pending_terminal { + if current != &pending { + conn.commit().await.map_err(storage_error)?; + return Ok(PendingTerminalAdvanceOutcome::Conflict); + } + } else { + let row = sqlx::query( + "UPDATE moa.execution_run SET pending_terminal_status=$4, \ + pending_terminal_reason=$5, pending_terminal_cause=$6, \ + pending_terminal_output=$7, cancellation_reason=$8, \ + next_wake_at=NULL, \ + updated_at=$9 WHERE run_uid=$1 AND controller_generation=$2 \ + AND wake_epoch=$3 AND pending_terminal_status IS NULL \ + AND status NOT IN ('completed','partial','blocked','unsupported', \ + 'failed','cancelled','compensating') RETURNING *", + ) + .bind(run.run_uid) + .bind(to_i64(controller_generation, "controller generation")?) + .bind(to_i64(expected_wake_epoch, "expected wake epoch")?) + .bind(pending.status.as_str()) + .bind(pending.reason.as_str()) + .bind(serde_json::to_value(PendingTerminalEvidencePayload { + terminal_evidence: pending.terminal_evidence.clone(), + completion_check_results: pending.completion_check_results.clone(), + terminal_gaps: pending.terminal_gaps.clone(), + })?) + .bind(&pending.output) + .bind(&pending.cancellation_reason) + .bind(now) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(row) = row else { + conn.rollback().await.map_err(storage_error)?; + return Ok(PendingTerminalAdvanceOutcome::Conflict); + }; + run = run_from_row(&row)?; + } + } + let pending = run + .pending_terminal + .clone() + .ok_or_else(|| Error::InvalidRepositoryData { + message: "terminal drain lost its pending terminal intent".to_string(), + })?; + let cancel_reason = if pending.reason == ExecutionTerminalReason::DeadlineExceeded { + ExecutionAttemptCancelReason::DeadlineExceeded + } else { + ExecutionAttemptCancelReason::RunTerminal + }; + let task_rows = sqlx::query( + "SELECT task.* FROM moa.execution_task AS task WHERE task.run_uid=$1 \ + AND task.status NOT IN ('completed','skipped','failed','cancelled','unknown_outcome') \ + AND task.attempt_state <> 'cancelling' \ + ORDER BY CASE WHEN task.attempt_state IN ('dispatching','running') THEN 0 ELSE 1 END, \ + task.task_id LIMIT $2 FOR UPDATE", + ) + .bind(run.run_uid) + .bind(i64::from(page_limit)) + .fetch_all(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let tasks = task_rows + .iter() + .map(task_from_row) + .collect::>>()?; + let processed_task_count = + u32::try_from(tasks.len()).map_err(|_| Error::InvalidRepositoryData { + message: "terminal drain task page exceeds u32".to_string(), + })?; + let storage_task_ids = tasks + .iter() + .filter(|task| { + task.status != ExecutionTaskStatus::WaitingExternal + && !matches!( + task.attempt_state, + ExecutionAttemptState::Dispatching | ExecutionAttemptState::Running + ) + }) + .map(|task| task.task_id.as_uuid()) + .collect::>(); + supersede_storage_task_waits(&mut conn, run.run_uid, run.tenant_id.0, &storage_task_ids) + .await?; + let mut settled_task_count = 0_u64; + let mut cancellation_dispatches = Vec::with_capacity(tasks.len()); + for task in tasks { + if task.status == ExecutionTaskStatus::WaitingExternal { + let external_job_uid = + task.external_job_uid + .ok_or_else(|| Error::InvalidRepositoryData { + message: "waiting-external task lost its exact external job UID" + .to_string(), + })?; + let owner = ExecutionExternalJobOwner::Task { + task_id: task.task_id.as_uuid(), + attempt_generation: task.attempt_generation, + }; + match request_external_job_cancellation_in_conn( + &mut conn, + config, + external_job_uid, + owner, + now, + ) + .await? + { + ExecutionExternalJobCancellationRequestOutcome::Applied(dispatch) + | ExecutionExternalJobCancellationRequestOutcome::Replayed(dispatch) => { + cancellation_dispatches.push(dispatch); + } + ExecutionExternalJobCancellationRequestOutcome::UnboundPendingRecovery => {} + ExecutionExternalJobCancellationRequestOutcome::AlreadyTerminal => { + let job = load_external_job_for_update_in_conn(conn.as_mut(), external_job_uid) + .await? + .ok_or_else(|| Error::InvalidRepositoryData { + message: "terminal external job disappeared under its owner fence" + .to_string(), + })?; + settle_task_external_job_terminal_in_conn(&mut conn, &job, now).await?; + } + ExecutionExternalJobCancellationRequestOutcome::NotFound + | ExecutionExternalJobCancellationRequestOutcome::Stale => { + return Err(Error::InvalidRepositoryData { + message: "waiting-external task has a stale external job owner".to_string(), + }); + } + } + continue; + } + if matches!( + task.attempt_state, + ExecutionAttemptState::Dispatching | ExecutionAttemptState::Running + ) { + cancellation_dispatches.push( + enqueue_pending_terminal_task_cancellation( + &mut conn, + &run, + &task, + cancel_reason, + pending.reason, + now, + ) + .await?, + ); + continue; + } + let original_status = task.status; + match record_task_outcome_in_conn( + &mut conn, + run.run_uid, + task.task_id, + task.generation, + cancelled_task_outcome( + format!("run terminal fence: {}", pending.reason.as_str()), + task.actual.clone(), + ), + ) + .await? + { + TaskOutcomeWrite::Applied { task, .. } | TaskOutcomeWrite::Replayed { task, .. } => { + transition_node_counters_in_tx( + &mut conn, + run.run_uid, + &task.node_id, + &task.item_key, + original_status, + ExecutionTaskStatus::Cancelled, + ) + .await?; + } + TaskOutcomeWrite::Rejected { reason, .. } => { + return Err(Error::InvalidRepositoryData { + message: format!("terminal drain task settlement was rejected: {reason:?}"), + }); + } + TaskOutcomeWrite::NotFound => { + return Err(Error::InvalidRepositoryData { + message: "terminal drain lost a row-locked task".to_string(), + }); + } + } + settled_task_count = + settled_task_count + .checked_add(1) + .ok_or_else(|| Error::InvalidRepositoryData { + message: "terminal drain settled-task count overflow".to_string(), + })?; + } + + let task_dispatch_count = cancellation_dispatches.len(); + let remaining_slots = page_limit.saturating_sub(processed_task_count); + if remaining_slots > 0 && run.status != ExecutionRunStatus::Compensating { + let compensation_rows = sqlx::query( + "SELECT compensation.* FROM moa.execution_compensation AS compensation \ + WHERE compensation.run_uid=$1 \ + AND compensation.attempt_state IN ('dispatching','running') \ + ORDER BY compensation.registered_sequence DESC LIMIT $2 FOR UPDATE", + ) + .bind(run.run_uid) + .bind(i64::from(remaining_slots)) + .fetch_all(conn.as_mut()) + .await + .map_err(sqlx_error)?; + for row in compensation_rows { + cancellation_dispatches.push( + enqueue_pending_terminal_compensation_cancellation( + &mut conn, + &run, + &row, + cancel_reason, + pending.reason, + now, + ) + .await?, + ); + } + } + let compensation_cancellation_count = cancellation_dispatches + .len() + .checked_sub(task_dispatch_count) + .and_then(|count| u32::try_from(count).ok()) + .ok_or_else(|| Error::InvalidRepositoryData { + message: "terminal drain compensation cancellation count overflow".to_string(), + })?; + let charged_after_cancellations = processed_task_count + .checked_add(compensation_cancellation_count) + .ok_or_else(|| Error::InvalidRepositoryData { + message: "terminal drain page accounting overflow after cancellation".to_string(), + })?; + let trigger_slots = page_limit.saturating_sub(charged_after_cancellations); + + let nonterminal_forward_count: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM moa.execution_task WHERE run_uid=$1 \ + AND status NOT IN ('completed','skipped','failed','cancelled','unknown_outcome')", + ) + .bind(run.run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let actionable_forward_exists: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM moa.execution_task WHERE run_uid=$1 \ + AND status NOT IN ('completed','skipped','failed','cancelled','unknown_outcome') \ + AND attempt_state <> 'cancelling' AND status <> 'waiting_external')", + ) + .bind(run.run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let active_count = active_attempt_capacity_count(&mut conn, run.run_uid).await?; + let has_registrations: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM moa.execution_compensation WHERE run_uid=$1)", + ) + .bind(run.run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let retain_cancelled_effects = pending.status == ExecutionRunStatus::Cancelled + && run.active_plan.definition.cancel_policy == ExecutionCancelPolicy::RetainEffects; + let should_compensate = has_registrations && !retain_cancelled_effects; + let active_trigger_exists: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM moa.execution_trigger WHERE run_uid=$1 \ + AND state = 'pending')", + ) + .bind(run.run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let cleanup_triggers_now = nonterminal_forward_count == 0 + && active_count == 0 + && !should_compensate + && run.status != ExecutionRunStatus::Compensating; + let (mut drained_trigger_count, mut trigger_work_remaining) = + if cleanup_triggers_now && trigger_slots > 0 { + let page = drain_run_triggers_page_in_conn(&mut conn, &run, trigger_slots).await?; + (page.drained_trigger_count, page.work_remaining) + } else if cleanup_triggers_now { + (0, active_trigger_exists) + } else { + (0, false) + }; + let ready_count: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM moa.execution_task WHERE run_uid=$1 AND status='ready'", + ) + .bind(run.run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + + let mut stage = PendingTerminalAdvanceStage::Draining; + let mut work_remaining = nonterminal_forward_count > 0 || active_count > 0; + let mut continuation_payload = None; + let mut continuation_not_before = now; + let mut checkpoint_status = run.status; + let mut checkpoint_active_count = active_count; + let mut compensation_admission = None; + if actionable_forward_exists || trigger_work_remaining { + stage = PendingTerminalAdvanceStage::EnqueuedPage; + work_remaining = true; + continuation_payload = Some(json!({ + "reason":"pending_terminal_page", + "source_wake_epoch": expected_wake_epoch, + })); + } else if nonterminal_forward_count == 0 && active_count == 0 { + if should_compensate && run.status != ExecutionRunStatus::Compensating { + stage = PendingTerminalAdvanceStage::EnqueuedPage; + checkpoint_status = ExecutionRunStatus::Compensating; + work_remaining = true; + continuation_payload = Some(json!({ + "reason":"pending_terminal_compensation", + "source_wake_epoch": expected_wake_epoch, + })); + } else if run.status == ExecutionRunStatus::Compensating { + match drive_pending_terminal_compensation_in_conn(&mut conn, config, &run, now).await? { + PendingCompensationDrive::Admitted(admission) + | PendingCompensationDrive::Replayed(admission) => { + stage = PendingTerminalAdvanceStage::CompensationQueued; + work_remaining = true; + compensation_admission = Some(admission); + checkpoint_active_count = + active_attempt_capacity_count(&mut conn, run.run_uid).await?; + } + PendingCompensationDrive::CapacityUnavailable { retry_at } => { + stage = PendingTerminalAdvanceStage::EnqueuedPage; + work_remaining = true; + continuation_not_before = retry_at; + continuation_payload = Some(json!({ + "reason":"pending_terminal_compensation_capacity", + "source_wake_epoch": expected_wake_epoch, + })); + } + PendingCompensationDrive::ExternalCancellation(dispatch) => { + cancellation_dispatches.push(dispatch); + work_remaining = true; + } + PendingCompensationDrive::Parked => { + work_remaining = true; + } + PendingCompensationDrive::ManualRepair(registration) => { + if trigger_slots > 0 { + let page = + drain_run_triggers_page_in_conn(&mut conn, &run, trigger_slots).await?; + drained_trigger_count = page.drained_trigger_count; + trigger_work_remaining = page.work_remaining; + } else { + trigger_work_remaining = active_trigger_exists; + } + if trigger_work_remaining { + stage = PendingTerminalAdvanceStage::EnqueuedPage; + work_remaining = true; + continuation_payload = Some(json!({ + "reason":"pending_terminal_manual_repair_cleanup", + "source_wake_epoch": expected_wake_epoch, + })); + } else { + let non_lifetime_capacity_exists: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM moa.execution_capacity_reservation \ + WHERE run_uid=$1 AND resource_dimension IN \ + ('active_tasks','scheduled_triggers','external_jobs') \ + AND state IN ('reserved','reconciling'))", + ) + .bind(run.run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if non_lifetime_capacity_exists { + return Err(Error::InvalidRepositoryData { + message: "failed compensation retained non-lifetime capacity" + .to_string(), + }); + } + let failure = compensation_failure_pending(&pending, ®istration)?; + replace_pending_terminal_exact( + &mut conn, + &run, + &pending, + &failure, + controller_generation, + expected_wake_epoch, + now, + ) + .await?; + let finalized = finalize_pending_terminal_exact( + &mut conn, + &run, + &failure, + controller_generation, + expected_wake_epoch, + now, + ) + .await?; + conn.commit().await.map_err(storage_error)?; + return Ok(PendingTerminalAdvanceOutcome::Applied(Box::new( + PendingTerminalAdvanceCommit { + run: finalized, + stage: PendingTerminalAdvanceStage::ManualRepairRequired, + settled_task_count, + drained_trigger_count, + cancellation_dispatches, + compensation_admission: None, + continuation: None, + work_remaining: false, + }, + ))); + } + } + PendingCompensationDrive::Complete => { + if trigger_slots > 0 { + let page = + drain_run_triggers_page_in_conn(&mut conn, &run, trigger_slots).await?; + drained_trigger_count = page.drained_trigger_count; + trigger_work_remaining = page.work_remaining; + } else { + trigger_work_remaining = active_trigger_exists; + } + if trigger_work_remaining { + stage = PendingTerminalAdvanceStage::EnqueuedPage; + work_remaining = true; + continuation_payload = Some(json!({ + "reason":"pending_terminal_trigger_cleanup", + "source_wake_epoch": expected_wake_epoch, + })); + } else { + let non_lifetime_capacity_exists: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM moa.execution_capacity_reservation \ + WHERE run_uid=$1 AND resource_dimension IN \ + ('active_tasks','scheduled_triggers','external_jobs') \ + AND state IN ('reserved','reconciling'))", + ) + .bind(run.run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if non_lifetime_capacity_exists { + return Err(Error::InvalidRepositoryData { + message: "completed compensation retained non-lifetime capacity" + .to_string(), + }); + } + let finalized = finalize_pending_terminal_exact( + &mut conn, + &run, + &pending, + controller_generation, + expected_wake_epoch, + now, + ) + .await?; + conn.commit().await.map_err(storage_error)?; + return Ok(PendingTerminalAdvanceOutcome::Applied(Box::new( + PendingTerminalAdvanceCommit { + run: finalized, + stage: PendingTerminalAdvanceStage::Finalized, + settled_task_count, + drained_trigger_count, + cancellation_dispatches, + compensation_admission: None, + continuation: None, + work_remaining: false, + }, + ))); + } + } + } + } else { + let non_lifetime_capacity_exists: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM moa.execution_capacity_reservation WHERE run_uid=$1 \ + AND resource_dimension IN ('active_tasks','scheduled_triggers','external_jobs') \ + AND state IN ('reserved','reconciling'))", + ) + .bind(run.run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if non_lifetime_capacity_exists { + work_remaining = true; + } else { + let finalized = finalize_pending_terminal_exact( + &mut conn, + &run, + &pending, + controller_generation, + expected_wake_epoch, + now, + ) + .await?; + stage = if finalized.manual_repair_required { + PendingTerminalAdvanceStage::ManualRepairRequired + } else { + PendingTerminalAdvanceStage::Finalized + }; + conn.commit().await.map_err(storage_error)?; + return Ok(PendingTerminalAdvanceOutcome::Applied(Box::new( + PendingTerminalAdvanceCommit { + run: finalized, + stage, + settled_task_count, + drained_trigger_count, + cancellation_dispatches, + compensation_admission: None, + continuation: None, + work_remaining: false, + }, + ))); + } + } + } + + let checkpointed = checkpoint_pending_terminal_wake( + &mut conn, + run.run_uid, + controller_generation, + expected_wake_epoch, + checkpoint_status, + u64::try_from(ready_count).map_err(|_| Error::InvalidRepositoryData { + message: "terminal drain ready-task count is negative".to_string(), + })?, + checkpoint_active_count, + now, + ) + .await?; + let continuation = if let Some(payload) = continuation_payload { + Some(Box::new( + enqueue_run_activation_in_conn( + conn.as_mut(), + checkpointed.tenant_id, + checkpointed.run_uid, + checkpointed.controller_generation, + continuation_not_before, + payload, + ) + .await?, + )) + } else { + None + }; + let row = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) + .bind(run.run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + run = run_from_row(&row)?; + conn.commit().await.map_err(storage_error)?; + Ok(PendingTerminalAdvanceOutcome::Applied(Box::new( + PendingTerminalAdvanceCommit { + run, + stage, + settled_task_count, + drained_trigger_count, + cancellation_dispatches, + compensation_admission, + continuation, + work_remaining, + }, + ))) +} + +async fn enqueue_pending_terminal_task_cancellation( + conn: &mut ScopedConn<'_>, + run: &ExecutionRunRecord, + task: &ExecutionTaskRecord, + reason: ExecutionAttemptCancelReason, + terminal_reason: ExecutionTerminalReason, + now: DateTime, +) -> Result { + let row = sqlx::query( + "SELECT reservation.reservation_uid, trigger.trigger_uid \ + FROM moa.execution_capacity_reservation AS reservation \ + JOIN moa.execution_trigger AS trigger ON trigger.run_uid=reservation.run_uid \ + AND trigger.task_id=reservation.task_id \ + AND trigger.controller_generation=reservation.controller_generation \ + AND trigger.attempt_generation=reservation.attempt_generation \ + AND trigger.trigger_kind='task_watchdog' \ + AND trigger.state = 'pending' \ + WHERE reservation.run_uid=$1 AND reservation.task_id=$2 \ + AND reservation.controller_generation=$3 AND reservation.attempt_generation=$4 \ + AND reservation.resource_dimension='active_tasks' \ + AND reservation.state IN ('reserved','reconciling') FOR UPDATE OF reservation, trigger", + ) + .bind(run.run_uid) + .bind(task.task_id.as_uuid()) + .bind(to_i64(run.controller_generation, "controller generation")?) + .bind(to_i64(task.attempt_generation, "task attempt generation")?) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + .ok_or_else(|| Error::InvalidRepositoryData { + message: format!( + "active task {} is missing its exact capacity or watchdog receipt", + task.task_id + ), + })?; + let active_dispatch_uid = + task.active_dispatch_uid + .ok_or_else(|| Error::InvalidRepositoryData { + message: format!( + "active task {} is missing its dispatch identity", + task.task_id + ), + })?; + let capacity_reservation_uid: Uuid = row.try_get("reservation_uid").map_err(row_error)?; + let watchdog_trigger_uid: Uuid = row.try_get("trigger_uid").map_err(row_error)?; + let cancellation_dispatch_uid = pending_terminal_cancel_dispatch_uid( + active_dispatch_uid, + run.controller_generation, + terminal_reason, + ); + let cancelling = sqlx::query( + "UPDATE moa.execution_task SET attempt_state='cancelling', \ + last_progress_at=GREATEST(last_progress_at,$6), updated_at=NOW() \ + WHERE run_uid=$1 AND task_id=$2 \ + AND generation=$3 AND attempt_generation=$4 AND active_dispatch_uid=$5 \ + AND attempt_state IN ('dispatching','running')", + ) + .bind(run.run_uid) + .bind(task.task_id.as_uuid()) + .bind(to_i64(task.generation, "task generation")?) + .bind(to_i64(task.attempt_generation, "task attempt generation")?) + .bind(active_dispatch_uid) + .bind(now) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if cancelling.rows_affected() != 1 { + return Err(Error::InvalidRepositoryData { + message: format!("task {} lost its terminal cancellation fence", task.task_id), + }); + } + let reconciling = sqlx::query( + "UPDATE moa.execution_capacity_reservation SET state='reconciling', updated_at=$2 \ + WHERE reservation_uid=$1 AND state IN ('reserved','reconciling')", + ) + .bind(capacity_reservation_uid) + .bind(now) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if reconciling.rows_affected() != 1 { + return Err(Error::InvalidRepositoryData { + message: format!("task {} lost its active-capacity receipt", task.task_id), + }); + } + let payload = serde_json::to_value(ExecutionTaskAttemptCancelRequest { + cancellation_dispatch_uid, + tenant_id: run.tenant_id, + run_uid: run.run_uid, + task_id: task.task_id, + controller_generation: run.controller_generation, + attempt_controller_generation: run.controller_generation, + task_generation: task.generation, + attempt_generation: task.attempt_generation, + active_dispatch_uid, + capacity_reservation_uid, + watchdog_trigger_uid, + reason, + })?; + enqueue_dispatch_in_conn( + conn.as_mut(), + &NewExecutionDispatch { + dispatch_uid: cancellation_dispatch_uid, + tenant_id: run.tenant_id, + run_uid: Some(run.run_uid), + task_id: Some(task.task_id.as_uuid()), + compensation_id: None, + trigger_uid: None, + external_job_uid: task.external_job_uid, + kind: ExecutionDispatchKind::TaskAttemptCancel, + controller_generation: Some(run.controller_generation), + wake_epoch: None, + attempt_generation: Some(task.attempt_generation), + compensation_generation: None, + compensation_attempt_generation: None, + not_before_at: now, + payload, + }, + ) + .await +} + +async fn enqueue_pending_terminal_compensation_cancellation( + conn: &mut ScopedConn<'_>, + run: &ExecutionRunRecord, + compensation_row: &PgRow, + reason: ExecutionAttemptCancelReason, + terminal_reason: ExecutionTerminalReason, + now: DateTime, +) -> Result { + let registration = compensation_from_row(compensation_row)?; + let attempt_generation = required_u64(compensation_row, "attempt_generation")?; + let active_dispatch_uid: Uuid = compensation_row + .try_get("active_dispatch_uid") + .map_err(row_error)?; + let receipt = sqlx::query( + "SELECT reservation.reservation_uid, trigger.trigger_uid \ + FROM moa.execution_capacity_reservation AS reservation \ + JOIN moa.execution_trigger AS trigger ON trigger.run_uid=reservation.run_uid \ + AND trigger.compensation_id=reservation.compensation_id \ + AND trigger.controller_generation=reservation.controller_generation \ + AND trigger.compensation_generation=reservation.compensation_generation \ + AND trigger.compensation_attempt_generation=reservation.compensation_attempt_generation \ + AND trigger.trigger_kind='compensation_watchdog' \ + AND trigger.state = 'pending' \ + WHERE reservation.run_uid=$1 AND reservation.compensation_id=$2 \ + AND reservation.controller_generation=$3 AND reservation.compensation_generation=$4 \ + AND reservation.compensation_attempt_generation=$5 \ + AND reservation.resource_dimension='active_tasks' \ + AND reservation.state IN ('reserved','reconciling') FOR UPDATE OF reservation, trigger", + ) + .bind(run.run_uid) + .bind(registration.compensation_id.as_uuid()) + .bind(to_i64(run.controller_generation, "controller generation")?) + .bind(to_i64(registration.generation, "compensation generation")?) + .bind(to_i64( + attempt_generation, + "compensation attempt generation", + )?) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + .ok_or_else(|| Error::InvalidRepositoryData { + message: format!( + "active compensation {} is missing its exact capacity or watchdog receipt", + registration.compensation_id + ), + })?; + let capacity_reservation_uid: Uuid = receipt.try_get("reservation_uid").map_err(row_error)?; + let watchdog_trigger_uid: Uuid = receipt.try_get("trigger_uid").map_err(row_error)?; + let cancellation_dispatch_uid = pending_terminal_cancel_dispatch_uid( + active_dispatch_uid, + run.controller_generation, + terminal_reason, + ); + let intent = compensation_release_intent(reason); + let cancelling = sqlx::query( + "UPDATE moa.execution_compensation SET attempt_state='cancelling', \ + release_intent=$7, last_progress_at=GREATEST(last_progress_at,$6), \ + updated_at=NOW() \ + WHERE run_uid=$1 AND compensation_id=$2 \ + AND generation=$3 AND attempt_generation=$4 AND active_dispatch_uid=$5 \ + AND attempt_state IN ('dispatching','running')", + ) + .bind(run.run_uid) + .bind(registration.compensation_id.as_uuid()) + .bind(to_i64(registration.generation, "compensation generation")?) + .bind(to_i64( + attempt_generation, + "compensation attempt generation", + )?) + .bind(active_dispatch_uid) + .bind(now) + .bind(compensation_release_intent_label(intent)) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if cancelling.rows_affected() != 1 { + return Err(Error::InvalidRepositoryData { + message: format!( + "compensation {} lost its terminal cancellation fence", + registration.compensation_id + ), + }); + } + let reconciling = sqlx::query( + "UPDATE moa.execution_capacity_reservation SET state='reconciling', updated_at=$2 \ + WHERE reservation_uid=$1 AND state IN ('reserved','reconciling')", + ) + .bind(capacity_reservation_uid) + .bind(now) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if reconciling.rows_affected() != 1 { + return Err(Error::InvalidRepositoryData { + message: format!( + "compensation {} lost its active-capacity receipt", + registration.compensation_id + ), + }); + } + let payload = serde_json::to_value(ExecutionCompensationAttemptCancelRequest { + cancellation_dispatch_uid, + tenant_id: run.tenant_id, + run_uid: run.run_uid, + compensation_id: registration.compensation_id, + controller_generation: run.controller_generation, + attempt_controller_generation: run.controller_generation, + compensation_generation: registration.generation, + compensation_attempt_generation: attempt_generation, + active_dispatch_uid, + capacity_reservation_uid, + watchdog_trigger_uid, + intent, + })?; + enqueue_dispatch_in_conn( + conn.as_mut(), + &NewExecutionDispatch { + dispatch_uid: cancellation_dispatch_uid, + tenant_id: run.tenant_id, + run_uid: Some(run.run_uid), + task_id: None, + compensation_id: Some(registration.compensation_id.as_uuid()), + trigger_uid: None, + external_job_uid: None, + kind: ExecutionDispatchKind::CompensationAttemptCancel, + controller_generation: Some(run.controller_generation), + wake_epoch: None, + attempt_generation: None, + compensation_generation: Some(registration.generation), + compensation_attempt_generation: Some(attempt_generation), + not_before_at: now, + payload, + }, + ) + .await +} + +fn pending_terminal_cancel_dispatch_uid( + active_dispatch_uid: Uuid, + controller_generation: u64, + terminal_reason: ExecutionTerminalReason, +) -> Uuid { + let name = format!( + "{active_dispatch_uid}:{controller_generation}:{}", + terminal_reason.as_str() + ); + Uuid::new_v5(&PENDING_TERMINAL_CANCEL_NAMESPACE, name.as_bytes()) +} + +fn compensation_release_intent( + reason: ExecutionAttemptCancelReason, +) -> ExecutionCompensationReleaseIntent { + match reason { + ExecutionAttemptCancelReason::DeadlineExceeded => { + ExecutionCompensationReleaseIntent::Deadline + } + ExecutionAttemptCancelReason::RunTerminal => { + ExecutionCompensationReleaseIntent::RunTerminal + } + ExecutionAttemptCancelReason::PauseRequested => ExecutionCompensationReleaseIntent::Pause, + ExecutionAttemptCancelReason::ExternalJobStarted => { + ExecutionCompensationReleaseIntent::ExternalJob + } + } +} + +async fn supersede_storage_task_waits( + conn: &mut ScopedConn<'_>, + run_uid: Uuid, + tenant_id: Uuid, + task_ids: &[Uuid], +) -> Result<()> { + if task_ids.is_empty() { + return Ok(()); + } + let trigger_uids = sqlx::query_scalar::<_, Uuid>( + "UPDATE moa.execution_trigger SET state='superseded', updated_at=NOW() \ + WHERE run_uid=$1 AND task_id = ANY($2::UUID[]) \ + AND trigger_kind <> 'task_watchdog' AND state = 'pending' \ + RETURNING trigger_uid", + ) + .bind(run_uid) + .bind(task_ids) + .fetch_all(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if trigger_uids.is_empty() { + return Ok(()); + } + sqlx::query( + "UPDATE moa.execution_dispatch_outbox \ + SET state='cancelled', claim_owner=NULL, claimed_at=NULL, claim_expires_at=NULL, \ + updated_at=NOW() \ + WHERE trigger_uid = ANY($1::UUID[]) AND state IN ('pending','dispatching')", + ) + .bind(&trigger_uids) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let receipt_count: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM moa.execution_capacity_reservation AS reservation \ + JOIN moa.execution_trigger AS trigger \ + ON trigger.trigger_uid=reservation.trigger_uid \ + AND trigger.tenant_id=reservation.tenant_id \ + AND trigger.run_uid IS NOT DISTINCT FROM reservation.run_uid \ + AND trigger.controller_generation IS NOT DISTINCT FROM reservation.controller_generation \ + WHERE reservation.trigger_uid = ANY($1::UUID[]) \ + AND reservation.tenant_id=$2 AND reservation.run_uid=$3 \ + AND reservation.resource_dimension='scheduled_triggers'", + ) + .bind(&trigger_uids) + .bind(tenant_id) + .bind(run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if usize::try_from(receipt_count).ok() != Some(trigger_uids.len()) { + return Err(Error::InvalidRepositoryData { + message: "storage-wait trigger capacity receipts do not match their exact owners" + .to_string(), + }); + } + let released_quantities = sqlx::query_scalar::<_, i64>( + "UPDATE moa.execution_capacity_reservation \ + SET state='released', released_at=NOW(), updated_at=NOW() \ + WHERE trigger_uid = ANY($1::UUID[]) AND tenant_id=$2 AND run_uid=$3 \ + AND resource_dimension='scheduled_triggers' \ + AND state IN ('reserved','reconciling') AND released_at IS NULL \ + RETURNING quantity", + ) + .bind(&trigger_uids) + .bind(tenant_id) + .bind(run_uid) + .fetch_all(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let released_quantity = released_quantities + .into_iter() + .try_fold(0_i64, i64::checked_add) + .ok_or_else(|| Error::InvalidRepositoryData { + message: "storage-wait trigger capacity quantity overflowed PostgreSQL BIGINT" + .to_string(), + })?; + if released_quantity == 0 { + return Ok(()); + } + let buckets = sqlx::query( + "UPDATE moa.execution_capacity_bucket \ + SET reserved_quantity=reserved_quantity-$2, version=version+1, updated_at=NOW() \ + WHERE resource_dimension='scheduled_triggers' AND reserved_quantity >= $2 \ + AND ((scope_kind='fleet' AND tenant_id IS NULL) \ + OR (scope_kind='tenant' AND tenant_id=$1))", + ) + .bind(tenant_id) + .bind(released_quantity) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if buckets.rows_affected() != 2 { + return Err(Error::InvalidRepositoryData { + message: "storage-wait trigger release did not decrement both capacity buckets" + .to_string(), + }); + } + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +async fn checkpoint_pending_terminal_wake( + conn: &mut ScopedConn<'_>, + run_uid: Uuid, + controller_generation: u64, + expected_wake_epoch: u64, + status: ExecutionRunStatus, + ready_task_count: u64, + active_task_count: u64, + now: DateTime, +) -> Result { + let row = sqlx::query( + "UPDATE moa.execution_run SET status=$4, activation_state='idle', \ + next_wake_at=NULL, waiting_since=NULL, ready_task_count=$5, \ + active_task_count=$6, processed_wake_epoch=$3, activation_failure_count=0, \ + last_progress_at=GREATEST(last_progress_at,$7), updated_at=NOW() \ + WHERE run_uid=$1 AND controller_generation=$2 AND wake_epoch >= $3 \ + AND processed_wake_epoch < $3 \ + AND activation_state IN ('queued','advancing','paused') RETURNING *", + ) + .bind(run_uid) + .bind(to_i64(controller_generation, "controller generation")?) + .bind(to_i64(expected_wake_epoch, "expected wake epoch")?) + .bind(status.as_str()) + .bind(to_i64(ready_task_count, "ready task count")?) + .bind(to_i64(active_task_count, "active task count")?) + .bind(now) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + .ok_or_else(|| Error::InvalidRepositoryData { + message: "terminal drain lost its controller wake checkpoint fence".to_string(), + })?; + run_from_row(&row) +} + +fn compensation_failure_pending( + original: &PendingExecutionTerminal, + registration: &CompensationRegistrationProjection, +) -> Result { + let outcome = registration + .outcome + .clone() + .ok_or_else(|| Error::InvalidRepositoryData { + message: "failed compensation is missing its terminal outcome".to_string(), + })?; + let pending = PendingExecutionTerminal { + status: ExecutionRunStatus::Failed, + reason: ExecutionTerminalReason::CompensationFailed, + terminal_evidence: ExecutionTerminalEvidence { + cause: ExecutionTerminalCause::CompensationFailure { + original_status: original.status, + original_reason: original.reason, + original_cause: Box::new(original.terminal_evidence.cause.clone()), + compensation_id: registration.compensation_id, + outcome, + }, + satisfied_requirement_count: original.terminal_evidence.satisfied_requirement_count, + requirement_count: original.terminal_evidence.requirement_count, + }, + completion_check_results: original.completion_check_results.clone(), + terminal_gaps: original.terminal_gaps.clone(), + output: original.output.clone(), + cancellation_reason: None, + }; + pending.validate()?; + Ok(pending) +} + +async fn replace_pending_terminal_exact( + conn: &mut ScopedConn<'_>, + run: &ExecutionRunRecord, + expected: &PendingExecutionTerminal, + replacement: &PendingExecutionTerminal, + controller_generation: u64, + expected_wake_epoch: u64, + now: DateTime, +) -> Result<()> { + let expected_payload = serde_json::to_value(PendingTerminalEvidencePayload { + terminal_evidence: expected.terminal_evidence.clone(), + completion_check_results: expected.completion_check_results.clone(), + terminal_gaps: expected.terminal_gaps.clone(), + })?; + let replacement_payload = serde_json::to_value(PendingTerminalEvidencePayload { + terminal_evidence: replacement.terminal_evidence.clone(), + completion_check_results: replacement.completion_check_results.clone(), + terminal_gaps: replacement.terminal_gaps.clone(), + })?; + let updated = sqlx::query( + "UPDATE moa.execution_run SET pending_terminal_status=$5, \ + pending_terminal_reason=$6, pending_terminal_cause=$7, pending_terminal_output=$8, \ + cancellation_reason=NULL, manual_repair_required=TRUE, updated_at=$9 \ + WHERE run_uid=$1 AND controller_generation=$2 AND wake_epoch >= $3 \ + AND pending_terminal_cause=$4 AND status='compensating'", + ) + .bind(run.run_uid) + .bind(to_i64(controller_generation, "controller generation")?) + .bind(to_i64(expected_wake_epoch, "expected wake epoch")?) + .bind(expected_payload) + .bind(replacement.status.as_str()) + .bind(replacement.reason.as_str()) + .bind(replacement_payload) + .bind(&replacement.output) + .bind(now) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if updated.rows_affected() != 1 { + return Err(Error::InvalidRepositoryData { + message: "compensation failure lost its exact pending-terminal replacement fence" + .to_string(), + }); + } + Ok(()) +} + +async fn finalize_pending_terminal_exact( + conn: &mut ScopedConn<'_>, + run: &ExecutionRunRecord, + pending: &PendingExecutionTerminal, + controller_generation: u64, + expected_wake_epoch: u64, + now: DateTime, +) -> Result { + release_owned_run_capacity_in_tx( + conn.as_mut(), + run.tenant_id, + run.run_uid, + run.controller_generation, + ) + .await?; + let evidence_payload = serde_json::to_value(PendingTerminalEvidencePayload { + terminal_evidence: pending.terminal_evidence.clone(), + completion_check_results: pending.completion_check_results.clone(), + terminal_gaps: pending.terminal_gaps.clone(), + })?; + let row = sqlx::query( + "UPDATE moa.execution_run SET status=$4, terminal_reason=$5, terminal_cause=$6, \ + terminal_satisfied_requirement_count=$7, terminal_requirement_count=$8, \ + completion_check_results=$9, terminal_gaps=$10, output=$11, \ + pending_terminal_status=NULL, pending_terminal_reason=NULL, \ + pending_terminal_cause=NULL, pending_terminal_output=NULL, \ + reserved_cost_microusd=0, reserved_tokens=0, reserved_tasks=0, \ + reserved_tool_calls=0, reserved_retrieved_bytes=0, \ + activation_state='terminal', waiting_reasons='[]'::JSONB, next_wake_at=NULL, \ + waiting_task_count=0, waiting_input_task_count=0, waiting_review_task_count=0, \ + waiting_signal_task_count=0, waiting_timer_task_count=0, \ + waiting_external_task_count=0, waiting_replan_task_count=0, \ + waiting_input_user_task_count=0, waiting_input_tenant_admin_task_count=0, \ + waiting_input_external_task_count=0, waiting_reasons_truncated=FALSE, \ + waiting_since=NULL, ready_task_count=0, active_task_count=0, \ + processed_wake_epoch=$3, activation_failure_count=0, completed_at=$12, \ + last_progress_at=GREATEST(last_progress_at,$12), updated_at=NOW() \ + WHERE run_uid=$1 AND controller_generation=$2 AND wake_epoch >= $3 \ + AND processed_wake_epoch < $3 AND pending_terminal_cause=$13 \ + AND NOT EXISTS (SELECT 1 FROM moa.execution_task WHERE run_uid=$1 \ + AND status NOT IN ('completed','skipped','failed','cancelled','unknown_outcome')) \ + AND NOT EXISTS (SELECT 1 FROM moa.execution_capacity_reservation WHERE run_uid=$1 \ + AND resource_dimension IN ('active_tasks','scheduled_triggers','external_jobs') \ + AND state IN ('reserved','reconciling')) \ + RETURNING *", + ) + .bind(run.run_uid) + .bind(to_i64(controller_generation, "controller generation")?) + .bind(to_i64(expected_wake_epoch, "expected wake epoch")?) + .bind(pending.status.as_str()) + .bind(pending.reason.as_str()) + .bind(serde_json::to_value(&pending.terminal_evidence.cause)?) + .bind(to_i64( + pending.terminal_evidence.satisfied_requirement_count, + "terminal satisfied requirement count", + )?) + .bind(to_i64( + pending.terminal_evidence.requirement_count, + "terminal requirement count", + )?) + .bind(serde_json::to_value(&pending.completion_check_results)?) + .bind(serde_json::to_value(&pending.terminal_gaps)?) + .bind(&pending.output) + .bind(now) + .bind(evidence_payload) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + .ok_or_else(|| Error::InvalidRepositoryData { + message: "terminal drain lost its final exact fence".to_string(), + })?; + run_from_row(&row) +} diff --git a/crates/moa-execution/tests/execution_db/completion_projection_db.rs b/crates/moa-execution/tests/execution_db/completion_projection_db.rs index b47ec666d..7c8088517 100644 --- a/crates/moa-execution/tests/execution_db/completion_projection_db.rs +++ b/crates/moa-execution/tests/execution_db/completion_projection_db.rs @@ -509,6 +509,19 @@ async fn replan_stop_completion_pages_rebind_exact_wake_without_duplicate_verifi message: "source unavailable".to_string(), }, }; + sqlx::query( + "UPDATE moa.execution_task SET status='reserved',updated_at=NOW() WHERE run_uid=$1", + ) + .bind(run.run_uid) + .execute(&pool) + .await?; + sqlx::query( + "UPDATE moa.execution_task SET status='running',attempt_state='running', \ + last_progress_at=NOW(),updated_at=NOW() WHERE run_uid=$1", + ) + .bind(run.run_uid) + .execute(&pool) + .await?; sqlx::query( "UPDATE moa.execution_task SET status='failed',attempt_state='terminal', \ current_outcome=$2,completed_at=NOW(),updated_at=NOW() \ @@ -537,6 +550,7 @@ async fn replan_stop_completion_pages_rebind_exact_wake_without_duplicate_verifi .bind(run.run_uid) .execute(&pool) .await?; + set_run_status_path(&pool, run.run_uid, &["running"]).await?; sqlx::query( "UPDATE moa.execution_run SET status='waiting_replan',ready_task_count=0, \ active_task_count=0,waiting_task_count=1,waiting_replan_task_count=1, \ diff --git a/crates/moa-execution/tests/execution_db/execution_capacity_db.rs b/crates/moa-execution/tests/execution_db/execution_capacity_db.rs index a0e22c6a5..504f9a08a 100644 --- a/crates/moa-execution/tests/execution_db/execution_capacity_db.rs +++ b/crates/moa-execution/tests/execution_db/execution_capacity_db.rs @@ -299,6 +299,73 @@ async fn weighted_admission_is_atomic_bounded_and_fleet_owned_db() -> TestResult Ok(()) } +#[tokio::test] +async fn same_tenant_batch_stops_at_capacity_ceiling_db() -> TestResult { + // Pins: one multi-item admission stops exactly at the tenant ceiling, leaves excess work + // ready, and keeps task state, receipts, and fleet/tenant counters consistent. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let run_uid = ready_run(&repository, tenant_id, "same-tenant-capacity", 5).await?; + let config = ExecutionConfig { + max_fleet_active_tasks: 5, + max_tenant_active_tasks: 3, + max_in_flight_tasks: 5, + ..ExecutionConfig::default() + }; + + let first = repository + .admit_ready_attempts(&config, 5, Utc::now()) + .await?; + assert_eq!(first.admitted.len(), 3); + assert!( + first + .admitted + .iter() + .all(|item| item.tenant_id == tenant_id), + "the batch must contain only the owning tenant's tasks" + ); + assert!(first.retry_after.is_some()); + + let task_counts: (i64, i64) = sqlx::query_as( + "SELECT count(*) FILTER (WHERE status='dispatching'), \ + count(*) FILTER (WHERE status='ready') \ + FROM moa.execution_task WHERE run_uid=$1", + ) + .bind(run_uid) + .fetch_one(&pool) + .await?; + assert_eq!(task_counts, (3, 2)); + let bucket_counts: (i64, i64) = sqlx::query_as( + "SELECT \ + max(reserved_quantity) FILTER (WHERE scope_kind='fleet'), \ + max(reserved_quantity) FILTER (WHERE scope_kind='tenant' AND tenant_id=$1) \ + FROM moa.execution_capacity_bucket WHERE resource_dimension='active_tasks'", + ) + .bind(tenant_id.0) + .fetch_one(&pool) + .await?; + assert_eq!(bucket_counts, (3, 3)); + let reservation_count: i64 = sqlx::query_scalar( + "SELECT count(*) FROM moa.execution_capacity_reservation \ + WHERE tenant_id=$1 AND run_uid=$2 AND resource_dimension='active_tasks' \ + AND state='reserved'", + ) + .bind(tenant_id.0) + .bind(run_uid) + .fetch_one(&pool) + .await?; + assert_eq!(reservation_count, 3); + + let second = repository + .admit_ready_attempts(&config, 5, Utc::now()) + .await?; + assert!(second.admitted.is_empty()); + assert!(second.retry_after.is_some()); + Ok(()) +} + #[tokio::test] async fn requested_admission_limit_is_a_hard_bound_independent_of_vec_capacity_db() -> TestResult { // Pins: a dispatcher request for one attempt admits exactly one durable task/outbox/watchdog diff --git a/crates/moa-hands/AGENTS.md b/crates/moa-hands/AGENTS.md new file mode 100644 index 000000000..18c0b6cb4 --- /dev/null +++ b/crates/moa-hands/AGENTS.md @@ -0,0 +1,12 @@ +# Hands Instructions + +Read `docs/06-hands-and-mcp.md`, `docs/08-security.md`, and +`docs/25-sandbox-workspaces.md`. This crate owns governed tool routing and the +sandbox-workspace domain, repositories, fences, ledgers, checkpoints, and +provider adapters. Preserve operation-ledger order, provider I/O boundaries, +commit-before-release, receipt fencing, and fail-closed policy intersections. + +Use `fast-pr`, `db-session`, and `db-memory` for focused checks. Sandbox service +and recovery lanes require their named Docker-backed fixture or E2E harness, +but deterministic lanes do not require live authorization. Set +`MOA_RUN_LIVE_E2E=1` only for an explicitly live target. diff --git a/crates/moa-hands/src/core/sandbox_workspace/lifecycle.rs b/crates/moa-hands/src/core/sandbox_workspace/lifecycle.rs index 07c034624..7218ca3c0 100644 --- a/crates/moa-hands/src/core/sandbox_workspace/lifecycle.rs +++ b/crates/moa-hands/src/core/sandbox_workspace/lifecycle.rs @@ -1,5 +1,10 @@ //! Managed durable sandbox-workspace lifecycle and commit barriers. +mod commit; +mod execution_release; +mod management; +mod materialization; + use chrono::{Duration as ChronoDuration, Utc}; use moa_core::{ error::{MoaError, Result}, @@ -73,2416 +78,7 @@ pub(in crate::core) struct WorkspaceCommitExecution<'a> { pub(in crate::core) release_compute: bool, } -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum ExecutionReleaseStep { - DurableReconciliation, - ProviderIo, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum PersistedLeaseReleaseState { - Missing, - Destroyed, - LiveOrAmbiguous, -} - -const fn compensation_release_identity_is_verified( - persisted_identity_present: bool, - lease_state: PersistedLeaseReleaseState, -) -> bool { - matches!( - (persisted_identity_present, lease_state), - (true, PersistedLeaseReleaseState::Destroyed) - | (false, PersistedLeaseReleaseState::Missing) - ) -} - -fn admit_execution_release_step( - scope: ToolCallScope<'_>, - step: ExecutionReleaseStep, -) -> Result<()> { - match step { - ExecutionReleaseStep::DurableReconciliation => Ok(()), - ExecutionReleaseStep::ProviderIo => scope.admit(), - } -} - impl ToolRouter { - /// Materializes the exact authorized worker workspace on its pinned provider. - pub async fn attach_managed_workspace( - &self, - session: &SessionMeta, - workspace_scope: &SandboxWorkspaceScope, - workspace_id: SandboxWorkspaceId, - ) -> Result<()> { - let workspace = self - .managed_workspace(session, workspace_scope, workspace_id) - .await?; - let route = self.management_route(&workspace.provider)?; - self.get_or_provision_hand_within( - &route, - session, - workspace_scope, - ToolCallScope::unbounded(), - ) - .await?; - let active = self - .managed_workspace(session, workspace_scope, workspace_id) - .await?; - if active.state != SandboxWorkspaceState::Active { - return Err(MoaError::StorageError( - "workspace attach completed without an active fenced writer".to_string(), - )); - } - Ok(()) - } - - /// Publishes one replay-stable explicit checkpoint through the durable commit barrier. - pub async fn checkpoint_managed_workspace( - &self, - session: &SessionMeta, - workspace_scope: &SandboxWorkspaceScope, - workspace_id: SandboxWorkspaceId, - operation_id: WorkspaceOperationId, - ) -> Result<()> { - let mut workspace = self - .managed_workspace(session, workspace_scope, workspace_id) - .await?; - if self - .confirmed_management_checkpoint_replay(&workspace, operation_id) - .await? - { - return Ok(()); - } - let hand = if matches!( - workspace.state, - SandboxWorkspaceState::Quiescing | SandboxWorkspaceState::Committing - ) { - let operations = self.hands.workspace_operations.as_ref().ok_or_else(|| { - MoaError::StorageError("workspace operation repository missing".to_string()) - })?; - let operation = operations - .get(session.tenant_id, operation_id) - .await? - .ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { - operation_id: operation_id.to_string(), - })?; - if operation.kind != WorkspaceOperationKind::Checkpoint - || operation.workspace_id != workspace_id - || operation.expected_writer_epoch != workspace.writer_epoch - || operation.expected_instance_generation != workspace.instance_generation - || operation.expected_checkpoint_generation != workspace.checkpoint_generation - { - return Err(MoaError::ExternalEffectUnknownOutcome { - operation_id: operation_id.to_string(), - }); - } - let lease_store = self.hands.hand_leases.as_ref().ok_or_else(|| { - MoaError::StorageError("durable hand lease store missing".to_string()) - })?; - lease_store - .get( - session.tenant_id, - session.id, - &workspace_lease_scope(workspace_scope), - &workspace.provider, - ) - .await? - .and_then(|lease| lease.handle.map(|handle| handle.handle)) - .ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { - operation_id: operation_id.to_string(), - })? - } else { - let route = self.management_route(&workspace.provider)?; - self.get_or_provision_hand_within( - &route, - session, - workspace_scope, - ToolCallScope::unbounded(), - ) - .await? - }; - workspace = self - .managed_workspace(session, workspace_scope, workspace_id) - .await?; - self.checkpoint_active_managed_workspace( - session, - workspace_scope, - &workspace, - operation_id, - &hand, - ) - .await - } - - /// Restores the exact current committed checkpoint into fresh provider compute. - /// - /// Historical revisions remain immutable retention records. Public restore - /// cannot silently move the monotonic workspace head backwards, so the - /// requested checkpoint must be the exact current recovery authority. - pub async fn restore_managed_workspace( - &self, - session: &SessionMeta, - workspace_scope: &SandboxWorkspaceScope, - workspace_id: SandboxWorkspaceId, - checkpoint_id: WorkspaceCheckpointId, - ) -> Result<()> { - let workspace = self - .managed_workspace(session, workspace_scope, workspace_id) - .await?; - let repository = - self.hands.workspace_repository.as_ref().ok_or_else(|| { - MoaError::StorageError("workspace repository missing".to_string()) - })?; - let checkpoint = repository - .get_checkpoint(session.tenant_id, workspace_id, checkpoint_id) - .await? - .ok_or_else(|| { - MoaError::ValidationError( - "restore checkpoint does not belong to the authorized workspace".to_string(), - ) - })?; - validate_managed_restore_target( - workspace.checkpoint_id, - workspace.checkpoint_generation, - checkpoint_id, - checkpoint.checkpoint_id, - checkpoint.generation, - checkpoint.state, - )?; - let route = self.management_route(&workspace.provider)?; - if workspace.state == SandboxWorkspaceState::Active { - self.reprovision_hand(session, workspace_scope, &route, ToolCallScope::unbounded()) - .await?; - } else { - self.get_or_provision_hand_within( - &route, - session, - workspace_scope, - ToolCallScope::unbounded(), - ) - .await?; - } - let restored = self - .managed_workspace(session, workspace_scope, workspace_id) - .await?; - if restored.state != SandboxWorkspaceState::Active - || restored.checkpoint_id != Some(checkpoint_id) - || restored.checkpoint_generation != checkpoint.generation - { - return Err(MoaError::StorageError( - "workspace restore completed without the exact committed checkpoint".to_string(), - )); - } - Ok(()) - } - - async fn managed_workspace( - &self, - session: &SessionMeta, - workspace_scope: &SandboxWorkspaceScope, - workspace_id: SandboxWorkspaceId, - ) -> Result { - let repository = - self.hands.workspace_repository.as_ref().ok_or_else(|| { - MoaError::StorageError("workspace repository missing".to_string()) - })?; - let workspace = repository - .get_by_scope(session.tenant_id, workspace_scope) - .await? - .ok_or_else(|| { - MoaError::PermissionDenied( - "sandbox workspace is not owned by the verified scope".to_string(), - ) - })?; - if workspace.workspace_id != workspace_id - || workspace.tenant_id != session.tenant_id - || workspace.scope != *workspace_scope - || workspace.access_fenced_at.is_some() - { - return Err(MoaError::PermissionDenied( - "sandbox workspace is not owned by the verified scope".to_string(), - )); - } - Ok(workspace) - } - - fn management_route(&self, provider: &str) -> Result { - self.catalog - .activated() - .capability_registrations() - .into_iter() - .find_map(|(_, execution)| match execution { - ToolExecution::Hand { routes } => { - routes.into_iter().find(|route| route.provider == provider) - } - _ => None, - }) - .ok_or_else(|| { - MoaError::ProviderError(format!( - "workspace provider {provider} has no configured hand route" - )) - }) - } - - async fn confirmed_management_checkpoint_replay( - &self, - workspace: &SandboxWorkspace, - operation_id: WorkspaceOperationId, - ) -> Result { - let operations = self.hands.workspace_operations.as_ref().ok_or_else(|| { - MoaError::StorageError("workspace operation repository missing".to_string()) - })?; - let repository = - self.hands.workspace_repository.as_ref().ok_or_else(|| { - MoaError::StorageError("workspace repository missing".to_string()) - })?; - let Some(operation) = operations.get(workspace.tenant_id, operation_id).await? else { - return Ok(false); - }; - if operation.outcome != WorkspaceOperationOutcome::Confirmed { - return Ok(false); - } - let checkpoint = repository - .get_checkpoint_for_operation(workspace.tenant_id, workspace.workspace_id, operation_id) - .await? - .ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { - operation_id: operation_id.to_string(), - })?; - let committed_generation = operation - .expected_checkpoint_generation - .checked_add(1) - .ok_or_else(|| { - MoaError::StorageError("workspace checkpoint generation overflowed".to_string()) - })?; - let parent_revision = revision_from_checkpoint_parent( - operation.expected_checkpoint_generation, - checkpoint.parent_checkpoint_id, - )?; - let mut original_binding = workspace.binding()?; - original_binding.current_revision = parent_revision; - let request_hash = management_checkpoint_request_hash(&original_binding, operation_id)?; - if operation.kind != WorkspaceOperationKind::Checkpoint - || operation.workspace_id != workspace.workspace_id - || operation.provider_account_id != workspace.provider_account_id - || operation.provider_account_generation != workspace.provider_account_generation - || operation.expected_writer_epoch != workspace.writer_epoch - || operation.expected_instance_generation != workspace.instance_generation - || operation.request_hash != request_hash - || operation.confirmed_disposition - != Some(WorkspaceConfirmedDisposition::ResourcePresent) - || checkpoint.state != WorkspaceCheckpointState::Available - || checkpoint.checkpoint_id != WorkspaceCheckpointId(operation_id.0) - || checkpoint.generation != committed_generation - || checkpoint.source_writer_epoch != workspace.writer_epoch - || checkpoint.source_instance_generation != workspace.instance_generation - || workspace.checkpoint_id != Some(checkpoint.checkpoint_id) - || workspace.checkpoint_generation != committed_generation - { - return Err(MoaError::ExternalEffectUnknownOutcome { - operation_id: operation_id.to_string(), - }); - } - Ok(true) - } - - async fn checkpoint_active_managed_workspace( - &self, - session: &SessionMeta, - workspace_scope: &SandboxWorkspaceScope, - workspace: &SandboxWorkspace, - operation_id: WorkspaceOperationId, - hand: &HandHandle, - ) -> Result<()> { - if !matches!( - workspace.state, - SandboxWorkspaceState::Active - | SandboxWorkspaceState::Quiescing - | SandboxWorkspaceState::Committing - ) { - return Err(MoaError::StorageError( - "workspace must be active before checkpoint publication".to_string(), - )); - } - let repository = - self.hands.workspace_repository.as_ref().ok_or_else(|| { - MoaError::StorageError("workspace repository missing".to_string()) - })?; - let operations = self.hands.workspace_operations.as_ref().ok_or_else(|| { - MoaError::StorageError("workspace operation repository missing".to_string()) - })?; - let storage_provider = self - .hands - .storage_providers - .get(&workspace.provider) - .ok_or_else(|| { - MoaError::ProviderError(format!( - "workspace storage provider {} is not registered", - workspace.provider - )) - })?; - let lease_store = self.hands.hand_leases.as_ref().ok_or_else(|| { - MoaError::StorageError("durable hand lease store missing".to_string()) - })?; - let binding = workspace.binding()?; - let lease_scope = workspace_lease_scope(workspace_scope); - let lease = lease_store - .get( - session.tenant_id, - session.id, - &lease_scope, - &workspace.provider, - ) - .await? - .ok_or_else(|| { - MoaError::StorageError( - "active workspace lease is missing before checkpoint".to_string(), - ) - })?; - if lease.status != HandLeaseStatus::Active - || lease.handle.as_ref().map(|lease| &lease.handle) != Some(hand) - || lease.attachment != Some(lease_attachment(&binding)?) - { - return Err(MoaError::ExternalEffectUnknownOutcome { - operation_id: operation_id.to_string(), - }); - } - - let checkpoint_id = WorkspaceCheckpointId(operation_id.0); - let existing_operation = operations.get(binding.tenant_id, operation_id).await?; - let deadline_at = existing_operation.as_ref().map_or_else( - || Utc::now() + ChronoDuration::minutes(5), - |operation| operation.deadline_at, - ); - let request_hash = management_checkpoint_request_hash(&binding, operation_id)?; - let expected_writer_epoch = i64::try_from(binding.writer_epoch) - .map_err(|_| MoaError::StorageError("workspace writer epoch is invalid".to_string()))?; - let expected_instance_generation = - i64::try_from(binding.instance_generation).map_err(|_| { - MoaError::StorageError("workspace instance generation is invalid".to_string()) - })?; - let expected_checkpoint_generation = - binding - .current_revision - .as_ref() - .map_or(Ok(0_i64), |revision| { - i64::try_from(revision.generation).map_err(|_| { - MoaError::StorageError( - "workspace checkpoint generation is invalid".to_string(), - ) - }) - })?; - let operation = operations - .persist_intent(&WorkspaceOperationIntent { - operation_id, - tenant_id: binding.tenant_id, - workspace_id: binding.workspace_id, - provider_account_id: binding.provider_account_id, - provider_account_generation: i64::try_from(binding.provider_account_generation) - .map_err(|_| { - MoaError::StorageError( - "workspace provider-account generation is invalid".to_string(), - ) - })?, - kind: WorkspaceOperationKind::Checkpoint, - request_hash: request_hash.clone(), - expected_writer_epoch, - expected_instance_generation, - expected_checkpoint_generation, - deadline_at, - reconcile_not_before: deadline_at + ChronoDuration::seconds(30), - }) - .await?; - if operation.outcome == WorkspaceOperationOutcome::Confirmed { - let current = self - .managed_workspace(session, workspace_scope, workspace.workspace_id) - .await?; - return self - .confirmed_management_checkpoint_replay(¤t, operation_id) - .await? - .then_some(()) - .ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { - operation_id: operation_id.to_string(), - }); - } - - let transitioned = match workspace.state { - SandboxWorkspaceState::Active => { - repository - .transition(WorkspaceTransition { - tenant_id: binding.tenant_id, - workspace_id: binding.workspace_id, - from: SandboxWorkspaceState::Active, - to: SandboxWorkspaceState::Quiescing, - writer_epoch: expected_writer_epoch, - instance_generation: expected_instance_generation, - }) - .await? - && repository - .transition(WorkspaceTransition { - tenant_id: binding.tenant_id, - workspace_id: binding.workspace_id, - from: SandboxWorkspaceState::Quiescing, - to: SandboxWorkspaceState::Committing, - writer_epoch: expected_writer_epoch, - instance_generation: expected_instance_generation, - }) - .await? - } - SandboxWorkspaceState::Quiescing => { - repository - .transition(WorkspaceTransition { - tenant_id: binding.tenant_id, - workspace_id: binding.workspace_id, - from: SandboxWorkspaceState::Quiescing, - to: SandboxWorkspaceState::Committing, - writer_epoch: expected_writer_epoch, - instance_generation: expected_instance_generation, - }) - .await? - } - SandboxWorkspaceState::Committing => true, - _ => false, - }; - if !transitioned { - operations - .mark_unknown(binding.tenant_id, operation_id) - .await?; - return Err(MoaError::ExternalEffectUnknownOutcome { - operation_id: operation_id.to_string(), - }); - } - if repository - .create_checkpoint(CreateCheckpointRequest { - checkpoint_id, - tenant_id: binding.tenant_id, - workspace_id: binding.workspace_id, - parent_checkpoint_id: binding - .current_revision - .as_ref() - .map(|revision| revision.checkpoint_id), - operation_id, - expected_writer_epoch, - expected_instance_generation, - expected_checkpoint_generation, - }) - .await? - .is_none() - { - operations - .mark_unknown(binding.tenant_id, operation_id) - .await?; - return Err(MoaError::ExternalEffectUnknownOutcome { - operation_id: operation_id.to_string(), - }); - } - let storage_operation = WorkspaceStorageOperation { - operation_id, - kind: WorkspaceOperationKind::Checkpoint, - binding: binding.clone(), - deadline: deadline_at, - request_hash, - }; - let provider_result = if operation.outcome == WorkspaceOperationOutcome::Unknown { - let storage = self.hands.checkpoint_store.as_ref().map(|store| { - store.storage_reference( - crate::core::sandbox_workspace::checkpoint::store::CheckpointStoreContext { - tenant_id: binding.tenant_id, - workspace_id: binding.workspace_id, - checkpoint_id, - provider_account_id: binding.provider_account_id, - provider_account_generation: binding.provider_account_generation, - }, - ) - }); - storage_provider - .reconcile_workspace_operation(WorkspaceReconcileRequest::new( - storage_operation, - Some(hand.clone()), - storage, - )?) - .await - } else { - if !operations - .begin_provider_attempt(binding.tenant_id, operation_id) - .await? - { - return Err(MoaError::ExternalEffectUnknownOutcome { - operation_id: operation_id.to_string(), - }); - } - storage_provider - .publish_workspace_checkpoint(WorkspaceCheckpointPublishRequest { - operation: storage_operation, - hand: hand.clone(), - parent_revision: binding.current_revision.clone(), - release_compute: false, - }) - .await - }; - let result = match provider_result { - Ok(result) => result, - Err(error) => { - tracing::warn!( - operation_id = %operation_id, - error = %error, - "workspace checkpoint provider outcome is ambiguous" - ); - operations - .mark_unknown(binding.tenant_id, operation_id) - .await?; - return Err(MoaError::ExternalEffectUnknownOutcome { - operation_id: operation_id.to_string(), - }); - } - }; - let (publication, post_commit_state) = match ( - result.outcome, - result.confirmed_disposition, - result.checkpoint_publication.as_ref(), - result.post_commit_state, - ) { - ( - WorkspaceOperationOutcome::Confirmed, - Some(WorkspaceConfirmedDisposition::ResourcePresent), - Some(publication), - Some(post_commit_state), - ) => (publication, post_commit_state), - _ => { - operations - .mark_unknown(binding.tenant_id, operation_id) - .await?; - return Err(MoaError::ExternalEffectUnknownOutcome { - operation_id: operation_id.to_string(), - }); - } - }; - if !repository - .publish_workspace_checkpoint(PublishCheckpointCommitRequest { - binding: &binding, - operation_id, - publication, - post_commit_state, - lease: &lease, - }) - .await? - { - self.delete_abandoned_checkpoint_prefix(&binding, publication.revision.checkpoint_id) - .await?; - operations - .mark_unknown(binding.tenant_id, operation_id) - .await?; - return Err(MoaError::ExternalEffectUnknownOutcome { - operation_id: operation_id.to_string(), - }); - } - if post_commit_state != WorkspacePostCommitState::AttachmentRetained { - let key = session_provider_key(session, Some(&lease_scope), &workspace.provider); - self.remove_cached_binding_if_matches(&key, hand, Some(lease.generation)) - .await; - self.remove_installed_marker( - manifest_scope_key(session, Some(&lease_scope)), - &workspace.provider, - ) - .await; - } - Ok(()) - } - - /// Resolves or creates the exact durable binding used for provisioning. - pub(in crate::core) async fn prepare_workspace_for_provision( - &self, - route: &HandRoute, - session: &SessionMeta, - workspace_scope: &SandboxWorkspaceScope, - call_scope: ToolCallScope<'_>, - ) -> Result { - let Some(repository) = self.hands.workspace_repository.as_ref() else { - return Ok(workspace_binding_for_hand( - session, - workspace_scope, - &route.provider, - )); - }; - let mut workspace = repository - .get_by_scope(session.tenant_id, workspace_scope) - .await? - .ok_or_else(|| { - MoaError::PermissionDenied( - "authorized sandbox workspace has not been resolved for this execution scope" - .to_string(), - ) - })?; - if workspace.provider != route.provider { - return Err(MoaError::ProviderError(format!( - "workspace is pinned to provider {}; cross-provider recovery is disabled", - workspace.provider - ))); - } - if workspace.access_fenced_at.is_some() - || matches!( - workspace.state, - moa_core::types::sandbox_workspace::SandboxWorkspaceState::Deleting - | moa_core::types::sandbox_workspace::SandboxWorkspaceState::Deleted - | moa_core::types::sandbox_workspace::SandboxWorkspaceState::Reconciling - | moa_core::types::sandbox_workspace::SandboxWorkspaceState::Failed - ) - { - return Err(MoaError::PermissionDenied( - "sandbox workspace is fenced or requires reconciliation".to_string(), - )); - } - if workspace.state == SandboxWorkspaceState::Creating { - call_scope.admit()?; - self.prepare_initial_workspace_storage(&workspace, call_scope) - .await?; - if !repository - .transition(WorkspaceTransition { - tenant_id: workspace.tenant_id, - workspace_id: workspace.workspace_id, - from: SandboxWorkspaceState::Creating, - to: SandboxWorkspaceState::Ready, - writer_epoch: workspace.writer_epoch, - instance_generation: workspace.instance_generation, - }) - .await? - { - workspace = repository - .get_by_scope(session.tenant_id, workspace_scope) - .await? - .ok_or_else(|| { - MoaError::StorageError( - "workspace disappeared while storage preparation completed".to_string(), - ) - })?; - } else { - workspace.state = SandboxWorkspaceState::Ready; - } - } - if workspace.state == SandboxWorkspaceState::Ready { - workspace = repository - .claim_writer(WorkspaceWriterClaim { - tenant_id: workspace.tenant_id, - workspace_id: workspace.workspace_id, - expected_state: workspace.state, - expected_writer_epoch: workspace.writer_epoch, - expected_instance_generation: workspace.instance_generation, - }) - .await? - .ok_or_else(|| { - MoaError::StorageError( - "workspace writer claim lost its lifecycle fence".to_string(), - ) - })?; - } - if !matches!( - workspace.state, - SandboxWorkspaceState::Active | SandboxWorkspaceState::Restoring - ) { - return Err(MoaError::StorageError(format!( - "workspace is not dispatchable while in state {}", - workspace.state.as_str() - ))); - } - workspace.binding() - } - - async fn prepare_initial_workspace_storage( - &self, - workspace: &SandboxWorkspace, - call_scope: ToolCallScope<'_>, - ) -> Result<()> { - let prepare_started_at = std::time::Instant::now(); - let operations = self.hands.workspace_operations.as_ref().ok_or_else(|| { - MoaError::StorageError("workspace operation repository missing".to_string()) - })?; - let storage_provider = self - .hands - .storage_providers - .get(&workspace.provider) - .ok_or_else(|| { - MoaError::ProviderError(format!( - "workspace storage provider {} is not registered", - workspace.provider - )) - })?; - let binding = workspace.binding()?; - let operation_id = moa_core::types::identifiers::WorkspaceOperationId(Uuid::new_v5( - &workspace.workspace_id.0, - b"prepare-initial-storage-v1", - )); - let existing = operations.get(workspace.tenant_id, operation_id).await?; - let deadline_at = existing.as_ref().map_or_else( - || { - call_scope - .budget - .deadline - .unwrap_or_else(|| Utc::now() + ChronoDuration::minutes(5)) - }, - |operation| operation.deadline_at, - ); - let reconcile_not_before = existing.as_ref().map_or_else( - || deadline_at + ChronoDuration::seconds(30), - |operation| operation.reconcile_not_before, - ); - let hash_bytes = serde_json::to_vec(&binding)?; - let request_hash = format!("sha256:{}", hex::encode(Sha256::digest(hash_bytes))); - let intent = WorkspaceOperationIntent { - operation_id, - tenant_id: workspace.tenant_id, - workspace_id: workspace.workspace_id, - provider_account_id: workspace.provider_account_id, - provider_account_generation: workspace.provider_account_generation, - kind: WorkspaceOperationKind::Create, - request_hash: request_hash.clone(), - expected_writer_epoch: workspace.writer_epoch, - expected_instance_generation: workspace.instance_generation, - expected_checkpoint_generation: workspace.checkpoint_generation, - deadline_at, - reconcile_not_before, - }; - let operation = operations.persist_intent(&intent).await?; - match (operation.outcome, operation.confirmed_disposition) { - ( - WorkspaceOperationOutcome::Confirmed, - Some(WorkspaceConfirmedDisposition::ResourcePresent), - ) => return Ok(()), - (WorkspaceOperationOutcome::Confirmed, _) => { - return Err(MoaError::ProviderError( - "workspace storage preparation was durably confirmed absent".to_string(), - )); - } - (WorkspaceOperationOutcome::Unknown, _) => { - return Err(MoaError::ExternalEffectUnknownOutcome { - operation_id: operation_id.to_string(), - }); - } - (WorkspaceOperationOutcome::NotSent, None) => {} - _ => { - return Err(MoaError::StorageError( - "workspace storage preparation has an inconsistent durable outcome".to_string(), - )); - } - } - failpoints::hit("post_reservation_pre_provider_create").await?; - call_scope.admit()?; - if !operations - .begin_provider_attempt(workspace.tenant_id, operation_id) - .await? - { - return Err(MoaError::ExternalEffectUnknownOutcome { - operation_id: operation_id.to_string(), - }); - } - let result = match storage_provider - .prepare_workspace_storage(WorkspaceStoragePrepareRequest { - operation: WorkspaceStorageOperation { - operation_id, - kind: WorkspaceOperationKind::Create, - binding, - deadline: deadline_at, - request_hash, - }, - }) - .await - { - Ok(result) => result, - Err(error) => { - tracing::warn!( - operation_id = %operation_id, - error = %error, - "workspace storage preparation outcome is ambiguous" - ); - operations - .mark_unknown(workspace.tenant_id, operation_id) - .await?; - return Err(MoaError::ExternalEffectUnknownOutcome { - operation_id: operation_id.to_string(), - }); - } - }; - // Every arm records an outcome, so the lifecycle counter carries the real - // success/ambiguous ratio rather than only the happy path. - match (result.outcome, result.confirmed_disposition) { - ( - WorkspaceOperationOutcome::Confirmed, - Some(WorkspaceConfirmedDisposition::ResourcePresent), - ) => { - if !operations - .confirm_disposition( - workspace.tenant_id, - operation_id, - WorkspaceConfirmedDisposition::ResourcePresent, - ) - .await? - { - return Err(MoaError::ExternalEffectUnknownOutcome { - operation_id: operation_id.to_string(), - }); - } - record_workspace_lifecycle( - &workspace.provider, - SandboxWorkspaceLifecycleOperation::Create, - SandboxWorkspaceMetricResult::Succeeded, - prepare_started_at.elapsed(), - ); - Ok(()) - } - ( - WorkspaceOperationOutcome::Confirmed, - Some(WorkspaceConfirmedDisposition::ResourceAbsent), - ) => { - if !operations - .confirm_disposition( - workspace.tenant_id, - operation_id, - WorkspaceConfirmedDisposition::ResourceAbsent, - ) - .await? - { - return Err(MoaError::ExternalEffectUnknownOutcome { - operation_id: operation_id.to_string(), - }); - } - record_workspace_lifecycle( - &workspace.provider, - SandboxWorkspaceLifecycleOperation::Create, - SandboxWorkspaceMetricResult::Failed, - prepare_started_at.elapsed(), - ); - Err(MoaError::ProviderError( - "workspace storage preparation was confirmed absent".to_string(), - )) - } - (WorkspaceOperationOutcome::Unknown, None) => { - operations - .mark_unknown(workspace.tenant_id, operation_id) - .await?; - record_workspace_lifecycle( - &workspace.provider, - SandboxWorkspaceLifecycleOperation::Create, - SandboxWorkspaceMetricResult::Ambiguous, - prepare_started_at.elapsed(), - ); - Err(MoaError::ExternalEffectUnknownOutcome { - operation_id: operation_id.to_string(), - }) - } - _ => { - record_workspace_lifecycle( - &workspace.provider, - SandboxWorkspaceLifecycleOperation::Create, - SandboxWorkspaceMetricResult::Failed, - prepare_started_at.elapsed(), - ); - Err(MoaError::ProviderError( - "workspace storage provider returned an inconsistent preparation result" - .to_string(), - )) - } - } - } - - /// Restores and verifies durable workspace bytes before lease activation. - pub(in crate::core) async fn hydrate_provisioned_workspace( - &self, - binding: &WorkspaceBinding, - claim: &HandLease, - hand: &HandHandle, - call_scope: ToolCallScope<'_>, - ) -> Result<()> { - let Some(repository) = self.hands.workspace_repository.as_ref() else { - return Ok(()); - }; - let operations = self.hands.workspace_operations.as_ref().ok_or_else(|| { - MoaError::StorageError("workspace operation repository missing".to_string()) - })?; - let provider = self - .hands - .storage_providers - .get(&claim.provider) - .ok_or_else(|| { - MoaError::ProviderError(format!( - "workspace storage provider {} is not registered", - claim.provider - )) - })?; - let hydration_started_at = std::time::Instant::now(); - let kind = if binding.current_revision.is_some() { - WorkspaceOperationKind::Restore - } else { - WorkspaceOperationKind::Attach - }; - let operation_id = WorkspaceOperationId(Uuid::new_v5( - &binding.workspace_id.0, - format!( - "hydrate-v1:{}:{}", - claim.provisioning_operation_id, - kind.as_str() - ) - .as_bytes(), - )); - let request_bytes = serde_json::to_vec(&(binding, hand, kind))?; - let request_hash = format!("sha256:{}", hex::encode(Sha256::digest(request_bytes))); - let intent = WorkspaceOperationIntent { - operation_id, - tenant_id: binding.tenant_id, - workspace_id: binding.workspace_id, - provider_account_id: binding.provider_account_id, - provider_account_generation: i64::try_from(binding.provider_account_generation) - .map_err(|_| { - MoaError::StorageError( - "workspace provider-account generation is invalid".to_string(), - ) - })?, - kind, - request_hash: request_hash.clone(), - expected_writer_epoch: i64::try_from(binding.writer_epoch).map_err(|_| { - MoaError::StorageError("workspace writer epoch is invalid".to_string()) - })?, - expected_instance_generation: i64::try_from(binding.instance_generation).map_err( - |_| MoaError::StorageError("workspace instance generation is invalid".to_string()), - )?, - expected_checkpoint_generation: binding.current_revision.as_ref().map_or( - Ok(0_i64), - |revision| { - i64::try_from(revision.generation).map_err(|_| { - MoaError::StorageError( - "workspace checkpoint generation is invalid".to_string(), - ) - }) - }, - )?, - deadline_at: claim.provisioning_deadline_at, - reconcile_not_before: claim.provisioning_deadline_at + ChronoDuration::seconds(30), - }; - let persisted = operations.persist_intent(&intent).await?; - match (persisted.outcome, persisted.confirmed_disposition) { - ( - WorkspaceOperationOutcome::Confirmed, - Some(WorkspaceConfirmedDisposition::ResourcePresent), - ) => return Ok(()), - (WorkspaceOperationOutcome::Confirmed, _) => { - return Err(MoaError::ProviderError(format!( - "workspace {} was durably confirmed absent", - kind.as_str() - ))); - } - (WorkspaceOperationOutcome::Unknown, _) => { - return Err(MoaError::ExternalEffectUnknownOutcome { - operation_id: operation_id.to_string(), - }); - } - (WorkspaceOperationOutcome::NotSent, None) => {} - _ => { - return Err(MoaError::StorageError(format!( - "workspace {} has an inconsistent durable outcome", - kind.as_str() - ))); - } - } - call_scope.admit()?; - if !operations - .begin_provider_attempt(binding.tenant_id, operation_id) - .await? - { - return Err(MoaError::ExternalEffectUnknownOutcome { - operation_id: operation_id.to_string(), - }); - } - let operation = WorkspaceStorageOperation { - operation_id, - kind, - binding: binding.clone(), - deadline: claim.provisioning_deadline_at, - request_hash, - }; - let provider_result = match binding.current_revision.as_ref() { - None => { - self.run_within_scope( - call_scope, - provider.attach_workspace(WorkspaceAttachRequest { - operation, - hand: hand.clone(), - storage: None, - }), - ) - .await - } - Some(revision) => { - let checkpoint = repository - .get_checkpoint( - binding.tenant_id, - binding.workspace_id, - revision.checkpoint_id, - ) - .await? - .ok_or_else(|| { - MoaError::StorageError( - "workspace head checkpoint is missing during restore".to_string(), - ) - })?; - if checkpoint.state - != moa_core::types::sandbox_workspace::WorkspaceCheckpointState::Available - || checkpoint.generation - != i64::try_from(revision.generation).map_err(|_| { - MoaError::StorageError( - "workspace checkpoint generation is invalid".to_string(), - ) - })? - { - return Err(MoaError::StorageError( - "workspace head checkpoint is not an exact available revision".to_string(), - )); - } - let resource_id = checkpoint.object_reference.ok_or_else(|| { - MoaError::StorageError( - "workspace head checkpoint has no portable object reference".to_string(), - ) - })?; - self.run_within_scope( - call_scope, - provider.restore_workspace(WorkspaceRestoreRequest { - operation, - hand: hand.clone(), - revision: revision.clone(), - checkpoint: ProviderStorageRef { - provider_account_id: binding.provider_account_id, - provider_account_generation: binding.provider_account_generation, - kind: ProviderStorageKind::PortableCheckpoint, - resource_id, - workspace_locator: None, - }, - }), - ) - .await - } - }; - let result = match provider_result { - Ok(result) => result, - Err(error) => { - tracing::warn!( - operation_id = %operation_id, - error = %error, - "workspace hydration provider outcome is ambiguous" - ); - operations - .mark_unknown(binding.tenant_id, operation_id) - .await?; - return Err(MoaError::ExternalEffectUnknownOutcome { - operation_id: operation_id.to_string(), - }); - } - }; - match (result.outcome, result.confirmed_disposition) { - ( - WorkspaceOperationOutcome::Confirmed, - Some(WorkspaceConfirmedDisposition::ResourcePresent), - ) => { - if !operations - .confirm_disposition( - binding.tenant_id, - operation_id, - WorkspaceConfirmedDisposition::ResourcePresent, - ) - .await? - { - return Err(MoaError::StorageError( - "workspace hydration lost its durable operation fence".to_string(), - )); - } - // Only a confirmed restore counts: an ambiguous or failed provider result - // leaves no verified checkpoint in fresh compute, so counting it here - // would overstate successful restores. - if kind == WorkspaceOperationKind::Restore { - record_workspace_restore(&claim.provider); - record_workspace_checkpoint( - &claim.provider, - SandboxWorkspaceCheckpointOperation::Restore, - SandboxWorkspaceMetricResult::Succeeded, - 0, - hydration_started_at.elapsed(), - ); - } - Ok(()) - } - ( - WorkspaceOperationOutcome::Confirmed, - Some(WorkspaceConfirmedDisposition::ResourceAbsent), - ) => { - if !operations - .confirm_disposition( - binding.tenant_id, - operation_id, - WorkspaceConfirmedDisposition::ResourceAbsent, - ) - .await? - { - return Err(MoaError::ExternalEffectUnknownOutcome { - operation_id: operation_id.to_string(), - }); - } - Err(MoaError::ProviderError(format!( - "workspace {} was confirmed absent", - kind.as_str() - ))) - } - (WorkspaceOperationOutcome::Unknown, None) => { - operations - .mark_unknown(binding.tenant_id, operation_id) - .await?; - Err(MoaError::ExternalEffectUnknownOutcome { - operation_id: operation_id.to_string(), - }) - } - _ => Err(MoaError::ProviderError( - "workspace storage provider returned an inconsistent hydration result".to_string(), - )), - } - } - - /// Reinstalls the current trusted manifest before publishing an active lease. - pub(in crate::core) async fn reinstall_trusted_files_before_activation( - &self, - session: &SessionMeta, - worker_id: &str, - provider: &str, - hand: &HandHandle, - call_scope: ToolCallScope<'_>, - ) -> Result>> { - let provider_impl = - self.hands.providers.get(provider).ok_or_else(|| { - MoaError::ProviderError(format!("unknown hand provider: {provider}")) - })?; - let manifest_key = manifest_scope_key(session, Some(worker_id)); - loop { - call_scope.admit()?; - let manifest = self - .hands - .trusted_sandbox_files - .read() - .await - .get(&manifest_key) - .cloned(); - let Some(manifest) = manifest else { - return Ok(None); - }; - self.run_within_scope( - call_scope, - provider_impl.install_files(hand, manifest.files.as_ref()), - ) - .await?; - call_scope.admit()?; - if self - .hands - .trusted_sandbox_files - .read() - .await - .get(&manifest_key) - .is_some_and(|current| std::sync::Arc::ptr_eq(current, &manifest)) - { - return Ok(Some(manifest)); - } - } - } - - /// Records a trusted manifest installed on the exact preactivation hand. - pub(in crate::core) async fn remember_preactivation_manifest_install( - &self, - session: &SessionMeta, - worker_id: &str, - provider: &str, - cache_key: &HandProviderCacheKey, - active: &ActiveHand, - manifest: Option<&std::sync::Arc>, - ) { - let Some(manifest) = manifest else { - return; - }; - let manifest_key = manifest_scope_key(session, Some(worker_id)); - let binding_is_current = self - .hands - .active_hands - .read() - .await - .get(cache_key) - .is_some_and(|current| current == active); - if binding_is_current - && self - .hands - .trusted_sandbox_files - .read() - .await - .get(&manifest_key) - .is_some_and(|current| std::sync::Arc::ptr_eq(current, manifest)) - { - self.hands - .installed_files - .write() - .await - .entry(manifest_key) - .or_default() - .insert( - provider.to_string(), - InstalledManifestMarker { - manifest_identity: manifest.identity, - handle: active.handle.clone(), - generation: active.generation, - }, - ); - } - } - - async fn confirmed_workspace_commit_replay( - &self, - workspace: &SandboxWorkspace, - tool_call_id: ToolCallId, - ) -> Result { - let operations = self.hands.workspace_operations.as_ref().ok_or_else(|| { - MoaError::StorageError("workspace operation repository missing".to_string()) - })?; - let repository = - self.hands.workspace_repository.as_ref().ok_or_else(|| { - MoaError::StorageError("workspace repository missing".to_string()) - })?; - let operation_id = WorkspaceOperationId(Uuid::new_v5( - &workspace.workspace_id.0, - format!("tool-commit-v1:{tool_call_id}").as_bytes(), - )); - let Some(operation) = operations.get(workspace.tenant_id, operation_id).await? else { - return Ok(false); - }; - if operation.outcome == WorkspaceOperationOutcome::NotSent { - return Ok(false); - } - if operation.outcome == WorkspaceOperationOutcome::Unknown { - return Ok(false); - } - let checkpoint = repository - .get_checkpoint_for_operation(workspace.tenant_id, workspace.workspace_id, operation_id) - .await? - .ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { - operation_id: operation_id.to_string(), - })?; - let committed_generation = operation - .expected_checkpoint_generation - .checked_add(1) - .ok_or_else(|| { - MoaError::StorageError("workspace checkpoint generation overflowed".to_string()) - })?; - let parent_revision = match ( - operation.expected_checkpoint_generation, - checkpoint.parent_checkpoint_id, - ) { - (0, None) => None, - (generation, Some(checkpoint_id)) if generation > 0 => { - Some(moa_core::types::sandbox_workspace::WorkspaceRevisionRef { - checkpoint_id, - generation: u64::try_from(generation).map_err(|_| { - MoaError::StorageError( - "workspace checkpoint generation is invalid".to_string(), - ) - })?, - format_version: CHECKPOINT_ARCHIVE_FORMAT_VERSION, - }) - } - _ => { - return Err(MoaError::ExternalEffectUnknownOutcome { - operation_id: operation_id.to_string(), - }); - } - }; - let mut original_binding = workspace.binding()?; - original_binding.current_revision = parent_revision; - let request_bytes = serde_json::to_vec(&(&original_binding, tool_call_id))?; - let request_hash = format!("sha256:{}", hex::encode(Sha256::digest(request_bytes))); - if operation.kind != WorkspaceOperationKind::Commit - || operation.workspace_id != workspace.workspace_id - || operation.provider_account_id != workspace.provider_account_id - || operation.provider_account_generation != workspace.provider_account_generation - || operation.expected_writer_epoch != workspace.writer_epoch - || operation.expected_instance_generation != workspace.instance_generation - || operation.request_hash != request_hash - || operation.confirmed_disposition - != Some(WorkspaceConfirmedDisposition::ResourcePresent) - || checkpoint.state != WorkspaceCheckpointState::Available - || checkpoint.checkpoint_id != WorkspaceCheckpointId(operation_id.0) - || checkpoint.generation != committed_generation - || checkpoint.source_writer_epoch != workspace.writer_epoch - || checkpoint.source_instance_generation != workspace.instance_generation - || workspace.checkpoint_id != Some(checkpoint.checkpoint_id) - || workspace.checkpoint_generation != committed_generation - { - return Err(MoaError::ExternalEffectUnknownOutcome { - operation_id: operation_id.to_string(), - }); - } - Ok(true) - } - - /// Publishes the mutable workspace for one already-journaled sandbox command. - /// - /// This never dispatches the command. It reloads the exact workspace, - /// active lease, and hand, then starts or resumes the deterministic commit. - pub async fn commit_authorized_workspace_after_tool( - &self, - request: JournaledWorkspaceCommit<'_>, - ) -> Result<()> { - request.scope.admit()?; - let workspace_scope = request.workspace_scope; - let repository = - self.hands.workspace_repository.as_ref().ok_or_else(|| { - MoaError::StorageError("workspace repository missing".to_string()) - })?; - let workspace = repository - .get_by_scope(request.session.tenant_id, workspace_scope) - .await? - .ok_or_else(|| { - MoaError::PermissionDenied( - "authorized sandbox workspace disappeared before commit".to_string(), - ) - })?; - if self - .confirmed_workspace_commit_replay(&workspace, request.tool_call_id) - .await? - { - return Ok(()); - } - let lease_scope = workspace_lease_scope(workspace_scope); - let lease_store = self.hands.hand_leases.as_ref().ok_or_else(|| { - MoaError::StorageError("durable hand lease store missing".to_string()) - })?; - let lease = lease_store - .get( - request.session.tenant_id, - request.session.id, - &lease_scope, - &workspace.provider, - ) - .await? - .ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { - operation_id: format!("workspace-tool-call:{}", request.tool_call_id), - })?; - let hand = lease - .handle - .as_ref() - .map(|handle| handle.handle.clone()) - .ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { - operation_id: format!("workspace-tool-call:{}", request.tool_call_id), - })?; - self.commit_workspace_after_tool(WorkspaceCommitExecution { - session: request.session, - workspace_scope, - tool_call_id: request.tool_call_id, - provider_name: &workspace.provider, - hand: &hand, - call_scope: request.scope, - release_compute: false, - }) - .await - } - - /// Checkpoints one execution-task workspace and releases its exact compute lease. - /// - /// The returned receipt is the durable proof required before a task may yield to - /// a timer, external callback, pause, or long backoff. Retries return the same - /// receipt. Provider teardown errors remain ambiguous and never produce release - /// proof; a later retry reconciles the checkpoint and repeats exact destruction. - pub async fn checkpoint_and_release_execution_hand( - &self, - request: ExecutionHandReleaseRequest<'_>, - ) -> Result { - if request.attempt_generation == 0 { - return Err(MoaError::ValidationError( - "execution task attempt generation must be positive".to_string(), - )); - } - let (task_id, logical_generation) = match request.owner { - ExecutionHandReleaseOwner::Task { - task_id, - logical_generation, - } if logical_generation > 0 => (task_id, logical_generation), - ExecutionHandReleaseOwner::Task { .. } => { - return Err(MoaError::ValidationError( - "execution task logical generation must be positive".to_string(), - )); - } - ExecutionHandReleaseOwner::Compensation { - compensation_id, - logical_generation, - } => { - return self - .release_execution_compensation_hand( - request, - compensation_id, - logical_generation, - ) - .await; - } - }; - let repository = - self.hands.workspace_repository.as_ref().ok_or_else(|| { - MoaError::StorageError("workspace repository missing".to_string()) - })?; - if let Some(receipt) = repository - .get_task_execution_hand_release_receipt( - request.session.tenant_id, - request.run_id, - task_id, - logical_generation, - request.attempt_generation, - ) - .await? - { - return Ok(receipt); - } - - let absence_receipt_id = Uuid::new_v5( - &request.run_id.0, - format!( - "execution-task-hand-absence-v1:{task_id}:{logical_generation}:{}", - request.attempt_generation - ) - .as_bytes(), - ); - match repository - .record_absent_task_execution_hand_release_receipt(AbsentTaskHandReleaseIntent { - receipt_id: absence_receipt_id, - tenant_id: request.session.tenant_id, - run_id: request.run_id, - task_id, - logical_generation, - attempt_generation: request.attempt_generation, - verified_at: Utc::now(), - }) - .await - { - Ok(receipt) => return Ok(receipt), - Err(MoaError::ExternalEffectUnknownOutcome { .. }) => {} - Err(error) => return Err(error), - } - - let workspace_scope = SandboxWorkspaceScope::ExecutionTask { - run_id: request.run_id, - task_id, - }; - let initial_workspace = repository - .get_by_scope(request.session.tenant_id, &workspace_scope) - .await? - .ok_or_else(|| { - MoaError::PermissionDenied( - "execution-task workspace disappeared before hand release".to_string(), - ) - })?; - let release_key = format!( - "execution-task-yield-v1:{}:{}:{}", - request.run_id, task_id, request.attempt_generation - ); - let tool_call_id = ToolCallId(Uuid::new_v5( - &initial_workspace.workspace_id.0, - release_key.as_bytes(), - )); - let candidate_receipt_id = Uuid::new_v5( - &initial_workspace.workspace_id.0, - format!("release-receipt-v1:{release_key}").as_bytes(), - ); - let lease_scope = workspace_lease_scope(&workspace_scope); - let lease_store = self.hands.hand_leases.as_ref().ok_or_else(|| { - MoaError::StorageError("durable hand lease store missing".to_string()) - })?; - let initial_lease = lease_store - .get( - request.session.tenant_id, - request.session.id, - &lease_scope, - &initial_workspace.provider, - ) - .await? - .ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { - operation_id: release_key.clone(), - })?; - let (receipt_id, release_claim_token, requested_at) = repository - .begin_task_execution_hand_release(TaskHandReleaseIntent { - receipt_id: candidate_receipt_id, - run_id: request.run_id, - task_id, - logical_generation, - attempt_generation: request.attempt_generation, - deadline_at: request - .scope - .budget - .deadline - .unwrap_or_else(|| Utc::now() + ChronoDuration::minutes(5)), - recovery_claim_expires_at: Utc::now() + ChronoDuration::minutes(5), - workspace: &initial_workspace, - lease: &initial_lease, - }) - .await?; - - admit_execution_release_step( - request.scope, - if initial_lease.status == HandLeaseStatus::Active { - ExecutionReleaseStep::ProviderIo - } else { - ExecutionReleaseStep::DurableReconciliation - }, - )?; - - match initial_lease.status { - HandLeaseStatus::Active => { - let hand = initial_lease - .handle - .as_ref() - .map(|handle| handle.handle.clone()) - .ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { - operation_id: release_key.clone(), - })?; - self.commit_workspace_after_tool(WorkspaceCommitExecution { - session: request.session, - workspace_scope: &workspace_scope, - tool_call_id, - provider_name: &initial_workspace.provider, - hand: &hand, - call_scope: request.scope, - release_compute: true, - }) - .await?; - } - HandLeaseStatus::Destroyed => { - if !self - .confirmed_workspace_commit_replay(&initial_workspace, tool_call_id) - .await? - { - return Err(MoaError::ExternalEffectUnknownOutcome { - operation_id: release_key.clone(), - }); - } - } - HandLeaseStatus::Provisioning - | HandLeaseStatus::Stale - | HandLeaseStatus::Failed - | HandLeaseStatus::Reaping => { - return Err(MoaError::ExternalEffectUnknownOutcome { - operation_id: release_key.clone(), - }); - } - } - - let mut final_workspace = repository - .get_by_scope(request.session.tenant_id, &workspace_scope) - .await? - .ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { - operation_id: release_key.clone(), - })?; - let mut final_lease = lease_store - .get( - request.session.tenant_id, - request.session.id, - &lease_scope, - &initial_workspace.provider, - ) - .await? - .ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { - operation_id: release_key.clone(), - })?; - - // An unknown provider outcome may reconcile the already-verified bytes - // while conservatively retaining the attachment. Finish the destroy as a - // separate exact step, then atomically release lease and capacity ownership. - if final_lease.status == HandLeaseStatus::Active { - if !self - .confirmed_workspace_commit_replay(&final_workspace, tool_call_id) - .await? - { - return Err(MoaError::ExternalEffectUnknownOutcome { - operation_id: release_key.clone(), - }); - } - let hand = final_lease - .handle - .as_ref() - .map(|handle| handle.handle.clone()) - .ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { - operation_id: release_key.clone(), - })?; - let provider = self - .hands - .providers - .get(&initial_workspace.provider) - .ok_or_else(|| { - MoaError::ProviderError(format!( - "hand provider {} is not registered", - initial_workspace.provider - )) - })?; - self.run_within_scope(request.scope, provider.destroy(&hand)) - .await - .map_err(|error| { - tracing::warn!( - operation_id = %release_key, - error = %error, - "execution-task hand destroy outcome is ambiguous" - ); - MoaError::ExternalEffectUnknownOutcome { - operation_id: release_key.clone(), - } - })?; - if !repository - .finalize_task_yield_destroy(&final_workspace.binding()?, &final_lease) - .await? - { - // The compute is gone but the durable release did not commit, so the - // charge is still held and a reconciler owns it. Recorded as ambiguous - // rather than succeeded so the two are distinguishable on the dashboard. - record_workspace_release( - &initial_workspace.provider, - SandboxWorkspaceMetricResult::Ambiguous, - ); - return Err(MoaError::ExternalEffectUnknownOutcome { - operation_id: release_key.clone(), - }); - } - // Counted only after provider destruction is verified AND the release - // receipt commits, which together are what actually free the capacity. - record_workspace_release( - &initial_workspace.provider, - SandboxWorkspaceMetricResult::Succeeded, - ); - let key = session_provider_key( - request.session, - Some(&lease_scope), - &initial_workspace.provider, - ); - self.remove_cached_binding_if_matches(&key, &hand, Some(initial_lease.generation)) - .await; - self.remove_installed_marker( - manifest_scope_key(request.session, Some(&lease_scope)), - &initial_workspace.provider, - ) - .await; - final_workspace = repository - .get_by_scope(request.session.tenant_id, &workspace_scope) - .await? - .ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { - operation_id: release_key.clone(), - })?; - final_lease = lease_store - .get( - request.session.tenant_id, - request.session.id, - &lease_scope, - &initial_workspace.provider, - ) - .await? - .ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { - operation_id: release_key.clone(), - })?; - } - - let operation_id = WorkspaceOperationId(Uuid::new_v5( - &initial_workspace.workspace_id.0, - format!("tool-commit-v1:{tool_call_id}").as_bytes(), - )); - let checkpoint_id = WorkspaceCheckpointId(operation_id.0); - let checkpoint = repository - .get_checkpoint( - request.session.tenant_id, - final_workspace.workspace_id, - checkpoint_id, - ) - .await? - .ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { - operation_id: release_key.clone(), - })?; - if final_workspace.state != SandboxWorkspaceState::Ready - || final_workspace.writer_epoch != initial_workspace.writer_epoch - || final_workspace.instance_generation != initial_workspace.instance_generation - || final_workspace.checkpoint_id != Some(checkpoint_id) - || final_workspace.checkpoint_generation != checkpoint.generation - || final_lease.status != HandLeaseStatus::Destroyed - || final_lease.handle.is_some() - || final_lease.generation != initial_lease.generation - || final_lease.provisioning_operation_id != initial_lease.provisioning_operation_id - || checkpoint.state != WorkspaceCheckpointState::Available - || checkpoint.manifest_digest.is_none() - || checkpoint.logical_bytes.is_none() - { - return Err(MoaError::ExternalEffectUnknownOutcome { - operation_id: release_key, - }); - } - let receipt = ExecutionHandReleaseReceipt { - receipt_id, - tenant_id: request.session.tenant_id, - run_id: request.run_id, - owner: request.owner, - attempt_generation: request.attempt_generation, - workspace_id: Some(final_workspace.workspace_id), - writer_epoch: Some(u64::try_from(final_workspace.writer_epoch).map_err(|_| { - MoaError::StorageError("workspace writer epoch is invalid".to_string()) - })?), - instance_generation: Some(u64::try_from(final_workspace.instance_generation).map_err( - |_| MoaError::StorageError("workspace instance generation is invalid".to_string()), - )?), - hand_provisioning_operation_id: Some(initial_lease.provisioning_operation_id), - hand_lease_generation: Some(u64::try_from(initial_lease.generation).map_err(|_| { - MoaError::StorageError("hand lease generation is invalid".to_string()) - })?), - checkpoint_id: Some(checkpoint_id), - checkpoint_generation: Some(u64::try_from(checkpoint.generation).map_err(|_| { - MoaError::StorageError("checkpoint generation is invalid".to_string()) - })?), - checkpoint_manifest_digest: Some(checkpoint.manifest_digest.ok_or_else(|| { - MoaError::StorageError("verified checkpoint digest is missing".to_string()) - })?), - checkpoint_logical_bytes: Some( - u64::try_from(checkpoint.logical_bytes.ok_or_else(|| { - MoaError::StorageError("verified checkpoint bytes are missing".to_string()) - })?) - .map_err(|_| MoaError::StorageError("checkpoint bytes are negative".to_string()))?, - ), - requested_at, - released_at: Utc::now(), - }; - repository - .record_task_execution_hand_release_receipt(&receipt, release_claim_token) - .await - } - - async fn release_execution_compensation_hand( - &self, - request: ExecutionHandReleaseRequest<'_>, - compensation_id: ExecutionCompensationScopeId, - logical_generation: u64, - ) -> Result { - if logical_generation == 0 { - return Err(MoaError::ValidationError( - "execution compensation logical generation must be positive".to_string(), - )); - } - let repository = - self.hands.workspace_repository.as_ref().ok_or_else(|| { - MoaError::StorageError("workspace repository missing".to_string()) - })?; - if let Some(receipt) = repository - .get_compensation_execution_hand_release_receipt( - request.session.tenant_id, - request.run_id, - compensation_id, - logical_generation, - request.attempt_generation, - ) - .await? - { - return Ok(receipt); - } - - let hand_scope = format!( - "execution_compensation:{}:{}", - request.run_id, compensation_id - ); - let lease_store = self.hands.hand_leases.as_ref().ok_or_else(|| { - MoaError::StorageError("durable hand lease store missing".to_string()) - })?; - if let Some(claim) = repository - .claim_pending_compensation_execution_hand_release( - request.session.tenant_id, - request.run_id, - compensation_id, - logical_generation, - request.attempt_generation, - Utc::now() + ChronoDuration::minutes(5), - ) - .await? - { - let persisted_identity = match ( - claim.hand_provisioning_operation_id, - claim.hand_lease_generation, - ) { - (Some(operation_id), Some(generation)) => Some((operation_id, generation)), - (None, None) => None, - _ => { - return Err(MoaError::StorageError( - "pending compensation release has a partial hand identity".to_string(), - )); - } - }; - let exact_lease = match persisted_identity { - Some((operation_id, generation)) => { - lease_store - .get_exact_generation( - request.session.tenant_id, - request.session.id, - &hand_scope, - operation_id, - generation, - ) - .await? - } - None => None, - }; - let provider_io_required = exact_lease - .as_ref() - .is_some_and(|lease| lease.status != HandLeaseStatus::Destroyed); - admit_execution_release_step( - request.scope, - if provider_io_required { - ExecutionReleaseStep::ProviderIo - } else { - ExecutionReleaseStep::DurableReconciliation - }, - )?; - if provider_io_required - && !self - .reclaim_hands( - request.session.tenant_id, - &request.session.id, - Some(&hand_scope), - ) - .await - { - return Err(MoaError::ExternalEffectUnknownOutcome { - operation_id: format!( - "execution-compensation-hand-release:{}:{compensation_id}:{logical_generation}:{}", - request.run_id, request.attempt_generation - ), - }); - } - let exact_lease = match persisted_identity { - Some((operation_id, generation)) => { - lease_store - .get_exact_generation( - request.session.tenant_id, - request.session.id, - &hand_scope, - operation_id, - generation, - ) - .await? - } - None => None, - }; - let lease_state = match exact_lease.as_ref() { - None => PersistedLeaseReleaseState::Missing, - Some(lease) - if lease.status == HandLeaseStatus::Destroyed && lease.handle.is_none() => - { - PersistedLeaseReleaseState::Destroyed - } - Some(_) => PersistedLeaseReleaseState::LiveOrAmbiguous, - }; - let exact_released = compensation_release_identity_is_verified( - persisted_identity.is_some(), - lease_state, - ); - let replacement = lease_store - .has_live_owner(request.session.tenant_id, request.session.id, &hand_scope) - .await?; - if !exact_released || replacement { - return Err(MoaError::ExternalEffectUnknownOutcome { - operation_id: format!( - "execution-compensation-hand-release:{}:{compensation_id}:{logical_generation}:{}", - request.run_id, request.attempt_generation - ), - }); - } - let hand_lease_generation = claim - .hand_lease_generation - .map(u64::try_from) - .transpose() - .map_err(|_| { - MoaError::StorageError("hand lease generation is invalid".to_string()) - })?; - return repository - .record_compensation_execution_hand_release_receipt( - &ExecutionHandReleaseReceipt { - receipt_id: claim.receipt_id, - tenant_id: request.session.tenant_id, - run_id: request.run_id, - owner: request.owner, - attempt_generation: request.attempt_generation, - workspace_id: None, - writer_epoch: None, - instance_generation: None, - hand_provisioning_operation_id: claim.hand_provisioning_operation_id, - hand_lease_generation, - checkpoint_id: None, - checkpoint_generation: None, - checkpoint_manifest_digest: None, - checkpoint_logical_bytes: None, - requested_at: claim.requested_at, - released_at: Utc::now(), - }, - request.session.id, - &hand_scope, - claim.claim_token, - ) - .await; - } - let leases = lease_store - .list_live_owner_candidates(request.session.tenant_id, request.session.id, &hand_scope) - .await?; - if leases.len() > 1 { - return Err(MoaError::ExternalEffectUnknownOutcome { - operation_id: format!( - "execution-compensation-hand-release:{}:{compensation_id}:{logical_generation}:{}", - request.run_id, request.attempt_generation - ), - }); - } - let initial_lease = leases.into_iter().next(); - - let release_key = format!( - "execution-compensation-release-v1:{}:{compensation_id}:{logical_generation}:{}", - request.run_id, request.attempt_generation - ); - let receipt_id = Uuid::new_v5(&request.run_id.0, release_key.as_bytes()); - let (receipt_id, claim_token, requested_at) = repository - .begin_compensation_execution_hand_release(CompensationHandReleaseIntent { - receipt_id, - tenant_id: request.session.tenant_id, - session_id: request.session.id, - run_id: request.run_id, - compensation_id, - logical_generation, - attempt_generation: request.attempt_generation, - hand_scope: &hand_scope, - lease: initial_lease.as_ref(), - deadline_at: request - .scope - .budget - .deadline - .unwrap_or_else(|| Utc::now() + ChronoDuration::minutes(5)), - recovery_claim_expires_at: Utc::now() + ChronoDuration::minutes(5), - }) - .await?; - admit_execution_release_step( - request.scope, - if initial_lease.is_some() { - ExecutionReleaseStep::ProviderIo - } else { - ExecutionReleaseStep::DurableReconciliation - }, - )?; - if initial_lease.is_some() - && !self - .reclaim_hands( - request.session.tenant_id, - &request.session.id, - Some(&hand_scope), - ) - .await - { - return Err(MoaError::ExternalEffectUnknownOutcome { - operation_id: release_key, - }); - } - if let Some(initial_lease) = initial_lease.as_ref() { - let exact_lease = lease_store - .get( - request.session.tenant_id, - request.session.id, - &hand_scope, - &initial_lease.provider, - ) - .await?; - let exact_destroyed = exact_lease.as_ref().is_some_and(|lease| { - lease.worker_id == hand_scope - && lease.provisioning_operation_id == initial_lease.provisioning_operation_id - && lease.generation == initial_lease.generation - && lease.status == HandLeaseStatus::Destroyed - && lease.handle.is_none() - }); - let replacement = lease_store - .has_live_owner(request.session.tenant_id, request.session.id, &hand_scope) - .await?; - if !exact_destroyed || replacement { - return Err(MoaError::ExternalEffectUnknownOutcome { - operation_id: release_key, - }); - } - } - let hand_lease_generation = initial_lease - .as_ref() - .map(|lease| { - u64::try_from(lease.generation).map_err(|_| { - MoaError::StorageError("hand lease generation is invalid".to_string()) - }) - }) - .transpose()?; - repository - .record_compensation_execution_hand_release_receipt( - &ExecutionHandReleaseReceipt { - receipt_id, - tenant_id: request.session.tenant_id, - run_id: request.run_id, - owner: request.owner, - attempt_generation: request.attempt_generation, - workspace_id: None, - writer_epoch: None, - instance_generation: None, - hand_provisioning_operation_id: initial_lease - .as_ref() - .map(|lease| lease.provisioning_operation_id), - hand_lease_generation, - checkpoint_id: None, - checkpoint_generation: None, - checkpoint_manifest_digest: None, - checkpoint_logical_bytes: None, - requested_at, - released_at: Utc::now(), - }, - request.session.id, - &hand_scope, - claim_token, - ) - .await - } - - pub(in crate::core) async fn commit_workspace_after_tool( - &self, - request: WorkspaceCommitExecution<'_>, - ) -> Result<()> { - let WorkspaceCommitExecution { - session, - workspace_scope, - tool_call_id, - provider_name, - hand, - call_scope, - release_compute, - } = request; - let Some(repository) = self.hands.workspace_repository.as_ref() else { - return Ok(()); - }; - let operations = self.hands.workspace_operations.as_ref().ok_or_else(|| { - MoaError::StorageError("workspace operation repository missing".to_string()) - })?; - let storage_provider = - self.hands - .storage_providers - .get(provider_name) - .ok_or_else(|| { - MoaError::ProviderError(format!( - "workspace storage provider {provider_name} is not registered" - )) - })?; - let lease_store = self.hands.hand_leases.as_ref().ok_or_else(|| { - MoaError::StorageError("durable hand lease store missing".to_string()) - })?; - let workspace = repository - .get_by_scope(session.tenant_id, workspace_scope) - .await? - .ok_or_else(|| { - MoaError::PermissionDenied( - "authorized sandbox workspace disappeared before commit".to_string(), - ) - })?; - if self - .confirmed_workspace_commit_replay(&workspace, tool_call_id) - .await? - { - return Ok(()); - } - if !matches!( - workspace.state, - SandboxWorkspaceState::Active - | SandboxWorkspaceState::Quiescing - | SandboxWorkspaceState::Committing - ) || workspace.provider != provider_name - || workspace.access_fenced_at.is_some() - { - return Err(MoaError::ExternalEffectUnknownOutcome { - operation_id: format!("workspace-tool-call:{tool_call_id}"), - }); - } - let binding = workspace.binding()?; - let lease_scope = workspace_lease_scope(workspace_scope); - let lease = lease_store - .get(session.tenant_id, session.id, &lease_scope, provider_name) - .await? - .ok_or_else(|| { - MoaError::StorageError( - "active workspace lease is missing before commit".to_string(), - ) - })?; - if lease.status != HandLeaseStatus::Active - || lease.handle.as_ref().map(|lease| &lease.handle) != Some(hand) - || lease.attachment != Some(lease_attachment(&binding)?) - { - return Err(MoaError::ExternalEffectUnknownOutcome { - operation_id: format!("workspace-tool-call:{tool_call_id}"), - }); - } - - let operation_id = WorkspaceOperationId(Uuid::new_v5( - &binding.workspace_id.0, - format!("tool-commit-v1:{tool_call_id}").as_bytes(), - )); - let checkpoint_id = WorkspaceCheckpointId(operation_id.0); - let existing_operation = operations.get(binding.tenant_id, operation_id).await?; - let mut deadline_at = existing_operation.as_ref().map_or_else( - || { - call_scope - .budget - .deadline - .unwrap_or_else(|| Utc::now() + ChronoDuration::minutes(5)) - }, - |operation| operation.deadline_at, - ); - let request_bytes = serde_json::to_vec(&(&binding, tool_call_id))?; - let request_hash = format!("sha256:{}", hex::encode(Sha256::digest(request_bytes))); - let expected_writer_epoch = i64::try_from(binding.writer_epoch) - .map_err(|_| MoaError::StorageError("workspace writer epoch is invalid".to_string()))?; - let expected_instance_generation = - i64::try_from(binding.instance_generation).map_err(|_| { - MoaError::StorageError("workspace instance generation is invalid".to_string()) - })?; - let expected_checkpoint_generation = - binding - .current_revision - .as_ref() - .map_or(Ok(0_i64), |revision| { - i64::try_from(revision.generation).map_err(|_| { - MoaError::StorageError( - "workspace checkpoint generation is invalid".to_string(), - ) - }) - })?; - let operation = operations - .persist_intent(&WorkspaceOperationIntent { - operation_id, - tenant_id: binding.tenant_id, - workspace_id: binding.workspace_id, - provider_account_id: binding.provider_account_id, - provider_account_generation: i64::try_from(binding.provider_account_generation) - .map_err(|_| { - MoaError::StorageError( - "workspace provider-account generation is invalid".to_string(), - ) - })?, - kind: WorkspaceOperationKind::Commit, - request_hash: request_hash.clone(), - expected_writer_epoch, - expected_instance_generation, - expected_checkpoint_generation, - deadline_at, - reconcile_not_before: deadline_at + ChronoDuration::seconds(30), - }) - .await?; - debug_assert_ne!(operation.outcome, WorkspaceOperationOutcome::Confirmed); - let transitioned = match workspace.state { - SandboxWorkspaceState::Active => { - repository - .transition(WorkspaceTransition { - tenant_id: binding.tenant_id, - workspace_id: binding.workspace_id, - from: SandboxWorkspaceState::Active, - to: SandboxWorkspaceState::Quiescing, - writer_epoch: expected_writer_epoch, - instance_generation: expected_instance_generation, - }) - .await? - && repository - .transition(WorkspaceTransition { - tenant_id: binding.tenant_id, - workspace_id: binding.workspace_id, - from: SandboxWorkspaceState::Quiescing, - to: SandboxWorkspaceState::Committing, - writer_epoch: expected_writer_epoch, - instance_generation: expected_instance_generation, - }) - .await? - } - SandboxWorkspaceState::Quiescing => { - repository - .transition(WorkspaceTransition { - tenant_id: binding.tenant_id, - workspace_id: binding.workspace_id, - from: SandboxWorkspaceState::Quiescing, - to: SandboxWorkspaceState::Committing, - writer_epoch: expected_writer_epoch, - instance_generation: expected_instance_generation, - }) - .await? - } - SandboxWorkspaceState::Committing => true, - _ => false, - }; - if !transitioned { - operations - .mark_unknown(binding.tenant_id, operation_id) - .await?; - return Err(MoaError::ExternalEffectUnknownOutcome { - operation_id: operation_id.to_string(), - }); - } - if let Err(error) = call_scope.admit() { - operations - .mark_unknown(binding.tenant_id, operation_id) - .await?; - return Err(error); - } - let checkpoint = repository - .create_checkpoint(CreateCheckpointRequest { - checkpoint_id, - tenant_id: binding.tenant_id, - workspace_id: binding.workspace_id, - parent_checkpoint_id: binding - .current_revision - .as_ref() - .map(|revision| revision.checkpoint_id), - operation_id, - expected_writer_epoch, - expected_instance_generation, - expected_checkpoint_generation, - }) - .await?; - if checkpoint.is_none() { - operations - .mark_unknown(binding.tenant_id, operation_id) - .await?; - return Err(MoaError::ExternalEffectUnknownOutcome { - operation_id: operation_id.to_string(), - }); - } - failpoints::hit("post_command_pre_checkpoint_publication").await?; - if operation.outcome == WorkspaceOperationOutcome::NotSent && deadline_at <= Utc::now() { - let renewed_deadline = call_scope - .budget - .deadline - .filter(|deadline| *deadline > Utc::now()) - .unwrap_or_else(|| Utc::now() + ChronoDuration::minutes(5)); - if !operations - .renew_not_sent_commit_deadline( - binding.tenant_id, - operation_id, - deadline_at, - renewed_deadline, - ) - .await? - { - return Err(MoaError::ExternalEffectUnknownOutcome { - operation_id: operation_id.to_string(), - }); - } - deadline_at = renewed_deadline; - } - let storage_operation = WorkspaceStorageOperation { - operation_id, - kind: WorkspaceOperationKind::Commit, - binding: binding.clone(), - deadline: deadline_at, - request_hash, - }; - let provider_result = if operation.outcome == WorkspaceOperationOutcome::Unknown { - let storage = self.hands.checkpoint_store.as_ref().map(|store| { - store.storage_reference( - crate::core::sandbox_workspace::checkpoint::store::CheckpointStoreContext { - tenant_id: binding.tenant_id, - workspace_id: binding.workspace_id, - checkpoint_id, - provider_account_id: binding.provider_account_id, - provider_account_generation: binding.provider_account_generation, - }, - ) - }); - let reconcile = - WorkspaceReconcileRequest::new(storage_operation, Some(hand.clone()), storage)?; - self.run_within_scope( - call_scope, - storage_provider.reconcile_workspace_operation(reconcile), - ) - .await - } else { - if !operations - .begin_provider_attempt(binding.tenant_id, operation_id) - .await? - { - return Err(MoaError::ExternalEffectUnknownOutcome { - operation_id: operation_id.to_string(), - }); - } - self.run_within_scope( - call_scope, - storage_provider.publish_workspace_checkpoint(WorkspaceCheckpointPublishRequest { - operation: storage_operation, - hand: hand.clone(), - parent_revision: binding.current_revision.clone(), - release_compute, - }), - ) - .await - }; - let result = match provider_result { - Ok(result) => result, - Err(error) => { - tracing::warn!( - operation_id = %operation_id, - error = %error, - "workspace tool commit provider outcome is ambiguous" - ); - operations - .mark_unknown(binding.tenant_id, operation_id) - .await?; - return Err(MoaError::ExternalEffectUnknownOutcome { - operation_id: operation_id.to_string(), - }); - } - }; - let (publication, post_commit_state) = match ( - result.outcome, - result.confirmed_disposition, - result.checkpoint_publication.as_ref(), - result.post_commit_state, - ) { - ( - WorkspaceOperationOutcome::Confirmed, - Some(WorkspaceConfirmedDisposition::ResourcePresent), - Some(publication), - Some(post_commit_state), - ) => (publication, post_commit_state), - _ => { - operations - .mark_unknown(binding.tenant_id, operation_id) - .await?; - return Err(MoaError::ExternalEffectUnknownOutcome { - operation_id: operation_id.to_string(), - }); - } - }; - if !repository - .publish_checkpoint_commit(PublishCheckpointCommitRequest { - binding: &binding, - operation_id, - publication, - post_commit_state, - lease: &lease, - }) - .await? - { - self.delete_abandoned_checkpoint_prefix(&binding, publication.revision.checkpoint_id) - .await?; - operations - .mark_unknown(binding.tenant_id, operation_id) - .await?; - return Err(MoaError::ExternalEffectUnknownOutcome { - operation_id: operation_id.to_string(), - }); - } - if post_commit_state != WorkspacePostCommitState::AttachmentRetained { - let key = session_provider_key(session, Some(&lease_scope), provider_name); - self.remove_cached_binding_if_matches(&key, hand, Some(lease.generation)) - .await; - self.remove_installed_marker( - manifest_scope_key(session, Some(&lease_scope)), - provider_name, - ) - .await; - } - Ok(()) - } - async fn delete_abandoned_checkpoint_prefix( &self, binding: &WorkspaceBinding, @@ -2507,35 +103,6 @@ impl ToolRouter { } } -fn revision_from_checkpoint_parent( - generation: i64, - checkpoint_id: Option, -) -> Result> { - match (generation, checkpoint_id) { - (0, None) => Ok(None), - (generation, Some(checkpoint_id)) if generation > 0 => Ok(Some( - moa_core::types::sandbox_workspace::WorkspaceRevisionRef { - checkpoint_id, - generation: u64::try_from(generation).map_err(|_| { - MoaError::StorageError("workspace checkpoint generation is invalid".to_string()) - })?, - format_version: CHECKPOINT_ARCHIVE_FORMAT_VERSION, - }, - )), - _ => Err(MoaError::StorageError( - "workspace checkpoint parent is inconsistent with its generation".to_string(), - )), - } -} - -fn management_checkpoint_request_hash( - binding: &WorkspaceBinding, - operation_id: WorkspaceOperationId, -) -> Result { - let bytes = serde_json::to_vec(&(binding, operation_id, WorkspaceOperationKind::Checkpoint))?; - Ok(format!("sha256:{}", hex::encode(Sha256::digest(bytes)))) -} - pub(in crate::core) fn validate_managed_restore_target( current_checkpoint_id: Option, current_generation: i64, @@ -2573,56 +140,3 @@ pub(in crate::core) fn lease_attachment( .map(|revision| revision.checkpoint_id), ) } - -#[cfg(test)] -mod tests { - use chrono::{Duration, Utc}; - use moa_core::{error::MoaError, types::resource::ResourceBudget}; - - use super::{ - ExecutionReleaseStep, PersistedLeaseReleaseState, ToolCallScope, - admit_execution_release_step, compensation_release_identity_is_verified, - }; - - #[test] - fn expired_release_budget_allows_reconciliation_but_rejects_new_provider_io() { - // Pins: retrying an exact release after its five-minute I/O window may return a durable - // receipt or finalize verified absence, but it must not start fresh provider operations. - let release_started_at = Utc::now() - Duration::minutes(6); - let scope = ToolCallScope::unbounded().with_budget(ResourceBudget::until( - release_started_at + Duration::minutes(5), - )); - - assert!( - admit_execution_release_step(scope, ExecutionReleaseStep::DurableReconciliation) - .is_ok() - ); - assert!(matches!( - admit_execution_release_step(scope, ExecutionReleaseStep::ProviderIo), - Err(MoaError::BudgetExhausted(_)) - )); - } - - #[test] - fn persisted_compensation_lease_identity_requires_the_exact_destroyed_row() { - // Pins: after provider teardown, a persisted op/generation may finalize only against its - // exact Destroyed row; a missing row is not interchangeable with an attempt that proved - // it never acquired a hand. - assert!(compensation_release_identity_is_verified( - true, - PersistedLeaseReleaseState::Destroyed, - )); - assert!(!compensation_release_identity_is_verified( - true, - PersistedLeaseReleaseState::Missing, - )); - assert!(!compensation_release_identity_is_verified( - true, - PersistedLeaseReleaseState::LiveOrAmbiguous, - )); - assert!(compensation_release_identity_is_verified( - false, - PersistedLeaseReleaseState::Missing, - )); - } -} diff --git a/crates/moa-hands/src/core/sandbox_workspace/lifecycle/commit.rs b/crates/moa-hands/src/core/sandbox_workspace/lifecycle/commit.rs new file mode 100644 index 000000000..ef301e4c5 --- /dev/null +++ b/crates/moa-hands/src/core/sandbox_workspace/lifecycle/commit.rs @@ -0,0 +1,498 @@ +//! Durable sandbox-workspace commit publication. + +use super::*; + +impl ToolRouter { + pub(super) async fn confirmed_workspace_commit_replay( + &self, + workspace: &SandboxWorkspace, + tool_call_id: ToolCallId, + ) -> Result { + let operations = self.hands.workspace_operations.as_ref().ok_or_else(|| { + MoaError::StorageError("workspace operation repository missing".to_string()) + })?; + let repository = + self.hands.workspace_repository.as_ref().ok_or_else(|| { + MoaError::StorageError("workspace repository missing".to_string()) + })?; + let operation_id = WorkspaceOperationId(Uuid::new_v5( + &workspace.workspace_id.0, + format!("tool-commit-v1:{tool_call_id}").as_bytes(), + )); + let Some(operation) = operations.get(workspace.tenant_id, operation_id).await? else { + return Ok(false); + }; + if operation.outcome == WorkspaceOperationOutcome::NotSent { + return Ok(false); + } + if operation.outcome == WorkspaceOperationOutcome::Unknown { + return Ok(false); + } + let checkpoint = repository + .get_checkpoint_for_operation(workspace.tenant_id, workspace.workspace_id, operation_id) + .await? + .ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_id.to_string(), + })?; + let committed_generation = operation + .expected_checkpoint_generation + .checked_add(1) + .ok_or_else(|| { + MoaError::StorageError("workspace checkpoint generation overflowed".to_string()) + })?; + let parent_revision = match ( + operation.expected_checkpoint_generation, + checkpoint.parent_checkpoint_id, + ) { + (0, None) => None, + (generation, Some(checkpoint_id)) if generation > 0 => { + Some(moa_core::types::sandbox_workspace::WorkspaceRevisionRef { + checkpoint_id, + generation: u64::try_from(generation).map_err(|_| { + MoaError::StorageError( + "workspace checkpoint generation is invalid".to_string(), + ) + })?, + format_version: CHECKPOINT_ARCHIVE_FORMAT_VERSION, + }) + } + _ => { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_id.to_string(), + }); + } + }; + let mut original_binding = workspace.binding()?; + original_binding.current_revision = parent_revision; + let request_bytes = serde_json::to_vec(&(&original_binding, tool_call_id))?; + let request_hash = format!("sha256:{}", hex::encode(Sha256::digest(request_bytes))); + if operation.kind != WorkspaceOperationKind::Commit + || operation.workspace_id != workspace.workspace_id + || operation.provider_account_id != workspace.provider_account_id + || operation.provider_account_generation != workspace.provider_account_generation + || operation.expected_writer_epoch != workspace.writer_epoch + || operation.expected_instance_generation != workspace.instance_generation + || operation.request_hash != request_hash + || operation.confirmed_disposition + != Some(WorkspaceConfirmedDisposition::ResourcePresent) + || checkpoint.state != WorkspaceCheckpointState::Available + || checkpoint.checkpoint_id != WorkspaceCheckpointId(operation_id.0) + || checkpoint.generation != committed_generation + || checkpoint.source_writer_epoch != workspace.writer_epoch + || checkpoint.source_instance_generation != workspace.instance_generation + || workspace.checkpoint_id != Some(checkpoint.checkpoint_id) + || workspace.checkpoint_generation != committed_generation + { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_id.to_string(), + }); + } + Ok(true) + } + + /// Publishes the mutable workspace for one already-journaled sandbox command. + /// + /// This never dispatches the command. It reloads the exact workspace, + /// active lease, and hand, then starts or resumes the deterministic commit. + pub async fn commit_authorized_workspace_after_tool( + &self, + request: JournaledWorkspaceCommit<'_>, + ) -> Result<()> { + request.scope.admit()?; + let workspace_scope = request.workspace_scope; + let repository = + self.hands.workspace_repository.as_ref().ok_or_else(|| { + MoaError::StorageError("workspace repository missing".to_string()) + })?; + let workspace = repository + .get_by_scope(request.session.tenant_id, workspace_scope) + .await? + .ok_or_else(|| { + MoaError::PermissionDenied( + "authorized sandbox workspace disappeared before commit".to_string(), + ) + })?; + if self + .confirmed_workspace_commit_replay(&workspace, request.tool_call_id) + .await? + { + return Ok(()); + } + let lease_scope = workspace_lease_scope(workspace_scope); + let lease_store = self.hands.hand_leases.as_ref().ok_or_else(|| { + MoaError::StorageError("durable hand lease store missing".to_string()) + })?; + let lease = lease_store + .get( + request.session.tenant_id, + request.session.id, + &lease_scope, + &workspace.provider, + ) + .await? + .ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { + operation_id: format!("workspace-tool-call:{}", request.tool_call_id), + })?; + let hand = lease + .handle + .as_ref() + .map(|handle| handle.handle.clone()) + .ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { + operation_id: format!("workspace-tool-call:{}", request.tool_call_id), + })?; + self.commit_workspace_after_tool(WorkspaceCommitExecution { + session: request.session, + workspace_scope, + tool_call_id: request.tool_call_id, + provider_name: &workspace.provider, + hand: &hand, + call_scope: request.scope, + release_compute: false, + }) + .await + } + + pub(in crate::core) async fn commit_workspace_after_tool( + &self, + request: WorkspaceCommitExecution<'_>, + ) -> Result<()> { + let WorkspaceCommitExecution { + session, + workspace_scope, + tool_call_id, + provider_name, + hand, + call_scope, + release_compute, + } = request; + let Some(repository) = self.hands.workspace_repository.as_ref() else { + return Ok(()); + }; + let operations = self.hands.workspace_operations.as_ref().ok_or_else(|| { + MoaError::StorageError("workspace operation repository missing".to_string()) + })?; + let storage_provider = + self.hands + .storage_providers + .get(provider_name) + .ok_or_else(|| { + MoaError::ProviderError(format!( + "workspace storage provider {provider_name} is not registered" + )) + })?; + let lease_store = self.hands.hand_leases.as_ref().ok_or_else(|| { + MoaError::StorageError("durable hand lease store missing".to_string()) + })?; + let workspace = repository + .get_by_scope(session.tenant_id, workspace_scope) + .await? + .ok_or_else(|| { + MoaError::PermissionDenied( + "authorized sandbox workspace disappeared before commit".to_string(), + ) + })?; + if self + .confirmed_workspace_commit_replay(&workspace, tool_call_id) + .await? + { + return Ok(()); + } + if !matches!( + workspace.state, + SandboxWorkspaceState::Active + | SandboxWorkspaceState::Quiescing + | SandboxWorkspaceState::Committing + ) || workspace.provider != provider_name + || workspace.access_fenced_at.is_some() + { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: format!("workspace-tool-call:{tool_call_id}"), + }); + } + let binding = workspace.binding()?; + let lease_scope = workspace_lease_scope(workspace_scope); + let lease = lease_store + .get(session.tenant_id, session.id, &lease_scope, provider_name) + .await? + .ok_or_else(|| { + MoaError::StorageError( + "active workspace lease is missing before commit".to_string(), + ) + })?; + if lease.status != HandLeaseStatus::Active + || lease.handle.as_ref().map(|lease| &lease.handle) != Some(hand) + || lease.attachment != Some(lease_attachment(&binding)?) + { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: format!("workspace-tool-call:{tool_call_id}"), + }); + } + + let operation_id = WorkspaceOperationId(Uuid::new_v5( + &binding.workspace_id.0, + format!("tool-commit-v1:{tool_call_id}").as_bytes(), + )); + let checkpoint_id = WorkspaceCheckpointId(operation_id.0); + let existing_operation = operations.get(binding.tenant_id, operation_id).await?; + let mut deadline_at = existing_operation.as_ref().map_or_else( + || { + call_scope + .budget + .deadline + .unwrap_or_else(|| Utc::now() + ChronoDuration::minutes(5)) + }, + |operation| operation.deadline_at, + ); + let request_bytes = serde_json::to_vec(&(&binding, tool_call_id))?; + let request_hash = format!("sha256:{}", hex::encode(Sha256::digest(request_bytes))); + let expected_writer_epoch = i64::try_from(binding.writer_epoch) + .map_err(|_| MoaError::StorageError("workspace writer epoch is invalid".to_string()))?; + let expected_instance_generation = + i64::try_from(binding.instance_generation).map_err(|_| { + MoaError::StorageError("workspace instance generation is invalid".to_string()) + })?; + let expected_checkpoint_generation = + binding + .current_revision + .as_ref() + .map_or(Ok(0_i64), |revision| { + i64::try_from(revision.generation).map_err(|_| { + MoaError::StorageError( + "workspace checkpoint generation is invalid".to_string(), + ) + }) + })?; + let operation = operations + .persist_intent(&WorkspaceOperationIntent { + operation_id, + tenant_id: binding.tenant_id, + workspace_id: binding.workspace_id, + provider_account_id: binding.provider_account_id, + provider_account_generation: i64::try_from(binding.provider_account_generation) + .map_err(|_| { + MoaError::StorageError( + "workspace provider-account generation is invalid".to_string(), + ) + })?, + kind: WorkspaceOperationKind::Commit, + request_hash: request_hash.clone(), + expected_writer_epoch, + expected_instance_generation, + expected_checkpoint_generation, + deadline_at, + reconcile_not_before: deadline_at + ChronoDuration::seconds(30), + }) + .await?; + debug_assert_ne!(operation.outcome, WorkspaceOperationOutcome::Confirmed); + let transitioned = match workspace.state { + SandboxWorkspaceState::Active => { + repository + .transition(WorkspaceTransition { + tenant_id: binding.tenant_id, + workspace_id: binding.workspace_id, + from: SandboxWorkspaceState::Active, + to: SandboxWorkspaceState::Quiescing, + writer_epoch: expected_writer_epoch, + instance_generation: expected_instance_generation, + }) + .await? + && repository + .transition(WorkspaceTransition { + tenant_id: binding.tenant_id, + workspace_id: binding.workspace_id, + from: SandboxWorkspaceState::Quiescing, + to: SandboxWorkspaceState::Committing, + writer_epoch: expected_writer_epoch, + instance_generation: expected_instance_generation, + }) + .await? + } + SandboxWorkspaceState::Quiescing => { + repository + .transition(WorkspaceTransition { + tenant_id: binding.tenant_id, + workspace_id: binding.workspace_id, + from: SandboxWorkspaceState::Quiescing, + to: SandboxWorkspaceState::Committing, + writer_epoch: expected_writer_epoch, + instance_generation: expected_instance_generation, + }) + .await? + } + SandboxWorkspaceState::Committing => true, + _ => false, + }; + if !transitioned { + operations + .mark_unknown(binding.tenant_id, operation_id) + .await?; + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_id.to_string(), + }); + } + if let Err(error) = call_scope.admit() { + operations + .mark_unknown(binding.tenant_id, operation_id) + .await?; + return Err(error); + } + let checkpoint = repository + .create_checkpoint(CreateCheckpointRequest { + checkpoint_id, + tenant_id: binding.tenant_id, + workspace_id: binding.workspace_id, + parent_checkpoint_id: binding + .current_revision + .as_ref() + .map(|revision| revision.checkpoint_id), + operation_id, + expected_writer_epoch, + expected_instance_generation, + expected_checkpoint_generation, + }) + .await?; + if checkpoint.is_none() { + operations + .mark_unknown(binding.tenant_id, operation_id) + .await?; + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_id.to_string(), + }); + } + failpoints::hit("post_command_pre_checkpoint_publication").await?; + if operation.outcome == WorkspaceOperationOutcome::NotSent && deadline_at <= Utc::now() { + let renewed_deadline = call_scope + .budget + .deadline + .filter(|deadline| *deadline > Utc::now()) + .unwrap_or_else(|| Utc::now() + ChronoDuration::minutes(5)); + if !operations + .renew_not_sent_commit_deadline( + binding.tenant_id, + operation_id, + deadline_at, + renewed_deadline, + ) + .await? + { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_id.to_string(), + }); + } + deadline_at = renewed_deadline; + } + let storage_operation = WorkspaceStorageOperation { + operation_id, + kind: WorkspaceOperationKind::Commit, + binding: binding.clone(), + deadline: deadline_at, + request_hash, + }; + let provider_result = if operation.outcome == WorkspaceOperationOutcome::Unknown { + let storage = self.hands.checkpoint_store.as_ref().map(|store| { + store.storage_reference( + crate::core::sandbox_workspace::checkpoint::store::CheckpointStoreContext { + tenant_id: binding.tenant_id, + workspace_id: binding.workspace_id, + checkpoint_id, + provider_account_id: binding.provider_account_id, + provider_account_generation: binding.provider_account_generation, + }, + ) + }); + let reconcile = + WorkspaceReconcileRequest::new(storage_operation, Some(hand.clone()), storage)?; + self.run_within_scope( + call_scope, + storage_provider.reconcile_workspace_operation(reconcile), + ) + .await + } else { + if !operations + .begin_provider_attempt(binding.tenant_id, operation_id) + .await? + { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_id.to_string(), + }); + } + self.run_within_scope( + call_scope, + storage_provider.publish_workspace_checkpoint(WorkspaceCheckpointPublishRequest { + operation: storage_operation, + hand: hand.clone(), + parent_revision: binding.current_revision.clone(), + release_compute, + }), + ) + .await + }; + let result = match provider_result { + Ok(result) => result, + Err(error) => { + tracing::warn!( + operation_id = %operation_id, + error = %error, + "workspace tool commit provider outcome is ambiguous" + ); + operations + .mark_unknown(binding.tenant_id, operation_id) + .await?; + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_id.to_string(), + }); + } + }; + let (publication, post_commit_state) = match ( + result.outcome, + result.confirmed_disposition, + result.checkpoint_publication.as_ref(), + result.post_commit_state, + ) { + ( + WorkspaceOperationOutcome::Confirmed, + Some(WorkspaceConfirmedDisposition::ResourcePresent), + Some(publication), + Some(post_commit_state), + ) => (publication, post_commit_state), + _ => { + operations + .mark_unknown(binding.tenant_id, operation_id) + .await?; + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_id.to_string(), + }); + } + }; + if !repository + .publish_checkpoint_commit(PublishCheckpointCommitRequest { + binding: &binding, + operation_id, + publication, + post_commit_state, + lease: &lease, + }) + .await? + { + self.delete_abandoned_checkpoint_prefix(&binding, publication.revision.checkpoint_id) + .await?; + operations + .mark_unknown(binding.tenant_id, operation_id) + .await?; + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_id.to_string(), + }); + } + if post_commit_state != WorkspacePostCommitState::AttachmentRetained { + let key = session_provider_key(session, Some(&lease_scope), provider_name); + self.remove_cached_binding_if_matches(&key, hand, Some(lease.generation)) + .await; + self.remove_installed_marker( + manifest_scope_key(session, Some(&lease_scope)), + provider_name, + ) + .await; + } + Ok(()) + } +} diff --git a/crates/moa-hands/src/core/sandbox_workspace/lifecycle/execution_release.rs b/crates/moa-hands/src/core/sandbox_workspace/lifecycle/execution_release.rs new file mode 100644 index 000000000..767dee99d --- /dev/null +++ b/crates/moa-hands/src/core/sandbox_workspace/lifecycle/execution_release.rs @@ -0,0 +1,749 @@ +//! Execution-task and compensation hand release recovery. + +use super::*; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum ExecutionReleaseStep { + DurableReconciliation, + ProviderIo, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum PersistedLeaseReleaseState { + Missing, + Destroyed, + LiveOrAmbiguous, +} + +const fn compensation_release_identity_is_verified( + persisted_identity_present: bool, + lease_state: PersistedLeaseReleaseState, +) -> bool { + matches!( + (persisted_identity_present, lease_state), + (true, PersistedLeaseReleaseState::Destroyed) + | (false, PersistedLeaseReleaseState::Missing) + ) +} + +fn admit_execution_release_step( + scope: ToolCallScope<'_>, + step: ExecutionReleaseStep, +) -> Result<()> { + match step { + ExecutionReleaseStep::DurableReconciliation => Ok(()), + ExecutionReleaseStep::ProviderIo => scope.admit(), + } +} + +impl ToolRouter { + /// Checkpoints one execution-task workspace and releases its exact compute lease. + /// + /// The returned receipt is the durable proof required before a task may yield to + /// a timer, external callback, pause, or long backoff. Retries return the same + /// receipt. Provider teardown errors remain ambiguous and never produce release + /// proof; a later retry reconciles the checkpoint and repeats exact destruction. + pub async fn checkpoint_and_release_execution_hand( + &self, + request: ExecutionHandReleaseRequest<'_>, + ) -> Result { + if request.attempt_generation == 0 { + return Err(MoaError::ValidationError( + "execution task attempt generation must be positive".to_string(), + )); + } + let (task_id, logical_generation) = match request.owner { + ExecutionHandReleaseOwner::Task { + task_id, + logical_generation, + } if logical_generation > 0 => (task_id, logical_generation), + ExecutionHandReleaseOwner::Task { .. } => { + return Err(MoaError::ValidationError( + "execution task logical generation must be positive".to_string(), + )); + } + ExecutionHandReleaseOwner::Compensation { + compensation_id, + logical_generation, + } => { + return self + .release_execution_compensation_hand( + request, + compensation_id, + logical_generation, + ) + .await; + } + }; + let repository = + self.hands.workspace_repository.as_ref().ok_or_else(|| { + MoaError::StorageError("workspace repository missing".to_string()) + })?; + if let Some(receipt) = repository + .get_task_execution_hand_release_receipt( + request.session.tenant_id, + request.run_id, + task_id, + logical_generation, + request.attempt_generation, + ) + .await? + { + return Ok(receipt); + } + + let absence_receipt_id = Uuid::new_v5( + &request.run_id.0, + format!( + "execution-task-hand-absence-v1:{task_id}:{logical_generation}:{}", + request.attempt_generation + ) + .as_bytes(), + ); + match repository + .record_absent_task_execution_hand_release_receipt(AbsentTaskHandReleaseIntent { + receipt_id: absence_receipt_id, + tenant_id: request.session.tenant_id, + run_id: request.run_id, + task_id, + logical_generation, + attempt_generation: request.attempt_generation, + verified_at: Utc::now(), + }) + .await + { + Ok(receipt) => return Ok(receipt), + Err(MoaError::ExternalEffectUnknownOutcome { .. }) => {} + Err(error) => return Err(error), + } + + let workspace_scope = SandboxWorkspaceScope::ExecutionTask { + run_id: request.run_id, + task_id, + }; + let initial_workspace = repository + .get_by_scope(request.session.tenant_id, &workspace_scope) + .await? + .ok_or_else(|| { + MoaError::PermissionDenied( + "execution-task workspace disappeared before hand release".to_string(), + ) + })?; + let release_key = format!( + "execution-task-yield-v1:{}:{}:{}", + request.run_id, task_id, request.attempt_generation + ); + let tool_call_id = ToolCallId(Uuid::new_v5( + &initial_workspace.workspace_id.0, + release_key.as_bytes(), + )); + let candidate_receipt_id = Uuid::new_v5( + &initial_workspace.workspace_id.0, + format!("release-receipt-v1:{release_key}").as_bytes(), + ); + let lease_scope = workspace_lease_scope(&workspace_scope); + let lease_store = self.hands.hand_leases.as_ref().ok_or_else(|| { + MoaError::StorageError("durable hand lease store missing".to_string()) + })?; + let initial_lease = lease_store + .get( + request.session.tenant_id, + request.session.id, + &lease_scope, + &initial_workspace.provider, + ) + .await? + .ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { + operation_id: release_key.clone(), + })?; + let (receipt_id, release_claim_token, requested_at) = repository + .begin_task_execution_hand_release(TaskHandReleaseIntent { + receipt_id: candidate_receipt_id, + run_id: request.run_id, + task_id, + logical_generation, + attempt_generation: request.attempt_generation, + deadline_at: request + .scope + .budget + .deadline + .unwrap_or_else(|| Utc::now() + ChronoDuration::minutes(5)), + recovery_claim_expires_at: Utc::now() + ChronoDuration::minutes(5), + workspace: &initial_workspace, + lease: &initial_lease, + }) + .await?; + + admit_execution_release_step( + request.scope, + if initial_lease.status == HandLeaseStatus::Active { + ExecutionReleaseStep::ProviderIo + } else { + ExecutionReleaseStep::DurableReconciliation + }, + )?; + + match initial_lease.status { + HandLeaseStatus::Active => { + let hand = initial_lease + .handle + .as_ref() + .map(|handle| handle.handle.clone()) + .ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { + operation_id: release_key.clone(), + })?; + self.commit_workspace_after_tool(WorkspaceCommitExecution { + session: request.session, + workspace_scope: &workspace_scope, + tool_call_id, + provider_name: &initial_workspace.provider, + hand: &hand, + call_scope: request.scope, + release_compute: true, + }) + .await?; + } + HandLeaseStatus::Destroyed => { + if !self + .confirmed_workspace_commit_replay(&initial_workspace, tool_call_id) + .await? + { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: release_key.clone(), + }); + } + } + HandLeaseStatus::Provisioning + | HandLeaseStatus::Stale + | HandLeaseStatus::Failed + | HandLeaseStatus::Reaping => { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: release_key.clone(), + }); + } + } + + let mut final_workspace = repository + .get_by_scope(request.session.tenant_id, &workspace_scope) + .await? + .ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { + operation_id: release_key.clone(), + })?; + let mut final_lease = lease_store + .get( + request.session.tenant_id, + request.session.id, + &lease_scope, + &initial_workspace.provider, + ) + .await? + .ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { + operation_id: release_key.clone(), + })?; + + // An unknown provider outcome may reconcile the already-verified bytes + // while conservatively retaining the attachment. Finish the destroy as a + // separate exact step, then atomically release lease and capacity ownership. + if final_lease.status == HandLeaseStatus::Active { + if !self + .confirmed_workspace_commit_replay(&final_workspace, tool_call_id) + .await? + { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: release_key.clone(), + }); + } + let hand = final_lease + .handle + .as_ref() + .map(|handle| handle.handle.clone()) + .ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { + operation_id: release_key.clone(), + })?; + let provider = self + .hands + .providers + .get(&initial_workspace.provider) + .ok_or_else(|| { + MoaError::ProviderError(format!( + "hand provider {} is not registered", + initial_workspace.provider + )) + })?; + self.run_within_scope(request.scope, provider.destroy(&hand)) + .await + .map_err(|error| { + tracing::warn!( + operation_id = %release_key, + error = %error, + "execution-task hand destroy outcome is ambiguous" + ); + MoaError::ExternalEffectUnknownOutcome { + operation_id: release_key.clone(), + } + })?; + if !repository + .finalize_task_yield_destroy(&final_workspace.binding()?, &final_lease) + .await? + { + // The compute is gone but the durable release did not commit, so the + // charge is still held and a reconciler owns it. Recorded as ambiguous + // rather than succeeded so the two are distinguishable on the dashboard. + record_workspace_release( + &initial_workspace.provider, + SandboxWorkspaceMetricResult::Ambiguous, + ); + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: release_key.clone(), + }); + } + // Counted only after provider destruction is verified AND the release + // receipt commits, which together are what actually free the capacity. + record_workspace_release( + &initial_workspace.provider, + SandboxWorkspaceMetricResult::Succeeded, + ); + let key = session_provider_key( + request.session, + Some(&lease_scope), + &initial_workspace.provider, + ); + self.remove_cached_binding_if_matches(&key, &hand, Some(initial_lease.generation)) + .await; + self.remove_installed_marker( + manifest_scope_key(request.session, Some(&lease_scope)), + &initial_workspace.provider, + ) + .await; + final_workspace = repository + .get_by_scope(request.session.tenant_id, &workspace_scope) + .await? + .ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { + operation_id: release_key.clone(), + })?; + final_lease = lease_store + .get( + request.session.tenant_id, + request.session.id, + &lease_scope, + &initial_workspace.provider, + ) + .await? + .ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { + operation_id: release_key.clone(), + })?; + } + + let operation_id = WorkspaceOperationId(Uuid::new_v5( + &initial_workspace.workspace_id.0, + format!("tool-commit-v1:{tool_call_id}").as_bytes(), + )); + let checkpoint_id = WorkspaceCheckpointId(operation_id.0); + let checkpoint = repository + .get_checkpoint( + request.session.tenant_id, + final_workspace.workspace_id, + checkpoint_id, + ) + .await? + .ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { + operation_id: release_key.clone(), + })?; + if final_workspace.state != SandboxWorkspaceState::Ready + || final_workspace.writer_epoch != initial_workspace.writer_epoch + || final_workspace.instance_generation != initial_workspace.instance_generation + || final_workspace.checkpoint_id != Some(checkpoint_id) + || final_workspace.checkpoint_generation != checkpoint.generation + || final_lease.status != HandLeaseStatus::Destroyed + || final_lease.handle.is_some() + || final_lease.generation != initial_lease.generation + || final_lease.provisioning_operation_id != initial_lease.provisioning_operation_id + || checkpoint.state != WorkspaceCheckpointState::Available + || checkpoint.manifest_digest.is_none() + || checkpoint.logical_bytes.is_none() + { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: release_key, + }); + } + let receipt = ExecutionHandReleaseReceipt { + receipt_id, + tenant_id: request.session.tenant_id, + run_id: request.run_id, + owner: request.owner, + attempt_generation: request.attempt_generation, + workspace_id: Some(final_workspace.workspace_id), + writer_epoch: Some(u64::try_from(final_workspace.writer_epoch).map_err(|_| { + MoaError::StorageError("workspace writer epoch is invalid".to_string()) + })?), + instance_generation: Some(u64::try_from(final_workspace.instance_generation).map_err( + |_| MoaError::StorageError("workspace instance generation is invalid".to_string()), + )?), + hand_provisioning_operation_id: Some(initial_lease.provisioning_operation_id), + hand_lease_generation: Some(u64::try_from(initial_lease.generation).map_err(|_| { + MoaError::StorageError("hand lease generation is invalid".to_string()) + })?), + checkpoint_id: Some(checkpoint_id), + checkpoint_generation: Some(u64::try_from(checkpoint.generation).map_err(|_| { + MoaError::StorageError("checkpoint generation is invalid".to_string()) + })?), + checkpoint_manifest_digest: Some(checkpoint.manifest_digest.ok_or_else(|| { + MoaError::StorageError("verified checkpoint digest is missing".to_string()) + })?), + checkpoint_logical_bytes: Some( + u64::try_from(checkpoint.logical_bytes.ok_or_else(|| { + MoaError::StorageError("verified checkpoint bytes are missing".to_string()) + })?) + .map_err(|_| MoaError::StorageError("checkpoint bytes are negative".to_string()))?, + ), + requested_at, + released_at: Utc::now(), + }; + repository + .record_task_execution_hand_release_receipt(&receipt, release_claim_token) + .await + } + + async fn release_execution_compensation_hand( + &self, + request: ExecutionHandReleaseRequest<'_>, + compensation_id: ExecutionCompensationScopeId, + logical_generation: u64, + ) -> Result { + if logical_generation == 0 { + return Err(MoaError::ValidationError( + "execution compensation logical generation must be positive".to_string(), + )); + } + let repository = + self.hands.workspace_repository.as_ref().ok_or_else(|| { + MoaError::StorageError("workspace repository missing".to_string()) + })?; + if let Some(receipt) = repository + .get_compensation_execution_hand_release_receipt( + request.session.tenant_id, + request.run_id, + compensation_id, + logical_generation, + request.attempt_generation, + ) + .await? + { + return Ok(receipt); + } + + let hand_scope = format!( + "execution_compensation:{}:{}", + request.run_id, compensation_id + ); + let lease_store = self.hands.hand_leases.as_ref().ok_or_else(|| { + MoaError::StorageError("durable hand lease store missing".to_string()) + })?; + if let Some(claim) = repository + .claim_pending_compensation_execution_hand_release( + request.session.tenant_id, + request.run_id, + compensation_id, + logical_generation, + request.attempt_generation, + Utc::now() + ChronoDuration::minutes(5), + ) + .await? + { + let persisted_identity = match ( + claim.hand_provisioning_operation_id, + claim.hand_lease_generation, + ) { + (Some(operation_id), Some(generation)) => Some((operation_id, generation)), + (None, None) => None, + _ => { + return Err(MoaError::StorageError( + "pending compensation release has a partial hand identity".to_string(), + )); + } + }; + let exact_lease = match persisted_identity { + Some((operation_id, generation)) => { + lease_store + .get_exact_generation( + request.session.tenant_id, + request.session.id, + &hand_scope, + operation_id, + generation, + ) + .await? + } + None => None, + }; + let provider_io_required = exact_lease + .as_ref() + .is_some_and(|lease| lease.status != HandLeaseStatus::Destroyed); + admit_execution_release_step( + request.scope, + if provider_io_required { + ExecutionReleaseStep::ProviderIo + } else { + ExecutionReleaseStep::DurableReconciliation + }, + )?; + if provider_io_required + && !self + .reclaim_hands( + request.session.tenant_id, + &request.session.id, + Some(&hand_scope), + ) + .await + { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: format!( + "execution-compensation-hand-release:{}:{compensation_id}:{logical_generation}:{}", + request.run_id, request.attempt_generation + ), + }); + } + let exact_lease = match persisted_identity { + Some((operation_id, generation)) => { + lease_store + .get_exact_generation( + request.session.tenant_id, + request.session.id, + &hand_scope, + operation_id, + generation, + ) + .await? + } + None => None, + }; + let lease_state = match exact_lease.as_ref() { + None => PersistedLeaseReleaseState::Missing, + Some(lease) + if lease.status == HandLeaseStatus::Destroyed && lease.handle.is_none() => + { + PersistedLeaseReleaseState::Destroyed + } + Some(_) => PersistedLeaseReleaseState::LiveOrAmbiguous, + }; + let exact_released = compensation_release_identity_is_verified( + persisted_identity.is_some(), + lease_state, + ); + let replacement = lease_store + .has_live_owner(request.session.tenant_id, request.session.id, &hand_scope) + .await?; + if !exact_released || replacement { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: format!( + "execution-compensation-hand-release:{}:{compensation_id}:{logical_generation}:{}", + request.run_id, request.attempt_generation + ), + }); + } + let hand_lease_generation = claim + .hand_lease_generation + .map(u64::try_from) + .transpose() + .map_err(|_| { + MoaError::StorageError("hand lease generation is invalid".to_string()) + })?; + return repository + .record_compensation_execution_hand_release_receipt( + &ExecutionHandReleaseReceipt { + receipt_id: claim.receipt_id, + tenant_id: request.session.tenant_id, + run_id: request.run_id, + owner: request.owner, + attempt_generation: request.attempt_generation, + workspace_id: None, + writer_epoch: None, + instance_generation: None, + hand_provisioning_operation_id: claim.hand_provisioning_operation_id, + hand_lease_generation, + checkpoint_id: None, + checkpoint_generation: None, + checkpoint_manifest_digest: None, + checkpoint_logical_bytes: None, + requested_at: claim.requested_at, + released_at: Utc::now(), + }, + request.session.id, + &hand_scope, + claim.claim_token, + ) + .await; + } + let leases = lease_store + .list_live_owner_candidates(request.session.tenant_id, request.session.id, &hand_scope) + .await?; + if leases.len() > 1 { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: format!( + "execution-compensation-hand-release:{}:{compensation_id}:{logical_generation}:{}", + request.run_id, request.attempt_generation + ), + }); + } + let initial_lease = leases.into_iter().next(); + + let release_key = format!( + "execution-compensation-release-v1:{}:{compensation_id}:{logical_generation}:{}", + request.run_id, request.attempt_generation + ); + let receipt_id = Uuid::new_v5(&request.run_id.0, release_key.as_bytes()); + let (receipt_id, claim_token, requested_at) = repository + .begin_compensation_execution_hand_release(CompensationHandReleaseIntent { + receipt_id, + tenant_id: request.session.tenant_id, + session_id: request.session.id, + run_id: request.run_id, + compensation_id, + logical_generation, + attempt_generation: request.attempt_generation, + hand_scope: &hand_scope, + lease: initial_lease.as_ref(), + deadline_at: request + .scope + .budget + .deadline + .unwrap_or_else(|| Utc::now() + ChronoDuration::minutes(5)), + recovery_claim_expires_at: Utc::now() + ChronoDuration::minutes(5), + }) + .await?; + admit_execution_release_step( + request.scope, + if initial_lease.is_some() { + ExecutionReleaseStep::ProviderIo + } else { + ExecutionReleaseStep::DurableReconciliation + }, + )?; + if initial_lease.is_some() + && !self + .reclaim_hands( + request.session.tenant_id, + &request.session.id, + Some(&hand_scope), + ) + .await + { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: release_key, + }); + } + if let Some(initial_lease) = initial_lease.as_ref() { + let exact_lease = lease_store + .get( + request.session.tenant_id, + request.session.id, + &hand_scope, + &initial_lease.provider, + ) + .await?; + let exact_destroyed = exact_lease.as_ref().is_some_and(|lease| { + lease.worker_id == hand_scope + && lease.provisioning_operation_id == initial_lease.provisioning_operation_id + && lease.generation == initial_lease.generation + && lease.status == HandLeaseStatus::Destroyed + && lease.handle.is_none() + }); + let replacement = lease_store + .has_live_owner(request.session.tenant_id, request.session.id, &hand_scope) + .await?; + if !exact_destroyed || replacement { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: release_key, + }); + } + } + let hand_lease_generation = initial_lease + .as_ref() + .map(|lease| { + u64::try_from(lease.generation).map_err(|_| { + MoaError::StorageError("hand lease generation is invalid".to_string()) + }) + }) + .transpose()?; + repository + .record_compensation_execution_hand_release_receipt( + &ExecutionHandReleaseReceipt { + receipt_id, + tenant_id: request.session.tenant_id, + run_id: request.run_id, + owner: request.owner, + attempt_generation: request.attempt_generation, + workspace_id: None, + writer_epoch: None, + instance_generation: None, + hand_provisioning_operation_id: initial_lease + .as_ref() + .map(|lease| lease.provisioning_operation_id), + hand_lease_generation, + checkpoint_id: None, + checkpoint_generation: None, + checkpoint_manifest_digest: None, + checkpoint_logical_bytes: None, + requested_at, + released_at: Utc::now(), + }, + request.session.id, + &hand_scope, + claim_token, + ) + .await + } +} + +#[cfg(test)] +mod tests { + use chrono::{Duration, Utc}; + use moa_core::{error::MoaError, types::resource::ResourceBudget}; + + use super::{ + ExecutionReleaseStep, PersistedLeaseReleaseState, ToolCallScope, + admit_execution_release_step, compensation_release_identity_is_verified, + }; + + #[test] + fn expired_release_budget_allows_reconciliation_but_rejects_new_provider_io() { + // Pins: retrying an exact release after its five-minute I/O window may return a durable + // receipt or finalize verified absence, but it must not start fresh provider operations. + let release_started_at = Utc::now() - Duration::minutes(6); + let scope = ToolCallScope::unbounded().with_budget(ResourceBudget::until( + release_started_at + Duration::minutes(5), + )); + + assert!( + admit_execution_release_step(scope, ExecutionReleaseStep::DurableReconciliation) + .is_ok() + ); + assert!(matches!( + admit_execution_release_step(scope, ExecutionReleaseStep::ProviderIo), + Err(MoaError::BudgetExhausted(_)) + )); + } + + #[test] + fn persisted_compensation_lease_identity_requires_the_exact_destroyed_row() { + // Pins: after provider teardown, a persisted op/generation may finalize only against its + // exact Destroyed row; a missing row is not interchangeable with an attempt that proved + // it never acquired a hand. + assert!(compensation_release_identity_is_verified( + true, + PersistedLeaseReleaseState::Destroyed, + )); + assert!(!compensation_release_identity_is_verified( + true, + PersistedLeaseReleaseState::Missing, + )); + assert!(!compensation_release_identity_is_verified( + true, + PersistedLeaseReleaseState::LiveOrAmbiguous, + )); + assert!(compensation_release_identity_is_verified( + false, + PersistedLeaseReleaseState::Missing, + )); + } +} diff --git a/crates/moa-hands/src/core/sandbox_workspace/lifecycle/management.rs b/crates/moa-hands/src/core/sandbox_workspace/lifecycle/management.rs new file mode 100644 index 000000000..b49cd5fcb --- /dev/null +++ b/crates/moa-hands/src/core/sandbox_workspace/lifecycle/management.rs @@ -0,0 +1,615 @@ +//! Public sandbox-workspace management operations. + +use super::*; + +impl ToolRouter { + /// Materializes the exact authorized worker workspace on its pinned provider. + pub async fn attach_managed_workspace( + &self, + session: &SessionMeta, + workspace_scope: &SandboxWorkspaceScope, + workspace_id: SandboxWorkspaceId, + ) -> Result<()> { + let workspace = self + .managed_workspace(session, workspace_scope, workspace_id) + .await?; + let route = self.management_route(&workspace.provider)?; + self.get_or_provision_hand_within( + &route, + session, + workspace_scope, + ToolCallScope::unbounded(), + ) + .await?; + let active = self + .managed_workspace(session, workspace_scope, workspace_id) + .await?; + if active.state != SandboxWorkspaceState::Active { + return Err(MoaError::StorageError( + "workspace attach completed without an active fenced writer".to_string(), + )); + } + Ok(()) + } + + /// Publishes one replay-stable explicit checkpoint through the durable commit barrier. + pub async fn checkpoint_managed_workspace( + &self, + session: &SessionMeta, + workspace_scope: &SandboxWorkspaceScope, + workspace_id: SandboxWorkspaceId, + operation_id: WorkspaceOperationId, + ) -> Result<()> { + let mut workspace = self + .managed_workspace(session, workspace_scope, workspace_id) + .await?; + if self + .confirmed_management_checkpoint_replay(&workspace, operation_id) + .await? + { + return Ok(()); + } + let hand = if matches!( + workspace.state, + SandboxWorkspaceState::Quiescing | SandboxWorkspaceState::Committing + ) { + let operations = self.hands.workspace_operations.as_ref().ok_or_else(|| { + MoaError::StorageError("workspace operation repository missing".to_string()) + })?; + let operation = operations + .get(session.tenant_id, operation_id) + .await? + .ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_id.to_string(), + })?; + if operation.kind != WorkspaceOperationKind::Checkpoint + || operation.workspace_id != workspace_id + || operation.expected_writer_epoch != workspace.writer_epoch + || operation.expected_instance_generation != workspace.instance_generation + || operation.expected_checkpoint_generation != workspace.checkpoint_generation + { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_id.to_string(), + }); + } + let lease_store = self.hands.hand_leases.as_ref().ok_or_else(|| { + MoaError::StorageError("durable hand lease store missing".to_string()) + })?; + lease_store + .get( + session.tenant_id, + session.id, + &workspace_lease_scope(workspace_scope), + &workspace.provider, + ) + .await? + .and_then(|lease| lease.handle.map(|handle| handle.handle)) + .ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_id.to_string(), + })? + } else { + let route = self.management_route(&workspace.provider)?; + self.get_or_provision_hand_within( + &route, + session, + workspace_scope, + ToolCallScope::unbounded(), + ) + .await? + }; + workspace = self + .managed_workspace(session, workspace_scope, workspace_id) + .await?; + self.checkpoint_active_managed_workspace( + session, + workspace_scope, + &workspace, + operation_id, + &hand, + ) + .await + } + + /// Restores the exact current committed checkpoint into fresh provider compute. + /// + /// Historical revisions remain immutable retention records. Public restore + /// cannot silently move the monotonic workspace head backwards, so the + /// requested checkpoint must be the exact current recovery authority. + pub async fn restore_managed_workspace( + &self, + session: &SessionMeta, + workspace_scope: &SandboxWorkspaceScope, + workspace_id: SandboxWorkspaceId, + checkpoint_id: WorkspaceCheckpointId, + ) -> Result<()> { + let workspace = self + .managed_workspace(session, workspace_scope, workspace_id) + .await?; + let repository = + self.hands.workspace_repository.as_ref().ok_or_else(|| { + MoaError::StorageError("workspace repository missing".to_string()) + })?; + let checkpoint = repository + .get_checkpoint(session.tenant_id, workspace_id, checkpoint_id) + .await? + .ok_or_else(|| { + MoaError::ValidationError( + "restore checkpoint does not belong to the authorized workspace".to_string(), + ) + })?; + validate_managed_restore_target( + workspace.checkpoint_id, + workspace.checkpoint_generation, + checkpoint_id, + checkpoint.checkpoint_id, + checkpoint.generation, + checkpoint.state, + )?; + let route = self.management_route(&workspace.provider)?; + if workspace.state == SandboxWorkspaceState::Active { + self.reprovision_hand(session, workspace_scope, &route, ToolCallScope::unbounded()) + .await?; + } else { + self.get_or_provision_hand_within( + &route, + session, + workspace_scope, + ToolCallScope::unbounded(), + ) + .await?; + } + let restored = self + .managed_workspace(session, workspace_scope, workspace_id) + .await?; + if restored.state != SandboxWorkspaceState::Active + || restored.checkpoint_id != Some(checkpoint_id) + || restored.checkpoint_generation != checkpoint.generation + { + return Err(MoaError::StorageError( + "workspace restore completed without the exact committed checkpoint".to_string(), + )); + } + Ok(()) + } + + async fn managed_workspace( + &self, + session: &SessionMeta, + workspace_scope: &SandboxWorkspaceScope, + workspace_id: SandboxWorkspaceId, + ) -> Result { + let repository = + self.hands.workspace_repository.as_ref().ok_or_else(|| { + MoaError::StorageError("workspace repository missing".to_string()) + })?; + let workspace = repository + .get_by_scope(session.tenant_id, workspace_scope) + .await? + .ok_or_else(|| { + MoaError::PermissionDenied( + "sandbox workspace is not owned by the verified scope".to_string(), + ) + })?; + if workspace.workspace_id != workspace_id + || workspace.tenant_id != session.tenant_id + || workspace.scope != *workspace_scope + || workspace.access_fenced_at.is_some() + { + return Err(MoaError::PermissionDenied( + "sandbox workspace is not owned by the verified scope".to_string(), + )); + } + Ok(workspace) + } + + fn management_route(&self, provider: &str) -> Result { + self.catalog + .activated() + .capability_registrations() + .into_iter() + .find_map(|(_, execution)| match execution { + ToolExecution::Hand { routes } => { + routes.into_iter().find(|route| route.provider == provider) + } + _ => None, + }) + .ok_or_else(|| { + MoaError::ProviderError(format!( + "workspace provider {provider} has no configured hand route" + )) + }) + } + + async fn confirmed_management_checkpoint_replay( + &self, + workspace: &SandboxWorkspace, + operation_id: WorkspaceOperationId, + ) -> Result { + let operations = self.hands.workspace_operations.as_ref().ok_or_else(|| { + MoaError::StorageError("workspace operation repository missing".to_string()) + })?; + let repository = + self.hands.workspace_repository.as_ref().ok_or_else(|| { + MoaError::StorageError("workspace repository missing".to_string()) + })?; + let Some(operation) = operations.get(workspace.tenant_id, operation_id).await? else { + return Ok(false); + }; + if operation.outcome != WorkspaceOperationOutcome::Confirmed { + return Ok(false); + } + let checkpoint = repository + .get_checkpoint_for_operation(workspace.tenant_id, workspace.workspace_id, operation_id) + .await? + .ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_id.to_string(), + })?; + let committed_generation = operation + .expected_checkpoint_generation + .checked_add(1) + .ok_or_else(|| { + MoaError::StorageError("workspace checkpoint generation overflowed".to_string()) + })?; + let parent_revision = revision_from_checkpoint_parent( + operation.expected_checkpoint_generation, + checkpoint.parent_checkpoint_id, + )?; + let mut original_binding = workspace.binding()?; + original_binding.current_revision = parent_revision; + let request_hash = management_checkpoint_request_hash(&original_binding, operation_id)?; + if operation.kind != WorkspaceOperationKind::Checkpoint + || operation.workspace_id != workspace.workspace_id + || operation.provider_account_id != workspace.provider_account_id + || operation.provider_account_generation != workspace.provider_account_generation + || operation.expected_writer_epoch != workspace.writer_epoch + || operation.expected_instance_generation != workspace.instance_generation + || operation.request_hash != request_hash + || operation.confirmed_disposition + != Some(WorkspaceConfirmedDisposition::ResourcePresent) + || checkpoint.state != WorkspaceCheckpointState::Available + || checkpoint.checkpoint_id != WorkspaceCheckpointId(operation_id.0) + || checkpoint.generation != committed_generation + || checkpoint.source_writer_epoch != workspace.writer_epoch + || checkpoint.source_instance_generation != workspace.instance_generation + || workspace.checkpoint_id != Some(checkpoint.checkpoint_id) + || workspace.checkpoint_generation != committed_generation + { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_id.to_string(), + }); + } + Ok(true) + } + + async fn checkpoint_active_managed_workspace( + &self, + session: &SessionMeta, + workspace_scope: &SandboxWorkspaceScope, + workspace: &SandboxWorkspace, + operation_id: WorkspaceOperationId, + hand: &HandHandle, + ) -> Result<()> { + if !matches!( + workspace.state, + SandboxWorkspaceState::Active + | SandboxWorkspaceState::Quiescing + | SandboxWorkspaceState::Committing + ) { + return Err(MoaError::StorageError( + "workspace must be active before checkpoint publication".to_string(), + )); + } + let repository = + self.hands.workspace_repository.as_ref().ok_or_else(|| { + MoaError::StorageError("workspace repository missing".to_string()) + })?; + let operations = self.hands.workspace_operations.as_ref().ok_or_else(|| { + MoaError::StorageError("workspace operation repository missing".to_string()) + })?; + let storage_provider = self + .hands + .storage_providers + .get(&workspace.provider) + .ok_or_else(|| { + MoaError::ProviderError(format!( + "workspace storage provider {} is not registered", + workspace.provider + )) + })?; + let lease_store = self.hands.hand_leases.as_ref().ok_or_else(|| { + MoaError::StorageError("durable hand lease store missing".to_string()) + })?; + let binding = workspace.binding()?; + let lease_scope = workspace_lease_scope(workspace_scope); + let lease = lease_store + .get( + session.tenant_id, + session.id, + &lease_scope, + &workspace.provider, + ) + .await? + .ok_or_else(|| { + MoaError::StorageError( + "active workspace lease is missing before checkpoint".to_string(), + ) + })?; + if lease.status != HandLeaseStatus::Active + || lease.handle.as_ref().map(|lease| &lease.handle) != Some(hand) + || lease.attachment != Some(lease_attachment(&binding)?) + { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_id.to_string(), + }); + } + + let checkpoint_id = WorkspaceCheckpointId(operation_id.0); + let existing_operation = operations.get(binding.tenant_id, operation_id).await?; + let deadline_at = existing_operation.as_ref().map_or_else( + || Utc::now() + ChronoDuration::minutes(5), + |operation| operation.deadline_at, + ); + let request_hash = management_checkpoint_request_hash(&binding, operation_id)?; + let expected_writer_epoch = i64::try_from(binding.writer_epoch) + .map_err(|_| MoaError::StorageError("workspace writer epoch is invalid".to_string()))?; + let expected_instance_generation = + i64::try_from(binding.instance_generation).map_err(|_| { + MoaError::StorageError("workspace instance generation is invalid".to_string()) + })?; + let expected_checkpoint_generation = + binding + .current_revision + .as_ref() + .map_or(Ok(0_i64), |revision| { + i64::try_from(revision.generation).map_err(|_| { + MoaError::StorageError( + "workspace checkpoint generation is invalid".to_string(), + ) + }) + })?; + let operation = operations + .persist_intent(&WorkspaceOperationIntent { + operation_id, + tenant_id: binding.tenant_id, + workspace_id: binding.workspace_id, + provider_account_id: binding.provider_account_id, + provider_account_generation: i64::try_from(binding.provider_account_generation) + .map_err(|_| { + MoaError::StorageError( + "workspace provider-account generation is invalid".to_string(), + ) + })?, + kind: WorkspaceOperationKind::Checkpoint, + request_hash: request_hash.clone(), + expected_writer_epoch, + expected_instance_generation, + expected_checkpoint_generation, + deadline_at, + reconcile_not_before: deadline_at + ChronoDuration::seconds(30), + }) + .await?; + if operation.outcome == WorkspaceOperationOutcome::Confirmed { + let current = self + .managed_workspace(session, workspace_scope, workspace.workspace_id) + .await?; + return self + .confirmed_management_checkpoint_replay(¤t, operation_id) + .await? + .then_some(()) + .ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_id.to_string(), + }); + } + + let transitioned = match workspace.state { + SandboxWorkspaceState::Active => { + repository + .transition(WorkspaceTransition { + tenant_id: binding.tenant_id, + workspace_id: binding.workspace_id, + from: SandboxWorkspaceState::Active, + to: SandboxWorkspaceState::Quiescing, + writer_epoch: expected_writer_epoch, + instance_generation: expected_instance_generation, + }) + .await? + && repository + .transition(WorkspaceTransition { + tenant_id: binding.tenant_id, + workspace_id: binding.workspace_id, + from: SandboxWorkspaceState::Quiescing, + to: SandboxWorkspaceState::Committing, + writer_epoch: expected_writer_epoch, + instance_generation: expected_instance_generation, + }) + .await? + } + SandboxWorkspaceState::Quiescing => { + repository + .transition(WorkspaceTransition { + tenant_id: binding.tenant_id, + workspace_id: binding.workspace_id, + from: SandboxWorkspaceState::Quiescing, + to: SandboxWorkspaceState::Committing, + writer_epoch: expected_writer_epoch, + instance_generation: expected_instance_generation, + }) + .await? + } + SandboxWorkspaceState::Committing => true, + _ => false, + }; + if !transitioned { + operations + .mark_unknown(binding.tenant_id, operation_id) + .await?; + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_id.to_string(), + }); + } + if repository + .create_checkpoint(CreateCheckpointRequest { + checkpoint_id, + tenant_id: binding.tenant_id, + workspace_id: binding.workspace_id, + parent_checkpoint_id: binding + .current_revision + .as_ref() + .map(|revision| revision.checkpoint_id), + operation_id, + expected_writer_epoch, + expected_instance_generation, + expected_checkpoint_generation, + }) + .await? + .is_none() + { + operations + .mark_unknown(binding.tenant_id, operation_id) + .await?; + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_id.to_string(), + }); + } + let storage_operation = WorkspaceStorageOperation { + operation_id, + kind: WorkspaceOperationKind::Checkpoint, + binding: binding.clone(), + deadline: deadline_at, + request_hash, + }; + let provider_result = if operation.outcome == WorkspaceOperationOutcome::Unknown { + let storage = self.hands.checkpoint_store.as_ref().map(|store| { + store.storage_reference( + crate::core::sandbox_workspace::checkpoint::store::CheckpointStoreContext { + tenant_id: binding.tenant_id, + workspace_id: binding.workspace_id, + checkpoint_id, + provider_account_id: binding.provider_account_id, + provider_account_generation: binding.provider_account_generation, + }, + ) + }); + storage_provider + .reconcile_workspace_operation(WorkspaceReconcileRequest::new( + storage_operation, + Some(hand.clone()), + storage, + )?) + .await + } else { + if !operations + .begin_provider_attempt(binding.tenant_id, operation_id) + .await? + { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_id.to_string(), + }); + } + storage_provider + .publish_workspace_checkpoint(WorkspaceCheckpointPublishRequest { + operation: storage_operation, + hand: hand.clone(), + parent_revision: binding.current_revision.clone(), + release_compute: false, + }) + .await + }; + let result = match provider_result { + Ok(result) => result, + Err(error) => { + tracing::warn!( + operation_id = %operation_id, + error = %error, + "workspace checkpoint provider outcome is ambiguous" + ); + operations + .mark_unknown(binding.tenant_id, operation_id) + .await?; + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_id.to_string(), + }); + } + }; + let (publication, post_commit_state) = match ( + result.outcome, + result.confirmed_disposition, + result.checkpoint_publication.as_ref(), + result.post_commit_state, + ) { + ( + WorkspaceOperationOutcome::Confirmed, + Some(WorkspaceConfirmedDisposition::ResourcePresent), + Some(publication), + Some(post_commit_state), + ) => (publication, post_commit_state), + _ => { + operations + .mark_unknown(binding.tenant_id, operation_id) + .await?; + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_id.to_string(), + }); + } + }; + if !repository + .publish_workspace_checkpoint(PublishCheckpointCommitRequest { + binding: &binding, + operation_id, + publication, + post_commit_state, + lease: &lease, + }) + .await? + { + self.delete_abandoned_checkpoint_prefix(&binding, publication.revision.checkpoint_id) + .await?; + operations + .mark_unknown(binding.tenant_id, operation_id) + .await?; + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_id.to_string(), + }); + } + if post_commit_state != WorkspacePostCommitState::AttachmentRetained { + let key = session_provider_key(session, Some(&lease_scope), &workspace.provider); + self.remove_cached_binding_if_matches(&key, hand, Some(lease.generation)) + .await; + self.remove_installed_marker( + manifest_scope_key(session, Some(&lease_scope)), + &workspace.provider, + ) + .await; + } + Ok(()) + } +} + +fn revision_from_checkpoint_parent( + generation: i64, + checkpoint_id: Option, +) -> Result> { + match (generation, checkpoint_id) { + (0, None) => Ok(None), + (generation, Some(checkpoint_id)) if generation > 0 => Ok(Some( + moa_core::types::sandbox_workspace::WorkspaceRevisionRef { + checkpoint_id, + generation: u64::try_from(generation).map_err(|_| { + MoaError::StorageError("workspace checkpoint generation is invalid".to_string()) + })?, + format_version: CHECKPOINT_ARCHIVE_FORMAT_VERSION, + }, + )), + _ => Err(MoaError::StorageError( + "workspace checkpoint parent is inconsistent with its generation".to_string(), + )), + } +} + +fn management_checkpoint_request_hash( + binding: &WorkspaceBinding, + operation_id: WorkspaceOperationId, +) -> Result { + let bytes = serde_json::to_vec(&(binding, operation_id, WorkspaceOperationKind::Checkpoint))?; + Ok(format!("sha256:{}", hex::encode(Sha256::digest(bytes)))) +} diff --git a/crates/moa-hands/src/core/sandbox_workspace/lifecycle/materialization.rs b/crates/moa-hands/src/core/sandbox_workspace/lifecycle/materialization.rs new file mode 100644 index 000000000..34d819f41 --- /dev/null +++ b/crates/moa-hands/src/core/sandbox_workspace/lifecycle/materialization.rs @@ -0,0 +1,649 @@ +//! Initial sandbox-workspace storage materialization and hand hydration. + +use super::*; + +impl ToolRouter { + /// Resolves or creates the exact durable binding used for provisioning. + pub(in crate::core) async fn prepare_workspace_for_provision( + &self, + route: &HandRoute, + session: &SessionMeta, + workspace_scope: &SandboxWorkspaceScope, + call_scope: ToolCallScope<'_>, + ) -> Result { + let Some(repository) = self.hands.workspace_repository.as_ref() else { + return Ok(workspace_binding_for_hand( + session, + workspace_scope, + &route.provider, + )); + }; + let mut workspace = repository + .get_by_scope(session.tenant_id, workspace_scope) + .await? + .ok_or_else(|| { + MoaError::PermissionDenied( + "authorized sandbox workspace has not been resolved for this execution scope" + .to_string(), + ) + })?; + if workspace.provider != route.provider { + return Err(MoaError::ProviderError(format!( + "workspace is pinned to provider {}; cross-provider recovery is disabled", + workspace.provider + ))); + } + if workspace.access_fenced_at.is_some() + || matches!( + workspace.state, + moa_core::types::sandbox_workspace::SandboxWorkspaceState::Deleting + | moa_core::types::sandbox_workspace::SandboxWorkspaceState::Deleted + | moa_core::types::sandbox_workspace::SandboxWorkspaceState::Reconciling + | moa_core::types::sandbox_workspace::SandboxWorkspaceState::Failed + ) + { + return Err(MoaError::PermissionDenied( + "sandbox workspace is fenced or requires reconciliation".to_string(), + )); + } + if workspace.state == SandboxWorkspaceState::Creating { + call_scope.admit()?; + self.prepare_initial_workspace_storage(&workspace, call_scope) + .await?; + if !repository + .transition(WorkspaceTransition { + tenant_id: workspace.tenant_id, + workspace_id: workspace.workspace_id, + from: SandboxWorkspaceState::Creating, + to: SandboxWorkspaceState::Ready, + writer_epoch: workspace.writer_epoch, + instance_generation: workspace.instance_generation, + }) + .await? + { + workspace = repository + .get_by_scope(session.tenant_id, workspace_scope) + .await? + .ok_or_else(|| { + MoaError::StorageError( + "workspace disappeared while storage preparation completed".to_string(), + ) + })?; + } else { + workspace.state = SandboxWorkspaceState::Ready; + } + } + if workspace.state == SandboxWorkspaceState::Ready { + workspace = repository + .claim_writer(WorkspaceWriterClaim { + tenant_id: workspace.tenant_id, + workspace_id: workspace.workspace_id, + expected_state: workspace.state, + expected_writer_epoch: workspace.writer_epoch, + expected_instance_generation: workspace.instance_generation, + }) + .await? + .ok_or_else(|| { + MoaError::StorageError( + "workspace writer claim lost its lifecycle fence".to_string(), + ) + })?; + } + if !matches!( + workspace.state, + SandboxWorkspaceState::Active | SandboxWorkspaceState::Restoring + ) { + return Err(MoaError::StorageError(format!( + "workspace is not dispatchable while in state {}", + workspace.state.as_str() + ))); + } + workspace.binding() + } + + async fn prepare_initial_workspace_storage( + &self, + workspace: &SandboxWorkspace, + call_scope: ToolCallScope<'_>, + ) -> Result<()> { + let prepare_started_at = std::time::Instant::now(); + let operations = self.hands.workspace_operations.as_ref().ok_or_else(|| { + MoaError::StorageError("workspace operation repository missing".to_string()) + })?; + let storage_provider = self + .hands + .storage_providers + .get(&workspace.provider) + .ok_or_else(|| { + MoaError::ProviderError(format!( + "workspace storage provider {} is not registered", + workspace.provider + )) + })?; + let binding = workspace.binding()?; + let operation_id = moa_core::types::identifiers::WorkspaceOperationId(Uuid::new_v5( + &workspace.workspace_id.0, + b"prepare-initial-storage-v1", + )); + let existing = operations.get(workspace.tenant_id, operation_id).await?; + let deadline_at = existing.as_ref().map_or_else( + || { + call_scope + .budget + .deadline + .unwrap_or_else(|| Utc::now() + ChronoDuration::minutes(5)) + }, + |operation| operation.deadline_at, + ); + let reconcile_not_before = existing.as_ref().map_or_else( + || deadline_at + ChronoDuration::seconds(30), + |operation| operation.reconcile_not_before, + ); + let hash_bytes = serde_json::to_vec(&binding)?; + let request_hash = format!("sha256:{}", hex::encode(Sha256::digest(hash_bytes))); + let intent = WorkspaceOperationIntent { + operation_id, + tenant_id: workspace.tenant_id, + workspace_id: workspace.workspace_id, + provider_account_id: workspace.provider_account_id, + provider_account_generation: workspace.provider_account_generation, + kind: WorkspaceOperationKind::Create, + request_hash: request_hash.clone(), + expected_writer_epoch: workspace.writer_epoch, + expected_instance_generation: workspace.instance_generation, + expected_checkpoint_generation: workspace.checkpoint_generation, + deadline_at, + reconcile_not_before, + }; + let operation = operations.persist_intent(&intent).await?; + match (operation.outcome, operation.confirmed_disposition) { + ( + WorkspaceOperationOutcome::Confirmed, + Some(WorkspaceConfirmedDisposition::ResourcePresent), + ) => return Ok(()), + (WorkspaceOperationOutcome::Confirmed, _) => { + return Err(MoaError::ProviderError( + "workspace storage preparation was durably confirmed absent".to_string(), + )); + } + (WorkspaceOperationOutcome::Unknown, _) => { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_id.to_string(), + }); + } + (WorkspaceOperationOutcome::NotSent, None) => {} + _ => { + return Err(MoaError::StorageError( + "workspace storage preparation has an inconsistent durable outcome".to_string(), + )); + } + } + failpoints::hit("post_reservation_pre_provider_create").await?; + call_scope.admit()?; + if !operations + .begin_provider_attempt(workspace.tenant_id, operation_id) + .await? + { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_id.to_string(), + }); + } + let result = match storage_provider + .prepare_workspace_storage(WorkspaceStoragePrepareRequest { + operation: WorkspaceStorageOperation { + operation_id, + kind: WorkspaceOperationKind::Create, + binding, + deadline: deadline_at, + request_hash, + }, + }) + .await + { + Ok(result) => result, + Err(error) => { + tracing::warn!( + operation_id = %operation_id, + error = %error, + "workspace storage preparation outcome is ambiguous" + ); + operations + .mark_unknown(workspace.tenant_id, operation_id) + .await?; + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_id.to_string(), + }); + } + }; + // Every arm records an outcome, so the lifecycle counter carries the real + // success/ambiguous ratio rather than only the happy path. + match (result.outcome, result.confirmed_disposition) { + ( + WorkspaceOperationOutcome::Confirmed, + Some(WorkspaceConfirmedDisposition::ResourcePresent), + ) => { + if !operations + .confirm_disposition( + workspace.tenant_id, + operation_id, + WorkspaceConfirmedDisposition::ResourcePresent, + ) + .await? + { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_id.to_string(), + }); + } + record_workspace_lifecycle( + &workspace.provider, + SandboxWorkspaceLifecycleOperation::Create, + SandboxWorkspaceMetricResult::Succeeded, + prepare_started_at.elapsed(), + ); + Ok(()) + } + ( + WorkspaceOperationOutcome::Confirmed, + Some(WorkspaceConfirmedDisposition::ResourceAbsent), + ) => { + if !operations + .confirm_disposition( + workspace.tenant_id, + operation_id, + WorkspaceConfirmedDisposition::ResourceAbsent, + ) + .await? + { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_id.to_string(), + }); + } + record_workspace_lifecycle( + &workspace.provider, + SandboxWorkspaceLifecycleOperation::Create, + SandboxWorkspaceMetricResult::Failed, + prepare_started_at.elapsed(), + ); + Err(MoaError::ProviderError( + "workspace storage preparation was confirmed absent".to_string(), + )) + } + (WorkspaceOperationOutcome::Unknown, None) => { + operations + .mark_unknown(workspace.tenant_id, operation_id) + .await?; + record_workspace_lifecycle( + &workspace.provider, + SandboxWorkspaceLifecycleOperation::Create, + SandboxWorkspaceMetricResult::Ambiguous, + prepare_started_at.elapsed(), + ); + Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_id.to_string(), + }) + } + _ => { + record_workspace_lifecycle( + &workspace.provider, + SandboxWorkspaceLifecycleOperation::Create, + SandboxWorkspaceMetricResult::Failed, + prepare_started_at.elapsed(), + ); + Err(MoaError::ProviderError( + "workspace storage provider returned an inconsistent preparation result" + .to_string(), + )) + } + } + } + + /// Restores and verifies durable workspace bytes before lease activation. + pub(in crate::core) async fn hydrate_provisioned_workspace( + &self, + binding: &WorkspaceBinding, + claim: &HandLease, + hand: &HandHandle, + call_scope: ToolCallScope<'_>, + ) -> Result<()> { + let Some(repository) = self.hands.workspace_repository.as_ref() else { + return Ok(()); + }; + let operations = self.hands.workspace_operations.as_ref().ok_or_else(|| { + MoaError::StorageError("workspace operation repository missing".to_string()) + })?; + let provider = self + .hands + .storage_providers + .get(&claim.provider) + .ok_or_else(|| { + MoaError::ProviderError(format!( + "workspace storage provider {} is not registered", + claim.provider + )) + })?; + let hydration_started_at = std::time::Instant::now(); + let kind = if binding.current_revision.is_some() { + WorkspaceOperationKind::Restore + } else { + WorkspaceOperationKind::Attach + }; + let operation_id = WorkspaceOperationId(Uuid::new_v5( + &binding.workspace_id.0, + format!( + "hydrate-v1:{}:{}", + claim.provisioning_operation_id, + kind.as_str() + ) + .as_bytes(), + )); + let request_bytes = serde_json::to_vec(&(binding, hand, kind))?; + let request_hash = format!("sha256:{}", hex::encode(Sha256::digest(request_bytes))); + let intent = WorkspaceOperationIntent { + operation_id, + tenant_id: binding.tenant_id, + workspace_id: binding.workspace_id, + provider_account_id: binding.provider_account_id, + provider_account_generation: i64::try_from(binding.provider_account_generation) + .map_err(|_| { + MoaError::StorageError( + "workspace provider-account generation is invalid".to_string(), + ) + })?, + kind, + request_hash: request_hash.clone(), + expected_writer_epoch: i64::try_from(binding.writer_epoch).map_err(|_| { + MoaError::StorageError("workspace writer epoch is invalid".to_string()) + })?, + expected_instance_generation: i64::try_from(binding.instance_generation).map_err( + |_| MoaError::StorageError("workspace instance generation is invalid".to_string()), + )?, + expected_checkpoint_generation: binding.current_revision.as_ref().map_or( + Ok(0_i64), + |revision| { + i64::try_from(revision.generation).map_err(|_| { + MoaError::StorageError( + "workspace checkpoint generation is invalid".to_string(), + ) + }) + }, + )?, + deadline_at: claim.provisioning_deadline_at, + reconcile_not_before: claim.provisioning_deadline_at + ChronoDuration::seconds(30), + }; + let persisted = operations.persist_intent(&intent).await?; + match (persisted.outcome, persisted.confirmed_disposition) { + ( + WorkspaceOperationOutcome::Confirmed, + Some(WorkspaceConfirmedDisposition::ResourcePresent), + ) => return Ok(()), + (WorkspaceOperationOutcome::Confirmed, _) => { + return Err(MoaError::ProviderError(format!( + "workspace {} was durably confirmed absent", + kind.as_str() + ))); + } + (WorkspaceOperationOutcome::Unknown, _) => { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_id.to_string(), + }); + } + (WorkspaceOperationOutcome::NotSent, None) => {} + _ => { + return Err(MoaError::StorageError(format!( + "workspace {} has an inconsistent durable outcome", + kind.as_str() + ))); + } + } + call_scope.admit()?; + if !operations + .begin_provider_attempt(binding.tenant_id, operation_id) + .await? + { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_id.to_string(), + }); + } + let operation = WorkspaceStorageOperation { + operation_id, + kind, + binding: binding.clone(), + deadline: claim.provisioning_deadline_at, + request_hash, + }; + let provider_result = match binding.current_revision.as_ref() { + None => { + self.run_within_scope( + call_scope, + provider.attach_workspace(WorkspaceAttachRequest { + operation, + hand: hand.clone(), + storage: None, + }), + ) + .await + } + Some(revision) => { + let checkpoint = repository + .get_checkpoint( + binding.tenant_id, + binding.workspace_id, + revision.checkpoint_id, + ) + .await? + .ok_or_else(|| { + MoaError::StorageError( + "workspace head checkpoint is missing during restore".to_string(), + ) + })?; + if checkpoint.state + != moa_core::types::sandbox_workspace::WorkspaceCheckpointState::Available + || checkpoint.generation + != i64::try_from(revision.generation).map_err(|_| { + MoaError::StorageError( + "workspace checkpoint generation is invalid".to_string(), + ) + })? + { + return Err(MoaError::StorageError( + "workspace head checkpoint is not an exact available revision".to_string(), + )); + } + let resource_id = checkpoint.object_reference.ok_or_else(|| { + MoaError::StorageError( + "workspace head checkpoint has no portable object reference".to_string(), + ) + })?; + self.run_within_scope( + call_scope, + provider.restore_workspace(WorkspaceRestoreRequest { + operation, + hand: hand.clone(), + revision: revision.clone(), + checkpoint: ProviderStorageRef { + provider_account_id: binding.provider_account_id, + provider_account_generation: binding.provider_account_generation, + kind: ProviderStorageKind::PortableCheckpoint, + resource_id, + workspace_locator: None, + }, + }), + ) + .await + } + }; + let result = match provider_result { + Ok(result) => result, + Err(error) => { + tracing::warn!( + operation_id = %operation_id, + error = %error, + "workspace hydration provider outcome is ambiguous" + ); + operations + .mark_unknown(binding.tenant_id, operation_id) + .await?; + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_id.to_string(), + }); + } + }; + match (result.outcome, result.confirmed_disposition) { + ( + WorkspaceOperationOutcome::Confirmed, + Some(WorkspaceConfirmedDisposition::ResourcePresent), + ) => { + if !operations + .confirm_disposition( + binding.tenant_id, + operation_id, + WorkspaceConfirmedDisposition::ResourcePresent, + ) + .await? + { + return Err(MoaError::StorageError( + "workspace hydration lost its durable operation fence".to_string(), + )); + } + // Only a confirmed restore counts: an ambiguous or failed provider result + // leaves no verified checkpoint in fresh compute, so counting it here + // would overstate successful restores. + if kind == WorkspaceOperationKind::Restore { + record_workspace_restore(&claim.provider); + record_workspace_checkpoint( + &claim.provider, + SandboxWorkspaceCheckpointOperation::Restore, + SandboxWorkspaceMetricResult::Succeeded, + 0, + hydration_started_at.elapsed(), + ); + } + Ok(()) + } + ( + WorkspaceOperationOutcome::Confirmed, + Some(WorkspaceConfirmedDisposition::ResourceAbsent), + ) => { + if !operations + .confirm_disposition( + binding.tenant_id, + operation_id, + WorkspaceConfirmedDisposition::ResourceAbsent, + ) + .await? + { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_id.to_string(), + }); + } + Err(MoaError::ProviderError(format!( + "workspace {} was confirmed absent", + kind.as_str() + ))) + } + (WorkspaceOperationOutcome::Unknown, None) => { + operations + .mark_unknown(binding.tenant_id, operation_id) + .await?; + Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_id.to_string(), + }) + } + _ => Err(MoaError::ProviderError( + "workspace storage provider returned an inconsistent hydration result".to_string(), + )), + } + } + + /// Reinstalls the current trusted manifest before publishing an active lease. + pub(in crate::core) async fn reinstall_trusted_files_before_activation( + &self, + session: &SessionMeta, + worker_id: &str, + provider: &str, + hand: &HandHandle, + call_scope: ToolCallScope<'_>, + ) -> Result>> { + let provider_impl = + self.hands.providers.get(provider).ok_or_else(|| { + MoaError::ProviderError(format!("unknown hand provider: {provider}")) + })?; + let manifest_key = manifest_scope_key(session, Some(worker_id)); + loop { + call_scope.admit()?; + let manifest = self + .hands + .trusted_sandbox_files + .read() + .await + .get(&manifest_key) + .cloned(); + let Some(manifest) = manifest else { + return Ok(None); + }; + self.run_within_scope( + call_scope, + provider_impl.install_files(hand, manifest.files.as_ref()), + ) + .await?; + call_scope.admit()?; + if self + .hands + .trusted_sandbox_files + .read() + .await + .get(&manifest_key) + .is_some_and(|current| std::sync::Arc::ptr_eq(current, &manifest)) + { + return Ok(Some(manifest)); + } + } + } + + /// Records a trusted manifest installed on the exact preactivation hand. + pub(in crate::core) async fn remember_preactivation_manifest_install( + &self, + session: &SessionMeta, + worker_id: &str, + provider: &str, + cache_key: &HandProviderCacheKey, + active: &ActiveHand, + manifest: Option<&std::sync::Arc>, + ) { + let Some(manifest) = manifest else { + return; + }; + let manifest_key = manifest_scope_key(session, Some(worker_id)); + let binding_is_current = self + .hands + .active_hands + .read() + .await + .get(cache_key) + .is_some_and(|current| current == active); + if binding_is_current + && self + .hands + .trusted_sandbox_files + .read() + .await + .get(&manifest_key) + .is_some_and(|current| std::sync::Arc::ptr_eq(current, manifest)) + { + self.hands + .installed_files + .write() + .await + .entry(manifest_key) + .or_default() + .insert( + provider.to_string(), + InstalledManifestMarker { + manifest_identity: manifest.identity, + handle: active.handle.clone(), + generation: active.generation, + }, + ); + } + } +} diff --git a/crates/moa-hands/src/core/sandbox_workspace/repository/checkpoints.rs b/crates/moa-hands/src/core/sandbox_workspace/repository/checkpoints.rs index e401d7f5a..80b1367fe 100644 --- a/crates/moa-hands/src/core/sandbox_workspace/repository/checkpoints.rs +++ b/crates/moa-hands/src/core/sandbox_workspace/repository/checkpoints.rs @@ -628,9 +628,18 @@ async fn checkpoint_capacity_matches( AND reservation.expected_writer_epoch = $6 AND reservation.expected_instance_generation = $7 AND reservation.resource_dimension IN ('checkpoints', 'logical_bytes') - AND reservation.reservation_state IN ('pending', 'committed', 'reconciling') AND operation.operation_kind = $9 - AND operation.outcome_class IN ('not_sent', 'unknown') + AND ( + ( + operation.outcome_class IN ('not_sent', 'unknown') + AND reservation.reservation_state IN ('pending', 'reconciling') + ) + OR ( + operation.outcome_class = 'confirmed' + AND operation.confirmed_disposition = 'resource_present' + AND reservation.reservation_state = 'committed' + ) + ) "#, ) .bind(binding.tenant_id) diff --git a/crates/moa-hands/tests/hands_db/sandbox_workspace/dispatch_db.rs b/crates/moa-hands/tests/hands_db/sandbox_workspace/dispatch_db.rs index 11428d6d0..1b242b4a7 100644 --- a/crates/moa-hands/tests/hands_db/sandbox_workspace/dispatch_db.rs +++ b/crates/moa-hands/tests/hands_db/sandbox_workspace/dispatch_db.rs @@ -51,7 +51,7 @@ use moa_hands::{ HandLeaseWorkspaceAttachment, LeaseHandle, PostgresHandLeaseStore, }, sandbox_workspace::{ - capacity::PostgresWorkspaceCapacityRepository, + capacity::{ActiveHandCapacityRequest, PostgresWorkspaceCapacityRepository}, checkpoint::model::{CreateCheckpointRequest, PublishCheckpointCommitRequest}, model::{ ActivateHydratedWorkspaceRequest, CreateWorkspaceRequest, SandboxWorkspace, @@ -353,6 +353,21 @@ async fn hydration_and_checkpoint_commits_are_atomic_replay_safe_and_generation_ provisioning.provisioning_operation_id, HandHandle::local(PathBuf::from(format!("/tmp/{workspace_id}"))), ); + let active_capacity = ActiveHandCapacityRequest { + tenant_id, + workspace_id, + provider_account_id: account_id, + provider_account_generation: 1, + provisioning_operation_id: provisioning.provisioning_operation_id, + hand_lease_generation: provisioning.generation, + expected_writer_epoch: restoring.writer_epoch, + expected_instance_generation: restoring.instance_generation, + }; + let capacity = PostgresWorkspaceCapacityRepository::new(pool.clone()); + capacity + .reserve_active_hand(&active_capacity) + .await + .expect("reserve exact active hand before activation"); assert!( workspaces .activate_hydrated(ActivateHydratedWorkspaceRequest { @@ -363,6 +378,12 @@ async fn hydration_and_checkpoint_commits_are_atomic_replay_safe_and_generation_ .await .expect("activate exact hydrated lease") ); + assert!( + capacity + .commit_active_hand(&active_capacity) + .await + .expect("commit exact active hand after activation") + ); let mut active_workspace = workspaces .get(tenant_id, workspace_id) .await @@ -701,6 +722,20 @@ async fn hydration_and_checkpoint_commits_are_atomic_replay_safe_and_generation_ abandoned_state, ("failed".to_string(), "released".to_string()) ); + let failed_revival = sqlx::query( + "UPDATE moa.sandbox_workspace_checkpoints SET lifecycle_state='creating' \ + WHERE checkpoint_id=$1", + ) + .bind(abandoned_checkpoint_id) + .execute(&pool) + .await + .expect_err("failed checkpoint audit must not re-enter the live generation index"); + assert!( + failed_revival + .to_string() + .contains("failed sandbox checkpoint audit is immutable"), + "unexpected failed-checkpoint mutation error: {failed_revival}" + ); } else { assert_eq!(active_workspace.state, SandboxWorkspaceState::Ready); assert_eq!(active_lease.status, HandLeaseStatus::Destroyed); diff --git a/crates/moa-memory/AGENTS.md b/crates/moa-memory/AGENTS.md new file mode 100644 index 000000000..f5689e571 --- /dev/null +++ b/crates/moa-memory/AGENTS.md @@ -0,0 +1,11 @@ +# Memory Instructions + +Read `docs/04-memory-architecture.md`, `docs/07-context-pipeline.md`, and +`docs/15-architecture-policy.md`. Graph storage is canonical; vector and +sidecar indexes are derived. Preserve tenant/storage-partition scope, RLS through +`ScopedConn`, sensitivity handling, ingestion provenance, and graph/vector +write ordering. Do not move memory-owned types into `moa-core`. + +Use `fast-pr` for pure logic and `db-memory` for scoped storage behavior. Live +retrieval/provider evaluations remain ignored until their named authorization, +credential, and budget gates are explicitly granted. diff --git a/crates/moa-migrations/migrations/postgres/V000058__sandbox_workspaces.sql b/crates/moa-migrations/migrations/postgres/V000058__sandbox_workspaces.sql index 3ba7febdc..8dafbcb67 100644 --- a/crates/moa-migrations/migrations/postgres/V000058__sandbox_workspaces.sql +++ b/crates/moa-migrations/migrations/postgres/V000058__sandbox_workspaces.sql @@ -402,8 +402,6 @@ CREATE TABLE moa.sandbox_workspace_checkpoints ( UNIQUE (checkpoint_id, workspace_id, tenant_id), CONSTRAINT sandbox_workspace_checkpoints_identity_generation_key UNIQUE (checkpoint_id, workspace_id, tenant_id, generation), - CONSTRAINT sandbox_workspace_checkpoints_generation_key - UNIQUE (tenant_id, workspace_id, generation), CONSTRAINT sandbox_workspace_checkpoints_workspace_fk FOREIGN KEY (workspace_id, tenant_id) REFERENCES moa.sandbox_workspaces (workspace_id, tenant_id) ON DELETE RESTRICT, @@ -486,6 +484,12 @@ CREATE TABLE moa.sandbox_workspace_checkpoints ( ) ); +-- Failed attempts remain immutable audit rows but do not consume the next +-- committed revision number forever. +CREATE UNIQUE INDEX sandbox_workspace_checkpoints_generation_key + ON moa.sandbox_workspace_checkpoints (tenant_id, workspace_id, generation) + WHERE lifecycle_state IN ('creating', 'available', 'deleting', 'deleted'); + CREATE INDEX sandbox_workspace_checkpoints_gc_candidates_idx ON moa.sandbox_workspace_checkpoints ( tenant_id, retention_state, gc_retry_not_before, created_at, generation @@ -568,6 +572,10 @@ BEGIN RAISE EXCEPTION 'sandbox checkpoint tombstone is immutable' USING ERRCODE = 'check_violation'; END IF; + IF OLD.lifecycle_state = 'failed' AND NEW IS DISTINCT FROM OLD THEN + RAISE EXCEPTION 'failed sandbox checkpoint audit is immutable' + USING ERRCODE = 'check_violation'; + END IF; RETURN NEW; END; $$; diff --git a/crates/moa-migrations/tests/run_idempotency_db/hand_leases.rs b/crates/moa-migrations/tests/run_idempotency_db/hand_leases.rs index 0d1c02ca8..1c64623d6 100644 --- a/crates/moa-migrations/tests/run_idempotency_db/hand_leases.rs +++ b/crates/moa-migrations/tests/run_idempotency_db/hand_leases.rs @@ -925,6 +925,20 @@ async fn hand_storage_v58_requires_legacy_drain_and_installs_tenant_schema_db() ) .fetch_one(&pool) .await?; + let checkpoint_generation_index = sqlx::query_scalar::<_, String>( + "SELECT indexdef FROM pg_indexes \ + WHERE schemaname='moa' AND tablename='sandbox_workspace_checkpoints' \ + AND indexname='sandbox_workspace_checkpoints_generation_key'", + ) + .fetch_one(&pool) + .await?; + let checkpoint_generation_constraint = sqlx::query_scalar::<_, bool>( + "SELECT EXISTS (SELECT 1 FROM pg_constraint \ + WHERE conrelid='moa.sandbox_workspace_checkpoints'::regclass \ + AND conname='sandbox_workspace_checkpoints_generation_key')", + ) + .fetch_one(&pool) + .await?; pool.close().await; Ok::<_, Box>(( first_error, @@ -935,6 +949,8 @@ async fn hand_storage_v58_requires_legacy_drain_and_installs_tenant_schema_db() composite_fk, release_receipt_fks, release_receipt_retention_guards, + checkpoint_generation_index, + checkpoint_generation_constraint, )) } .await; @@ -948,6 +964,8 @@ async fn hand_storage_v58_requires_legacy_drain_and_installs_tenant_schema_db() composite_fk, release_receipt_fks, release_receipt_retention_guards, + checkpoint_generation_index, + checkpoint_generation_constraint, ) = outcome.expect("V58 should apply after legacy compute is drained"); assert!( first_error.contains("legacy hands remain live"), @@ -980,4 +998,14 @@ async fn hand_storage_v58_requires_legacy_drain_and_installs_tenant_schema_db() release_receipt_retention_guards, "task hand release receipts must reject post-archive writes and fence retention deletes" ); + assert!( + checkpoint_generation_index.contains( + "WHERE (lifecycle_state = ANY (ARRAY['creating'::text, 'available'::text, 'deleting'::text, 'deleted'::text]))" + ), + "only live checkpoint rows may reserve a committed generation: {checkpoint_generation_index}" + ); + assert!( + !checkpoint_generation_constraint, + "failed checkpoint attempts must not permanently consume the next committed generation" + ); } diff --git a/crates/moa-orchestrator/AGENTS.md b/crates/moa-orchestrator/AGENTS.md new file mode 100644 index 000000000..a5029aaaf --- /dev/null +++ b/crates/moa-orchestrator/AGENTS.md @@ -0,0 +1,13 @@ +# Orchestrator Instructions + +Read `docs/02-brain-orchestration.md`, `docs/05-session-event-log.md`, and +`docs/12-restate-architecture.md`. Keep this crate at the Restate transport, +authorization, workflow, and composition boundary; domain decisions and SQL +belong in their owning services or repositories. Preserve journal step names, +serialized state, replay behavior, and authorization-before-read ordering. + +Use `fast-pr`, `db-session`, or `db-memory` for focused checks. Deterministic +service and recovery profiles require their named repository fixture or E2E +harness, but not live authorization. Set `MOA_RUN_LIVE_E2E=1` only for an +explicitly live target; provider credentials do not belong in deterministic +lanes. diff --git a/crates/moa-orchestrator/src/workflows/execution_task_attempt.rs b/crates/moa-orchestrator/src/workflows/execution_task_attempt.rs index 28925264f..80bc2071b 100644 --- a/crates/moa-orchestrator/src/workflows/execution_task_attempt.rs +++ b/crates/moa-orchestrator/src/workflows/execution_task_attempt.rs @@ -1,6 +1,7 @@ //! One immutable, bounded task-attempt workflow per durable dispatch identity. mod active; +mod continuation; mod external; mod watchdog; mod yielding; diff --git a/crates/moa-orchestrator/src/workflows/execution_task_attempt/active.rs b/crates/moa-orchestrator/src/workflows/execution_task_attempt/active.rs index 62e5ca163..c6e9c83c9 100644 --- a/crates/moa-orchestrator/src/workflows/execution_task_attempt/active.rs +++ b/crates/moa-orchestrator/src/workflows/execution_task_attempt/active.rs @@ -1,216 +1,35 @@ -//! Typed exits produced by one bounded active task slice. +//! Typed exits and shared mechanics for one bounded active task slice. -use std::collections::{BTreeMap, BTreeSet}; - -use chrono::{DateTime, Utc}; +mod agent; +mod capability; +mod heartbeat; use moa_artifacts::execution_plan::{ - CapabilityReference, ExecutionFailureClass, ExecutionTaskOutcome, ExecutionTaskResult, - ExecutionUsage, -}; -use moa_core::{ - traits::SessionStore as _, - types::{ - action_policy::{ActionRuleScope, CapabilityProvenance}, - completion::{CompletionContent, ToolCallContent, ToolInvocation}, - context::ContextMessage, - identifiers::ToolCallId, - resource::ResourceBudget, - security::{ - SecurityCircuitOwner, SecurityCircuitStage, SecurityCircuitState, ToolCapabilityId, - }, - tools::{AsyncToolJobTerminalOutcome, IdempotencyClass, ToolAsyncMode}, - }, + CapabilityReference, ExecutionFailureClass, ExecutionTaskOutcome, }; +use moa_core::traits::SessionStore as _; use moa_execution::{ capability::{CapabilitySource, ExecutionCapability}, repository::task::{ NewTaskAttemptCheckpoint, TaskAttemptCheckpointKind, TaskAttemptCheckpointRecord, - TaskAttemptCheckpointWriteOutcome, TaskAttemptProgressOutcome, TaskAttemptRecord, + TaskAttemptCheckpointWriteOutcome, TaskAttemptRecord, }, schema::validate_instance, state::{LogicalTaskKind, completed_task_outcome, failed_task_outcome}, wire::{ExecutionTaskAttemptRequest, ExecutionToolDispatchRejection}, }; use restate_sdk::prelude::*; -use serde::{Deserialize, Serialize}; -use serde_json::{Value, json}; +use serde::Serialize; use uuid::Uuid; -use crate::{ - services::llm_gateway::{ - BoundedCompletionRequest, LLMCompletionAction, LLMCompletionOwner, LLMGatewayClient, - attach_completion_owner, completion_idempotency_key, - }, - tool_invocation::governed::{ - GovernedInvocationDisposition, GovernedInvocationOrigin, GovernedInvocationOutcome, - GovernedInvocationRequest, invoke_governed_tool, - }, - workflows::{ - durable_utc_now, - errors::moa_error_to_handler_error, - execution_task_attempt::{ - ExecutionTaskAttemptImpl, capability_tool_name, task_attempt_fence, - }, +use crate::workflows::{ + durable_utc_now, + errors::moa_error_to_handler_error, + execution_task_attempt::{ + ExecutionTaskAttemptImpl, continuation::TaskAttemptContinuation, task_attempt_fence, }, }; -/// Current durable schema for a bounded task-agent continuation. -pub(super) const TASK_ATTEMPT_CONTINUATION_SCHEMA_VERSION: u32 = 1; - -/// Maximum canonical continuation payload accepted by persistence. -pub(super) const MAX_TASK_ATTEMPT_CONTINUATION_BYTES: usize = 1024 * 1024; - -/// Canonical state needed to resume an agent without replaying an external effect. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub(super) struct TaskAttemptContinuation { - /// Durable schema version. - pub schema_version: u32, - /// Exact bounded execution state. - pub state: TaskAttemptContinuationState, - /// Exact storage-only action-review resolution consumed by the next attempt. - pub review_resolution: Option, - /// Exact terminal provider outcome consumed by a resumed agent external effect. - pub external_job_resolution: Option, - /// Release receipt that proves sandbox compute is asleep before this wait was published. - pub workspace_release_receipt_id: Option, -} - -/// Supported bounded continuation points. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] -pub(super) enum TaskAttemptContinuationState { - /// Task-local agent state after a complete model/tool boundary. - Agent { - /// Complete bounded conversation required by the next model turn. - messages: Vec, - /// Zero-based model turn to execute next. - next_turn: u32, - /// Cumulative durable task usage. - usage: moa_artifacts::execution_plan::ExecutionUsage, - /// Prompt-injection circuit state owned by this exact task generation. - security_circuit: SecurityCircuitState, - /// Capabilities fenced by the persisted circuit. - disabled_capabilities: std::collections::BTreeMap, - /// Exact effect waiting on a storage-only action review, when present. - pending_review: Option>, - /// Model-emitted tool effects not yet dispatched by a bounded slice. - pending_tool_calls: Vec, - /// Exact agent tool invocation currently owned by an asynchronous provider job. - pending_external: Option, - }, - /// Direct capability effect waiting on a storage-only action review. - CapabilityReview { - /// Exact reviewed effect; resumption consumes its persisted resolution. - pending_review: PendingReviewedToolInvocation, - /// Cumulative durable task usage. - usage: moa_artifacts::execution_plan::ExecutionUsage, - }, - /// Direct async-capable effect reserved before its provider start. - CapabilityExternalStart { - /// Stable tool-call identity reused if recovery proves the provider did not start. - tool_id: ToolCallId, - /// Cumulative durable task usage before provider dispatch. - usage: moa_artifacts::execution_plan::ExecutionUsage, - }, -} - -/// Reviewed provider effect that must never be reconstructed from a fresh model turn. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub(super) struct PendingReviewedToolInvocation { - /// Stable action-review identity. - pub review_uid: Uuid, - /// Exact durable review expiry returned by action-review admission. - pub expires_at: chrono::DateTime, - /// Exact provider invocation accepted by policy. - pub invocation: ToolInvocation, - /// Compiler/catalog-pinned replay semantics for watchdog classification. - pub effect_idempotency: IdempotencyClass, -} - -/// Agent effect that was durably handed to an asynchronous provider. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub(super) struct PendingExternalToolInvocation { - /// Stable MOA external-job identity bound before sandbox release. - pub external_job_uid: Option, - /// Exact model-emitted invocation awaiting the terminal provider result. - pub invocation: ToolInvocation, - /// Compiler/catalog-pinned replay semantics. - pub effect_idempotency: IdempotencyClass, -} - -struct AgentTaskSpec<'a> { - instructions: &'a str, - skill_refs: &'a [moa_artifacts::reference::ArtifactRef], - capability_refs: &'a [CapabilityReference], - max_turns: u32, -} - -struct AgentPending { - review: Option, - tool_calls: Vec, - external: Option, -} - -impl TaskAttemptContinuation { - /// Returns the exact action-review identity carried by a parked continuation. - pub(super) const fn pending_review_uid(&self) -> Option { - match &self.state { - TaskAttemptContinuationState::Agent { pending_review, .. } => match pending_review { - Some(pending) => Some(pending.review_uid), - None => None, - }, - TaskAttemptContinuationState::CapabilityReview { pending_review, .. } => { - Some(pending_review.review_uid) - } - TaskAttemptContinuationState::CapabilityExternalStart { .. } => None, - } - } - - /// Binds the deterministic MOA external-job identity before checkpoint persistence. - pub(super) fn bind_external_job(&mut self, external_job_uid: Uuid) -> Result<(), String> { - let TaskAttemptContinuationState::Agent { - pending_external: Some(pending), - .. - } = &mut self.state - else { - return Err("agent external continuation is missing its pending effect".to_string()); - }; - if pending - .external_job_uid - .is_some_and(|current| current != external_job_uid) - { - return Err("agent external continuation is bound to another job".to_string()); - } - pending.external_job_uid = Some(external_job_uid); - Ok(()) - } - - /// Serializes and enforces the hard continuation-size bound before any DB write. - pub(super) fn to_bounded_json(&self) -> Result { - if self.schema_version != TASK_ATTEMPT_CONTINUATION_SCHEMA_VERSION { - return Err(format!( - "unsupported task continuation schema version {}", - self.schema_version - )); - } - let bytes = serde_json::to_vec(self) - .map_err(|error| format!("serialize task continuation: {error}"))?; - if bytes.len() > MAX_TASK_ATTEMPT_CONTINUATION_BYTES { - return Err(format!( - "task continuation is {} bytes; maximum is {} and the task must be decomposed or replanned", - bytes.len(), - MAX_TASK_ATTEMPT_CONTINUATION_BYTES - )); - } - serde_json::from_slice(&bytes) - .map_err(|error| format!("decode canonical task continuation: {error}")) - } -} - /// Complete set of boundaries at which an active task workflow must return. #[derive(Clone, Debug, PartialEq)] pub(super) enum ActiveTaskAttemptExit { @@ -283,7 +102,7 @@ pub(super) async fn execute_task_attempt( Ok(ActiveTaskAttemptExit::Outcome(outcome)) } LogicalTaskKind::Capability { reference } => { - execute_direct_capability( + capability::execute_direct_capability( workflow, ctx, request, @@ -299,12 +118,12 @@ pub(super) async fn execute_task_attempt( capability_refs, max_turns, } => { - execute_agent_turn( + agent::execute_agent_turn( workflow, ctx, request, started, - AgentTaskSpec { + agent::AgentTaskSpec { instructions, skill_refs, capability_refs, @@ -319,12 +138,12 @@ pub(super) async fn execute_task_attempt( max_turns, .. } => { - execute_agent_turn( + agent::execute_agent_turn( workflow, ctx, request, started, - AgentTaskSpec { + agent::AgentTaskSpec { instructions, skill_refs: &[], capability_refs: &[], @@ -343,177 +162,6 @@ pub(super) async fn execute_task_attempt( } } -/// Durable step boundary at which an active attempt reports progress. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum AttemptHeartbeat { - /// One model completion is about to start. The bounded gateway budget cannot outlive the - /// attempt deadline, so the persisted stall window covers that exact call. - ModelTurnStart, - /// One model turn returned, so the following tool dispatch starts its own stall window. - ModelTurn, - /// One governed tool invocation is about to start with its declared stall bound. - ToolCallStart { bound: Option }, - /// One governed tool invocation returned, so sandbox release and continuation persistence - /// start their own stall window. - ToolCall, -} - -impl AttemptHeartbeat { - /// Deterministic journal step name for this boundary. - const fn observation_step(self) -> &'static str { - match self { - Self::ModelTurnStart => "task_attempt_model_turn_start_progress_at", - Self::ModelTurn => "task_attempt_model_turn_progress_at", - Self::ToolCallStart { .. } => "task_attempt_tool_call_start_progress_at", - Self::ToolCall => "task_attempt_tool_call_progress_at", - } - } - - /// Deterministic journal step name for the persisted heartbeat. - const fn write_step(self) -> &'static str { - match self { - Self::ModelTurnStart => "record_task_attempt_model_turn_start_progress", - Self::ModelTurn => "record_task_attempt_model_turn_progress", - Self::ToolCallStart { .. } => "record_task_attempt_tool_call_start_progress", - Self::ToolCall => "record_task_attempt_tool_call_progress", - } - } - - /// Upper bound of the step this boundary opens, when that step declares one. - /// - /// Post-return boundaries clear the bound back to the configured heartbeat floor. - fn step_bound_seconds( - self, - request: &ExecutionTaskAttemptRequest, - observed_at: DateTime, - ) -> Option { - match self { - Self::ModelTurnStart => Some(AttemptStepBound::UntilAttemptDeadline) - .and_then(|bound| bound.seconds(request, observed_at)), - Self::ToolCallStart { bound } => { - bound.and_then(|bound| bound.seconds(request, observed_at)) - } - Self::ModelTurn | Self::ToolCall => None, - } - } -} - -/// Returns the bound to record for the capability step this attempt is about to dispatch. -/// -/// An external provider start remains bounded by the attempt deadline so its recovery trigger, -/// rather than the task watchdog, retains authority over an ambiguous start. -fn capability_step_bound( - requires_sandbox: bool, - async_mode: &ToolAsyncMode, - tool_call: &ToolCallContent, -) -> Option { - if requires_sandbox || matches!(async_mode, ToolAsyncMode::MayReturnExternalJob { .. }) { - return Some(AttemptStepBound::UntilAttemptDeadline); - } - moa_hands::tools::bash::declared_tool_step_bound( - &tool_call.invocation.name, - &tool_call.invocation.input, - ) - .and_then(|bound| u32::try_from(bound.as_secs()).ok()) - .map(AttemptStepBound::Declared) -} - -/// Upper bound a dispatching step declares for itself. -/// -/// `UntilAttemptDeadline` is resolved against the journaled heartbeat instant rather than a -/// fresh clock read, because a workflow that reads the wall clock outside the journal -/// produces a different value on replay. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum AttemptStepBound { - /// The step named its own ceiling in seconds. - Declared(u32), - /// The step runs until the attempt deadline and must not be cut short before it. - UntilAttemptDeadline, -} - -impl AttemptStepBound { - /// Resolves this bound to seconds against one journaled observation instant. - fn seconds( - self, - request: &ExecutionTaskAttemptRequest, - observed_at: DateTime, - ) -> Option { - match self { - Self::Declared(seconds) => Some(seconds), - // Rounded up so a sub-second remainder still outlasts the deadline it covers. - Self::UntilAttemptDeadline => u32::try_from( - request - .attempt_deadline_at - .signed_duration_since(observed_at) - .num_seconds() - .saturating_add(1), - ) - .ok() - .filter(|seconds| *seconds > 0), - } - } -} - -/// Advances the active attempt's durable progress clock and returns whether it still owns it. -/// -/// The exact dispatch fence prevents a parked, superseded, or settled attempt from regaining -/// dispatch authority. The observation time is journaled for replay stability. -async fn record_attempt_heartbeat( - workflow: &ExecutionTaskAttemptImpl, - ctx: &WorkflowContext<'_>, - request: &ExecutionTaskAttemptRequest, - boundary: AttemptHeartbeat, -) -> Result { - let observed_at = durable_utc_now(ctx, boundary.observation_step()).await?; - let repository = workflow.repository.clone(); - let fence = task_attempt_fence(request); - Ok(ctx - .run(|| async move { - repository - .record_task_attempt_progress( - fence, - observed_at, - boundary.step_bound_seconds(request, observed_at), - ) - .await - .map(|outcome| Json::from(attempt_progress_retains_ownership(outcome))) - .map_err(crate::workflows::errors::execution_error_to_handler_error) - }) - .name(boundary.write_step()) - .await? - .into_inner()) -} - -const fn attempt_progress_retains_ownership(outcome: TaskAttemptProgressOutcome) -> bool { - matches!( - outcome, - TaskAttemptProgressOutcome::Applied | TaskAttemptProgressOutcome::Replayed - ) -} - -/// Records the exact capability bound and confirms ownership before provider dispatch. -async fn begin_capability_dispatch( - workflow: &ExecutionTaskAttemptImpl, - ctx: &WorkflowContext<'_>, - request: &ExecutionTaskAttemptRequest, - capability: &ExecutionCapability, - tool_call: &ToolCallContent, -) -> Result { - record_attempt_heartbeat( - workflow, - ctx, - request, - AttemptHeartbeat::ToolCallStart { - bound: capability_step_bound( - capability.requires_sandbox, - &capability.async_mode, - tool_call, - ), - }, - ) - .await -} - async fn persist_external_start_checkpoint( workflow: &ExecutionTaskAttemptImpl, ctx: &WorkflowContext<'_>, @@ -555,1093 +203,6 @@ async fn persist_external_start_checkpoint( .into_inner()) } -async fn execute_direct_capability( - workflow: &ExecutionTaskAttemptImpl, - ctx: &WorkflowContext<'_>, - request: &ExecutionTaskAttemptRequest, - started: &TaskAttemptRecord, - reference: &CapabilityReference, - continuation: Option<&TaskAttemptContinuation>, -) -> Result { - let capability = find_capability(&started.run, reference)?; - let (tool_id, mut usage) = match continuation { - Some( - continuation @ TaskAttemptContinuation { - state: TaskAttemptContinuationState::CapabilityReview { .. }, - .. - }, - ) => return resume_reviewed_capability(capability, continuation), - Some(TaskAttemptContinuation { - state: TaskAttemptContinuationState::CapabilityExternalStart { tool_id, usage }, - review_resolution: None, - external_job_resolution: None, - .. - }) => (*tool_id, usage.clone()), - Some(_) => { - return Err(TerminalError::new( - "direct capability received an incompatible continuation", - ) - .into()); - } - None => { - if let Err(error) = validate_instance( - &capability.input_schema, - &started.task.input, - "execution_task.capability_input", - ) { - return Ok(ActiveTaskAttemptExit::Outcome(failed_task_outcome( - ExecutionFailureClass::InvalidInput, - error.to_string(), - started.task.actual.clone(), - ))); - } - ( - ToolCallId(Uuid::new_v5( - &request.dispatch_uid, - format!("task-capability:{}", started.task.generation).as_bytes(), - )), - started.task.actual.clone(), - ) - } - }; - let session = load_session(workflow, ctx, &started.run, &started.task).await?; - let tool_name = capability_tool_name(capability)?; - let tool_call = ToolCallContent { - invocation: ToolInvocation { - id: Some(tool_id.to_string()), - name: tool_name.clone(), - input: started.task.input.clone(), - }, - provider_metadata: None, - }; - let allowed_tools = BTreeSet::from([tool_name]); - let provenance = CapabilityProvenance { - kind: Some(capability_source_kind(&capability.source).to_string()), - id: Some(format!( - "{}@{}", - capability.reference.name, capability.reference.version - )), - step_id: Some(started.task.node_id.clone()), - }; - if matches!( - capability.async_mode, - ToolAsyncMode::MayReturnExternalJob { .. } - ) { - let provisional = TaskAttemptContinuation { - schema_version: TASK_ATTEMPT_CONTINUATION_SCHEMA_VERSION, - state: TaskAttemptContinuationState::CapabilityExternalStart { - tool_id, - usage: usage.clone(), - }, - review_resolution: None, - external_job_resolution: None, - workspace_release_receipt_id: None, - }; - if !persist_external_start_checkpoint( - workflow, - ctx, - request, - started, - TaskAttemptCheckpointKind::CapabilityExternalStart, - &provisional, - ) - .await? - { - return Ok(ActiveTaskAttemptExit::OwnershipLost); - } - } - if !begin_capability_dispatch(workflow, ctx, request, capability, &tool_call).await? { - return Ok(ActiveTaskAttemptExit::OwnershipLost); - } - let governed = invoke_governed_tool( - ctx, - GovernedInvocationRequest { - session: &session, - identity: &started.run.admitted_identity, - session_id: started.run.session_id, - tool_id, - tool_call: &tool_call, - allowed_tools: &allowed_tools, - expected_tool_contract_revision: Some(&capability.contract_revision), - active_canary: None, - trusted_sandbox_manifest: None, - origin: GovernedInvocationOrigin::ExecutionTask { - run_uid: started.run.run_uid, - task_uid: started.task.task_id.as_uuid(), - generation: started.task.generation, - attempt_generation: request.attempt_generation, - }, - capability_provenance: Some(&provenance), - capability_policy_context: Some(&capability.policy_context), - resource_budget: ResourceBudget::until(request.attempt_deadline_at), - }, - &workflow.session_limits, - workflow.session_store.clone(), - workflow.channel_adapters.as_ref(), - ) - .await?; - if !record_attempt_heartbeat(workflow, ctx, request, AttemptHeartbeat::ToolCall).await? { - return Ok(ActiveTaskAttemptExit::OwnershipLost); - } - usage.tool_calls = usage.tool_calls.saturating_add(1); - classify_capability_outcome(capability, governed, usage) -} - -fn classify_capability_outcome( - capability: &ExecutionCapability, - outcome: GovernedInvocationOutcome, - mut usage: ExecutionUsage, -) -> Result { - match outcome { - GovernedInvocationOutcome::Completed(result) - if result.disposition == GovernedInvocationDisposition::ReviewPending => - { - let review = result.review.ok_or_else(|| { - TerminalError::new( - "review-pending governed result is missing durable review identity", - ) - })?; - Ok(ActiveTaskAttemptExit::ReviewPending { - continuation: TaskAttemptContinuation { - schema_version: TASK_ATTEMPT_CONTINUATION_SCHEMA_VERSION, - state: TaskAttemptContinuationState::CapabilityReview { - pending_review: PendingReviewedToolInvocation { - review_uid: review.review_uid, - expires_at: review.expires_at, - invocation: result.invocation, - effect_idempotency: capability.idempotency_class, - }, - usage, - }, - review_resolution: None, - external_job_resolution: None, - workspace_release_receipt_id: None, - }, - }) - } - GovernedInvocationOutcome::Completed(result) => { - usage.retrieved_bytes = usage.retrieved_bytes.saturating_add(serialized_len( - &result.output.safe_output.structured_payload(), - )); - let task_outcome = if result.output.is_error() { - if capability.idempotency_class == IdempotencyClass::Idempotent { - failed_task_outcome( - ExecutionFailureClass::Retryable, - result.output.safe_output.to_text(), - usage, - ) - } else if capability.action_class - != moa_core::types::action_policy::ActionClass::Read - { - ExecutionTaskOutcome { - schema_version: 1, - usage, - result: ExecutionTaskResult::UnknownOutcome { - message: format!( - "non-idempotent side effect returned an error after possible commit: {}", - result.output.safe_output.to_text() - ), - }, - } - } else { - failed_task_outcome( - ExecutionFailureClass::Terminal, - result.output.safe_output.to_text(), - usage, - ) - } - } else { - let value = result - .output - .safe_output - .structured_payload() - .cloned() - .unwrap_or_else(|| Value::String(result.output.safe_output.to_text())); - if let Err(error) = validate_instance( - &capability.output_schema, - &value, - "execution_task.capability_output", - ) { - if capability.action_class == moa_core::types::action_policy::ActionClass::Read - { - failed_task_outcome( - ExecutionFailureClass::InvalidOutput, - error.to_string(), - usage, - ) - } else { - ExecutionTaskOutcome { - schema_version: 1, - usage, - result: ExecutionTaskResult::UnknownOutcome { - message: format!( - "side effect returned invalid output after possible commit: {error}" - ), - }, - } - } - } else { - completed_task_outcome(value, usage) - } - }; - Ok(ActiveTaskAttemptExit::Outcome(task_outcome)) - } - GovernedInvocationOutcome::ExternalJob { - external_job_uid, .. - } => Ok(ActiveTaskAttemptExit::ExternalJob { - external_job_uid, - continuation: None, - }), - GovernedInvocationOutcome::UnknownOutcome { message, .. } => { - Ok(ActiveTaskAttemptExit::Outcome(ExecutionTaskOutcome { - schema_version: 1, - usage, - result: ExecutionTaskResult::UnknownOutcome { message }, - })) - } - GovernedInvocationOutcome::NotDispatched { reason, .. } => { - Ok(ActiveTaskAttemptExit::Outcome(failed_task_outcome( - ExecutionFailureClass::Terminal, - execution_dispatch_rejection_message(reason), - usage, - ))) - } - GovernedInvocationOutcome::Delegation { .. } => { - Err(TerminalError::new("execution tasks cannot invoke delegation capabilities").into()) - } - } -} - -fn resume_reviewed_capability( - capability: &ExecutionCapability, - continuation: &TaskAttemptContinuation, -) -> Result { - let TaskAttemptContinuationState::CapabilityReview { - pending_review: _, - usage, - } = &continuation.state - else { - return Err(TerminalError::new( - "direct capability received an incompatible agent continuation", - ) - .into()); - }; - let resolution = continuation.review_resolution.as_ref().ok_or_else(|| { - TerminalError::new("reviewed capability continuation has no durable resolution") - })?; - let exit = match resolution { - moa_execution::wire::ExecutionActionReviewResolution::Completed { tool_output } => { - match serde_json::from_value::( - tool_output.clone(), - ) { - Ok(output) => ActiveTaskAttemptExit::Outcome(capability_output_outcome( - capability, - output, - usage.clone(), - )), - Err(error) => ActiveTaskAttemptExit::Outcome(ExecutionTaskOutcome { - schema_version: 1, - usage: usage.clone(), - result: ExecutionTaskResult::UnknownOutcome { - message: format!( - "reviewed capability returned invalid output after possible commit: {error}" - ), - }, - }), - } - } - moa_execution::wire::ExecutionActionReviewResolution::ExternalJob { - external_job_uid, - .. - } => ActiveTaskAttemptExit::ExternalJob { - external_job_uid: *external_job_uid, - continuation: None, - }, - moa_execution::wire::ExecutionActionReviewResolution::Failed { class, message } => { - ActiveTaskAttemptExit::Outcome(ExecutionTaskOutcome { - schema_version: 1, - usage: usage.clone(), - result: ExecutionTaskResult::Failed { - class: class.clone(), - message: message.clone(), - }, - }) - } - moa_execution::wire::ExecutionActionReviewResolution::UnknownOutcome { message } => { - ActiveTaskAttemptExit::Outcome(ExecutionTaskOutcome { - schema_version: 1, - usage: usage.clone(), - result: ExecutionTaskResult::UnknownOutcome { - message: message.clone(), - }, - }) - } - moa_execution::wire::ExecutionActionReviewResolution::NotDispatched { reason } => { - ActiveTaskAttemptExit::Outcome(failed_task_outcome( - ExecutionFailureClass::Terminal, - execution_dispatch_rejection_message(*reason), - usage.clone(), - )) - } - moa_execution::wire::ExecutionActionReviewResolution::Denied { reason } => { - ActiveTaskAttemptExit::Outcome(failed_task_outcome( - ExecutionFailureClass::AuthorizationDenied, - reason.clone(), - usage.clone(), - )) - } - moa_execution::wire::ExecutionActionReviewResolution::TimedOut { reason } => { - ActiveTaskAttemptExit::Outcome(failed_task_outcome( - ExecutionFailureClass::DeadlineExceeded, - reason.clone(), - usage.clone(), - )) - } - }; - Ok(exit) -} - -fn capability_output_outcome( - capability: &ExecutionCapability, - output: moa_core::types::tools::SecuredToolOutput, - usage: ExecutionUsage, -) -> ExecutionTaskOutcome { - if output.is_error() { - if capability.idempotency_class == IdempotencyClass::Idempotent { - return failed_task_outcome( - ExecutionFailureClass::Retryable, - output.safe_output.to_text(), - usage, - ); - } - if capability.action_class != moa_core::types::action_policy::ActionClass::Read { - return ExecutionTaskOutcome { - schema_version: 1, - usage, - result: ExecutionTaskResult::UnknownOutcome { - message: format!( - "non-idempotent side effect returned an error after possible commit: {}", - output.safe_output.to_text() - ), - }, - }; - } - return failed_task_outcome( - ExecutionFailureClass::Terminal, - output.safe_output.to_text(), - usage, - ); - } - let value = output - .safe_output - .structured_payload() - .cloned() - .unwrap_or_else(|| Value::String(output.safe_output.to_text())); - if let Err(error) = validate_instance( - &capability.output_schema, - &value, - "execution_task.capability_output", - ) { - if capability.action_class == moa_core::types::action_policy::ActionClass::Read { - failed_task_outcome( - ExecutionFailureClass::InvalidOutput, - error.to_string(), - usage, - ) - } else { - ExecutionTaskOutcome { - schema_version: 1, - usage, - result: ExecutionTaskResult::UnknownOutcome { - message: format!( - "side effect returned invalid output after possible commit: {error}" - ), - }, - } - } - } else { - completed_task_outcome(value, usage) - } -} - -async fn execute_agent_turn( - workflow: &ExecutionTaskAttemptImpl, - ctx: &WorkflowContext<'_>, - request: &ExecutionTaskAttemptRequest, - started: &TaskAttemptRecord, - spec: AgentTaskSpec<'_>, - continuation: Option<&TaskAttemptContinuation>, -) -> Result { - let AgentTaskSpec { - instructions, - skill_refs, - capability_refs, - max_turns, - } = spec; - if max_turns == 0 { - return Ok(ActiveTaskAttemptExit::Outcome(failed_task_outcome( - ExecutionFailureClass::InvalidInput, - "agent max_turns must be positive".to_string(), - started.task.actual.clone(), - ))); - } - let mut capabilities = BTreeMap::::new(); - for reference in capability_refs { - let capability = find_capability(&started.run, reference)?; - let tool_name = capability_tool_name(capability)?; - if capabilities.insert(tool_name.clone(), capability).is_some() { - return Err(TerminalError::new(format!( - "task-local agent has ambiguous capability tool `{tool_name}`" - )) - .into()); - } - } - let circuit_owner = SecurityCircuitOwner::ExecutionTask { - run_uid: started.run.run_uid, - task_uid: started.task.task_id.as_uuid(), - generation: started.task.generation, - }; - let ( - mut messages, - mut next_turn, - mut usage, - mut security_circuit, - mut disabled_capabilities, - mut pending_review, - mut pending_tool_calls, - mut pending_external, - ) = match continuation { - Some(TaskAttemptContinuation { - state: - TaskAttemptContinuationState::Agent { - messages, - next_turn, - usage, - security_circuit, - disabled_capabilities, - pending_review, - pending_tool_calls, - pending_external, - }, - .. - }) => ( - messages.clone(), - *next_turn, - usage.clone(), - security_circuit.clone(), - disabled_capabilities.clone(), - pending_review.as_deref().cloned(), - pending_tool_calls.clone(), - pending_external.clone(), - ), - Some(_) => { - return Err(TerminalError::new( - "task-local agent received an incompatible continuation", - ) - .into()); - } - None => { - let skills = load_pinned_skills(workflow, ctx, started, skill_refs).await?; - let mut circuit = SecurityCircuitState::default(); - circuit.adopt_owner(&circuit_owner); - ( - vec![ - ContextMessage::system(agent_system_prompt(instructions, &skills)), - ContextMessage::user( - json!({ - "resolved_input": started.task.input, - "resume_inputs": started.task.resume_input_history, - }) - .to_string(), - ), - ], - 0, - started.task.actual.clone(), - circuit, - BTreeMap::new(), - None, - Vec::new(), - None, - ) - } - }; - security_circuit.adopt_owner(&circuit_owner); - - if let Some(external) = pending_external.take() { - if let Some(external_job_uid) = external.external_job_uid { - let resolution = continuation - .and_then(|continuation| continuation.external_job_resolution.as_ref()) - .ok_or_else(|| { - TerminalError::new("agent external continuation has no terminal resolution") - })?; - let tool_use_id = external - .invocation - .id - .clone() - .unwrap_or_else(|| format!("external-job-{external_job_uid}")); - match resolution { - AsyncToolJobTerminalOutcome::Completed { output } => { - messages.push(ContextMessage::tool_result( - tool_use_id, - output.to_string(), - None, - )); - } - AsyncToolJobTerminalOutcome::Failed { error } => { - messages.push(ContextMessage::tool_result( - tool_use_id, - format!("external tool failed: {error}"), - None, - )); - } - AsyncToolJobTerminalOutcome::Cancelled => { - messages.push(ContextMessage::tool_result( - tool_use_id, - "external tool was cancelled", - None, - )); - } - AsyncToolJobTerminalOutcome::UnknownOutcome { error } => { - return Ok(ActiveTaskAttemptExit::Outcome(ExecutionTaskOutcome { - schema_version: 1, - usage, - result: ExecutionTaskResult::UnknownOutcome { - message: format!("external agent effect outcome is unknown: {error}"), - }, - })); - } - } - } else { - // Provider start recovery proved NotStarted and re-admitted the exact continuation. - // Reinsert the original model invocation so its stable tool id/idempotency key is - // dispatched again without asking the model or repeating prior tool effects. - pending_tool_calls.insert(0, external.invocation); - } - } - - if let Some(reviewed) = pending_review.take() { - let resolution = continuation - .and_then(|continuation| continuation.review_resolution.as_ref()) - .ok_or_else(|| { - TerminalError::new("agent review continuation has no durable resolution") - })?; - match resolution { - moa_execution::wire::ExecutionActionReviewResolution::Completed { tool_output } => { - let output = serde_json::from_value::( - tool_output.clone(), - ) - .map_err(|error| { - TerminalError::new(format!("decode reviewed agent capability output: {error}")) - })?; - append_agent_tool_output(&mut messages, &reviewed.invocation, &output); - usage.retrieved_bytes = usage - .retrieved_bytes - .saturating_add(serialized_len(&output.safe_output.structured_payload())); - } - moa_execution::wire::ExecutionActionReviewResolution::ExternalJob { - external_job_uid, - .. - } => { - return Ok(ActiveTaskAttemptExit::ExternalJob { - external_job_uid: *external_job_uid, - continuation: Some(agent_continuation( - messages, - next_turn, - usage, - security_circuit, - disabled_capabilities, - AgentPending { - review: None, - tool_calls: pending_tool_calls, - external: Some(PendingExternalToolInvocation { - external_job_uid: None, - invocation: reviewed.invocation, - effect_idempotency: reviewed.effect_idempotency, - }), - }, - )), - }); - } - moa_execution::wire::ExecutionActionReviewResolution::Failed { class, message } => { - return Ok(ActiveTaskAttemptExit::Outcome(failed_task_outcome( - class.clone(), - message.clone(), - usage, - ))); - } - moa_execution::wire::ExecutionActionReviewResolution::UnknownOutcome { message } => { - return Ok(ActiveTaskAttemptExit::Outcome(ExecutionTaskOutcome { - schema_version: 1, - usage, - result: ExecutionTaskResult::UnknownOutcome { - message: message.clone(), - }, - })); - } - moa_execution::wire::ExecutionActionReviewResolution::NotDispatched { reason } => { - return Ok(ActiveTaskAttemptExit::Outcome(failed_task_outcome( - ExecutionFailureClass::Terminal, - execution_dispatch_rejection_message(*reason), - usage, - ))); - } - moa_execution::wire::ExecutionActionReviewResolution::Denied { reason } => { - return Ok(ActiveTaskAttemptExit::Outcome(failed_task_outcome( - ExecutionFailureClass::AuthorizationDenied, - reason.clone(), - usage, - ))); - } - moa_execution::wire::ExecutionActionReviewResolution::TimedOut { reason } => { - return Ok(ActiveTaskAttemptExit::Outcome(failed_task_outcome( - ExecutionFailureClass::DeadlineExceeded, - reason.clone(), - usage, - ))); - } - } - } - - if pending_tool_calls.is_empty() { - if next_turn >= max_turns { - return Ok(ActiveTaskAttemptExit::Outcome(failed_task_outcome( - ExecutionFailureClass::Terminal, - format!("task-local agent exhausted max_turns={max_turns}"), - usage, - ))); - } - let mut completion = moa_core::types::completion::CompletionRequest { - model: None, - messages: messages.clone(), - tools: capabilities - .iter() - .filter(|(name, _)| !disabled_capabilities.contains_key(*name)) - .map(|(name, capability)| agent_tool_schema(name, capability)) - .collect(), - max_output_tokens: None, - temperature: None, - response_format: None, - native_web_search: Default::default(), - metadata: std::collections::HashMap::new(), - }; - let owner = LLMCompletionOwner::execution_task_attempt(request.dispatch_uid); - attach_completion_owner(&mut completion, &owner); - if !record_attempt_heartbeat(workflow, ctx, request, AttemptHeartbeat::ModelTurnStart) - .await? - { - return Ok(ActiveTaskAttemptExit::OwnershipLost); - } - let response = crate::restate_identity::replay_safe_request( - ctx.service_client::() - .complete_bounded(Json::from(BoundedCompletionRequest { - request: completion, - budget: ResourceBudget::until(request.attempt_deadline_at), - })) - .idempotency_key(completion_idempotency_key( - ctx.invocation_id(), - LLMCompletionAction::ExecutionTaskModel { - generation: started.task.generation, - turn: next_turn, - }, - )), - ) - .call() - .await? - .into_inner(); - if !record_attempt_heartbeat(workflow, ctx, request, AttemptHeartbeat::ModelTurn).await? { - return Ok(ActiveTaskAttemptExit::OwnershipLost); - } - usage.tokens = usage - .tokens - .saturating_add(response.usage.total_input_tokens() as u64) - .saturating_add(response.usage.output_tokens as u64); - usage.cost_microusd = usage.cost_microusd.saturating_add( - moa_providers::pricing_for_model(response.model.as_str()) - .map(|pricing| pricing.cost_micros(&response.usage)) - .unwrap_or_default(), - ); - let tool_calls = response - .content - .iter() - .filter_map(|content| match content { - CompletionContent::ToolCall(call) => Some(call.invocation.clone()), - CompletionContent::Text(_) | CompletionContent::ProviderToolResult { .. } => None, - }) - .collect::>(); - if tool_calls.is_empty() { - let outcome = moa_execution::state::parse_agent_task_outcome(&response.text, usage); - if matches!(outcome.result, ExecutionTaskResult::NeedsInput { .. }) { - messages.push(ContextMessage::assistant_with_thought_signature( - response.text, - response.thought_signature, - )); - return Ok(ActiveTaskAttemptExit::InputPending { - continuation: agent_continuation( - messages, - next_turn.saturating_add(1), - outcome.usage.clone(), - security_circuit, - disabled_capabilities, - AgentPending { - review: None, - tool_calls: pending_tool_calls, - external: pending_external, - }, - ), - outcome, - }); - } - return Ok(ActiveTaskAttemptExit::Outcome(outcome)); - } - for (index, invocation) in tool_calls.iter().cloned().enumerate() { - messages.push(ContextMessage::assistant_tool_call_with_thought_signature( - invocation, - if index == 0 { - response.text.clone() - } else { - String::new() - }, - (index == 0) - .then(|| response.thought_signature.clone()) - .flatten(), - )); - } - pending_tool_calls = tool_calls; - next_turn = next_turn.saturating_add(1); - return Ok(ActiveTaskAttemptExit::Continue { - continuation: agent_continuation( - messages, - next_turn, - usage, - security_circuit, - disabled_capabilities, - AgentPending { - review: None, - tool_calls: pending_tool_calls, - external: pending_external, - }, - ), - }); - } - - let invocation = pending_tool_calls.remove(0); - let capability = capabilities.get(&invocation.name).copied().ok_or_else(|| { - TerminalError::new(format!( - "agent emitted undeclared capability `{}`", - invocation.name - )) - })?; - if disabled_capabilities.contains_key(&invocation.name) { - let tool_use_id = invocation - .id - .clone() - .unwrap_or_else(|| format!("execution-{}-{next_turn}", started.task.task_id)); - messages.push(ContextMessage::tool_result( - tool_use_id, - "This tool capability is disabled for this task by the security circuit.", - None, - )); - return Ok(ActiveTaskAttemptExit::Continue { - continuation: agent_continuation( - messages, - next_turn, - usage, - security_circuit, - disabled_capabilities, - AgentPending { - review: None, - tool_calls: pending_tool_calls, - external: pending_external, - }, - ), - }); - } - let session = load_session(workflow, ctx, &started.run, &started.task).await?; - let tool_id = ToolCallId(Uuid::new_v5( - &started.task.task_id.as_uuid(), - format!( - "agent-tool:{}:{}:{}", - started.task.generation, - next_turn, - invocation.id.as_deref().unwrap_or(&invocation.name) - ) - .as_bytes(), - )); - let tool_call = ToolCallContent { - invocation: invocation.clone(), - provider_metadata: None, - }; - let allowed_tools = capabilities.keys().cloned().collect::>(); - let provenance = CapabilityProvenance { - kind: Some(capability_source_kind(&capability.source).to_string()), - id: Some(format!( - "{}@{}", - capability.reference.name, capability.reference.version - )), - step_id: Some(started.task.node_id.clone()), - }; - if matches!( - capability.async_mode, - ToolAsyncMode::MayReturnExternalJob { .. } - ) { - let provisional = agent_continuation( - messages.clone(), - next_turn, - usage.clone(), - security_circuit.clone(), - disabled_capabilities.clone(), - AgentPending { - review: None, - tool_calls: pending_tool_calls.clone(), - external: Some(PendingExternalToolInvocation { - external_job_uid: None, - invocation: invocation.clone(), - effect_idempotency: capability.idempotency_class, - }), - }, - ); - if !persist_external_start_checkpoint( - workflow, - ctx, - request, - started, - TaskAttemptCheckpointKind::AgentContinuation, - &provisional, - ) - .await? - { - return Ok(ActiveTaskAttemptExit::OwnershipLost); - } - } - if !begin_capability_dispatch(workflow, ctx, request, capability, &tool_call).await? { - return Ok(ActiveTaskAttemptExit::OwnershipLost); - } - let governed = invoke_governed_tool( - ctx, - GovernedInvocationRequest { - session: &session, - identity: &started.run.admitted_identity, - session_id: started.run.session_id, - tool_id, - tool_call: &tool_call, - allowed_tools: &allowed_tools, - expected_tool_contract_revision: Some(&capability.contract_revision), - active_canary: None, - trusted_sandbox_manifest: None, - origin: GovernedInvocationOrigin::ExecutionTask { - run_uid: started.run.run_uid, - task_uid: started.task.task_id.as_uuid(), - generation: started.task.generation, - attempt_generation: request.attempt_generation, - }, - capability_provenance: Some(&provenance), - capability_policy_context: Some(&capability.policy_context), - resource_budget: ResourceBudget::until(request.attempt_deadline_at), - }, - &workflow.session_limits, - workflow.session_store.clone(), - workflow.channel_adapters.as_ref(), - ) - .await?; - if !record_attempt_heartbeat(workflow, ctx, request, AttemptHeartbeat::ToolCall).await? { - return Ok(ActiveTaskAttemptExit::OwnershipLost); - } - usage.tool_calls = usage.tool_calls.saturating_add(1); - match governed { - GovernedInvocationOutcome::Completed(result) - if result.disposition == GovernedInvocationDisposition::ReviewPending => - { - let review = result.review.ok_or_else(|| { - TerminalError::new("review-pending agent result is missing durable review identity") - })?; - Ok(ActiveTaskAttemptExit::ReviewPending { - continuation: agent_continuation( - messages, - next_turn, - usage, - security_circuit, - disabled_capabilities, - AgentPending { - review: Some(PendingReviewedToolInvocation { - review_uid: review.review_uid, - expires_at: review.expires_at, - invocation: result.invocation, - effect_idempotency: capability.idempotency_class, - }), - tool_calls: pending_tool_calls, - external: pending_external, - }, - ), - }) - } - GovernedInvocationOutcome::Completed(result) => { - let output = result.output; - usage.retrieved_bytes = usage - .retrieved_bytes - .saturating_add(serialized_len(&output.safe_output.structured_payload())); - if !output.assessment.is_safe() { - moa_security::apply_owner_assessment( - &mut security_circuit, - moa_security::CircuitTarget { - session_id: session.id, - owner: &circuit_owner, - capability: &output.capability, - tool_call_id: tool_id, - }, - &output.assessment, - ) - .map_err(|_| TerminalError::new("agent security assessment owner mismatch"))?; - let stage = security_circuit.stage(&circuit_owner, &output.capability); - if !stage.permits_dispatch() { - disabled_capabilities - .insert(invocation.name.clone(), output.capability.clone()); - } - if stage == SecurityCircuitStage::Halted { - return Ok(ActiveTaskAttemptExit::Outcome(failed_task_outcome( - ExecutionFailureClass::Terminal, - "task stopped after unsafe capability output".to_string(), - usage, - ))); - } - if stage == SecurityCircuitStage::SuspendedForInput { - append_agent_tool_output(&mut messages, &invocation, &output); - let outcome = ExecutionTaskOutcome { - schema_version: 1, - usage: usage.clone(), - result: ExecutionTaskResult::NeedsInput { - question: "A capability returned potentially unsafe content. Continue?" - .to_string(), - audience: moa_artifacts::execution_plan::InputAudience::User, - }, - }; - return Ok(ActiveTaskAttemptExit::InputPending { - outcome, - continuation: agent_continuation( - messages, - next_turn, - usage, - security_circuit, - disabled_capabilities, - AgentPending { - review: None, - tool_calls: pending_tool_calls, - external: pending_external, - }, - ), - }); - } - } - append_agent_tool_output(&mut messages, &invocation, &output); - Ok(ActiveTaskAttemptExit::Continue { - continuation: agent_continuation( - messages, - next_turn, - usage, - security_circuit, - disabled_capabilities, - AgentPending { - review: None, - tool_calls: pending_tool_calls, - external: pending_external, - }, - ), - }) - } - GovernedInvocationOutcome::ExternalJob { - external_job_uid, .. - } => Ok(ActiveTaskAttemptExit::ExternalJob { - external_job_uid, - continuation: Some(agent_continuation( - messages, - next_turn, - usage, - security_circuit, - disabled_capabilities, - AgentPending { - review: None, - tool_calls: pending_tool_calls, - external: Some(PendingExternalToolInvocation { - external_job_uid: None, - invocation, - effect_idempotency: capability.idempotency_class, - }), - }, - )), - }), - GovernedInvocationOutcome::UnknownOutcome { message, .. } => { - Ok(ActiveTaskAttemptExit::Outcome(ExecutionTaskOutcome { - schema_version: 1, - usage, - result: ExecutionTaskResult::UnknownOutcome { message }, - })) - } - GovernedInvocationOutcome::NotDispatched { reason, .. } => { - Ok(ActiveTaskAttemptExit::Outcome(failed_task_outcome( - ExecutionFailureClass::Terminal, - execution_dispatch_rejection_message(reason), - usage, - ))) - } - GovernedInvocationOutcome::Delegation { .. } => Err(TerminalError::new( - "execution task agents cannot invoke delegation capabilities", - ) - .into()), - } -} - -fn agent_continuation( - messages: Vec, - next_turn: u32, - usage: ExecutionUsage, - security_circuit: SecurityCircuitState, - disabled_capabilities: BTreeMap, - pending: AgentPending, -) -> TaskAttemptContinuation { - TaskAttemptContinuation { - schema_version: TASK_ATTEMPT_CONTINUATION_SCHEMA_VERSION, - state: TaskAttemptContinuationState::Agent { - messages, - next_turn, - usage, - security_circuit, - disabled_capabilities, - pending_review: pending.review.map(Box::new), - pending_tool_calls: pending.tool_calls, - pending_external: pending.external, - }, - review_resolution: None, - external_job_resolution: None, - workspace_release_receipt_id: None, - } -} - -fn agent_tool_schema(name: &str, capability: &ExecutionCapability) -> Value { - json!({ - "name": name, - "description": capability.description, - "input_schema": capability.input_schema, - }) -} - -fn append_agent_tool_output( - messages: &mut Vec, - invocation: &ToolInvocation, - output: &moa_core::types::tools::SecuredToolOutput, -) { - let tool_use_id = invocation.id.clone().unwrap_or_else(|| { - Uuid::new_v5( - &Uuid::NAMESPACE_OID, - format!("{}:{}", invocation.name, invocation.input).as_bytes(), - ) - .to_string() - }); - messages.push(ContextMessage::tool_result( - tool_use_id, - output.safe_output.to_text(), - Some(output.safe_output.content.clone()), - )); -} - async fn load_session( workflow: &ExecutionTaskAttemptImpl, ctx: &WorkflowContext<'_>, @@ -1666,53 +227,6 @@ async fn load_session( .into_inner()) } -async fn load_pinned_skills( - workflow: &ExecutionTaskAttemptImpl, - ctx: &WorkflowContext<'_>, - started: &TaskAttemptRecord, - skill_refs: &[moa_artifacts::reference::ArtifactRef], -) -> Result, HandlerError> { - let mut markdown = Vec::with_capacity(skill_refs.len()); - let scope = started.run.contact_id.map_or( - ActionRuleScope::Tenant { - tenant_id: started.run.tenant_id, - }, - |contact_id| ActionRuleScope::Contact { - tenant_id: started.run.tenant_id, - contact_id, - }, - ); - for (index, skill_ref) in skill_refs.iter().enumerate() { - if !started.run.authorization.skill_refs.contains(skill_ref) { - return Err(TerminalError::new( - "task requested a skill outside its authorization envelope", - ) - .into()); - } - let pinned = started - .run - .pinned_instruction_skills - .iter() - .find(|pinned| pinned.skill_ref == *skill_ref) - .ok_or_else(|| TerminalError::new("task requested an unpinned skill"))?; - let pool = workflow.pool.clone(); - let revision_uid = pinned.revision_uid; - let loaded = ctx - .run(|| async move { - moa_skills::registry::SkillRegistry::new(pool) - .load_skill_markdown(&scope, revision_uid) - .await - .map(Json::from) - .map_err(moa_error_to_handler_error) - }) - .name(format!("task_attempt_skill:{index}:{revision_uid}")) - .await? - .into_inner(); - markdown.push(loaded); - } - Ok(markdown) -} - fn find_capability<'a>( run: &'a moa_execution::repository::ExecutionRunRecord, reference: &CapabilityReference, @@ -1756,391 +270,8 @@ fn execution_dispatch_rejection_message(reason: ExecutionToolDispatchRejection) format!("execution effect was not dispatched: {label}") } -fn agent_system_prompt(instructions: &str, skills: &[String]) -> String { - format!( - "{instructions}\n\nPinned instruction skills:\n{}\n\nReturn only JSON.", - skills.join("\n\n---\n\n") - ) -} - fn serialized_len(value: &T) -> u64 { serde_json::to_vec(value) .map(|bytes| bytes.len() as u64) .unwrap_or_default() } - -#[cfg(test)] -mod tests { - use chrono::{Duration, TimeZone, Utc}; - use moa_artifacts::execution_plan::ExecutionUsage; - use moa_core::types::{ - completion::ToolInvocation, context::ContextMessage, identifiers::TenantId, - }; - use moa_execution::state::ExecutionTaskId; - - use super::*; - - // Pins: every pre-provider heartbeat is an ownership check, not telemetry. A stale, - // absent, or non-running attempt must stop before model or tool dispatch, while exact - // replay of an already-journaled heartbeat retains authority. - #[test] - fn heartbeat_verdict_stops_dispatch_after_ownership_loss_offline() { - assert!(attempt_progress_retains_ownership( - TaskAttemptProgressOutcome::Applied - )); - assert!(attempt_progress_retains_ownership( - TaskAttemptProgressOutcome::Replayed - )); - for lost in [ - TaskAttemptProgressOutcome::NotFound, - TaskAttemptProgressOutcome::Stale, - TaskAttemptProgressOutcome::InvalidState, - ] { - assert!(!attempt_progress_retains_ownership(lost)); - } - } - - // Pins: a healthy model or tool call whose declared duration exceeds the configured - // heartbeat floor remains live for that exact step, and the first post-return heartbeat - // clears the widened bound back to the ordinary orchestration floor. - #[test] - fn model_and_tool_steps_are_bounded_before_dispatch_and_cleared_after_return_offline() { - let observed_at = Utc - .with_ymd_and_hms(2026, 8, 13, 12, 0, 0) - .single() - .expect("fixture timestamp is valid"); - let request = ExecutionTaskAttemptRequest { - dispatch_uid: Uuid::from_u128(1), - capacity_reservation_uid: Uuid::from_u128(2), - watchdog_trigger_uid: Uuid::from_u128(3), - watchdog_dispatch_uid: Uuid::from_u128(4), - run_uid: Uuid::from_u128(5), - task_id: ExecutionTaskId::from_uuid(Uuid::from_u128(6)), - controller_generation: 7, - attempt_generation: 8, - attempt_deadline_at: observed_at + Duration::seconds(121), - tenant_id: TenantId(Uuid::from_u128(9)), - }; - - assert_eq!( - AttemptHeartbeat::ModelTurnStart.step_bound_seconds(&request, observed_at), - Some(122), - ); - assert_eq!( - AttemptHeartbeat::ToolCallStart { - bound: Some(AttemptStepBound::Declared(90)), - } - .step_bound_seconds(&request, observed_at), - Some(90), - ); - assert_eq!( - AttemptHeartbeat::ModelTurn.step_bound_seconds(&request, observed_at), - None, - ); - assert_eq!( - AttemptHeartbeat::ToolCall.step_bound_seconds(&request, observed_at), - None, - ); - } - - // Pins: sandbox lifecycle work shares the active attempt deadline because provisioning, - // restore, install, execution, and commit can outlive the command timeout alone. A - // non-sandbox synchronous call keeps its narrower declared execution bound. - #[test] - fn sandbox_capability_uses_attempt_bound_while_non_sandbox_keeps_tool_bound_offline() { - let tool_call = ToolCallContent { - invocation: ToolInvocation { - id: Some("bounded-bash".to_string()), - name: "bash".to_string(), - input: json!({"cmd": "sleep 1", "timeout_secs": 90}), - }, - provider_metadata: None, - }; - - assert_eq!( - capability_step_bound(true, &ToolAsyncMode::SynchronousOnly, &tool_call), - Some(AttemptStepBound::UntilAttemptDeadline), - ); - assert_eq!( - capability_step_bound(false, &ToolAsyncMode::SynchronousOnly, &tool_call), - Some(AttemptStepBound::Declared(90)), - ); - assert_eq!( - capability_step_bound( - false, - &ToolAsyncMode::MayReturnExternalJob { - provider: "fixture".to_string(), - }, - &tool_call, - ), - Some(AttemptStepBound::UntilAttemptDeadline), - ); - } - - // Pins: a continuation that cannot fit in the bounded durable payload is rejected - // before persistence so callers must decompose or request a replan. - #[test] - fn oversized_agent_continuation_requires_decomposition_offline() { - let continuation = TaskAttemptContinuation { - schema_version: TASK_ATTEMPT_CONTINUATION_SCHEMA_VERSION, - state: TaskAttemptContinuationState::Agent { - messages: vec![ContextMessage::user( - "x".repeat(MAX_TASK_ATTEMPT_CONTINUATION_BYTES), - )], - next_turn: 1, - usage: ExecutionUsage { - cost_microusd: 0, - tokens: 0, - tool_calls: 0, - retrieved_bytes: 0, - }, - security_circuit: SecurityCircuitState::default(), - disabled_capabilities: std::collections::BTreeMap::new(), - pending_review: None, - pending_tool_calls: Vec::new(), - pending_external: None, - }, - review_resolution: None, - external_job_resolution: None, - workspace_release_receipt_id: None, - }; - - let error = continuation - .to_bounded_json() - .expect_err("oversized continuation must fail closed"); - assert!(error.contains("must be decomposed or replanned")); - } - - // Pins: once an asynchronous provider start commits, the durable checkpoint - // retains the exact model invocation, effect semantics, and MOA job identity; - // decoding the checkpoint must not reconstruct or resend that effect. - #[test] - fn agent_external_continuation_round_trips_exact_effect_owner_offline() { - let external_job_uid = Uuid::from_u128(41); - let invocation = ToolInvocation { - id: Some("provider-call-7".to_string()), - name: "render_video".to_string(), - input: json!({"prompt": "durable sunrise"}), - }; - let mut continuation = agent_continuation( - vec![ContextMessage::user("render a durable sunrise")], - 3, - zero_usage(), - SecurityCircuitState::default(), - BTreeMap::new(), - AgentPending { - review: None, - tool_calls: Vec::new(), - external: Some(PendingExternalToolInvocation { - external_job_uid: None, - invocation: invocation.clone(), - effect_idempotency: IdempotencyClass::NonIdempotent, - }), - }, - ); - - continuation - .bind_external_job(external_job_uid) - .expect("fresh external continuation must accept its durable job identity"); - let persisted = continuation - .to_bounded_json() - .expect("exact continuation must fit the durable bound"); - let decoded: TaskAttemptContinuation = - serde_json::from_value(persisted).expect("persisted continuation must decode"); - - let TaskAttemptContinuationState::Agent { - pending_external: Some(pending), - next_turn, - .. - } = decoded.state - else { - panic!("external continuation lost its exact pending effect"); - }; - assert_eq!(next_turn, 3); - assert_eq!(pending.external_job_uid, Some(external_job_uid)); - assert_eq!(pending.invocation, invocation); - assert_eq!(pending.effect_idempotency, IdempotencyClass::NonIdempotent); - } - - // Pins: a storage-only review checkpoint retains the exact reviewed - // invocation and expiry across serialization, so a resumed attempt consumes - // the decision without regenerating the provider effect. - #[test] - fn agent_review_continuation_round_trips_exact_effect_fence_offline() { - let review_uid = Uuid::from_u128(51); - let expires_at = Utc - .with_ymd_and_hms(2030, 5, 6, 7, 8, 9) - .single() - .expect("fixed review expiry"); - let invocation = ToolInvocation { - id: Some("reviewed-call-2".to_string()), - name: "publish_release".to_string(), - input: json!({"version": "2.0.0"}), - }; - let continuation = agent_continuation( - vec![ContextMessage::user("publish only after review")], - 2, - zero_usage(), - SecurityCircuitState::default(), - BTreeMap::new(), - AgentPending { - review: Some(PendingReviewedToolInvocation { - review_uid, - expires_at, - invocation: invocation.clone(), - effect_idempotency: IdempotencyClass::NonIdempotent, - }), - tool_calls: Vec::new(), - external: None, - }, - ); - - let decoded: TaskAttemptContinuation = serde_json::from_value( - continuation - .to_bounded_json() - .expect("review continuation must fit the durable bound"), - ) - .expect("persisted review continuation must decode"); - assert_eq!(decoded.pending_review_uid(), Some(review_uid)); - let TaskAttemptContinuationState::Agent { - pending_review: Some(pending), - .. - } = decoded.state - else { - panic!("review continuation lost its exact pending effect"); - }; - assert_eq!(pending.expires_at, expires_at); - assert_eq!(pending.invocation, invocation); - } - - // Pins: an input boundary keeps the already-completed model turn and circuit - // state in the bounded checkpoint; resumption starts at the following turn - // instead of calling the model again for the same prompt. - #[test] - fn agent_input_continuation_round_trips_next_turn_and_messages_offline() { - let continuation = agent_continuation( - vec![ - ContextMessage::user("inspect the unsafe payload"), - ContextMessage::assistant("May I continue with the unsafe payload?"), - ], - 4, - ExecutionUsage { - cost_microusd: 17, - tokens: 23, - tool_calls: 2, - retrieved_bytes: 31, - }, - SecurityCircuitState::default(), - BTreeMap::new(), - AgentPending { - review: None, - tool_calls: Vec::new(), - external: None, - }, - ); - - let decoded: TaskAttemptContinuation = serde_json::from_value( - continuation - .to_bounded_json() - .expect("input continuation must fit the durable bound"), - ) - .expect("persisted input continuation must decode"); - let TaskAttemptContinuationState::Agent { - messages, - next_turn, - usage, - .. - } = decoded.state - else { - panic!("input continuation changed state kind"); - }; - assert_eq!(next_turn, 4); - assert_eq!(messages.len(), 2); - assert_eq!(usage.tokens, 23); - assert_eq!(usage.tool_calls, 2); - } - - // Pins: provider recovery may prove that a reserved start never happened; - // the current checkpoint must retain the exact invocation with no job UID so - // the successor attempt can replay that call without repeating the model turn. - #[test] - fn provisional_agent_external_start_round_trips_without_a_job_uid_offline() { - let invocation = ToolInvocation { - id: Some("stable-provider-call".to_string()), - name: "render_video".to_string(), - input: json!({"prompt": "recover this exact effect"}), - }; - let continuation = agent_continuation( - vec![ContextMessage::user("render once")], - 2, - zero_usage(), - SecurityCircuitState::default(), - BTreeMap::new(), - AgentPending { - review: None, - tool_calls: Vec::new(), - external: Some(PendingExternalToolInvocation { - external_job_uid: None, - invocation: invocation.clone(), - effect_idempotency: IdempotencyClass::NonIdempotent, - }), - }, - ); - - let decoded: TaskAttemptContinuation = serde_json::from_value( - continuation - .to_bounded_json() - .expect("provisional continuation must fit"), - ) - .expect("provisional continuation must decode"); - let TaskAttemptContinuationState::Agent { - pending_external: Some(pending), - .. - } = decoded.state - else { - panic!("provisional external start lost its pending invocation"); - }; - assert_eq!(pending.external_job_uid, None); - assert_eq!(pending.invocation, invocation); - } - - // Pins: a direct async capability resumes with the same stable tool-call ID - // after a NotStarted recovery instead of creating a second provider identity. - #[test] - fn direct_external_start_checkpoint_round_trips_stable_tool_id_offline() { - let tool_id = ToolCallId(Uuid::from_u128(77)); - let continuation = TaskAttemptContinuation { - schema_version: TASK_ATTEMPT_CONTINUATION_SCHEMA_VERSION, - state: TaskAttemptContinuationState::CapabilityExternalStart { - tool_id, - usage: zero_usage(), - }, - review_resolution: None, - external_job_resolution: None, - workspace_release_receipt_id: None, - }; - - let decoded: TaskAttemptContinuation = serde_json::from_value( - continuation - .to_bounded_json() - .expect("direct provisional continuation must fit"), - ) - .expect("direct provisional continuation must decode"); - assert!(matches!( - decoded.state, - TaskAttemptContinuationState::CapabilityExternalStart { - tool_id: decoded_tool_id, - .. - } if decoded_tool_id == tool_id - )); - } - - const fn zero_usage() -> ExecutionUsage { - ExecutionUsage { - cost_microusd: 0, - tokens: 0, - tool_calls: 0, - retrieved_bytes: 0, - } - } -} diff --git a/crates/moa-orchestrator/src/workflows/execution_task_attempt/active/agent.rs b/crates/moa-orchestrator/src/workflows/execution_task_attempt/active/agent.rs new file mode 100644 index 000000000..f7ae10975 --- /dev/null +++ b/crates/moa-orchestrator/src/workflows/execution_task_attempt/active/agent.rs @@ -0,0 +1,1018 @@ +//! Bounded task-local agent execution and continuation transitions. + +use std::collections::{BTreeMap, BTreeSet}; + +use moa_artifacts::execution_plan::{ + CapabilityReference, ExecutionFailureClass, ExecutionTaskOutcome, ExecutionTaskResult, + ExecutionUsage, +}; +use moa_core::types::{ + action_policy::{ActionRuleScope, CapabilityProvenance}, + completion::{CompletionContent, ToolCallContent, ToolInvocation}, + context::ContextMessage, + identifiers::ToolCallId, + resource::ResourceBudget, + security::{ + SecurityCircuitOwner, SecurityCircuitStage, SecurityCircuitState, ToolCapabilityId, + }, + tools::{AsyncToolJobTerminalOutcome, ToolAsyncMode}, +}; +use moa_execution::{ + capability::ExecutionCapability, + repository::task::{TaskAttemptCheckpointKind, TaskAttemptRecord}, + state::failed_task_outcome, + wire::ExecutionTaskAttemptRequest, +}; +use restate_sdk::prelude::*; +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::{ + services::llm_gateway::{ + BoundedCompletionRequest, LLMCompletionAction, LLMCompletionOwner, LLMGatewayClient, + attach_completion_owner, completion_idempotency_key, + }, + tool_invocation::governed::{ + GovernedInvocationDisposition, GovernedInvocationOrigin, GovernedInvocationOutcome, + GovernedInvocationRequest, invoke_governed_tool, + }, + workflows::{ + errors::moa_error_to_handler_error, + execution_task_attempt::{ + ExecutionTaskAttemptImpl, capability_tool_name, + continuation::{ + PendingExternalToolInvocation, PendingReviewedToolInvocation, + TASK_ATTEMPT_CONTINUATION_SCHEMA_VERSION, TaskAttemptContinuation, + TaskAttemptContinuationState, + }, + }, + }, +}; + +use super::{ + ActiveTaskAttemptExit, capability_source_kind, execution_dispatch_rejection_message, + find_capability, + heartbeat::{AttemptHeartbeat, begin_capability_dispatch, record_attempt_heartbeat}, + load_session, persist_external_start_checkpoint, serialized_len, +}; + +/// Immutable task-local agent definition selected by the logical task. +pub(super) struct AgentTaskSpec<'a> { + /// Task-local instructions. + pub(super) instructions: &'a str, + /// Pinned instruction-only skills. + pub(super) skill_refs: &'a [moa_artifacts::reference::ArtifactRef], + /// Governed capabilities available to the task-local agent. + pub(super) capability_refs: &'a [CapabilityReference], + /// Maximum model turns admitted for this task. + pub(super) max_turns: u32, +} + +struct AgentPending { + review: Option, + tool_calls: Vec, + external: Option, +} + +/// Executes one bounded task-local agent model or tool boundary. +pub(super) async fn execute_agent_turn( + workflow: &ExecutionTaskAttemptImpl, + ctx: &WorkflowContext<'_>, + request: &ExecutionTaskAttemptRequest, + started: &TaskAttemptRecord, + spec: AgentTaskSpec<'_>, + continuation: Option<&TaskAttemptContinuation>, +) -> Result { + let AgentTaskSpec { + instructions, + skill_refs, + capability_refs, + max_turns, + } = spec; + if max_turns == 0 { + return Ok(ActiveTaskAttemptExit::Outcome(failed_task_outcome( + ExecutionFailureClass::InvalidInput, + "agent max_turns must be positive".to_string(), + started.task.actual.clone(), + ))); + } + let mut capabilities = BTreeMap::::new(); + for reference in capability_refs { + let capability = find_capability(&started.run, reference)?; + let tool_name = capability_tool_name(capability)?; + if capabilities.insert(tool_name.clone(), capability).is_some() { + return Err(TerminalError::new(format!( + "task-local agent has ambiguous capability tool `{tool_name}`" + )) + .into()); + } + } + let circuit_owner = SecurityCircuitOwner::ExecutionTask { + run_uid: started.run.run_uid, + task_uid: started.task.task_id.as_uuid(), + generation: started.task.generation, + }; + let ( + mut messages, + mut next_turn, + mut usage, + mut security_circuit, + mut disabled_capabilities, + mut pending_review, + mut pending_tool_calls, + mut pending_external, + ) = match continuation { + Some(TaskAttemptContinuation { + state: + TaskAttemptContinuationState::Agent { + messages, + next_turn, + usage, + security_circuit, + disabled_capabilities, + pending_review, + pending_tool_calls, + pending_external, + }, + .. + }) => ( + messages.clone(), + *next_turn, + usage.clone(), + security_circuit.clone(), + disabled_capabilities.clone(), + pending_review.as_deref().cloned(), + pending_tool_calls.clone(), + pending_external.clone(), + ), + Some(_) => { + return Err(TerminalError::new( + "task-local agent received an incompatible continuation", + ) + .into()); + } + None => { + let skills = load_pinned_skills(workflow, ctx, started, skill_refs).await?; + let mut circuit = SecurityCircuitState::default(); + circuit.adopt_owner(&circuit_owner); + ( + vec![ + ContextMessage::system(agent_system_prompt(instructions, &skills)), + ContextMessage::user( + json!({ + "resolved_input": started.task.input, + "resume_inputs": started.task.resume_input_history, + }) + .to_string(), + ), + ], + 0, + started.task.actual.clone(), + circuit, + BTreeMap::new(), + None, + Vec::new(), + None, + ) + } + }; + security_circuit.adopt_owner(&circuit_owner); + + if let Some(external) = pending_external.take() { + if let Some(external_job_uid) = external.external_job_uid { + let resolution = continuation + .and_then(|continuation| continuation.external_job_resolution.as_ref()) + .ok_or_else(|| { + TerminalError::new("agent external continuation has no terminal resolution") + })?; + let tool_use_id = external + .invocation + .id + .clone() + .unwrap_or_else(|| format!("external-job-{external_job_uid}")); + match resolution { + AsyncToolJobTerminalOutcome::Completed { output } => { + messages.push(ContextMessage::tool_result( + tool_use_id, + output.to_string(), + None, + )); + } + AsyncToolJobTerminalOutcome::Failed { error } => { + messages.push(ContextMessage::tool_result( + tool_use_id, + format!("external tool failed: {error}"), + None, + )); + } + AsyncToolJobTerminalOutcome::Cancelled => { + messages.push(ContextMessage::tool_result( + tool_use_id, + "external tool was cancelled", + None, + )); + } + AsyncToolJobTerminalOutcome::UnknownOutcome { error } => { + return Ok(ActiveTaskAttemptExit::Outcome(ExecutionTaskOutcome { + schema_version: 1, + usage, + result: ExecutionTaskResult::UnknownOutcome { + message: format!("external agent effect outcome is unknown: {error}"), + }, + })); + } + } + } else { + // Provider start recovery proved NotStarted and re-admitted the exact continuation. + // Reinsert the original model invocation so its stable tool id/idempotency key is + // dispatched again without asking the model or repeating prior tool effects. + pending_tool_calls.insert(0, external.invocation); + } + } + + if let Some(reviewed) = pending_review.take() { + let resolution = continuation + .and_then(|continuation| continuation.review_resolution.as_ref()) + .ok_or_else(|| { + TerminalError::new("agent review continuation has no durable resolution") + })?; + match resolution { + moa_execution::wire::ExecutionActionReviewResolution::Completed { tool_output } => { + let output = serde_json::from_value::( + tool_output.clone(), + ) + .map_err(|error| { + TerminalError::new(format!("decode reviewed agent capability output: {error}")) + })?; + append_agent_tool_output(&mut messages, &reviewed.invocation, &output); + usage.retrieved_bytes = usage + .retrieved_bytes + .saturating_add(serialized_len(&output.safe_output.structured_payload())); + } + moa_execution::wire::ExecutionActionReviewResolution::ExternalJob { + external_job_uid, + .. + } => { + return Ok(ActiveTaskAttemptExit::ExternalJob { + external_job_uid: *external_job_uid, + continuation: Some(agent_continuation( + messages, + next_turn, + usage, + security_circuit, + disabled_capabilities, + AgentPending { + review: None, + tool_calls: pending_tool_calls, + external: Some(PendingExternalToolInvocation { + external_job_uid: None, + invocation: reviewed.invocation, + effect_idempotency: reviewed.effect_idempotency, + }), + }, + )), + }); + } + moa_execution::wire::ExecutionActionReviewResolution::Failed { class, message } => { + return Ok(ActiveTaskAttemptExit::Outcome(failed_task_outcome( + class.clone(), + message.clone(), + usage, + ))); + } + moa_execution::wire::ExecutionActionReviewResolution::UnknownOutcome { message } => { + return Ok(ActiveTaskAttemptExit::Outcome(ExecutionTaskOutcome { + schema_version: 1, + usage, + result: ExecutionTaskResult::UnknownOutcome { + message: message.clone(), + }, + })); + } + moa_execution::wire::ExecutionActionReviewResolution::NotDispatched { reason } => { + return Ok(ActiveTaskAttemptExit::Outcome(failed_task_outcome( + ExecutionFailureClass::Terminal, + execution_dispatch_rejection_message(*reason), + usage, + ))); + } + moa_execution::wire::ExecutionActionReviewResolution::Denied { reason } => { + return Ok(ActiveTaskAttemptExit::Outcome(failed_task_outcome( + ExecutionFailureClass::AuthorizationDenied, + reason.clone(), + usage, + ))); + } + moa_execution::wire::ExecutionActionReviewResolution::TimedOut { reason } => { + return Ok(ActiveTaskAttemptExit::Outcome(failed_task_outcome( + ExecutionFailureClass::DeadlineExceeded, + reason.clone(), + usage, + ))); + } + } + } + + if pending_tool_calls.is_empty() { + if next_turn >= max_turns { + return Ok(ActiveTaskAttemptExit::Outcome(failed_task_outcome( + ExecutionFailureClass::Terminal, + format!("task-local agent exhausted max_turns={max_turns}"), + usage, + ))); + } + let mut completion = moa_core::types::completion::CompletionRequest { + model: None, + messages: messages.clone(), + tools: capabilities + .iter() + .filter(|(name, _)| !disabled_capabilities.contains_key(*name)) + .map(|(name, capability)| agent_tool_schema(name, capability)) + .collect(), + max_output_tokens: None, + temperature: None, + response_format: None, + native_web_search: Default::default(), + metadata: std::collections::HashMap::new(), + }; + let owner = LLMCompletionOwner::execution_task_attempt(request.dispatch_uid); + attach_completion_owner(&mut completion, &owner); + if !record_attempt_heartbeat(workflow, ctx, request, AttemptHeartbeat::ModelTurnStart) + .await? + { + return Ok(ActiveTaskAttemptExit::OwnershipLost); + } + let response = crate::restate_identity::replay_safe_request( + ctx.service_client::() + .complete_bounded(Json::from(BoundedCompletionRequest { + request: completion, + budget: ResourceBudget::until(request.attempt_deadline_at), + })) + .idempotency_key(completion_idempotency_key( + ctx.invocation_id(), + LLMCompletionAction::ExecutionTaskModel { + generation: started.task.generation, + turn: next_turn, + }, + )), + ) + .call() + .await? + .into_inner(); + if !record_attempt_heartbeat(workflow, ctx, request, AttemptHeartbeat::ModelTurn).await? { + return Ok(ActiveTaskAttemptExit::OwnershipLost); + } + usage.tokens = usage + .tokens + .saturating_add(response.usage.total_input_tokens() as u64) + .saturating_add(response.usage.output_tokens as u64); + usage.cost_microusd = usage.cost_microusd.saturating_add( + moa_providers::pricing_for_model(response.model.as_str()) + .map(|pricing| pricing.cost_micros(&response.usage)) + .unwrap_or_default(), + ); + let tool_calls = response + .content + .iter() + .filter_map(|content| match content { + CompletionContent::ToolCall(call) => Some(call.invocation.clone()), + CompletionContent::Text(_) | CompletionContent::ProviderToolResult { .. } => None, + }) + .collect::>(); + if tool_calls.is_empty() { + let outcome = moa_execution::state::parse_agent_task_outcome(&response.text, usage); + if matches!(outcome.result, ExecutionTaskResult::NeedsInput { .. }) { + messages.push(ContextMessage::assistant_with_thought_signature( + response.text, + response.thought_signature, + )); + return Ok(ActiveTaskAttemptExit::InputPending { + continuation: agent_continuation( + messages, + next_turn.saturating_add(1), + outcome.usage.clone(), + security_circuit, + disabled_capabilities, + AgentPending { + review: None, + tool_calls: pending_tool_calls, + external: pending_external, + }, + ), + outcome, + }); + } + return Ok(ActiveTaskAttemptExit::Outcome(outcome)); + } + for (index, invocation) in tool_calls.iter().cloned().enumerate() { + messages.push(ContextMessage::assistant_tool_call_with_thought_signature( + invocation, + if index == 0 { + response.text.clone() + } else { + String::new() + }, + (index == 0) + .then(|| response.thought_signature.clone()) + .flatten(), + )); + } + pending_tool_calls = tool_calls; + next_turn = next_turn.saturating_add(1); + return Ok(ActiveTaskAttemptExit::Continue { + continuation: agent_continuation( + messages, + next_turn, + usage, + security_circuit, + disabled_capabilities, + AgentPending { + review: None, + tool_calls: pending_tool_calls, + external: pending_external, + }, + ), + }); + } + + let invocation = pending_tool_calls.remove(0); + let capability = capabilities.get(&invocation.name).copied().ok_or_else(|| { + TerminalError::new(format!( + "agent emitted undeclared capability `{}`", + invocation.name + )) + })?; + if disabled_capabilities.contains_key(&invocation.name) { + let tool_use_id = invocation + .id + .clone() + .unwrap_or_else(|| format!("execution-{}-{next_turn}", started.task.task_id)); + messages.push(ContextMessage::tool_result( + tool_use_id, + "This tool capability is disabled for this task by the security circuit.", + None, + )); + return Ok(ActiveTaskAttemptExit::Continue { + continuation: agent_continuation( + messages, + next_turn, + usage, + security_circuit, + disabled_capabilities, + AgentPending { + review: None, + tool_calls: pending_tool_calls, + external: pending_external, + }, + ), + }); + } + let session = load_session(workflow, ctx, &started.run, &started.task).await?; + let tool_id = ToolCallId(Uuid::new_v5( + &started.task.task_id.as_uuid(), + format!( + "agent-tool:{}:{}:{}", + started.task.generation, + next_turn, + invocation.id.as_deref().unwrap_or(&invocation.name) + ) + .as_bytes(), + )); + let tool_call = ToolCallContent { + invocation: invocation.clone(), + provider_metadata: None, + }; + let allowed_tools = capabilities.keys().cloned().collect::>(); + let provenance = CapabilityProvenance { + kind: Some(capability_source_kind(&capability.source).to_string()), + id: Some(format!( + "{}@{}", + capability.reference.name, capability.reference.version + )), + step_id: Some(started.task.node_id.clone()), + }; + if matches!( + capability.async_mode, + ToolAsyncMode::MayReturnExternalJob { .. } + ) { + let provisional = agent_continuation( + messages.clone(), + next_turn, + usage.clone(), + security_circuit.clone(), + disabled_capabilities.clone(), + AgentPending { + review: None, + tool_calls: pending_tool_calls.clone(), + external: Some(PendingExternalToolInvocation { + external_job_uid: None, + invocation: invocation.clone(), + effect_idempotency: capability.idempotency_class, + }), + }, + ); + if !persist_external_start_checkpoint( + workflow, + ctx, + request, + started, + TaskAttemptCheckpointKind::AgentContinuation, + &provisional, + ) + .await? + { + return Ok(ActiveTaskAttemptExit::OwnershipLost); + } + } + if !begin_capability_dispatch(workflow, ctx, request, capability, &tool_call).await? { + return Ok(ActiveTaskAttemptExit::OwnershipLost); + } + let governed = invoke_governed_tool( + ctx, + GovernedInvocationRequest { + session: &session, + identity: &started.run.admitted_identity, + session_id: started.run.session_id, + tool_id, + tool_call: &tool_call, + allowed_tools: &allowed_tools, + expected_tool_contract_revision: Some(&capability.contract_revision), + active_canary: None, + trusted_sandbox_manifest: None, + origin: GovernedInvocationOrigin::ExecutionTask { + run_uid: started.run.run_uid, + task_uid: started.task.task_id.as_uuid(), + generation: started.task.generation, + attempt_generation: request.attempt_generation, + }, + capability_provenance: Some(&provenance), + capability_policy_context: Some(&capability.policy_context), + resource_budget: ResourceBudget::until(request.attempt_deadline_at), + }, + &workflow.session_limits, + workflow.session_store.clone(), + workflow.channel_adapters.as_ref(), + ) + .await?; + if !record_attempt_heartbeat(workflow, ctx, request, AttemptHeartbeat::ToolCall).await? { + return Ok(ActiveTaskAttemptExit::OwnershipLost); + } + usage.tool_calls = usage.tool_calls.saturating_add(1); + match governed { + GovernedInvocationOutcome::Completed(result) + if result.disposition == GovernedInvocationDisposition::ReviewPending => + { + let review = result.review.ok_or_else(|| { + TerminalError::new("review-pending agent result is missing durable review identity") + })?; + Ok(ActiveTaskAttemptExit::ReviewPending { + continuation: agent_continuation( + messages, + next_turn, + usage, + security_circuit, + disabled_capabilities, + AgentPending { + review: Some(PendingReviewedToolInvocation { + review_uid: review.review_uid, + expires_at: review.expires_at, + invocation: result.invocation, + effect_idempotency: capability.idempotency_class, + }), + tool_calls: pending_tool_calls, + external: pending_external, + }, + ), + }) + } + GovernedInvocationOutcome::Completed(result) => { + let output = result.output; + usage.retrieved_bytes = usage + .retrieved_bytes + .saturating_add(serialized_len(&output.safe_output.structured_payload())); + if !output.assessment.is_safe() { + moa_security::apply_owner_assessment( + &mut security_circuit, + moa_security::CircuitTarget { + session_id: session.id, + owner: &circuit_owner, + capability: &output.capability, + tool_call_id: tool_id, + }, + &output.assessment, + ) + .map_err(|_| TerminalError::new("agent security assessment owner mismatch"))?; + let stage = security_circuit.stage(&circuit_owner, &output.capability); + if !stage.permits_dispatch() { + disabled_capabilities + .insert(invocation.name.clone(), output.capability.clone()); + } + if stage == SecurityCircuitStage::Halted { + return Ok(ActiveTaskAttemptExit::Outcome(failed_task_outcome( + ExecutionFailureClass::Terminal, + "task stopped after unsafe capability output".to_string(), + usage, + ))); + } + if stage == SecurityCircuitStage::SuspendedForInput { + append_agent_tool_output(&mut messages, &invocation, &output); + let outcome = ExecutionTaskOutcome { + schema_version: 1, + usage: usage.clone(), + result: ExecutionTaskResult::NeedsInput { + question: "A capability returned potentially unsafe content. Continue?" + .to_string(), + audience: moa_artifacts::execution_plan::InputAudience::User, + }, + }; + return Ok(ActiveTaskAttemptExit::InputPending { + outcome, + continuation: agent_continuation( + messages, + next_turn, + usage, + security_circuit, + disabled_capabilities, + AgentPending { + review: None, + tool_calls: pending_tool_calls, + external: pending_external, + }, + ), + }); + } + } + append_agent_tool_output(&mut messages, &invocation, &output); + Ok(ActiveTaskAttemptExit::Continue { + continuation: agent_continuation( + messages, + next_turn, + usage, + security_circuit, + disabled_capabilities, + AgentPending { + review: None, + tool_calls: pending_tool_calls, + external: pending_external, + }, + ), + }) + } + GovernedInvocationOutcome::ExternalJob { + external_job_uid, .. + } => Ok(ActiveTaskAttemptExit::ExternalJob { + external_job_uid, + continuation: Some(agent_continuation( + messages, + next_turn, + usage, + security_circuit, + disabled_capabilities, + AgentPending { + review: None, + tool_calls: pending_tool_calls, + external: Some(PendingExternalToolInvocation { + external_job_uid: None, + invocation, + effect_idempotency: capability.idempotency_class, + }), + }, + )), + }), + GovernedInvocationOutcome::UnknownOutcome { message, .. } => { + Ok(ActiveTaskAttemptExit::Outcome(ExecutionTaskOutcome { + schema_version: 1, + usage, + result: ExecutionTaskResult::UnknownOutcome { message }, + })) + } + GovernedInvocationOutcome::NotDispatched { reason, .. } => { + Ok(ActiveTaskAttemptExit::Outcome(failed_task_outcome( + ExecutionFailureClass::Terminal, + execution_dispatch_rejection_message(reason), + usage, + ))) + } + GovernedInvocationOutcome::Delegation { .. } => Err(TerminalError::new( + "execution task agents cannot invoke delegation capabilities", + ) + .into()), + } +} + +fn agent_continuation( + messages: Vec, + next_turn: u32, + usage: ExecutionUsage, + security_circuit: SecurityCircuitState, + disabled_capabilities: BTreeMap, + pending: AgentPending, +) -> TaskAttemptContinuation { + TaskAttemptContinuation { + schema_version: TASK_ATTEMPT_CONTINUATION_SCHEMA_VERSION, + state: TaskAttemptContinuationState::Agent { + messages, + next_turn, + usage, + security_circuit, + disabled_capabilities, + pending_review: pending.review.map(Box::new), + pending_tool_calls: pending.tool_calls, + pending_external: pending.external, + }, + review_resolution: None, + external_job_resolution: None, + workspace_release_receipt_id: None, + } +} + +fn agent_tool_schema(name: &str, capability: &ExecutionCapability) -> Value { + json!({ + "name": name, + "description": capability.description, + "input_schema": capability.input_schema, + }) +} + +fn append_agent_tool_output( + messages: &mut Vec, + invocation: &ToolInvocation, + output: &moa_core::types::tools::SecuredToolOutput, +) { + let tool_use_id = invocation.id.clone().unwrap_or_else(|| { + Uuid::new_v5( + &Uuid::NAMESPACE_OID, + format!("{}:{}", invocation.name, invocation.input).as_bytes(), + ) + .to_string() + }); + messages.push(ContextMessage::tool_result( + tool_use_id, + output.safe_output.to_text(), + Some(output.safe_output.content.clone()), + )); +} + +async fn load_pinned_skills( + workflow: &ExecutionTaskAttemptImpl, + ctx: &WorkflowContext<'_>, + started: &TaskAttemptRecord, + skill_refs: &[moa_artifacts::reference::ArtifactRef], +) -> Result, HandlerError> { + let mut markdown = Vec::with_capacity(skill_refs.len()); + let scope = started.run.contact_id.map_or( + ActionRuleScope::Tenant { + tenant_id: started.run.tenant_id, + }, + |contact_id| ActionRuleScope::Contact { + tenant_id: started.run.tenant_id, + contact_id, + }, + ); + for (index, skill_ref) in skill_refs.iter().enumerate() { + if !started.run.authorization.skill_refs.contains(skill_ref) { + return Err(TerminalError::new( + "task requested a skill outside its authorization envelope", + ) + .into()); + } + let pinned = started + .run + .pinned_instruction_skills + .iter() + .find(|pinned| pinned.skill_ref == *skill_ref) + .ok_or_else(|| TerminalError::new("task requested an unpinned skill"))?; + let pool = workflow.pool.clone(); + let revision_uid = pinned.revision_uid; + let loaded = ctx + .run(|| async move { + moa_skills::registry::SkillRegistry::new(pool) + .load_skill_markdown(&scope, revision_uid) + .await + .map(Json::from) + .map_err(moa_error_to_handler_error) + }) + .name(format!("task_attempt_skill:{index}:{revision_uid}")) + .await? + .into_inner(); + markdown.push(loaded); + } + Ok(markdown) +} + +fn agent_system_prompt(instructions: &str, skills: &[String]) -> String { + format!( + "{instructions}\n\nPinned instruction skills:\n{}\n\nReturn only JSON.", + skills.join("\n\n---\n\n") + ) +} + +#[cfg(test)] +mod tests { + use chrono::{TimeZone, Utc}; + use moa_core::types::tools::IdempotencyClass; + + use super::*; + + // Pins: once an asynchronous provider start commits, the durable checkpoint + // retains the exact model invocation, effect semantics, and MOA job identity; + // decoding the checkpoint must not reconstruct or resend that effect. + #[test] + fn agent_external_continuation_round_trips_exact_effect_owner_offline() { + let external_job_uid = Uuid::from_u128(41); + let invocation = ToolInvocation { + id: Some("provider-call-7".to_string()), + name: "render_video".to_string(), + input: json!({"prompt": "durable sunrise"}), + }; + let mut continuation = agent_continuation( + vec![ContextMessage::user("render a durable sunrise")], + 3, + zero_usage(), + SecurityCircuitState::default(), + BTreeMap::new(), + AgentPending { + review: None, + tool_calls: Vec::new(), + external: Some(PendingExternalToolInvocation { + external_job_uid: None, + invocation: invocation.clone(), + effect_idempotency: IdempotencyClass::NonIdempotent, + }), + }, + ); + + continuation + .bind_external_job(external_job_uid) + .expect("fresh external continuation must accept its durable job identity"); + let persisted = continuation + .to_bounded_json() + .expect("exact continuation must fit the durable bound"); + let decoded: TaskAttemptContinuation = + serde_json::from_value(persisted).expect("persisted continuation must decode"); + + let TaskAttemptContinuationState::Agent { + pending_external: Some(pending), + next_turn, + .. + } = decoded.state + else { + panic!("external continuation lost its exact pending effect"); + }; + assert_eq!(next_turn, 3); + assert_eq!(pending.external_job_uid, Some(external_job_uid)); + assert_eq!(pending.invocation, invocation); + assert_eq!(pending.effect_idempotency, IdempotencyClass::NonIdempotent); + } + + // Pins: a storage-only review checkpoint retains the exact reviewed + // invocation and expiry across serialization, so a resumed attempt consumes + // the decision without regenerating the provider effect. + #[test] + fn agent_review_continuation_round_trips_exact_effect_fence_offline() { + let review_uid = Uuid::from_u128(51); + let expires_at = Utc + .with_ymd_and_hms(2030, 5, 6, 7, 8, 9) + .single() + .expect("fixed review expiry"); + let invocation = ToolInvocation { + id: Some("reviewed-call-2".to_string()), + name: "publish_release".to_string(), + input: json!({"version": "2.0.0"}), + }; + let continuation = agent_continuation( + vec![ContextMessage::user("publish only after review")], + 2, + zero_usage(), + SecurityCircuitState::default(), + BTreeMap::new(), + AgentPending { + review: Some(PendingReviewedToolInvocation { + review_uid, + expires_at, + invocation: invocation.clone(), + effect_idempotency: IdempotencyClass::NonIdempotent, + }), + tool_calls: Vec::new(), + external: None, + }, + ); + + let decoded: TaskAttemptContinuation = serde_json::from_value( + continuation + .to_bounded_json() + .expect("review continuation must fit the durable bound"), + ) + .expect("persisted review continuation must decode"); + assert_eq!(decoded.pending_review_uid(), Some(review_uid)); + let TaskAttemptContinuationState::Agent { + pending_review: Some(pending), + .. + } = decoded.state + else { + panic!("review continuation lost its exact pending effect"); + }; + assert_eq!(pending.expires_at, expires_at); + assert_eq!(pending.invocation, invocation); + } + + // Pins: an input boundary keeps the already-completed model turn and circuit + // state in the bounded checkpoint; resumption starts at the following turn + // instead of calling the model again for the same prompt. + #[test] + fn agent_input_continuation_round_trips_next_turn_and_messages_offline() { + let continuation = agent_continuation( + vec![ + ContextMessage::user("inspect the unsafe payload"), + ContextMessage::assistant("May I continue with the unsafe payload?"), + ], + 4, + ExecutionUsage { + cost_microusd: 17, + tokens: 23, + tool_calls: 2, + retrieved_bytes: 31, + }, + SecurityCircuitState::default(), + BTreeMap::new(), + AgentPending { + review: None, + tool_calls: Vec::new(), + external: None, + }, + ); + + let decoded: TaskAttemptContinuation = serde_json::from_value( + continuation + .to_bounded_json() + .expect("input continuation must fit the durable bound"), + ) + .expect("persisted input continuation must decode"); + let TaskAttemptContinuationState::Agent { + messages, + next_turn, + usage, + .. + } = decoded.state + else { + panic!("input continuation changed state kind"); + }; + assert_eq!(next_turn, 4); + assert_eq!(messages.len(), 2); + assert_eq!(usage.tokens, 23); + assert_eq!(usage.tool_calls, 2); + } + + // Pins: provider recovery may prove that a reserved start never happened; + // the current checkpoint must retain the exact invocation with no job UID so + // the successor attempt can replay that call without repeating the model turn. + #[test] + fn provisional_agent_external_start_round_trips_without_a_job_uid_offline() { + let invocation = ToolInvocation { + id: Some("stable-provider-call".to_string()), + name: "render_video".to_string(), + input: json!({"prompt": "recover this exact effect"}), + }; + let continuation = agent_continuation( + vec![ContextMessage::user("render once")], + 2, + zero_usage(), + SecurityCircuitState::default(), + BTreeMap::new(), + AgentPending { + review: None, + tool_calls: Vec::new(), + external: Some(PendingExternalToolInvocation { + external_job_uid: None, + invocation: invocation.clone(), + effect_idempotency: IdempotencyClass::NonIdempotent, + }), + }, + ); + + let decoded: TaskAttemptContinuation = serde_json::from_value( + continuation + .to_bounded_json() + .expect("provisional continuation must fit"), + ) + .expect("provisional continuation must decode"); + let TaskAttemptContinuationState::Agent { + pending_external: Some(pending), + .. + } = decoded.state + else { + panic!("provisional external start lost its pending invocation"); + }; + assert_eq!(pending.external_job_uid, None); + assert_eq!(pending.invocation, invocation); + } + + const fn zero_usage() -> ExecutionUsage { + ExecutionUsage { + cost_microusd: 0, + tokens: 0, + tool_calls: 0, + retrieved_bytes: 0, + } + } +} diff --git a/crates/moa-orchestrator/src/workflows/execution_task_attempt/active/capability.rs b/crates/moa-orchestrator/src/workflows/execution_task_attempt/active/capability.rs new file mode 100644 index 000000000..9f09b5045 --- /dev/null +++ b/crates/moa-orchestrator/src/workflows/execution_task_attempt/active/capability.rs @@ -0,0 +1,456 @@ +//! Direct governed-capability execution for one active task attempt. + +use std::collections::BTreeSet; + +use moa_artifacts::execution_plan::{ + CapabilityReference, ExecutionFailureClass, ExecutionTaskOutcome, ExecutionTaskResult, + ExecutionUsage, +}; +use moa_core::types::{ + action_policy::CapabilityProvenance, + completion::{ToolCallContent, ToolInvocation}, + identifiers::ToolCallId, + resource::ResourceBudget, + tools::{IdempotencyClass, ToolAsyncMode}, +}; +use moa_execution::{ + capability::ExecutionCapability, + repository::task::{TaskAttemptCheckpointKind, TaskAttemptRecord}, + schema::validate_instance, + state::{completed_task_outcome, failed_task_outcome}, + wire::ExecutionTaskAttemptRequest, +}; +use restate_sdk::prelude::*; +use serde_json::Value; +use uuid::Uuid; + +use crate::{ + tool_invocation::governed::{ + GovernedInvocationDisposition, GovernedInvocationOrigin, GovernedInvocationOutcome, + GovernedInvocationRequest, invoke_governed_tool, + }, + workflows::execution_task_attempt::{ + ExecutionTaskAttemptImpl, capability_tool_name, + continuation::{ + PendingReviewedToolInvocation, TASK_ATTEMPT_CONTINUATION_SCHEMA_VERSION, + TaskAttemptContinuation, TaskAttemptContinuationState, + }, + }, +}; + +use super::{ + ActiveTaskAttemptExit, capability_source_kind, execution_dispatch_rejection_message, + find_capability, + heartbeat::{AttemptHeartbeat, begin_capability_dispatch, record_attempt_heartbeat}, + load_session, persist_external_start_checkpoint, serialized_len, +}; + +/// Executes one direct governed capability without waiting on a future event. +pub(super) async fn execute_direct_capability( + workflow: &ExecutionTaskAttemptImpl, + ctx: &WorkflowContext<'_>, + request: &ExecutionTaskAttemptRequest, + started: &TaskAttemptRecord, + reference: &CapabilityReference, + continuation: Option<&TaskAttemptContinuation>, +) -> Result { + let capability = find_capability(&started.run, reference)?; + let (tool_id, mut usage) = match continuation { + Some( + continuation @ TaskAttemptContinuation { + state: TaskAttemptContinuationState::CapabilityReview { .. }, + .. + }, + ) => return resume_reviewed_capability(capability, continuation), + Some(TaskAttemptContinuation { + state: TaskAttemptContinuationState::CapabilityExternalStart { tool_id, usage }, + review_resolution: None, + external_job_resolution: None, + .. + }) => (*tool_id, usage.clone()), + Some(_) => { + return Err(TerminalError::new( + "direct capability received an incompatible continuation", + ) + .into()); + } + None => { + if let Err(error) = validate_instance( + &capability.input_schema, + &started.task.input, + "execution_task.capability_input", + ) { + return Ok(ActiveTaskAttemptExit::Outcome(failed_task_outcome( + ExecutionFailureClass::InvalidInput, + error.to_string(), + started.task.actual.clone(), + ))); + } + ( + ToolCallId(Uuid::new_v5( + &request.dispatch_uid, + format!("task-capability:{}", started.task.generation).as_bytes(), + )), + started.task.actual.clone(), + ) + } + }; + let session = load_session(workflow, ctx, &started.run, &started.task).await?; + let tool_name = capability_tool_name(capability)?; + let tool_call = ToolCallContent { + invocation: ToolInvocation { + id: Some(tool_id.to_string()), + name: tool_name.clone(), + input: started.task.input.clone(), + }, + provider_metadata: None, + }; + let allowed_tools = BTreeSet::from([tool_name]); + let provenance = CapabilityProvenance { + kind: Some(capability_source_kind(&capability.source).to_string()), + id: Some(format!( + "{}@{}", + capability.reference.name, capability.reference.version + )), + step_id: Some(started.task.node_id.clone()), + }; + if matches!( + capability.async_mode, + ToolAsyncMode::MayReturnExternalJob { .. } + ) { + let provisional = TaskAttemptContinuation { + schema_version: TASK_ATTEMPT_CONTINUATION_SCHEMA_VERSION, + state: TaskAttemptContinuationState::CapabilityExternalStart { + tool_id, + usage: usage.clone(), + }, + review_resolution: None, + external_job_resolution: None, + workspace_release_receipt_id: None, + }; + if !persist_external_start_checkpoint( + workflow, + ctx, + request, + started, + TaskAttemptCheckpointKind::CapabilityExternalStart, + &provisional, + ) + .await? + { + return Ok(ActiveTaskAttemptExit::OwnershipLost); + } + } + if !begin_capability_dispatch(workflow, ctx, request, capability, &tool_call).await? { + return Ok(ActiveTaskAttemptExit::OwnershipLost); + } + let governed = invoke_governed_tool( + ctx, + GovernedInvocationRequest { + session: &session, + identity: &started.run.admitted_identity, + session_id: started.run.session_id, + tool_id, + tool_call: &tool_call, + allowed_tools: &allowed_tools, + expected_tool_contract_revision: Some(&capability.contract_revision), + active_canary: None, + trusted_sandbox_manifest: None, + origin: GovernedInvocationOrigin::ExecutionTask { + run_uid: started.run.run_uid, + task_uid: started.task.task_id.as_uuid(), + generation: started.task.generation, + attempt_generation: request.attempt_generation, + }, + capability_provenance: Some(&provenance), + capability_policy_context: Some(&capability.policy_context), + resource_budget: ResourceBudget::until(request.attempt_deadline_at), + }, + &workflow.session_limits, + workflow.session_store.clone(), + workflow.channel_adapters.as_ref(), + ) + .await?; + if !record_attempt_heartbeat(workflow, ctx, request, AttemptHeartbeat::ToolCall).await? { + return Ok(ActiveTaskAttemptExit::OwnershipLost); + } + usage.tool_calls = usage.tool_calls.saturating_add(1); + classify_capability_outcome(capability, governed, usage) +} + +fn classify_capability_outcome( + capability: &ExecutionCapability, + outcome: GovernedInvocationOutcome, + mut usage: ExecutionUsage, +) -> Result { + match outcome { + GovernedInvocationOutcome::Completed(result) + if result.disposition == GovernedInvocationDisposition::ReviewPending => + { + let review = result.review.ok_or_else(|| { + TerminalError::new( + "review-pending governed result is missing durable review identity", + ) + })?; + Ok(ActiveTaskAttemptExit::ReviewPending { + continuation: TaskAttemptContinuation { + schema_version: TASK_ATTEMPT_CONTINUATION_SCHEMA_VERSION, + state: TaskAttemptContinuationState::CapabilityReview { + pending_review: PendingReviewedToolInvocation { + review_uid: review.review_uid, + expires_at: review.expires_at, + invocation: result.invocation, + effect_idempotency: capability.idempotency_class, + }, + usage, + }, + review_resolution: None, + external_job_resolution: None, + workspace_release_receipt_id: None, + }, + }) + } + GovernedInvocationOutcome::Completed(result) => { + usage.retrieved_bytes = usage.retrieved_bytes.saturating_add(serialized_len( + &result.output.safe_output.structured_payload(), + )); + let task_outcome = if result.output.is_error() { + if capability.idempotency_class == IdempotencyClass::Idempotent { + failed_task_outcome( + ExecutionFailureClass::Retryable, + result.output.safe_output.to_text(), + usage, + ) + } else if capability.action_class + != moa_core::types::action_policy::ActionClass::Read + { + ExecutionTaskOutcome { + schema_version: 1, + usage, + result: ExecutionTaskResult::UnknownOutcome { + message: format!( + "non-idempotent side effect returned an error after possible commit: {}", + result.output.safe_output.to_text() + ), + }, + } + } else { + failed_task_outcome( + ExecutionFailureClass::Terminal, + result.output.safe_output.to_text(), + usage, + ) + } + } else { + let value = result + .output + .safe_output + .structured_payload() + .cloned() + .unwrap_or_else(|| Value::String(result.output.safe_output.to_text())); + if let Err(error) = validate_instance( + &capability.output_schema, + &value, + "execution_task.capability_output", + ) { + if capability.action_class == moa_core::types::action_policy::ActionClass::Read + { + failed_task_outcome( + ExecutionFailureClass::InvalidOutput, + error.to_string(), + usage, + ) + } else { + ExecutionTaskOutcome { + schema_version: 1, + usage, + result: ExecutionTaskResult::UnknownOutcome { + message: format!( + "side effect returned invalid output after possible commit: {error}" + ), + }, + } + } + } else { + completed_task_outcome(value, usage) + } + }; + Ok(ActiveTaskAttemptExit::Outcome(task_outcome)) + } + GovernedInvocationOutcome::ExternalJob { + external_job_uid, .. + } => Ok(ActiveTaskAttemptExit::ExternalJob { + external_job_uid, + continuation: None, + }), + GovernedInvocationOutcome::UnknownOutcome { message, .. } => { + Ok(ActiveTaskAttemptExit::Outcome(ExecutionTaskOutcome { + schema_version: 1, + usage, + result: ExecutionTaskResult::UnknownOutcome { message }, + })) + } + GovernedInvocationOutcome::NotDispatched { reason, .. } => { + Ok(ActiveTaskAttemptExit::Outcome(failed_task_outcome( + ExecutionFailureClass::Terminal, + execution_dispatch_rejection_message(reason), + usage, + ))) + } + GovernedInvocationOutcome::Delegation { .. } => { + Err(TerminalError::new("execution tasks cannot invoke delegation capabilities").into()) + } + } +} + +fn resume_reviewed_capability( + capability: &ExecutionCapability, + continuation: &TaskAttemptContinuation, +) -> Result { + let TaskAttemptContinuationState::CapabilityReview { + pending_review: _, + usage, + } = &continuation.state + else { + return Err(TerminalError::new( + "direct capability received an incompatible agent continuation", + ) + .into()); + }; + let resolution = continuation.review_resolution.as_ref().ok_or_else(|| { + TerminalError::new("reviewed capability continuation has no durable resolution") + })?; + let exit = match resolution { + moa_execution::wire::ExecutionActionReviewResolution::Completed { tool_output } => { + match serde_json::from_value::( + tool_output.clone(), + ) { + Ok(output) => ActiveTaskAttemptExit::Outcome(capability_output_outcome( + capability, + output, + usage.clone(), + )), + Err(error) => ActiveTaskAttemptExit::Outcome(ExecutionTaskOutcome { + schema_version: 1, + usage: usage.clone(), + result: ExecutionTaskResult::UnknownOutcome { + message: format!( + "reviewed capability returned invalid output after possible commit: {error}" + ), + }, + }), + } + } + moa_execution::wire::ExecutionActionReviewResolution::ExternalJob { + external_job_uid, + .. + } => ActiveTaskAttemptExit::ExternalJob { + external_job_uid: *external_job_uid, + continuation: None, + }, + moa_execution::wire::ExecutionActionReviewResolution::Failed { class, message } => { + ActiveTaskAttemptExit::Outcome(ExecutionTaskOutcome { + schema_version: 1, + usage: usage.clone(), + result: ExecutionTaskResult::Failed { + class: class.clone(), + message: message.clone(), + }, + }) + } + moa_execution::wire::ExecutionActionReviewResolution::UnknownOutcome { message } => { + ActiveTaskAttemptExit::Outcome(ExecutionTaskOutcome { + schema_version: 1, + usage: usage.clone(), + result: ExecutionTaskResult::UnknownOutcome { + message: message.clone(), + }, + }) + } + moa_execution::wire::ExecutionActionReviewResolution::NotDispatched { reason } => { + ActiveTaskAttemptExit::Outcome(failed_task_outcome( + ExecutionFailureClass::Terminal, + execution_dispatch_rejection_message(*reason), + usage.clone(), + )) + } + moa_execution::wire::ExecutionActionReviewResolution::Denied { reason } => { + ActiveTaskAttemptExit::Outcome(failed_task_outcome( + ExecutionFailureClass::AuthorizationDenied, + reason.clone(), + usage.clone(), + )) + } + moa_execution::wire::ExecutionActionReviewResolution::TimedOut { reason } => { + ActiveTaskAttemptExit::Outcome(failed_task_outcome( + ExecutionFailureClass::DeadlineExceeded, + reason.clone(), + usage.clone(), + )) + } + }; + Ok(exit) +} + +fn capability_output_outcome( + capability: &ExecutionCapability, + output: moa_core::types::tools::SecuredToolOutput, + usage: ExecutionUsage, +) -> ExecutionTaskOutcome { + if output.is_error() { + if capability.idempotency_class == IdempotencyClass::Idempotent { + return failed_task_outcome( + ExecutionFailureClass::Retryable, + output.safe_output.to_text(), + usage, + ); + } + if capability.action_class != moa_core::types::action_policy::ActionClass::Read { + return ExecutionTaskOutcome { + schema_version: 1, + usage, + result: ExecutionTaskResult::UnknownOutcome { + message: format!( + "non-idempotent side effect returned an error after possible commit: {}", + output.safe_output.to_text() + ), + }, + }; + } + return failed_task_outcome( + ExecutionFailureClass::Terminal, + output.safe_output.to_text(), + usage, + ); + } + let value = output + .safe_output + .structured_payload() + .cloned() + .unwrap_or_else(|| Value::String(output.safe_output.to_text())); + if let Err(error) = validate_instance( + &capability.output_schema, + &value, + "execution_task.capability_output", + ) { + if capability.action_class == moa_core::types::action_policy::ActionClass::Read { + failed_task_outcome( + ExecutionFailureClass::InvalidOutput, + error.to_string(), + usage, + ) + } else { + ExecutionTaskOutcome { + schema_version: 1, + usage, + result: ExecutionTaskResult::UnknownOutcome { + message: format!( + "side effect returned invalid output after possible commit: {error}" + ), + }, + } + } + } else { + completed_task_outcome(value, usage) + } +} diff --git a/crates/moa-orchestrator/src/workflows/execution_task_attempt/active/heartbeat.rs b/crates/moa-orchestrator/src/workflows/execution_task_attempt/active/heartbeat.rs new file mode 100644 index 000000000..1bb3bcb3b --- /dev/null +++ b/crates/moa-orchestrator/src/workflows/execution_task_attempt/active/heartbeat.rs @@ -0,0 +1,293 @@ +//! Durable heartbeat fencing for active execution-task steps. + +use chrono::{DateTime, Utc}; +use moa_core::types::{completion::ToolCallContent, tools::ToolAsyncMode}; +use moa_execution::{ + capability::ExecutionCapability, repository::task::TaskAttemptProgressOutcome, + wire::ExecutionTaskAttemptRequest, +}; +use restate_sdk::prelude::*; + +use crate::workflows::{ + durable_utc_now, + execution_task_attempt::{ExecutionTaskAttemptImpl, task_attempt_fence}, +}; + +/// Durable step boundary at which an active attempt reports progress. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum AttemptHeartbeat { + /// One model completion is about to start. The bounded gateway budget cannot outlive the + /// attempt deadline, so the persisted stall window covers that exact call. + ModelTurnStart, + /// One model turn returned, so the following tool dispatch starts its own stall window. + ModelTurn, + /// One governed tool invocation is about to start with its declared stall bound. + ToolCallStart { bound: Option }, + /// One governed tool invocation returned, so sandbox release and continuation persistence + /// start their own stall window. + ToolCall, +} + +impl AttemptHeartbeat { + /// Deterministic journal step name for this boundary. + const fn observation_step(self) -> &'static str { + match self { + Self::ModelTurnStart => "task_attempt_model_turn_start_progress_at", + Self::ModelTurn => "task_attempt_model_turn_progress_at", + Self::ToolCallStart { .. } => "task_attempt_tool_call_start_progress_at", + Self::ToolCall => "task_attempt_tool_call_progress_at", + } + } + + /// Deterministic journal step name for the persisted heartbeat. + const fn write_step(self) -> &'static str { + match self { + Self::ModelTurnStart => "record_task_attempt_model_turn_start_progress", + Self::ModelTurn => "record_task_attempt_model_turn_progress", + Self::ToolCallStart { .. } => "record_task_attempt_tool_call_start_progress", + Self::ToolCall => "record_task_attempt_tool_call_progress", + } + } + + /// Upper bound of the step this boundary opens, when that step declares one. + /// + /// Post-return boundaries clear the bound back to the configured heartbeat floor. + fn step_bound_seconds( + self, + request: &ExecutionTaskAttemptRequest, + observed_at: DateTime, + ) -> Option { + match self { + Self::ModelTurnStart => Some(AttemptStepBound::UntilAttemptDeadline) + .and_then(|bound| bound.seconds(request, observed_at)), + Self::ToolCallStart { bound } => { + bound.and_then(|bound| bound.seconds(request, observed_at)) + } + Self::ModelTurn | Self::ToolCall => None, + } + } +} + +/// Returns the bound to record for the capability step this attempt is about to dispatch. +/// +/// An external provider start remains bounded by the attempt deadline so its recovery trigger, +/// rather than the task watchdog, retains authority over an ambiguous start. +fn capability_step_bound( + requires_sandbox: bool, + async_mode: &ToolAsyncMode, + tool_call: &ToolCallContent, +) -> Option { + if requires_sandbox || matches!(async_mode, ToolAsyncMode::MayReturnExternalJob { .. }) { + return Some(AttemptStepBound::UntilAttemptDeadline); + } + moa_hands::tools::bash::declared_tool_step_bound( + &tool_call.invocation.name, + &tool_call.invocation.input, + ) + .and_then(|bound| u32::try_from(bound.as_secs()).ok()) + .map(AttemptStepBound::Declared) +} + +/// Upper bound a dispatching step declares for itself. +/// +/// `UntilAttemptDeadline` is resolved against the journaled heartbeat instant rather than a +/// fresh clock read, because a workflow that reads the wall clock outside the journal +/// produces a different value on replay. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum AttemptStepBound { + /// The step named its own ceiling in seconds. + Declared(u32), + /// The step runs until the attempt deadline and must not be cut short before it. + UntilAttemptDeadline, +} + +impl AttemptStepBound { + /// Resolves this bound to seconds against one journaled observation instant. + fn seconds( + self, + request: &ExecutionTaskAttemptRequest, + observed_at: DateTime, + ) -> Option { + match self { + Self::Declared(seconds) => Some(seconds), + // Rounded up so a sub-second remainder still outlasts the deadline it covers. + Self::UntilAttemptDeadline => u32::try_from( + request + .attempt_deadline_at + .signed_duration_since(observed_at) + .num_seconds() + .saturating_add(1), + ) + .ok() + .filter(|seconds| *seconds > 0), + } + } +} + +/// Advances the active attempt's durable progress clock and returns whether it still owns it. +/// +/// The exact dispatch fence prevents a parked, superseded, or settled attempt from regaining +/// dispatch authority. The observation time is journaled for replay stability. +pub(super) async fn record_attempt_heartbeat( + workflow: &ExecutionTaskAttemptImpl, + ctx: &WorkflowContext<'_>, + request: &ExecutionTaskAttemptRequest, + boundary: AttemptHeartbeat, +) -> Result { + let observed_at = durable_utc_now(ctx, boundary.observation_step()).await?; + let repository = workflow.repository.clone(); + let fence = task_attempt_fence(request); + Ok(ctx + .run(|| async move { + repository + .record_task_attempt_progress( + fence, + observed_at, + boundary.step_bound_seconds(request, observed_at), + ) + .await + .map(|outcome| Json::from(attempt_progress_retains_ownership(outcome))) + .map_err(crate::workflows::errors::execution_error_to_handler_error) + }) + .name(boundary.write_step()) + .await? + .into_inner()) +} + +const fn attempt_progress_retains_ownership(outcome: TaskAttemptProgressOutcome) -> bool { + matches!( + outcome, + TaskAttemptProgressOutcome::Applied | TaskAttemptProgressOutcome::Replayed + ) +} + +/// Records the exact capability bound and confirms ownership before provider dispatch. +pub(super) async fn begin_capability_dispatch( + workflow: &ExecutionTaskAttemptImpl, + ctx: &WorkflowContext<'_>, + request: &ExecutionTaskAttemptRequest, + capability: &ExecutionCapability, + tool_call: &ToolCallContent, +) -> Result { + record_attempt_heartbeat( + workflow, + ctx, + request, + AttemptHeartbeat::ToolCallStart { + bound: capability_step_bound( + capability.requires_sandbox, + &capability.async_mode, + tool_call, + ), + }, + ) + .await +} + +#[cfg(test)] +mod tests { + use chrono::{Duration, TimeZone, Utc}; + use moa_core::types::{completion::ToolInvocation, identifiers::TenantId}; + use moa_execution::state::ExecutionTaskId; + use serde_json::json; + use uuid::Uuid; + + use super::*; + + // Pins: every pre-provider heartbeat is an ownership check, not telemetry. A stale, + // absent, or non-running attempt must stop before model or tool dispatch, while exact + // replay of an already-journaled heartbeat retains authority. + #[test] + fn heartbeat_verdict_stops_dispatch_after_ownership_loss_offline() { + assert!(attempt_progress_retains_ownership( + TaskAttemptProgressOutcome::Applied + )); + assert!(attempt_progress_retains_ownership( + TaskAttemptProgressOutcome::Replayed + )); + for lost in [ + TaskAttemptProgressOutcome::NotFound, + TaskAttemptProgressOutcome::Stale, + TaskAttemptProgressOutcome::InvalidState, + ] { + assert!(!attempt_progress_retains_ownership(lost)); + } + } + + // Pins: a healthy model or tool call whose declared duration exceeds the configured + // heartbeat floor remains live for that exact step, and the first post-return heartbeat + // clears the widened bound back to the ordinary orchestration floor. + #[test] + fn model_and_tool_steps_are_bounded_before_dispatch_and_cleared_after_return_offline() { + let observed_at = Utc + .with_ymd_and_hms(2026, 8, 13, 12, 0, 0) + .single() + .expect("fixture timestamp is valid"); + let request = ExecutionTaskAttemptRequest { + dispatch_uid: Uuid::from_u128(1), + capacity_reservation_uid: Uuid::from_u128(2), + watchdog_trigger_uid: Uuid::from_u128(3), + watchdog_dispatch_uid: Uuid::from_u128(4), + run_uid: Uuid::from_u128(5), + task_id: ExecutionTaskId::from_uuid(Uuid::from_u128(6)), + controller_generation: 7, + attempt_generation: 8, + attempt_deadline_at: observed_at + Duration::seconds(121), + tenant_id: TenantId(Uuid::from_u128(9)), + }; + + assert_eq!( + AttemptHeartbeat::ModelTurnStart.step_bound_seconds(&request, observed_at), + Some(122), + ); + assert_eq!( + AttemptHeartbeat::ToolCallStart { + bound: Some(AttemptStepBound::Declared(90)), + } + .step_bound_seconds(&request, observed_at), + Some(90), + ); + assert_eq!( + AttemptHeartbeat::ModelTurn.step_bound_seconds(&request, observed_at), + None, + ); + assert_eq!( + AttemptHeartbeat::ToolCall.step_bound_seconds(&request, observed_at), + None, + ); + } + + // Pins: sandbox lifecycle work shares the active attempt deadline because provisioning, + // restore, install, execution, and commit can outlive the command timeout alone. A + // non-sandbox synchronous call keeps its narrower declared execution bound. + #[test] + fn sandbox_capability_uses_attempt_bound_while_non_sandbox_keeps_tool_bound_offline() { + let tool_call = ToolCallContent { + invocation: ToolInvocation { + id: Some("bounded-bash".to_string()), + name: "bash".to_string(), + input: json!({"cmd": "sleep 1", "timeout_secs": 90}), + }, + provider_metadata: None, + }; + + assert_eq!( + capability_step_bound(true, &ToolAsyncMode::SynchronousOnly, &tool_call), + Some(AttemptStepBound::UntilAttemptDeadline), + ); + assert_eq!( + capability_step_bound(false, &ToolAsyncMode::SynchronousOnly, &tool_call), + Some(AttemptStepBound::Declared(90)), + ); + assert_eq!( + capability_step_bound( + false, + &ToolAsyncMode::MayReturnExternalJob { + provider: "fixture".to_string(), + }, + &tool_call, + ), + Some(AttemptStepBound::UntilAttemptDeadline), + ); + } +} diff --git a/crates/moa-orchestrator/src/workflows/execution_task_attempt/continuation.rs b/crates/moa-orchestrator/src/workflows/execution_task_attempt/continuation.rs new file mode 100644 index 000000000..979554db2 --- /dev/null +++ b/crates/moa-orchestrator/src/workflows/execution_task_attempt/continuation.rs @@ -0,0 +1,235 @@ +//! Persisted continuation schema for bounded execution-task attempts. + +use moa_core::types::{ + completion::ToolInvocation, + context::ContextMessage, + identifiers::ToolCallId, + security::{SecurityCircuitState, ToolCapabilityId}, + tools::{AsyncToolJobTerminalOutcome, IdempotencyClass}, +}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +/// Current durable schema for a bounded task-agent continuation. +pub(super) const TASK_ATTEMPT_CONTINUATION_SCHEMA_VERSION: u32 = 1; + +/// Maximum canonical continuation payload accepted by persistence. +pub(super) const MAX_TASK_ATTEMPT_CONTINUATION_BYTES: usize = 1024 * 1024; + +/// Canonical state needed to resume an agent without replaying an external effect. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct TaskAttemptContinuation { + /// Durable schema version. + pub schema_version: u32, + /// Exact bounded execution state. + pub state: TaskAttemptContinuationState, + /// Exact storage-only action-review resolution consumed by the next attempt. + pub review_resolution: Option, + /// Exact terminal provider outcome consumed by a resumed agent external effect. + pub external_job_resolution: Option, + /// Release receipt that proves sandbox compute is asleep before this wait was published. + pub workspace_release_receipt_id: Option, +} + +/// Supported bounded continuation points. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub(super) enum TaskAttemptContinuationState { + /// Task-local agent state after a complete model/tool boundary. + Agent { + /// Complete bounded conversation required by the next model turn. + messages: Vec, + /// Zero-based model turn to execute next. + next_turn: u32, + /// Cumulative durable task usage. + usage: moa_artifacts::execution_plan::ExecutionUsage, + /// Prompt-injection circuit state owned by this exact task generation. + security_circuit: SecurityCircuitState, + /// Capabilities fenced by the persisted circuit. + disabled_capabilities: std::collections::BTreeMap, + /// Exact effect waiting on a storage-only action review, when present. + pending_review: Option>, + /// Model-emitted tool effects not yet dispatched by a bounded slice. + pending_tool_calls: Vec, + /// Exact agent tool invocation currently owned by an asynchronous provider job. + pending_external: Option, + }, + /// Direct capability effect waiting on a storage-only action review. + CapabilityReview { + /// Exact reviewed effect; resumption consumes its persisted resolution. + pending_review: PendingReviewedToolInvocation, + /// Cumulative durable task usage. + usage: moa_artifacts::execution_plan::ExecutionUsage, + }, + /// Direct async-capable effect reserved before its provider start. + CapabilityExternalStart { + /// Stable tool-call identity reused if recovery proves the provider did not start. + tool_id: ToolCallId, + /// Cumulative durable task usage before provider dispatch. + usage: moa_artifacts::execution_plan::ExecutionUsage, + }, +} + +/// Reviewed provider effect that must never be reconstructed from a fresh model turn. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct PendingReviewedToolInvocation { + /// Stable action-review identity. + pub review_uid: Uuid, + /// Exact durable review expiry returned by action-review admission. + pub expires_at: chrono::DateTime, + /// Exact provider invocation accepted by policy. + pub invocation: ToolInvocation, + /// Compiler/catalog-pinned replay semantics for watchdog classification. + pub effect_idempotency: IdempotencyClass, +} + +/// Agent effect that was durably handed to an asynchronous provider. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct PendingExternalToolInvocation { + /// Stable MOA external-job identity bound before sandbox release. + pub external_job_uid: Option, + /// Exact model-emitted invocation awaiting the terminal provider result. + pub invocation: ToolInvocation, + /// Compiler/catalog-pinned replay semantics. + pub effect_idempotency: IdempotencyClass, +} + +impl TaskAttemptContinuation { + /// Returns the exact action-review identity carried by a parked continuation. + pub(super) const fn pending_review_uid(&self) -> Option { + match &self.state { + TaskAttemptContinuationState::Agent { pending_review, .. } => match pending_review { + Some(pending) => Some(pending.review_uid), + None => None, + }, + TaskAttemptContinuationState::CapabilityReview { pending_review, .. } => { + Some(pending_review.review_uid) + } + TaskAttemptContinuationState::CapabilityExternalStart { .. } => None, + } + } + + /// Binds the deterministic MOA external-job identity before checkpoint persistence. + pub(super) fn bind_external_job(&mut self, external_job_uid: Uuid) -> Result<(), String> { + let TaskAttemptContinuationState::Agent { + pending_external: Some(pending), + .. + } = &mut self.state + else { + return Err("agent external continuation is missing its pending effect".to_string()); + }; + if pending + .external_job_uid + .is_some_and(|current| current != external_job_uid) + { + return Err("agent external continuation is bound to another job".to_string()); + } + pending.external_job_uid = Some(external_job_uid); + Ok(()) + } + + /// Serializes and enforces the hard continuation-size bound before any DB write. + pub(super) fn to_bounded_json(&self) -> Result { + if self.schema_version != TASK_ATTEMPT_CONTINUATION_SCHEMA_VERSION { + return Err(format!( + "unsupported task continuation schema version {}", + self.schema_version + )); + } + let bytes = serde_json::to_vec(self) + .map_err(|error| format!("serialize task continuation: {error}"))?; + if bytes.len() > MAX_TASK_ATTEMPT_CONTINUATION_BYTES { + return Err(format!( + "task continuation is {} bytes; maximum is {} and the task must be decomposed or replanned", + bytes.len(), + MAX_TASK_ATTEMPT_CONTINUATION_BYTES + )); + } + serde_json::from_slice(&bytes) + .map_err(|error| format!("decode canonical task continuation: {error}")) + } +} + +#[cfg(test)] +mod tests { + use moa_artifacts::execution_plan::ExecutionUsage; + + use super::*; + + // Pins: a continuation that cannot fit in the bounded durable payload is rejected + // before persistence so callers must decompose or request a replan. + #[test] + fn oversized_agent_continuation_requires_decomposition_offline() { + let continuation = TaskAttemptContinuation { + schema_version: TASK_ATTEMPT_CONTINUATION_SCHEMA_VERSION, + state: TaskAttemptContinuationState::Agent { + messages: vec![ContextMessage::user( + "x".repeat(MAX_TASK_ATTEMPT_CONTINUATION_BYTES), + )], + next_turn: 1, + usage: ExecutionUsage { + cost_microusd: 0, + tokens: 0, + tool_calls: 0, + retrieved_bytes: 0, + }, + security_circuit: SecurityCircuitState::default(), + disabled_capabilities: std::collections::BTreeMap::new(), + pending_review: None, + pending_tool_calls: Vec::new(), + pending_external: None, + }, + review_resolution: None, + external_job_resolution: None, + workspace_release_receipt_id: None, + }; + + let error = continuation + .to_bounded_json() + .expect_err("oversized continuation must fail closed"); + assert!(error.contains("must be decomposed or replanned")); + } + + // Pins: a direct async capability resumes with the same stable tool-call ID + // after a NotStarted recovery instead of creating a second provider identity. + #[test] + fn direct_external_start_checkpoint_round_trips_stable_tool_id_offline() { + let tool_id = ToolCallId(Uuid::from_u128(77)); + let continuation = TaskAttemptContinuation { + schema_version: TASK_ATTEMPT_CONTINUATION_SCHEMA_VERSION, + state: TaskAttemptContinuationState::CapabilityExternalStart { + tool_id, + usage: zero_usage(), + }, + review_resolution: None, + external_job_resolution: None, + workspace_release_receipt_id: None, + }; + + let decoded: TaskAttemptContinuation = serde_json::from_value( + continuation + .to_bounded_json() + .expect("direct provisional continuation must fit"), + ) + .expect("direct provisional continuation must decode"); + assert!(matches!( + decoded.state, + TaskAttemptContinuationState::CapabilityExternalStart { + tool_id: decoded_tool_id, + .. + } if decoded_tool_id == tool_id + )); + } + + const fn zero_usage() -> ExecutionUsage { + ExecutionUsage { + cost_microusd: 0, + tokens: 0, + tool_calls: 0, + retrieved_bytes: 0, + } + } +} diff --git a/crates/moa-orchestrator/src/workflows/execution_task_attempt/external.rs b/crates/moa-orchestrator/src/workflows/execution_task_attempt/external.rs index cb5e90c56..0165e4fae 100644 --- a/crates/moa-orchestrator/src/workflows/execution_task_attempt/external.rs +++ b/crates/moa-orchestrator/src/workflows/execution_task_attempt/external.rs @@ -14,7 +14,7 @@ use crate::workflows::{ durable_utc_now, errors::execution_error_to_handler_error, execution_task_attempt::{ - ExecutionTaskAttemptImpl, active::TaskAttemptContinuation, task_attempt_fence, + ExecutionTaskAttemptImpl, continuation::TaskAttemptContinuation, task_attempt_fence, yielding::checkpoint_task_hands_workflow, }, }; diff --git a/crates/moa-orchestrator/src/workflows/execution_task_attempt/watchdog.rs b/crates/moa-orchestrator/src/workflows/execution_task_attempt/watchdog.rs index 3731ca9fe..d04e7dfdd 100644 --- a/crates/moa-orchestrator/src/workflows/execution_task_attempt/watchdog.rs +++ b/crates/moa-orchestrator/src/workflows/execution_task_attempt/watchdog.rs @@ -30,7 +30,7 @@ use crate::{ errors::execution_error_to_handler_error, execution_task_attempt::{ ExecutionTaskAttemptImpl, - active::{TaskAttemptContinuation, TaskAttemptContinuationState}, + continuation::{TaskAttemptContinuation, TaskAttemptContinuationState}, task_attempt_fence, yielding::{begin_release_shared, checkpoint_task_hands_shared}, }, diff --git a/crates/moa-orchestrator/src/workflows/execution_task_attempt/yielding.rs b/crates/moa-orchestrator/src/workflows/execution_task_attempt/yielding.rs index 41585048f..85ebd7712 100644 --- a/crates/moa-orchestrator/src/workflows/execution_task_attempt/yielding.rs +++ b/crates/moa-orchestrator/src/workflows/execution_task_attempt/yielding.rs @@ -36,7 +36,7 @@ use crate::{ errors::execution_error_to_handler_error, execution_task_attempt::{ ExecutionTaskAttemptImpl, - active::{TaskAttemptContinuation, TaskAttemptContinuationState}, + continuation::{TaskAttemptContinuation, TaskAttemptContinuationState}, task_attempt_fence, }, }, diff --git a/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/burst_admission.rs b/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/burst_admission.rs index 9480a79b1..72c121bc3 100644 --- a/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/burst_admission.rs +++ b/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e/burst_admission.rs @@ -15,6 +15,8 @@ async fn one_thousand_common_wakes_bound_capacity_invocations_and_oldest_ready_a const RUN_COUNT: usize = 1_000; const FLEET_CAP: usize = 32; const ADMISSION_CONCURRENCY: usize = 8; + // Matches `MoaExecutionOldestReadyAgeHigh` in the checked-in alert rules. + const OLDEST_READY_ALERT_SECONDS: f64 = 120.0; // Seconds between admission and the shared absolute wake, and the margin by // which every run must already be parked before that wake arrives. const PRE_WAKE_SECONDS: i64 = 180; @@ -195,7 +197,7 @@ async fn one_thousand_common_wakes_bound_capacity_invocations_and_oldest_ready_a && metric .data_points() .iter() - .any(|point| point.value() > 0.0 && point.value() <= 60.0) + .any(|point| point.value() > 0.0 && point.value() <= OLDEST_READY_ALERT_SECONDS) }) .await .context("observe checked production oldest-ready-age metric")?; @@ -203,11 +205,11 @@ async fn one_thousand_common_wakes_bound_capacity_invocations_and_oldest_ready_a oldest_ready_metric .data_points() .iter() - .any(|point| point.value() > 0.0 && point.value() <= 60.0) + .any(|point| { point.value() > 0.0 && point.value() <= OLDEST_READY_ALERT_SECONDS }) ); let mut released = 0; - let mut maximum_oldest_ready_seconds = 0.0_f64; + let mut maximum_raw_oldest_ready_seconds = 0.0_f64; let drain_deadline = Instant::now() + DRAIN_BUDGET; while released < RUN_COUNT { let next = (released + FLEET_CAP).min(RUN_COUNT); @@ -218,28 +220,29 @@ async fn one_thousand_common_wakes_bound_capacity_invocations_and_oldest_ready_a ); } controller.wait_for_calls(next, remaining).await?; - let wave_active: i64 = sqlx::query_scalar( - "SELECT COALESCE(SUM(quantity), 0)::BIGINT FROM moa.execution_capacity_reservation \ - WHERE tenant_id = $1 AND resource_dimension = 'active_tasks' AND state <> 'released'", + // Observe both invariants from one statement snapshot so diagnostics do not hold every + // fleet-capped wave across a second database round trip. + let (wave_active, oldest): (i64, f64) = sqlx::query_as( + "SELECT \ + (SELECT COALESCE(SUM(quantity), 0)::BIGINT \ + FROM moa.execution_capacity_reservation \ + WHERE tenant_id = $1 AND resource_dimension = 'active_tasks' \ + AND state <> 'released'), \ + (SELECT COALESCE(EXTRACT(EPOCH FROM (now() - MIN(ready_at))), 0)::DOUBLE PRECISION \ + FROM moa.execution_task WHERE tenant_id = $1 AND status = 'ready')", ) .bind(tenant_id.0) .fetch_one(&pool) .await?; assert!(wave_active <= FLEET_CAP as i64); - let oldest: f64 = sqlx::query_scalar( - "SELECT COALESCE(EXTRACT(EPOCH FROM (now() - MIN(ready_at))), 0)::DOUBLE PRECISION \ - FROM moa.execution_task WHERE tenant_id = $1 AND status = 'ready'", - ) - .bind(tenant_id.0) - .fetch_one(&pool) - .await?; - maximum_oldest_ready_seconds = maximum_oldest_ready_seconds.max(oldest); + maximum_raw_oldest_ready_seconds = maximum_raw_oldest_ready_seconds.max(oldest); controller.release(next - released); released = next; } assert!( - maximum_oldest_ready_seconds <= 60.0, - "oldest ready task exceeded bounded age: {maximum_oldest_ready_seconds}s" + maximum_raw_oldest_ready_seconds <= OLDEST_READY_ALERT_SECONDS, + "raw oldest-ready age exceeded the production alert threshold: \ + {maximum_raw_oldest_ready_seconds}s" ); // The terminal settle is the tail of the same fleet-capped drain the loop above // budgets `DRAIN_BUDGET` for: every one of `RUN_COUNT` runs still has to finish diff --git a/crates/moa-providers/Cargo.toml b/crates/moa-providers/Cargo.toml index 5076fa68c..ec4bdc71d 100644 --- a/crates/moa-providers/Cargo.toml +++ b/crates/moa-providers/Cargo.toml @@ -39,6 +39,7 @@ workspace-hack = { workspace = true } [dev-dependencies] insta.workspace = true +moa-eval-core = { workspace = true } moa-memory-graph = { workspace = true } moa-runtime-store = { workspace = true, features = ["redis"] } opentelemetry_sdk = { workspace = true, features = ["testing"] } diff --git a/crates/moa-providers/tests/openai_responses_envelope.rs b/crates/moa-providers/tests/openai_responses_envelope.rs index cb69b4943..64b2c8b9d 100644 --- a/crates/moa-providers/tests/openai_responses_envelope.rs +++ b/crates/moa-providers/tests/openai_responses_envelope.rs @@ -2,13 +2,13 @@ use std::collections::HashMap; -use moa_core::transcript::ProviderEvent; use moa_core::{ traits::LLMProvider, types::completion::CompletionContent, types::completion::CompletionRequest, types::completion::StopReason, types::completion::TokenUsage, types::completion::ToolCallContent, types::completion::ToolInvocation, types::context::ContextMessage, types::identifiers::ModelId, }; +use moa_eval_core::transcript::ProviderEvent; use moa_providers::{OpenAIProvider, debug_build_openai_request_body}; use serde_json::{Value, json}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; diff --git a/crates/moa-test-support/Cargo.toml b/crates/moa-test-support/Cargo.toml index 8b7e41b49..35ef749b8 100644 --- a/crates/moa-test-support/Cargo.toml +++ b/crates/moa-test-support/Cargo.toml @@ -83,3 +83,6 @@ tracing.workspace = true url = { workspace = true, optional = true } uuid.workspace = true workspace-hack = { workspace = true } + +[dev-dependencies] +moa-eval-core = { workspace = true } diff --git a/crates/moa-test-support/README.md b/crates/moa-test-support/README.md index c44e0e71e..46b31103f 100644 --- a/crates/moa-test-support/README.md +++ b/crates/moa-test-support/README.md @@ -15,10 +15,10 @@ let cents = table.cost_cents("anthropic", "claude-sonnet-4", 125_000, 20_000, 50 ## Recorded Transcripts -`moa_core::transcript` reads and writes JSONL transcripts with one metadata line followed by one turn per line. Every turn must end with a terminal provider event. +`moa_eval_core::transcript` reads and writes JSONL transcripts with one metadata line followed by one turn per line. Every turn must end with a terminal provider event. ```rust -use moa_core::transcript::Transcript; +use moa_eval_core::transcript::Transcript; let transcript = Transcript::read_jsonl("crates/moa-test-support/fixtures/transcripts/example_minimal.jsonl".as_ref())?; transcript.validate()?; diff --git a/crates/moa-test-support/tests/fixtures_round_trip.rs b/crates/moa-test-support/tests/fixtures_round_trip.rs index f87099070..976c6073c 100644 --- a/crates/moa-test-support/tests/fixtures_round_trip.rs +++ b/crates/moa-test-support/tests/fixtures_round_trip.rs @@ -2,7 +2,7 @@ use std::path::Path; -use moa_core::transcript::{ProviderEvent, Transcript, TranscriptError, Turn, UserUtterance}; +use moa_eval_core::transcript::{ProviderEvent, Transcript, TranscriptError, Turn, UserUtterance}; use moa_test_support::postgres::bootstrap_test_db; use moa_test_support::pricing::PricingTable; use sqlx::Row; diff --git a/crates/xtask/README.md b/crates/xtask/README.md index ff14301f6..7203007e2 100644 --- a/crates/xtask/README.md +++ b/crates/xtask/README.md @@ -14,6 +14,21 @@ Default commands: - `check-migrations` — enforce the flat canonical `V000001..V00000N` central sequence, ban non-central `migrations/` directories, and require exact table ownership. +- `check-subsystems` — validate the path, owner, architecture-doc, local-agent, + nextest/Make, live-prerequisite, and workspace-member coverage in + `.agents/subsystems.toml`. +- `plan-subsystem-audit` — resolve a base revision or explicit paths through the + validated subsystem map and write at most four deterministic, read-only + context packets under `target/agent-audits/`. It never launches agents or runs + the referenced test/live gates. + +Plan an affected audit with checked-in identifiers rather than copied commands: + +```bash +cargo xtask plan-subsystem-audit --base origin/main --max-agents 4 +cargo xtask plan-subsystem-audit --path crates/moa-hands/src/lib.rs \ + --output target/agent-audits/hands-review +``` ## Features diff --git a/crates/xtask/src/execution_trace_manifest.rs b/crates/xtask/src/execution_trace_manifest.rs index cef1fbb58..f2dfdf58a 100644 --- a/crates/xtask/src/execution_trace_manifest.rs +++ b/crates/xtask/src/execution_trace_manifest.rs @@ -606,6 +606,27 @@ const SENDERS: &[SenderManifestEntry] = &[ "ExecutionDispatcherClient", "dispatch" ), + sender!( + "crates/moa-orchestrator/src/services/execution_amendment_planner.rs", + "complete", + TRACE_HELPER, + "LLMGatewayClient", + "complete_bounded" + ), + sender!( + "crates/moa-orchestrator/src/services/execution_amendment_planner.rs", + "dispatch_parked_replan_planning", + TRACE_HELPER, + "ExecutionAmendmentPlannerClient", + "plan" + ), + sender!( + "crates/moa-orchestrator/src/services/execution_amendment_planner.rs", + "submit_amendment", + IDENTITY_TRACE_HELPER, + "ExecutionClient", + "apply_amendment" + ), sender!( "crates/moa-orchestrator/src/services/execution_dispatcher.rs", "accept_target", @@ -994,19 +1015,12 @@ const SENDERS: &[SenderManifestEntry] = &[ "mark_consolidation_started" ), sender!( - "crates/moa-orchestrator/src/workflows/execution_compensation_attempt.rs", + "crates/moa-orchestrator/src/workflows/attempt_slice.rs", "kick_dispatcher", TRACE_HELPER, "ExecutionDispatcherClient", "dispatch" ), - sender!( - "crates/moa-orchestrator/src/workflows/execution_compensation_attempt.rs", - "kick_dispatcher_shared", - TRACE_HELPER, - "ExecutionDispatcherClient", - "dispatch" - ), sender!( "crates/moa-orchestrator/src/workflows/execution_compensation_attempt/external.rs", "yield_external_job", @@ -1036,21 +1050,7 @@ const SENDERS: &[SenderManifestEntry] = &[ "checkpoint_and_release_execution_hands" ), sender!( - "crates/moa-orchestrator/src/workflows/execution_task_attempt.rs", - "kick_dispatcher", - TRACE_HELPER, - "ExecutionDispatcherClient", - "dispatch" - ), - sender!( - "crates/moa-orchestrator/src/workflows/execution_task_attempt.rs", - "kick_dispatcher_shared", - TRACE_HELPER, - "ExecutionDispatcherClient", - "dispatch" - ), - sender!( - "crates/moa-orchestrator/src/workflows/execution_task_attempt/active.rs", + "crates/moa-orchestrator/src/workflows/execution_task_attempt/active/agent.rs", "execute_agent_turn", TRACE_HELPER, "LLMGatewayClient", @@ -1628,6 +1628,14 @@ const RECEIVERS: &[ReceiverManifestEntry] = &[ adoption_symbol: "crate::ctx::adopt_incoming_trace_parent", }, }, + ReceiverManifestEntry { + client: "ExecutionAmendmentPlannerClient", + receiver: ReceiverKind::MoaHandler { + path: "crates/moa-orchestrator/src/services/execution_amendment_planner.rs", + symbol: "*", + adoption_symbol: "crate::ctx::adopt_incoming_trace_parent", + }, + }, ReceiverManifestEntry { client: "ExecutionCompensationAttemptClient", receiver: ReceiverKind::MoaHandler { diff --git a/crates/xtask/src/main.rs b/crates/xtask/src/main.rs index 7efe70b4e..4c7a4c06b 100644 --- a/crates/xtask/src/main.rs +++ b/crates/xtask/src/main.rs @@ -39,6 +39,7 @@ mod record_memory_merges; mod run_external_memory_eval; #[cfg(feature = "eval-tools")] mod run_memory_retrieval_eval; +mod subsystem_map; #[cfg(feature = "eval-tools")] mod wixqa_rag_eval; @@ -96,6 +97,8 @@ fn main() -> Result<()> { Some("audit-paths") => cmd_audit_paths(), Some("check-architecture-boundaries") => check_architecture_boundaries::run(), Some("check-migrations") => cmd_check_migrations(), + Some("check-subsystems") => subsystem_map::check(), + Some("plan-subsystem-audit") => subsystem_map::plan(args), #[cfg(feature = "eval-tools")] Some("check-eval-budgets") => check_eval_budgets::run(args), #[cfg(feature = "eval-tools")] @@ -130,7 +133,7 @@ fn main() -> Result<()> { "xtask command `{command}` requires `cargo run -p xtask --features eval-tools -- {command}`" ), Some(command) => bail!("unknown xtask command: {command}"), - None => bail!("missing xtask command; try `cargo xtask audit-paths`"), + None => bail!("missing xtask command; try `cargo xtask check-subsystems`"), } } diff --git a/crates/xtask/src/subsystem_map.rs b/crates/xtask/src/subsystem_map.rs new file mode 100644 index 000000000..90e704569 --- /dev/null +++ b/crates/xtask/src/subsystem_map.rs @@ -0,0 +1,1249 @@ +//! Validated subsystem routing and bounded audit-packet planning. + +use std::collections::{BTreeMap, BTreeSet}; +use std::env; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use anyhow::{Context, Result, anyhow, bail}; +use serde::{Deserialize, Serialize}; + +const REGISTRY_PATH: &str = ".agents/subsystems.toml"; +const NEXTEST_CONFIG_PATH: &str = ".config/nextest.toml"; +const MAKEFILE_PATH: &str = "Makefile"; +const AUDIT_ARTIFACT_ROOT: &str = "target/agent-audits"; +const REGISTRY_VERSION: u8 = 1; +const HARD_MAX_AGENTS: usize = 4; + +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct SubsystemRegistry { + version: u8, + audit: AuditPolicy, + subsystem: Vec, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct AuditPolicy { + max_agents: usize, + report_word_limit: usize, + artifact_root: String, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct Subsystem { + id: String, + owner: String, + path_prefixes: Vec, + docs: Vec, + agent_files: Vec, + #[serde(default)] + local_agents: Vec, + #[serde(default)] + test_profiles: Vec, + #[serde(default)] + make_targets: Vec, + #[serde(default)] + live_gates: Vec, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct LocalAgentFile { + path_prefix: String, + file: String, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct LiveGate { + id: String, + make_target: String, + authorization_env: Vec, + #[serde(default)] + credentials_any_of: Vec, + #[serde(default)] + budget_env: Vec, + #[serde(default)] + services: Vec, + billed: bool, +} + +#[derive(Debug)] +struct WorkspaceInfo { + package_names: BTreeSet, + member_paths: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +struct AuditPlan { + schema_version: u8, + reviewer_cap: usize, + report_word_limit: usize, + packet_count: usize, + packets: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +struct AuditPacket { + id: String, + subsystem_ids: Vec, + paths: Vec, + docs: Vec, + agent_files: Vec, + test_profiles: Vec, + make_targets: Vec, + live_gates: Vec, +} + +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize)] +struct PacketLiveGate { + subsystem_id: String, + id: String, + make_target: String, + authorization_env: Vec, + credentials_any_of: Vec, + budget_env: Vec, + services: Vec, + billed: bool, +} + +#[derive(Debug, Default)] +struct PacketBuilder { + subsystem_ids: BTreeSet, + paths: BTreeSet, + docs: BTreeSet, + agent_files: BTreeSet, + test_profiles: BTreeSet, + make_targets: BTreeSet, + live_gates: BTreeSet, +} + +#[derive(Debug, Default)] +struct PlanOptions { + base: Option, + paths: Vec, + max_agents: Option, + output: Option, +} + +/// Validates the checked-in subsystem registry against the current workspace. +pub(crate) fn check() -> Result<()> { + let root = repository_root()?; + let registry = load_registry(&root)?; + let workspace = load_workspace_info(&root)?; + let profiles = load_nextest_profiles(&root)?; + let make_targets = load_make_targets(&root)?; + validate_registry(&root, ®istry, &workspace, &profiles, &make_targets)?; + println!( + "subsystem registry clean: {} groups cover {} workspace members", + registry.subsystem.len(), + workspace.member_paths.len() + ); + Ok(()) +} + +/// Plans bounded, read-only subsystem audit packets without launching agents. +pub(crate) fn plan(args: impl Iterator) -> Result<()> { + let root = repository_root()?; + let Some(options) = parse_plan_options(args)? else { + return Ok(()); + }; + let registry = load_registry(&root)?; + let workspace = load_workspace_info(&root)?; + let profiles = load_nextest_profiles(&root)?; + let make_targets = load_make_targets(&root)?; + validate_registry(&root, ®istry, &workspace, &profiles, &make_targets)?; + + let paths = collect_plan_paths(&root, &options)?; + let plan = build_audit_plan(®istry, paths, options.max_agents)?; + let output = resolve_audit_output( + &root, + ®istry.audit.artifact_root, + options.output.as_deref(), + )?; + write_audit_plan(&output, &plan)?; + println!( + "planned {} bounded audit packet(s) under {}", + plan.packet_count, + output.display() + ); + Ok(()) +} + +fn repository_root() -> Result { + let current = env::current_dir().context("resolve current directory")?; + current + .ancestors() + .find(|path| { + path.join("Cargo.toml").is_file() && path.join("crates/xtask/Cargo.toml").is_file() + }) + .map(Path::to_path_buf) + .ok_or_else(|| { + anyhow!( + "could not locate the MOA repository root from {}", + current.display() + ) + }) +} + +fn load_registry(root: &Path) -> Result { + let path = root.join(REGISTRY_PATH); + let body = fs::read_to_string(&path).with_context(|| format!("read {}", path.display()))?; + toml::from_str(&body).with_context(|| format!("parse {}", path.display())) +} + +fn load_workspace_info(root: &Path) -> Result { + let output = Command::new("cargo") + .args(["metadata", "--format-version", "1", "--no-deps", "--locked"]) + .current_dir(root) + .output() + .context("run cargo metadata for subsystem coverage")?; + if !output.status.success() { + bail!( + "cargo metadata failed while checking subsystem coverage: {}", + String::from_utf8_lossy(&output.stderr).trim() + ); + } + + #[derive(Deserialize)] + struct Metadata { + packages: Vec, + workspace_members: Vec, + } + + #[derive(Deserialize)] + struct Package { + id: String, + name: String, + manifest_path: PathBuf, + } + + let metadata: Metadata = serde_json::from_slice(&output.stdout) + .context("parse cargo metadata for subsystem coverage")?; + let member_ids = metadata + .workspace_members + .into_iter() + .collect::>(); + let mut package_names = BTreeSet::new(); + let mut member_paths = Vec::new(); + for package in metadata + .packages + .into_iter() + .filter(|package| member_ids.contains(&package.id)) + { + package_names.insert(package.name); + let package_dir = package + .manifest_path + .parent() + .context("workspace package manifest has no parent directory")?; + let relative = package_dir.strip_prefix(root).with_context(|| { + format!( + "workspace package {} is outside repository root {}", + package_dir.display(), + root.display() + ) + })?; + member_paths.push(format!("{}/", path_to_slashes(relative))); + } + member_paths.sort(); + Ok(WorkspaceInfo { + package_names, + member_paths, + }) +} + +fn load_nextest_profiles(root: &Path) -> Result> { + let path = root.join(NEXTEST_CONFIG_PATH); + let body = fs::read_to_string(&path).with_context(|| format!("read {}", path.display()))?; + Ok(body + .lines() + .filter_map(|line| { + line.trim() + .strip_prefix("[profile.") + .and_then(|value| value.strip_suffix(']')) + .map(str::to_string) + }) + .collect()) +} + +fn load_make_targets(root: &Path) -> Result> { + let path = root.join(MAKEFILE_PATH); + let body = fs::read_to_string(&path).with_context(|| format!("read {}", path.display()))?; + Ok(body + .lines() + .filter_map(|line| { + if line.starts_with(|character: char| character.is_whitespace() || character == '#') { + return None; + } + let (candidate, _) = line.split_once(':')?; + let candidate = candidate.trim(); + (!candidate.is_empty() + && !candidate.starts_with('.') + && !candidate.contains(char::is_whitespace) + && !candidate.contains('=')) + .then(|| candidate.to_string()) + }) + .collect()) +} + +fn validate_registry( + root: &Path, + registry: &SubsystemRegistry, + workspace: &WorkspaceInfo, + profiles: &BTreeSet, + make_targets: &BTreeSet, +) -> Result<()> { + if registry.version != REGISTRY_VERSION { + bail!( + "unsupported subsystem registry version {}; expected {REGISTRY_VERSION}", + registry.version + ); + } + if !(1..=HARD_MAX_AGENTS).contains(®istry.audit.max_agents) { + bail!( + "audit.max_agents must be between 1 and {HARD_MAX_AGENTS}; saw {}", + registry.audit.max_agents + ); + } + if registry.audit.report_word_limit == 0 { + bail!("audit.report_word_limit must be positive"); + } + validate_relative_path(®istry.audit.artifact_root, "audit.artifact_root")?; + if registry.audit.artifact_root != AUDIT_ARTIFACT_ROOT { + bail!("audit.artifact_root must be `{AUDIT_ARTIFACT_ROOT}`"); + } + + let mut ids = BTreeSet::new(); + let mut prefixes = BTreeMap::::new(); + for subsystem in ®istry.subsystem { + validate_identifier(&subsystem.id, "subsystem id")?; + if !ids.insert(subsystem.id.clone()) { + bail!("duplicate subsystem id `{}`", subsystem.id); + } + if !workspace.package_names.contains(&subsystem.owner) { + bail!( + "subsystem `{}` names unknown workspace owner `{}`", + subsystem.id, + subsystem.owner + ); + } + if subsystem.path_prefixes.is_empty() { + bail!("subsystem `{}` has no path_prefixes", subsystem.id); + } + for prefix in &subsystem.path_prefixes { + validate_relative_path(prefix, "path prefix")?; + validate_configured_path(root, prefix, "path prefix", &subsystem.id)?; + if let Some(previous) = prefixes.insert(prefix.clone(), subsystem.id.clone()) { + bail!( + "ambiguous path prefix `{prefix}` is declared by `{previous}` and `{}`", + subsystem.id + ); + } + } + + if subsystem.docs.is_empty() { + bail!("subsystem `{}` has no canonical docs", subsystem.id); + } + for doc in &subsystem.docs { + validate_relative_path(doc, "doc path")?; + validate_configured_path(root, doc, "doc", &subsystem.id)?; + } + if subsystem.agent_files.is_empty() { + bail!("subsystem `{}` has no agent_files", subsystem.id); + } + for agent_file in &subsystem.agent_files { + validate_agent_file(root, agent_file, &subsystem.id)?; + } + for local in &subsystem.local_agents { + validate_relative_path(&local.path_prefix, "local agent path prefix")?; + if !subsystem + .path_prefixes + .iter() + .any(|prefix| local.path_prefix.starts_with(prefix)) + { + bail!( + "subsystem `{}` local agent prefix `{}` is outside its routed prefixes", + subsystem.id, + local.path_prefix + ); + } + validate_agent_file(root, &local.file, &subsystem.id)?; + } + + if subsystem.test_profiles.is_empty() && subsystem.make_targets.is_empty() { + bail!( + "subsystem `{}` must reference at least one nextest profile or Make target", + subsystem.id + ); + } + for profile in &subsystem.test_profiles { + if !profiles.contains(profile) { + bail!( + "subsystem `{}` references unknown nextest profile `{profile}`", + subsystem.id + ); + } + } + for target in &subsystem.make_targets { + if !make_targets.contains(target) { + bail!( + "subsystem `{}` references unknown Make target `{target}`", + subsystem.id + ); + } + } + validate_live_gates(subsystem, make_targets)?; + } + + for member in &workspace.member_paths { + if route_subsystem(®istry.subsystem, member)?.is_none() { + bail!("workspace member `{member}` is not covered by any subsystem prefix"); + } + } + Ok(()) +} + +fn validate_live_gates(subsystem: &Subsystem, make_targets: &BTreeSet) -> Result<()> { + let mut ids = BTreeSet::new(); + for gate in &subsystem.live_gates { + validate_identifier(&gate.id, "live gate id")?; + if !ids.insert(&gate.id) { + bail!( + "subsystem `{}` has duplicate live gate id `{}`", + subsystem.id, + gate.id + ); + } + if gate.authorization_env.is_empty() { + bail!( + "subsystem `{}` live gate `{}` has no explicit authorization_env", + subsystem.id, + gate.id + ); + } + if !make_targets.contains(&gate.make_target) + || !subsystem.make_targets.contains(&gate.make_target) + { + bail!( + "subsystem `{}` live gate `{}` must reference one of its checked-in Make targets; saw `{}`", + subsystem.id, + gate.id, + gate.make_target + ); + } + for variable in gate + .authorization_env + .iter() + .chain(&gate.credentials_any_of) + .chain(&gate.budget_env) + { + if !valid_env_name(variable) { + bail!( + "subsystem `{}` live gate `{}` has invalid environment variable `{variable}`", + subsystem.id, + gate.id + ); + } + } + if gate.billed && gate.credentials_any_of.is_empty() { + bail!( + "subsystem `{}` billed live gate `{}` must name credentials_any_of", + subsystem.id, + gate.id + ); + } + if gate + .services + .iter() + .any(|service| service.trim().is_empty()) + { + bail!( + "subsystem `{}` live gate `{}` contains an empty service name", + subsystem.id, + gate.id + ); + } + } + Ok(()) +} + +fn validate_agent_file(root: &Path, path: &str, subsystem_id: &str) -> Result<()> { + validate_relative_path(path, "agent file")?; + if Path::new(path).file_name().and_then(|name| name.to_str()) != Some("AGENTS.md") { + bail!("subsystem `{subsystem_id}` agent file must be named AGENTS.md; saw `{path}`"); + } + validate_configured_path(root, path, "agent file", subsystem_id) +} + +fn validate_configured_path(root: &Path, value: &str, kind: &str, owner: &str) -> Result<()> { + let path = root.join(value.trim_end_matches('/')); + if !path.exists() { + bail!("subsystem `{owner}` {kind} does not exist: `{value}`"); + } + Ok(()) +} + +fn validate_relative_path(value: &str, label: &str) -> Result<()> { + if value.is_empty() || Path::new(value).is_absolute() || value.starts_with("./") { + bail!("{label} must be a non-empty normalized repository-relative path: `{value}`"); + } + if Path::new(value) + .components() + .any(|component| matches!(component, std::path::Component::ParentDir)) + { + bail!("{label} must not contain parent traversal: `{value}`"); + } + Ok(()) +} + +fn validate_identifier(value: &str, label: &str) -> Result<()> { + if value.is_empty() + || !value + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-') + { + bail!("{label} must use lowercase kebab-case: `{value}`"); + } + Ok(()) +} + +fn valid_env_name(value: &str) -> bool { + !value.is_empty() + && value + .bytes() + .all(|byte| byte.is_ascii_uppercase() || byte.is_ascii_digit() || byte == b'_') +} + +fn route_subsystem<'a>(subsystems: &'a [Subsystem], path: &str) -> Result> { + let mut best: Option<(&Subsystem, usize)> = None; + for subsystem in subsystems { + for prefix in &subsystem.path_prefixes { + if !path_matches_prefix(path, prefix) { + continue; + } + match best { + None => best = Some((subsystem, prefix.len())), + Some((_, length)) if prefix.len() > length => { + best = Some((subsystem, prefix.len())); + } + Some((current, length)) if prefix.len() == length && current.id != subsystem.id => { + bail!( + "path `{path}` is ambiguous between `{}` and `{}` at prefix length {length}", + current.id, + subsystem.id + ); + } + Some(_) => {} + } + } + } + Ok(best.map(|(subsystem, _)| subsystem)) +} + +fn path_matches_prefix(path: &str, prefix: &str) -> bool { + if prefix.ends_with('/') { + path.starts_with(prefix) + } else { + path == prefix + } +} + +fn parse_plan_options(mut args: impl Iterator) -> Result> { + let mut options = PlanOptions::default(); + while let Some(argument) = args.next() { + match argument.as_str() { + "--base" => { + options.base = Some(next_value(&mut args, "--base")?); + } + "--path" => options.paths.push(next_value(&mut args, "--path")?), + "--max-agents" => { + let value = next_value(&mut args, "--max-agents")?; + options.max_agents = Some( + value + .parse::() + .with_context(|| format!("parse --max-agents value `{value}`"))?, + ); + } + "--output" => { + options.output = Some(PathBuf::from(next_value(&mut args, "--output")?)); + } + "-h" | "--help" => { + println!( + "usage: cargo xtask plan-subsystem-audit (--base REV | --path PATH...) [--max-agents N] [--output DIR]" + ); + return Ok(None); + } + _ => bail!("unknown plan-subsystem-audit argument `{argument}`"), + } + } + if options.base.is_none() && options.paths.is_empty() { + bail!("plan-subsystem-audit requires --base REV or at least one --path PATH"); + } + Ok(Some(options)) +} + +fn next_value(args: &mut impl Iterator, option: &str) -> Result { + args.next() + .ok_or_else(|| anyhow!("{option} requires a value")) +} + +fn collect_plan_paths(root: &Path, options: &PlanOptions) -> Result> { + let mut paths = BTreeSet::new(); + if let Some(base) = &options.base { + for path in git_lines(root, &["diff", "--name-only", base, "--"])? { + paths.insert(normalize_input_path(&path)?); + } + for path in git_lines(root, &["ls-files", "--others", "--exclude-standard"])? { + paths.insert(normalize_input_path(&path)?); + } + } + for path in &options.paths { + paths.insert(normalize_input_path(path)?); + } + Ok(paths.into_iter().collect()) +} + +fn git_lines(root: &Path, args: &[&str]) -> Result> { + let output = Command::new("git") + .args(args) + .current_dir(root) + .output() + .with_context(|| format!("run git {}", args.join(" ")))?; + if !output.status.success() { + bail!( + "git {} failed: {}", + args.join(" "), + String::from_utf8_lossy(&output.stderr).trim() + ); + } + Ok(String::from_utf8(output.stdout) + .context("git output was not UTF-8")? + .lines() + .filter(|line| !line.is_empty()) + .map(str::to_string) + .collect()) +} + +fn normalize_input_path(path: &str) -> Result { + let normalized = path.strip_prefix("./").unwrap_or(path); + validate_relative_path(normalized, "audit input path")?; + Ok(normalized.to_string()) +} + +fn build_audit_plan( + registry: &SubsystemRegistry, + paths: Vec, + requested_max_agents: Option, +) -> Result { + let reviewer_cap = requested_max_agents + .unwrap_or(registry.audit.max_agents) + .min(registry.audit.max_agents) + .min(HARD_MAX_AGENTS); + if reviewer_cap == 0 { + bail!("--max-agents must be positive"); + } + + let mut routed = BTreeMap::)>::new(); + let mut uncovered = Vec::new(); + for path in paths { + match route_subsystem(®istry.subsystem, &path)? { + Some(subsystem) => { + routed + .entry(subsystem.id.clone()) + .or_insert_with(|| (subsystem, BTreeSet::new())) + .1 + .insert(path); + } + None => uncovered.push(path), + } + } + if !uncovered.is_empty() { + bail!( + "audit paths are not covered by the subsystem registry:\n{}", + uncovered.join("\n") + ); + } + + let packet_count = routed.len().min(reviewer_cap); + let mut builders = (0..packet_count) + .map(|_| PacketBuilder::default()) + .collect::>(); + for (index, (_, (subsystem, subsystem_paths))) in routed.into_iter().enumerate() { + let builder = &mut builders[index % packet_count]; + builder.subsystem_ids.insert(subsystem.id.clone()); + builder.paths.extend(subsystem_paths.iter().cloned()); + builder.docs.extend(subsystem.docs.iter().cloned()); + builder + .agent_files + .extend(subsystem.agent_files.iter().cloned()); + for local in &subsystem.local_agents { + if subsystem_paths + .iter() + .any(|path| path_matches_prefix(path, &local.path_prefix)) + { + builder.agent_files.insert(local.file.clone()); + } + } + builder + .test_profiles + .extend(subsystem.test_profiles.iter().cloned()); + builder + .make_targets + .extend(subsystem.make_targets.iter().cloned()); + builder + .live_gates + .extend(subsystem.live_gates.iter().map(|gate| PacketLiveGate { + subsystem_id: subsystem.id.clone(), + id: gate.id.clone(), + make_target: gate.make_target.clone(), + authorization_env: gate.authorization_env.clone(), + credentials_any_of: gate.credentials_any_of.clone(), + budget_env: gate.budget_env.clone(), + services: gate.services.clone(), + billed: gate.billed, + })); + } + + let packets = builders + .into_iter() + .enumerate() + .map(|(index, builder)| AuditPacket { + id: format!("packet-{:02}", index + 1), + subsystem_ids: builder.subsystem_ids.into_iter().collect(), + paths: builder.paths.into_iter().collect(), + docs: builder.docs.into_iter().collect(), + agent_files: builder.agent_files.into_iter().collect(), + test_profiles: builder.test_profiles.into_iter().collect(), + make_targets: builder.make_targets.into_iter().collect(), + live_gates: builder.live_gates.into_iter().collect(), + }) + .collect::>(); + Ok(AuditPlan { + schema_version: 1, + reviewer_cap, + report_word_limit: registry.audit.report_word_limit, + packet_count: packets.len(), + packets, + }) +} + +fn current_revision(root: &Path) -> Result { + git_lines(root, &["rev-parse", "--short", "HEAD"])? + .into_iter() + .next() + .ok_or_else(|| anyhow!("git rev-parse returned no revision")) +} + +fn resolve_audit_output( + root: &Path, + artifact_root: &str, + requested: Option<&Path>, +) -> Result { + let relative = match requested { + Some(path) => { + let value = path + .to_str() + .context("audit output path must be valid UTF-8")?; + validate_relative_path(value, "audit output path")?; + let path = Path::new(value); + let suffix = path.strip_prefix(artifact_root).with_context(|| { + format!("audit output must stay beneath `{artifact_root}`: `{value}`") + })?; + if suffix.as_os_str().is_empty() { + bail!("audit output must name a run directory beneath `{artifact_root}`"); + } + path.to_path_buf() + } + None => Path::new(artifact_root).join(current_revision(root)?), + }; + reject_symlinked_output_components(root, &relative)?; + Ok(root.join(relative)) +} + +fn reject_symlinked_output_components(root: &Path, relative: &Path) -> Result<()> { + let mut current = root.to_path_buf(); + for component in relative.components() { + current.push(component.as_os_str()); + match fs::symlink_metadata(¤t) { + Ok(metadata) if metadata.file_type().is_symlink() => { + bail!( + "audit output may not traverse symlinked path component `{}`", + current.display() + ); + } + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => break, + Err(error) => { + return Err(error) + .with_context(|| format!("inspect audit output path {}", current.display())); + } + } + } + Ok(()) +} + +fn write_audit_plan(output: &Path, plan: &AuditPlan) -> Result<()> { + fs::create_dir_all(output.join("reports")) + .with_context(|| format!("create audit output directory {}", output.display()))?; + remove_stale_generated_files(output)?; + let mut json = serde_json::to_string_pretty(plan).context("serialize audit plan")?; + json.push('\n'); + fs::write(output.join("plan.json"), json) + .with_context(|| format!("write {}/plan.json", output.display()))?; + for packet in &plan.packets { + fs::write( + output.join(format!("{}.md", packet.id)), + render_packet(packet, plan.report_word_limit), + ) + .with_context(|| format!("write audit packet {}", packet.id))?; + } + fs::write( + output.join("checkpoint.md"), + "# Audit Checkpoint\n\n- Status: planned\n- Integration owner: unassigned\n- Completed packets: none\n- Live authorization: not granted\n", + ) + .with_context(|| format!("write {}/checkpoint.md", output.display()))?; + Ok(()) +} + +fn remove_stale_generated_files(output: &Path) -> Result<()> { + remove_generated_file(output, "plan.json")?; + remove_generated_file(output, "checkpoint.md")?; + for entry in fs::read_dir(output) + .with_context(|| format!("read audit output directory {}", output.display()))? + { + let entry = entry.with_context(|| format!("read entry under {}", output.display()))?; + let name = entry.file_name(); + let Some(name) = name.to_str() else { + continue; + }; + let Some(sequence) = name + .strip_prefix("packet-") + .and_then(|value| value.strip_suffix(".md")) + else { + continue; + }; + if sequence.is_empty() || !sequence.bytes().all(|byte| byte.is_ascii_digit()) { + continue; + } + let file_type = entry + .file_type() + .with_context(|| format!("inspect stale audit packet {}", entry.path().display()))?; + if !file_type.is_file() && !file_type.is_symlink() { + bail!( + "refusing to replace non-file audit packet path `{}`", + entry.path().display() + ); + } + fs::remove_file(entry.path()) + .with_context(|| format!("remove stale audit packet {name}"))?; + } + Ok(()) +} + +fn remove_generated_file(output: &Path, name: &str) -> Result<()> { + let path = output.join(name); + let metadata = match fs::symlink_metadata(&path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => { + return Err(error).with_context(|| format!("inspect generated audit file {name}")); + } + }; + if !metadata.file_type().is_file() && !metadata.file_type().is_symlink() { + bail!( + "refusing to replace non-file generated audit path `{}`", + path.display() + ); + } + fs::remove_file(path).with_context(|| format!("remove generated audit file {name}")) +} + +fn render_packet(packet: &AuditPacket, report_word_limit: usize) -> String { + let mut body = String::new(); + body.push_str(&format!("# {}\n\n", packet.id)); + body.push_str("- Mode: read-only discovery\n"); + body.push_str(&format!("- Report limit: {report_word_limit} words\n")); + body.push_str(&format!( + "- Subsystems: {}\n", + packet.subsystem_ids.join(", ") + )); + append_section(&mut body, "Paths", &packet.paths); + append_section(&mut body, "Canonical docs", &packet.docs); + append_section(&mut body, "Agent instructions", &packet.agent_files); + append_section(&mut body, "Nextest profiles", &packet.test_profiles); + append_section(&mut body, "Make targets", &packet.make_targets); + body.push_str("\n## Live gates\n\n"); + if packet.live_gates.is_empty() { + body.push_str("- None.\n"); + } else { + body.push_str("Do not run these gates without their explicit authorization:\n\n"); + for gate in &packet.live_gates { + body.push_str(&format!( + "- `{}` / `{}`: authorization `{}`; billed `{}`\n", + gate.subsystem_id, + gate.id, + gate.authorization_env.join(", "), + gate.billed + )); + } + } + body.push_str( + "\nReturn exact path evidence, unresolved gaps, and the smallest safe next change.\n", + ); + body +} + +fn append_section(body: &mut String, heading: &str, values: &[String]) { + body.push_str(&format!("\n## {heading}\n\n")); + for value in values { + body.push_str(&format!("- `{value}`\n")); + } +} + +fn path_to_slashes(path: &Path) -> String { + path.components() + .map(|component| component.as_os_str().to_string_lossy()) + .collect::>() + .join("/") +} + +#[cfg(test)] +mod tests { + use super::*; + + fn subsystem(id: &str, prefix: &str) -> Subsystem { + Subsystem { + id: id.to_string(), + owner: "owner".to_string(), + path_prefixes: vec![prefix.to_string()], + docs: vec!["docs/01.md".to_string()], + agent_files: vec!["AGENTS.md".to_string()], + local_agents: Vec::new(), + test_profiles: vec!["fast-pr".to_string()], + make_targets: Vec::new(), + live_gates: Vec::new(), + } + } + + fn registry(subsystems: Vec) -> SubsystemRegistry { + SubsystemRegistry { + version: REGISTRY_VERSION, + audit: AuditPolicy { + max_agents: HARD_MAX_AGENTS, + report_word_limit: 600, + artifact_root: AUDIT_ARTIFACT_ROOT.to_string(), + }, + subsystem: subsystems, + } + } + + fn workspace(paths: &[&str]) -> WorkspaceInfo { + WorkspaceInfo { + package_names: BTreeSet::from(["owner".to_string()]), + member_paths: paths.iter().map(|path| (*path).to_string()).collect(), + } + } + + fn create_validation_tree(root: &Path, prefixes: &[&str]) { + fs::create_dir_all(root.join("docs")).expect("create synthetic docs directory"); + fs::write(root.join("docs/01.md"), "# Architecture\n") + .expect("write synthetic architecture doc"); + fs::write(root.join("AGENTS.md"), "# Instructions\n") + .expect("write synthetic agent instructions"); + for prefix in prefixes { + fs::create_dir_all(root.join(prefix.trim_end_matches('/'))) + .expect("create synthetic routed prefix"); + } + } + + #[test] + fn longest_prefix_routes_to_the_most_specific_subsystem() { + // Pins: grouped parent routing never steals a path from a more-specific owner. + let subsystems = vec![ + subsystem("repository", "crates/"), + subsystem("memory", "crates/moa-memory/"), + ]; + + let routed = route_subsystem(&subsystems, "crates/moa-memory/graph/src/lib.rs") + .expect("longest-prefix routing should be unambiguous") + .expect("memory source should have a routed subsystem"); + + assert_eq!(routed.id, "memory"); + } + + #[test] + fn explicit_directory_path_preserves_prefix_routing() { + // Pins: an explicit crate directory remains routable instead of losing its trailing slash. + let normalized = normalize_input_path("./crates/moa-hands/") + .expect("normalize explicit crate directory"); + assert_eq!(normalized, "crates/moa-hands/"); + + let plan = build_audit_plan( + ®istry(vec![subsystem("hands", "crates/moa-hands/")]), + vec![normalized], + Some(1), + ) + .expect("explicit crate directory should route to its subsystem"); + + assert_eq!(plan.packet_count, 1); + assert_eq!(plan.packets[0].subsystem_ids, ["hands"]); + assert_eq!(plan.packets[0].paths, ["crates/moa-hands/"]); + } + + #[test] + fn duplicate_prefixes_are_rejected_as_ambiguous() { + // Pins: two owners cannot silently receive identical context for one path. + let temporary = tempfile::tempdir().expect("create validation tree"); + create_validation_tree(temporary.path(), &["crates/shared/"]); + let registry = registry(vec![ + subsystem("alpha", "crates/shared/"), + subsystem("beta", "crates/shared/"), + ]); + + let error = validate_registry( + temporary.path(), + ®istry, + &workspace(&["crates/shared/"]), + &BTreeSet::from(["fast-pr".to_string()]), + &BTreeSet::new(), + ) + .expect_err("duplicate prefixes must fail validation"); + + assert!(error.to_string().contains("ambiguous path prefix")); + } + + #[test] + fn missing_configured_paths_are_rejected() { + // Pins: stale registry paths fail before audit planning can omit their owner. + let temporary = tempfile::tempdir().expect("create validation tree"); + create_validation_tree(temporary.path(), &[]); + let registry = registry(vec![subsystem("missing", "crates/missing/")]); + + let error = validate_registry( + temporary.path(), + ®istry, + &workspace(&["crates/missing/"]), + &BTreeSet::from(["fast-pr".to_string()]), + &BTreeSet::new(), + ) + .expect_err("missing routed paths must fail validation"); + + assert!(error.to_string().contains("path prefix does not exist")); + } + + #[test] + fn uncovered_workspace_members_are_rejected() { + // Pins: adding a workspace crate requires assigning it to one subsystem. + let temporary = tempfile::tempdir().expect("create validation tree"); + create_validation_tree(temporary.path(), &["crates/covered/"]); + let registry = registry(vec![subsystem("covered", "crates/covered/")]); + + let error = validate_registry( + temporary.path(), + ®istry, + &workspace(&["crates/covered/", "crates/uncovered/"]), + &BTreeSet::from(["fast-pr".to_string()]), + &BTreeSet::new(), + ) + .expect_err("uncovered workspace members must fail validation"); + + assert!( + error + .to_string() + .contains("workspace member `crates/uncovered/` is not covered") + ); + } + + #[test] + fn reviewer_count_is_capped_by_registry_policy() { + // Pins: explicit requests cannot expand a broad audit beyond four reviewers. + let registry = registry( + (0..6) + .map(|index| subsystem(&format!("group-{index}"), &format!("group-{index}/"))) + .collect(), + ); + let paths = (0..6) + .map(|index| format!("group-{index}/file.rs")) + .collect(); + + let plan = build_audit_plan(®istry, paths, Some(99)) + .expect("covered paths should produce a bounded plan"); + + assert_eq!(plan.reviewer_cap, 4); + assert_eq!(plan.packet_count, 4); + assert_eq!( + plan.packets + .iter() + .flat_map(|packet| &packet.subsystem_ids) + .collect::>() + .len(), + 6 + ); + } + + #[test] + fn audit_packet_output_is_deterministic_for_input_order() { + // Pins: identical change sets produce byte-stable packets across sessions. + let registry = registry(vec![ + subsystem("alpha", "alpha/"), + subsystem("beta", "beta/"), + ]); + let first = build_audit_plan( + ®istry, + vec!["beta/z.rs".to_string(), "alpha/a.rs".to_string()], + Some(2), + ) + .expect("first plan should build"); + let second = build_audit_plan( + ®istry, + vec!["alpha/a.rs".to_string(), "beta/z.rs".to_string()], + Some(2), + ) + .expect("second plan should build"); + + assert_eq!(first, second); + assert_eq!( + serde_json::to_string_pretty(&first).expect("serialize first plan"), + serde_json::to_string_pretty(&second).expect("serialize second plan") + ); + assert_eq!( + render_packet(&first.packets[0], first.report_word_limit), + render_packet(&second.packets[0], second.report_word_limit) + ); + } + + #[test] + fn audit_output_rejects_absolute_traversal_and_tracked_paths() { + // Pins: --output cannot escape the ignored audit-artifact root or overwrite repo files. + let temporary = tempfile::tempdir().expect("create audit output root"); + let artifact_root = AUDIT_ARTIFACT_ROOT; + let allowed = resolve_audit_output( + temporary.path(), + artifact_root, + Some(Path::new("target/agent-audits/review-01")), + ) + .expect("nested audit output should be accepted"); + assert_eq!( + allowed, + temporary.path().join("target/agent-audits/review-01") + ); + + for rejected in [ + temporary.path().join("absolute-review"), + PathBuf::from("target/agent-audits/../tracked-review"), + PathBuf::from("docs/engineering-discipline/review"), + PathBuf::from("target/agent-audits"), + ] { + assert!( + resolve_audit_output(temporary.path(), artifact_root, Some(&rejected)).is_err(), + "unsafe audit output should be rejected: {}", + rejected.display() + ); + } + } + + #[test] + fn repeated_packet_write_removes_only_stale_owned_packets() { + // Pins: replanning one HEAD cannot leave obsolete packets or delete reviewer-owned files. + let temporary = tempfile::tempdir().expect("create audit output directory"); + let output = temporary.path().join("target/agent-audits/review-01"); + let registry = registry(vec![ + subsystem("alpha", "alpha/"), + subsystem("beta", "beta/"), + ]); + let first = build_audit_plan( + ®istry, + vec!["alpha/a.rs".to_string(), "beta/b.rs".to_string()], + Some(2), + ) + .expect("build initial two-packet plan"); + write_audit_plan(&output, &first).expect("write initial audit packets"); + assert!(output.join("packet-02.md").is_file()); + + fs::write(output.join("packet-notes.md"), "reviewer notes\n") + .expect("write reviewer-owned packet notes"); + fs::write(output.join("checkpoint.md"), "stale checkpoint\n") + .expect("write stale generated checkpoint"); + let second = build_audit_plan(®istry, vec!["alpha/a.rs".to_string()], Some(1)) + .expect("build replacement one-packet plan"); + write_audit_plan(&output, &second).expect("replace generated audit packets"); + + assert!(output.join("packet-01.md").is_file()); + assert!(!output.join("packet-02.md").exists()); + assert_eq!( + fs::read_to_string(output.join("packet-notes.md")) + .expect("read preserved reviewer notes"), + "reviewer notes\n" + ); + assert_eq!( + fs::read_to_string(output.join("checkpoint.md")).expect("read replacement checkpoint"), + "# Audit Checkpoint\n\n- Status: planned\n- Integration owner: unassigned\n- Completed packets: none\n- Live authorization: not granted\n" + ); + let expected_plan = format!( + "{}\n", + serde_json::to_string_pretty(&second).expect("serialize expected replacement plan") + ); + assert_eq!( + fs::read_to_string(output.join("plan.json")).expect("read replacement plan"), + expected_plan + ); + } + + #[cfg(unix)] + #[test] + fn packet_write_replaces_generated_symlink_without_following_it() { + // Pins: a stale generated filename cannot redirect planner writes outside the run folder. + use std::os::unix::fs::symlink; + + let temporary = tempfile::tempdir().expect("create audit output directory"); + let output = temporary.path().join("target/agent-audits/review-01"); + fs::create_dir_all(&output).expect("create audit run directory"); + let outside = temporary.path().join("tracked-plan.json"); + fs::write(&outside, "do not overwrite\n").expect("write protected outside file"); + symlink(&outside, output.join("plan.json")).expect("link stale generated plan"); + let plan = build_audit_plan( + ®istry(vec![subsystem("alpha", "alpha/")]), + vec!["alpha/a.rs".to_string()], + Some(1), + ) + .expect("build replacement plan"); + + write_audit_plan(&output, &plan).expect("replace generated symlink safely"); + + assert_eq!( + fs::read_to_string(&outside).expect("read protected outside file"), + "do not overwrite\n" + ); + assert!( + fs::symlink_metadata(output.join("plan.json")) + .expect("inspect replacement plan") + .file_type() + .is_file() + ); + } + + #[test] + fn billed_live_gate_requires_explicit_credentials() { + // Pins: a billed lane cannot be represented as an unauthenticated generic target. + let mut entry = subsystem("providers", "crates/providers/"); + entry.make_targets.push("test-provider-e2e".to_string()); + entry.live_gates.push(LiveGate { + id: "provider-e2e".to_string(), + make_target: "test-provider-e2e".to_string(), + authorization_env: vec!["MOA_RUN_LIVE_PROVIDER_TESTS".to_string()], + credentials_any_of: Vec::new(), + budget_env: Vec::new(), + services: vec!["restate".to_string()], + billed: true, + }); + + let error = validate_live_gates(&entry, &BTreeSet::from(["test-provider-e2e".to_string()])) + .expect_err("billed gates without credentials must fail"); + + assert!(error.to_string().contains("must name credentials_any_of")); + } +} diff --git a/docs/engineering-discipline/plans/2026-08-13-codex-efficient-repository-structure.md b/docs/engineering-discipline/plans/2026-08-13-codex-efficient-repository-structure.md new file mode 100644 index 000000000..f662b0f9b --- /dev/null +++ b/docs/engineering-discipline/plans/2026-08-13-codex-efficient-repository-structure.md @@ -0,0 +1,156 @@ +# Codex-Efficient Repository Structure + +## Objective + +Reduce the context, duplicated analysis, repeated compilation, and log volume required to work safely across the whole MOA repository. Improve routing and ownership without creating compatibility, network, or facade layers; fix only runtime defects directly exposed by certification. + +## Constraints + +- `docs/01-architecture-overview.md` and `docs/15-architecture-policy.md` remain the ownership sources of truth. +- Refactors follow durable behavior boundaries, never line count alone. +- Existing public paths and runtime contracts stay unchanged unless direct imports can be updated in the same private module boundary; no compatibility re-exports. +- Restate journal step names, serialization, SQL ordering, fencing, provider I/O ordering, and idempotency keys are preserved. +- Live, billed, credentialed, and 24h/7d checks remain explicitly authorized external gates. +- Parallel workers receive only mapped paths and documents. One integration owner runs broad validation. + +## Repository-Wide Design + +### Validated subsystem registry + +Add `.agents/subsystems.toml` as the single path-routing registry. Cover every workspace crate plus repository operations with grouped entries for: + +1. platform core and configuration; +2. execution domain and artifacts; +3. orchestration, edge, sessions, and messaging; +4. hands and sandbox workspaces; +5. connectors, knowledge, and outbound security; +6. providers and model governance; +7. memory, retrieval, and brain context; +8. auth, principals, and contacts; +9. lineage, observability, and analytics; +10. skills, experiments, eval, load test, and test support; +11. migrations and database operations; +12. repository tooling, deployment, and documentation. + +Each entry records stable ownership, exact path prefixes, canonical docs, applicable local `AGENTS.md`, deterministic test profiles or Make targets, and structured live prerequisites. Longest-prefix resolution is deterministic; the validator rejects ambiguous prefixes and uncovered workspace members. + +Add `xtask check-subsystems` to validate the registry and `xtask plan-subsystem-audit` to turn a base revision or explicit paths into bounded context packets under `target/agent-audits/`. The planner caps reviewers, emits selected paths/docs/tests/live gates, and does not launch agents itself. + +### Instruction hierarchy + +Extend root `AGENTS.md` with a bounded audit/implementation/certification workflow: + +- resolve the subsystem registry before broad reading; +- use a maximum of four read-only discovery agents by default; +- send minimal context and disjoint ownership; +- reconcile evidence before editing; +- use one integration owner for broad Cargo/E2E runs; +- cap command output and persist summaries/artifacts; +- checkpoint completed phases so later work can start in a fresh session; +- never infer authorization for billed/live gates. + +Add short local instruction files only where policy differs materially: + +- `crates/moa-orchestrator/AGENTS.md` +- `crates/moa-execution/AGENTS.md` +- `crates/moa-hands/AGENTS.md` +- `crates/moa-memory/AGENTS.md` +- `crates/moa-auth/AGENTS.md` +- `crates/moa-connectors/AGENTS.md` + +### MOA-wide structural inventory + +Add `docs/engineering-discipline/repository-structure-inventory.md`. Rank large and change-central production files as: + +- `split-now`: a verified behavior boundary, independent consumers/tests, and a safe atomic write set exist; +- `keep`: size is justified by one cohesive generated/table-driven/state-machine owner; +- `investigate`: size or fan-in is notable, but evidence is insufficient for a safe extraction. + +This tranche implements only the three independently verified `split-now` items below. Other entries become explicitly routed follow-up work instead of hidden debt or speculative churn. + +Certification exposed three adjacent defects that were repaired in their +existing owners: duplicate same-tenant admission locks, checkpoint replay and +failed-generation liveness, and a burst test that used a test-only 60-second +pseudo-SLO instead of the checked-in 120-second production alert contract. + +## Parallel Implementation Tasks + +### Task A - Repository-wide routing and bounded workflow + +Write set: + +- `.agents/subsystems.toml` +- root `AGENTS.md` +- six local `AGENTS.md` files listed above +- `crates/xtask/src/subsystem_map.rs` +- `crates/xtask/src/main.rs` +- `crates/xtask/README.md` +- `docs/engineering-discipline/repository-structure-inventory.md` + +Requirements: + +- Validate all configured paths, docs, local instructions, owners, profiles/targets, and live-gate structure. +- Unit-test longest-prefix routing, ambiguity rejection, missing-path rejection, reviewer capping, and deterministic packet output. +- Reuse checked-in nextest/Make identifiers; do not duplicate test implementation or create an autonomous Codex launcher. + +### Task B - Execution task-attempt behavior modules + +Write set: + +- `crates/moa-orchestrator/src/workflows/execution_task_attempt.rs` +- `crates/moa-orchestrator/src/workflows/execution_task_attempt/active.rs` +- `crates/moa-orchestrator/src/workflows/execution_task_attempt/{external,yielding,watchdog}.rs` +- new `execution_task_attempt/continuation.rs` +- new `execution_task_attempt/active/{heartbeat,capability,agent}.rs` + +Move continuation schema, heartbeat fencing, direct capability execution, and agent execution to their behavior owners. Keep `execute_task_attempt`, exit routing, and genuinely shared helpers in the thin parent. Preserve serialized shapes, model/tool progression, journal names, provider ordering, and sibling visibility. Move existing inline tests with their owners. + +### Task C - Pending-terminal compensation coordination + +Write set: + +- `crates/moa-execution/src/repository/compensation.rs` +- new `crates/moa-execution/src/repository/compensation/pending_terminal.rs` + +Move the terminal fence/drain/finalization state machine into a private child module while keeping public `ExecutionRepository` methods and result types at their existing paths. Keep shared compensation attempt primitives in the parent. Preserve SQL transaction order, row locks, capacity accounting, paging, replay, and terminal replacement behavior. + +### Task D - Sandbox workspace lifecycle behavior modules + +Write set: + +- `crates/moa-hands/src/core/sandbox_workspace/lifecycle.rs` +- new `crates/moa-hands/src/core/sandbox_workspace/lifecycle/{management,materialization,commit,execution_release}.rs` + +Move management operations, initial materialization/hydration, commit publication, and execution-release recovery into private child modules. Keep shared commit result, lease attachment, and abandoned-checkpoint cleanup in the parent. Preserve operation-ledger order, manifest locking, provider I/O boundaries, commit-before-release, exact receipt fencing, and current public `ToolRouter` methods. + +## Dependencies And Execution Order + +1. Complete the read-only seam and repository-wide inventory. +2. Run Tasks A-D in parallel because their Rust write sets do not overlap. +3. Reconcile visibility and formatting centrally after all workers stop editing. +4. Run focused tests once per touched crate, then strict Clippy and workspace build once. +5. Run independent read-only review against this plan. + +## Verification + +Deterministic gates: + +```bash +cargo run -p xtask --locked -- check-subsystems +cargo test -p xtask --locked subsystem_map +cargo test -p moa-orchestrator --lib --locked execution_task_attempt +cargo test -p moa-execution --lib --locked repository::compensation +cargo test -p moa-hands --lib --locked sandbox_workspace +cargo clippy -p xtask -p moa-orchestrator -p moa-execution -p moa-hands --all-targets --all-features --locked -- -D warnings +cargo build --workspace --locked +cargo fmt --all --check +git diff --check +``` + +Run the focused existing DB/service cases selected by the registry when local services are available. Compile ignored live targets if their code moved, but do not set live/billing flags. Report credentialed provider E2E and 24h/7d canaries as not run unless separately authorized. + +## Rollback And Review Boundaries + +- Each task is independently revertible by write set. +- If an extraction needs public compatibility exports, changes SQL/provider ordering, or expands beyond the listed files, stop and re-plan rather than widening the patch. +- A failing pre-existing test must be reproduced against the base revision before classification; do not weaken it. diff --git a/docs/engineering-discipline/repository-structure-inventory.md b/docs/engineering-discipline/repository-structure-inventory.md new file mode 100644 index 000000000..d71a64353 --- /dev/null +++ b/docs/engineering-discipline/repository-structure-inventory.md @@ -0,0 +1,62 @@ +# Repository Structure Inventory + +This inventory routes structural work by verified behavior seams, not line count +alone. `split-now` means an extraction boundary is evidenced and can be planned; +it does not mean the extraction is complete. `investigate` needs a bounded +read-only seam review first. `keep` records a cohesive owner that should not be +split merely because it is large. + +## Current tranche + +Only these private extractions are in the current implementation tranche: + +- `crates/moa-execution/src/repository/compensation.rs`: pending-terminal + fence, drain, and finalization state machine. +- `crates/moa-hands/src/core/sandbox_workspace/lifecycle.rs`: management, + materialization, commit, and execution-release behavior. +- `crates/moa-orchestrator/src/workflows/execution_task_attempt/active.rs`: + heartbeat, direct-capability, and agent-attempt behavior. + +These current-tranche extractions are complete and certified by focused +behavior/persistence tests, strict Clippy, the workspace build, and the +deterministic long-horizon service lane. Every other `split-now` row below is a +phased follow-up with a separate future write set. + +## Split now + +| Path | Verified seam | Status | +|---|---|---| +| `crates/moa-execution/src/repository/compensation.rs` | Pending-terminal coordination is a private transaction/fence state machine distinct from shared compensation-attempt primitives. | Current tranche; certified | +| `crates/moa-orchestrator/src/workflows/execution_task_attempt/active.rs` | Heartbeat fencing, direct capability execution, agent turns, and continuation state are distinct bounded-attempt behaviors. | Current tranche; certified | +| `crates/moa-execution/src/repository/task.rs` | Attempt admission/liveness, checkpoints, external jobs, settlement, and capacity accounting form independently testable repository behavior families. | Phased follow-up | +| `crates/moa-orchestrator/src/services/tool_executor.rs` | External-job adapters, scoped catalog/policy construction, governed dispatch, and callback/recovery behavior have separate consumers and fixtures. | Phased follow-up | +| `crates/moa-core/src/types/execution_planning.rs` | Routing evidence, durable-upgrade transitions, plan/goal contracts, task outcomes, and audit envelopes are stable type families inside one owning module. | Phased follow-up | +| `crates/moa-hands/src/core/sandbox_workspace/lifecycle.rs` | Management, hydration/materialization, commit publication, and execution-release recovery preserve distinct provider-I/O and ledger boundaries. | Current tranche; certified | +| `crates/moa-orchestrator/src/workflows/experiment_trial_run/target_execution.rs` | Target preparation, session ownership/resume, observation, usage, and terminal scoring are separable workflow behaviors with durable ordering constraints. | Phased follow-up | +| `crates/moa-retrieval/src/retrieval/legs.rs` | Graph expansion/policy, exact seeds, lexical/vector legs, temporal scoring, and diagnostic assembly are independently testable retrieval stages. | Phased follow-up | +| `crates/moa-eval/src/kernel/stats.rs` | Bootstrap estimation, paired tests, multiple-comparison correction, arm summaries, and deterministic sampling are distinct statistical algorithms. | Phased follow-up | +| `crates/xtask/src/execution_trace_manifest.rs` | Manifest data, discovery, source loading, sender/receiver audits, and diagnostics are separate repository-validation responsibilities. | Phased follow-up | + +## Investigate + +| Path | Question to resolve before extraction | +|---|---| +| `crates/moa-providers/src/registry.rs` | Determine whether model catalog data, provider construction, capability lookup, and governance policy have independent consumers or intentionally share one exhaustive registry. | +| `crates/moa-memory/ingest/src/slow_path.rs` | Trace transaction, contradiction, extraction, and graph/vector write ordering before proposing a seam; ingestion correctness may require one coordinated owner. | +| `crates/moa-artifacts/src/validation.rs` | Measure which validation families already delegate to child modules and whether another extraction reduces coupling without fragmenting one canonical artifact contract. | + +## Keep + +| Path | Reason | +|---|---| +| `crates/moa-brain/src/pipeline/memory.rs` | One cohesive context-pipeline memory stage owns retrieval request construction, degradation, rendering, and stage telemetry. | +| `crates/moa-connectors/src/executor.rs` | One constrained HTTP execution boundary intentionally keeps destination admission, credential application, request execution, and durable outcome normalization together. | +| `crates/moa-experiments/src/plan.rs` | One canonical experiment-plan contract and validation state machine benefits from exhaustive, colocated semantics. | + +## Review rule + +Promoting an `investigate` or `keep` entry to `split-now` requires a read-only +report naming the behavior owner, callers, tests, invariant-preserving move, and +disjoint write set. Completing a `split-now` row requires the focused tests plus +the repository integration owner’s final validation; a file move alone is not +completion. From 9409a693f122aa022a8efcf8898052f1b9bebe44 Mon Sep 17 00:00:00 2001 From: Hwuiwon Kim Date: Fri, 14 Aug 2026 07:56:12 -0400 Subject: [PATCH 06/21] make human input waits indefinite --- .config/nextest.toml | 21 +- crates/moa-artifacts/src/execution_plan.rs | 2 - crates/moa-artifacts/src/validation.rs | 6 - .../src/validation/execution_plan.rs | 111 ---- .../artifacts_offline/definition_roundtrip.rs | 9 - .../execution_plan_validation.rs | 73 +-- .../src/execution_planning/request.rs | 13 +- .../src/prompts/execution_planner.md | 5 +- crates/moa-brain/tests/brain_turn_offline.rs | 4 - crates/moa-config/src/context.rs | 20 - crates/moa-config/src/env_overlay/mod.rs | 4 - crates/moa-config/src/execution.rs | 21 +- .../examples/generate_execution_corpus.rs | 6 - .../execution/contract-recorded.jsonl | 160 ++--- .../scenarios/execution/manifest.toml | 2 +- .../tests/eval_offline/execution_snapshot.rs | 7 +- crates/moa-execution/src/compiler/mod.rs | 35 -- crates/moa-execution/src/compiler/tests.rs | 8 +- .../compiler/validation/wait_feasibility.rs | 5 +- crates/moa-execution/src/interpreter/tests.rs | 12 +- .../moa-execution/src/repository/capacity.rs | 14 + .../src/repository/compensation.rs | 10 +- .../compensation/pending_terminal.rs | 197 ++++++- .../src/repository/completion.rs | 95 +-- .../src/repository/completion/coverage.rs | 480 +++++++++++++++ crates/moa-execution/src/repository/mod.rs | 2 + crates/moa-execution/src/repository/outbox.rs | 1 + .../moa-execution/src/repository/outcome.rs | 22 +- .../outcome/amendment_reconciliation.rs | 196 +++++++ crates/moa-execution/src/repository/ready.rs | 61 +- .../src/repository/replan_stop.rs | 5 +- crates/moa-execution/src/repository/rows.rs | 3 + crates/moa-execution/src/repository/run.rs | 70 ++- crates/moa-execution/src/repository/sql.rs | 31 +- crates/moa-execution/src/repository/task.rs | 396 +++++++------ .../src/repository/transition.rs | 28 +- .../moa-execution/src/repository/trigger.rs | 35 +- crates/moa-execution/src/state.rs | 2 - crates/moa-execution/tests/compiler.rs | 19 - crates/moa-execution/tests/completion.rs | 10 +- .../execution_db/compensation_attempts_db.rs | 59 ++ .../execution_db/completion_projection_db.rs | 196 ++++++- .../execution_db/execution_capacity_db.rs | 96 +++ .../execution_db/long_horizon_state_db.rs | 434 +++++++++++++- .../execution_db/planning_and_audit_db.rs | 271 ++++++++- .../tests/execution_db/support.rs | 6 - .../tests/execution_db/trigger_outbox_db.rs | 107 ++++ .../execution_db/wait_entry_deadline_db.rs | 551 +++++++++++++++--- crates/moa-execution/tests/interpreter.rs | 11 +- crates/moa-hands/src/core/dispatch.rs | 38 +- crates/moa-hands/src/core/mod.rs | 3 +- .../src/core/sandbox_workspace/lifecycle.rs | 6 +- .../lifecycle/execution_release.rs | 33 +- .../lifecycle/worker_release.rs | 490 ++++++++++++++++ .../src/core/sandbox_workspace/model.rs | 25 + .../sandbox_workspace/repository/lifecycle.rs | 48 +- .../core/sandbox_workspace/repository/mod.rs | 23 +- crates/moa-hands/src/lib.rs | 6 +- .../hands_db/sandbox_workspace/dispatch_db.rs | 228 +++++++- .../sandbox_workspace/lifecycle_db.rs | 221 +++++-- .../V000059__long_horizon_execution.sql | 36 +- .../execution_and_security_catalog.rs | 48 +- .../execution_compensation.rs | 25 +- .../src/objects/session/handlers/progress.rs | 2 +- .../src/objects/session/handlers/turns.rs | 129 ++-- .../handlers/turns/coordinator_input.rs | 113 ++++ .../objects/session/handlers/turns/replies.rs | 63 ++ .../session/handlers/turns/turn_start.rs | 9 + .../src/objects/session/mod.rs | 71 +++ .../src/services/execution/handlers.rs | 47 +- .../src/services/execution/support.rs | 67 +++ .../src/services/execution/tests.rs | 28 +- .../execution_amendment_planner/tests.rs | 9 +- .../services/skill_regression/compilation.rs | 12 +- .../src/services/tool_executor.rs | 84 ++- .../execution_task_attempt/active/agent.rs | 56 +- .../src/workflows/turn_events.rs | 3 - .../src/workflows/turn_execution/mod.rs | 23 - .../src/workflows/turn_execution/tools.rs | 18 +- .../src/workflows/worker_turn_execution.rs | 91 +-- .../tests/analytics_parity_docker.rs | 8 +- ...oordinator_worker_behavior_provider_e2e.rs | 96 +-- .../tests/execution_run_service_e2e.rs | 12 - .../admission_replay.rs | 6 - .../bulk_and_recovery.rs | 6 - .../compensation_recovery.rs | 6 - .../execution_run_service_e2e/evaluation.rs | 26 +- .../observability.rs | 6 - .../replan_and_completion.rs | 139 +++-- .../execution_run_service_e2e/routing.rs | 114 +++- .../task_lifecycle.rs | 335 +++++------ .../terminal_matrix.rs | 12 - .../integration/action_policy_flow_e2e.rs | 343 +++++++---- .../long_horizon_execution_service_e2e.rs | 44 -- .../action_reviews_reaper_db.rs | 19 +- .../orchestrator_db/analytics_export_db.rs | 14 +- .../execution_dispatch_reconciliation_db.rs | 29 +- .../orchestrator_db/execution_schedule_db.rs | 52 +- .../orchestrator_db/execution_service_db.rs | 34 +- .../session_turn_lifecycle_service_e2e.rs | 139 ----- .../tests/worker_coordination_service_e2e.rs | 64 ++ .../moa-test-support/src/execution_audits.rs | 7 + crates/xtask/src/execution_trace_manifest.rs | 21 + docs/01-architecture-overview.md | 10 +- docs/02-brain-orchestration.md | 15 +- docs/12-restate-architecture.md | 15 +- docs/23-environment-variables.md | 2 - .../artifacts/damaged-food-order.skill.yaml | 6 - .../patterns/custom-logic.skill.yaml | 6 - .../patterns/human-approval.skill.yaml | 6 - .../patterns/parallel-review.skill.yaml | 6 - .../artifacts/patterns/react-agent.skill.yaml | 6 - .../artifacts/patterns/sequential.skill.yaml | 6 - docs/schemas/moa-skill-v1.schema.json | 33 -- 114 files changed, 5569 insertions(+), 2061 deletions(-) create mode 100644 crates/moa-execution/src/repository/completion/coverage.rs create mode 100644 crates/moa-execution/src/repository/outcome/amendment_reconciliation.rs create mode 100644 crates/moa-hands/src/core/sandbox_workspace/lifecycle/worker_release.rs create mode 100644 crates/moa-orchestrator/src/objects/session/handlers/turns/coordinator_input.rs diff --git a/.config/nextest.toml b/.config/nextest.toml index b572c2549..bcea6b143 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -80,6 +80,8 @@ inherits = "default" test-threads = 1 default-filter = ''' binary(/^(session_turn_lifecycle_service_e2e|turn_execution_smoke_service_e2e)$/) + | (binary(/^llm_gateway_provider_e2e$/) + & test(/^direct_llm_gateway_ingress_is_rejected$/)) - test(/recovery_matrix_/) ''' @@ -95,6 +97,8 @@ default-filter = ''' binary(/^orchestrator_fixture_service_e2e$/) | binary(/^(action_policy_service_e2e|guardrails_service_e2e|ingestion_service_e2e|memory_service_e2e|turn_responsiveness_service_e2e)$/) | binary(/^turn_terminal_failure_service_e2e$/) + | (binary(/^coordinator_worker_behavior_provider_e2e$/) + & test(/^duplicate_worker_rejection_persists_tool_result_and_turn_continues_service_e2e$/)) ''' # Accelerated eight-logical-day execution validation. The binary owns one @@ -221,18 +225,22 @@ binary(/^mock_loadtest_service_e2e$/) [profile.provider-e2e] inherits = "default" test-threads = 1 -# The coordinator provider binary also contains deterministic crash/recovery -# fixtures. Those fixtures own a dedicated Restate instance and run in the -# restate-recovery lane, so they cannot share the external Restate configured -# for provider E2E. The paid 20x5 execution evaluation has a separate budget -# authorization contract and must never be swept into the ordinary provider -# lane merely because its binary name ends in `_provider_e2e`. +# The coordinator provider binary also contains deterministic dedicated-fixture +# tests. They run in fixture/recovery lanes with external Restate discovery +# disabled, so they cannot share the external Restate configured for provider +# E2E. The paid 20x5 execution evaluation has a separate budget authorization +# contract and must never be swept into the ordinary provider lane merely +# because its binary name ends in `_provider_e2e`. default-filter = ''' ( binary(/_provider_e2e$/) | binary(/^(provider_matrix_live|query_rewrite_live)$/) ) - (binary(/^coordinator_worker_behavior_provider_e2e$/) & test(/^recovery_matrix_/)) + - (binary(/^coordinator_worker_behavior_provider_e2e$/) + & test(/^duplicate_worker_rejection_persists_tool_result_and_turn_continues_service_e2e$/)) + - (binary(/^llm_gateway_provider_e2e$/) + & test(/^direct_llm_gateway_ingress_is_rejected$/)) - binary(/^execution_eval_provider_e2e$/) ''' @@ -264,7 +272,6 @@ binary(/^execution_run_service_e2e$/) | test(/remove_only_amendment_is_no_progress_service_e2e/) | test(/useful_amendment_preserves_completed_work_service_e2e/) | test(/stale_generation_is_audit_only_service_e2e/) - | test(/duplicate_completion_does_not_double_account_service_e2e/) | (test(/execution_eval_/) - test(/execution_eval_silent_incomplete_500_item_universe/)) ) ''' diff --git a/crates/moa-artifacts/src/execution_plan.rs b/crates/moa-artifacts/src/execution_plan.rs index a3dca250e..87423954c 100644 --- a/crates/moa-artifacts/src/execution_plan.rs +++ b/crates/moa-artifacts/src/execution_plan.rs @@ -129,8 +129,6 @@ pub enum CompletionCheckKind { pub struct ExecutionPlanDefinition { /// Explicit policy for effects already committed when the run is cancelled. pub cancel_policy: ExecutionCancelPolicy, - /// Expiry behavior for runtime input requests returned by executable tasks. - pub input_wait_policy: ExecutionWaitPolicy, /// JSON Schema for run input. pub input_schema: Value, /// JSON Schema for terminal output. diff --git a/crates/moa-artifacts/src/validation.rs b/crates/moa-artifacts/src/validation.rs index 7c836f352..c2e4c998a 100644 --- a/crates/moa-artifacts/src/validation.rs +++ b/crates/moa-artifacts/src/validation.rs @@ -934,12 +934,6 @@ fn validate_execution_plan_at( allow_absolute_temporal_targets: bool, report: &mut ValidationReport, ) { - execution_plan::validate_input_wait_policy( - &format!("{root}.input_wait_policy"), - &definition.input_wait_policy, - allow_absolute_temporal_targets, - report, - ); validate_json_schema( &format!("{root}.input_schema"), &definition.input_schema, diff --git a/crates/moa-artifacts/src/validation/execution_plan.rs b/crates/moa-artifacts/src/validation/execution_plan.rs index 83d4f0b8b..b5dd374bb 100644 --- a/crates/moa-artifacts/src/validation/execution_plan.rs +++ b/crates/moa-artifacts/src/validation/execution_plan.rs @@ -234,36 +234,6 @@ fn validate_wait_policy( } } -/// Validates the plan-level expiry policy for runtime `NeedsInput` outcomes. -/// -/// This one policy settles whichever logical task returned `NeedsInput`, so a -/// declared `continue_with` output has no single node `output_schema` to be checked -/// against. It is rejected here rather than deferred to run materialization, where -/// the schema check is a non-retryable failure. -pub(super) fn validate_input_wait_policy( - root: &str, - policy: &ExecutionWaitPolicy, - allow_absolute_temporal_targets: bool, - report: &mut ValidationReport, -) { - validate_temporal_target( - &format!("{root}.expiry"), - &policy.expiry, - allow_absolute_temporal_targets, - report, - ); - if matches!( - policy.on_expiry, - ExecutionWaitExpiryAction::ContinueWith { .. } - ) { - report.push_error( - format!("{root}.on_expiry"), - "input wait expiry must fail the waiting task; continue_with cannot be validated \ - against the output schema of the node that requested input", - ); - } -} - pub(super) fn validate_temporal_target( root: &str, target: &ExecutionTemporalTarget, @@ -281,84 +251,3 @@ pub(super) fn validate_temporal_target( ExecutionTemporalTarget::At { .. } | ExecutionTemporalTarget::After { .. } => {} } } - -#[cfg(test)] -mod tests { - use serde_json::json; - - use crate::execution_plan::{ - ExecutionCancelPolicy, ExecutionNode, ExecutionOperation, ExecutionPlanDefinition, - ExecutionTemporalTarget, ExecutionWaitExpiryAction, ExecutionWaitPolicy, RetryPolicy, - }; - use crate::validation::validate_execution_plan_definition; - - // Pins: `input_wait_policy.on_expiry` is the one wait policy with no owning node, - // so a `continue_with` output has nothing to validate against. Before this check - // it compiled cleanly and the schema violation surfaced at run materialization as - // a non-retryable infrastructure error against whichever task happened to ask for - // input. The accepted direction proves the rejection is about `continue_with` - // and not the surrounding fixture; the removed `fail_run` wire spelling is - // rejected explicitly instead of pretending it remains a second valid action. - #[test] - fn input_wait_policy_accepts_fail_task_and_rejects_other_settlements() { - let continued = plan(ExecutionWaitExpiryAction::ContinueWith { - output: json!({ "approved": true }), - }); - - let report = validate_execution_plan_definition(&continued); - - assert!( - report.errors.iter().any(|error| { - error.path == "execution_plan.input_wait_policy.on_expiry" - && error.message.contains("must fail the waiting task") - }), - "continue_with must be refused for the plan-level input wait policy: {report:?}" - ); - - let report = validate_execution_plan_definition(&plan(ExecutionWaitExpiryAction::FailTask)); - assert!( - report.errors.is_empty(), - "fail_task must remain the valid input-wait expiry: {report:?}" - ); - - assert!( - serde_json::from_value::(json!({ - "kind": "fail_run" - })) - .is_err(), - "the removed fail_run wire spelling must fail closed" - ); - } - - fn plan(on_expiry: ExecutionWaitExpiryAction) -> ExecutionPlanDefinition { - ExecutionPlanDefinition { - cancel_policy: ExecutionCancelPolicy::RetainEffects, - input_wait_policy: ExecutionWaitPolicy { - expiry: ExecutionTemporalTarget::After { - delay_seconds: 3_600, - }, - on_expiry, - }, - input_schema: json!({ "type": "object" }), - output_schema: json!({ "type": "object" }), - nodes: vec![ExecutionNode { - id: "output".to_string(), - requirement_ids: vec!["req_output".to_string()], - depends_on: Vec::new(), - when: None, - input: json!({}), - output_schema: json!({ "type": "object" }), - operation: ExecutionOperation::Output { - value: json!({ "$ref": "$.input" }), - }, - compensation: None, - retry: RetryPolicy { - max_attempts: 1, - initial_backoff_ms: 0, - max_backoff_ms: 0, - }, - budget: None, - }], - } - } -} diff --git a/crates/moa-artifacts/tests/artifacts_offline/definition_roundtrip.rs b/crates/moa-artifacts/tests/artifacts_offline/definition_roundtrip.rs index 0e83adc8c..a749c40cf 100644 --- a/crates/moa-artifacts/tests/artifacts_offline/definition_roundtrip.rs +++ b/crates/moa-artifacts/tests/artifacts_offline/definition_roundtrip.rs @@ -527,9 +527,6 @@ definition: completion_checks: [] plan: cancel_policy: retain_effects - input_wait_policy: - expiry: { kind: after, delay_seconds: 3600 } - on_expiry: { kind: fail_task } input_schema: { type: object } output_schema: { type: object } nodes: @@ -619,9 +616,6 @@ definition: completion_checks: [] plan: cancel_policy: retain_effects - input_wait_policy: - expiry: { kind: after, delay_seconds: 3600 } - on_expiry: { kind: fail_task } input_schema: { type: object } output_schema: { type: object } nodes: @@ -696,9 +690,6 @@ definition: completion_checks: [] plan: cancel_policy: retain_effects - input_wait_policy: - expiry: { kind: after, delay_seconds: 3600 } - on_expiry: { kind: fail_task } input_schema: { type: object } output_schema: { type: object } nodes: diff --git a/crates/moa-artifacts/tests/artifacts_offline/execution_plan_validation.rs b/crates/moa-artifacts/tests/artifacts_offline/execution_plan_validation.rs index 8f7b0e820..18e8e0887 100644 --- a/crates/moa-artifacts/tests/artifacts_offline/execution_plan_validation.rs +++ b/crates/moa-artifacts/tests/artifacts_offline/execution_plan_validation.rs @@ -241,14 +241,6 @@ fn temporal_targets_round_trip_and_reject_zero_relative_delay() { ); } - let mut zero_input_wait = valid_plan(); - zero_input_wait.input_wait_policy.expiry = ExecutionTemporalTarget::After { delay_seconds: 0 }; - assert_error( - &validate_execution_plan_definition(&zero_input_wait), - "execution_plan.input_wait_policy.expiry", - "temporal delay_seconds must be at least one", - ); - let mut zero_timer = valid_plan(); zero_timer.nodes[0].operation = ExecutionOperation::WaitUntil { wake: ExecutionTemporalTarget::After { delay_seconds: 0 }, @@ -265,19 +257,7 @@ fn temporal_targets_round_trip_and_reject_zero_relative_delay() { fn reusable_plan_templates_reject_absolute_temporal_targets_at_every_wait_surface() { // Pins: reusable skill templates stay valid over time by expressing waits relative to when // each wait state is entered; exact UTC targets remain valid only for one-off plans. - let mut standalone = valid_plan(); - standalone.input_wait_policy.expiry = absolute_target(); - assert!( - validate_execution_plan_definition(&standalone).is_ok(), - "one-off plans may declare an exact UTC input-wait expiry" - ); - let mut cases = Vec::new(); - cases.push(( - standalone, - "definition.spec.execution_plan.plan.input_wait_policy.expiry", - )); - let mut review = valid_plan(); review.nodes[0].operation = ExecutionOperation::Review { prompt: "Approve?".to_string(), @@ -428,42 +408,20 @@ fn skill_schema_and_rust_types_require_the_same_wait_contract() { stale_plan .as_object_mut() .expect("plan is an object") - .remove("input_wait_policy"); + .insert( + "input_wait_policy".to_string(), + json!({ + "expiry": { "kind": "after", "delay_seconds": 3600 }, + "on_expiry": { "kind": "fail_task" } + }), + ); assert!( serde_json::from_value::(stale_plan.clone()).is_err(), - "Rust type must reject a plan without input_wait_policy" + "Rust type must reject the removed input_wait_policy field" ); assert!( !plan_validator.is_valid(&stale_plan), - "skill schema must reject a plan without input_wait_policy" - ); - - // The plan-level policy settles whichever task returned NeedsInput, so a declared - // continue_with output has no node output_schema to be checked against. The - // canonical validator refuses it, and the schema must refuse it at the same - // place — while still accepting continue_with on a node-owned wait, which does - // have an owning schema. - let mut continued_input_wait = plan_json; - continued_input_wait["input_wait_policy"]["on_expiry"] = - json!({ "kind": "continue_with", "output": { "approved": true } }); - assert!( - !plan_validator.is_valid(&continued_input_wait), - "skill schema must reject continue_with on the plan-level input wait policy" - ); - let mut definition = - serde_json::from_value::(continued_input_wait.clone()) - .expect("plan-level continue_with is still a well-formed value"); - assert!( - validate_execution_plan_definition(&definition) - .errors - .iter() - .any(|error| error.path == "execution_plan.input_wait_policy.on_expiry"), - "canonical validation must reject continue_with on the input wait policy" - ); - definition.input_wait_policy.on_expiry = ExecutionWaitExpiryAction::FailTask; - assert!( - plan_validator.is_valid(&serde_json::to_value(&definition).expect("serialize plan")), - "schema must still accept a failing input wait expiry" + "skill schema must reject the removed input_wait_policy field" ); } @@ -549,10 +507,6 @@ fn skill_reference_paths_cover_agent_map_and_reducer_agents_only() { }, "plan": { "cancel_policy": "retain_effects", - "input_wait_policy": { - "expiry": { "kind": "after", "delay_seconds": 3600 }, - "on_expiry": { "kind": "fail_task" } - }, "input_schema": { "type": "object" }, "output_schema": { "type": "object" }, "nodes": [ @@ -673,13 +627,7 @@ fn execution_plan_round_trips_without_a_nested_version() { let encoded = serde_json::to_value(&plan).expect("serialize plan"); assert!(encoded.get("schema_version").is_none()); assert_eq!(encoded["cancel_policy"], json!("retain_effects")); - assert_eq!( - encoded["input_wait_policy"], - json!({ - "expiry": { "kind": "after", "delay_seconds": 3600 }, - "on_expiry": { "kind": "fail_task" } - }) - ); + assert!(encoded.get("input_wait_policy").is_none()); assert_eq!( serde_json::from_value::(encoded).expect("deserialize exact plan"), plan @@ -1466,7 +1414,6 @@ fn task_outcome_variants_round_trip_without_extra_envelope_fields() { fn valid_plan() -> ExecutionPlanDefinition { ExecutionPlanDefinition { cancel_policy: ExecutionCancelPolicy::RetainEffects, - input_wait_policy: wait_policy(ExecutionWaitExpiryAction::FailTask), input_schema: json!({ "type": "object" }), output_schema: json!({ "type": "object" }), nodes: vec![ diff --git a/crates/moa-brain/src/execution_planning/request.rs b/crates/moa-brain/src/execution_planning/request.rs index 775c126fb..e4745ffc1 100644 --- a/crates/moa-brain/src/execution_planning/request.rs +++ b/crates/moa-brain/src/execution_planning/request.rs @@ -19,7 +19,7 @@ use serde::{Deserialize, Serialize}; use serde_json::Value; /// Stable execution-planner prompt identifier. -pub const EXECUTION_PLANNER_PROMPT_VERSION: &str = "execution-planner-v8"; +pub const EXECUTION_PLANNER_PROMPT_VERSION: &str = "execution-planner-v10"; /// Fixed maximum collected planner output tokens. pub const EXECUTION_PLANNER_MAX_OUTPUT_TOKENS: usize = 32_768; const EXECUTION_PLANNER_PROMPT: &str = include_str!("../prompts/execution_planner.md"); @@ -170,7 +170,7 @@ fn build_initial_request( compiler_report_json, } => { user_payload.push_str(&format!( - "\nRepair the candidate exactly once. Preserve immutable_goal byte-for-byte after canonicalization. Do not discover new authority or capabilities.\n{original_candidate_json}\n{immutable_goal_json}\n{compiler_report_json}" + "\nRepair the candidate exactly once. Preserve immutable_goal byte-for-byte after canonicalization. Do not discover new authority or capabilities. Return only the replacement JSON object.\n{original_candidate_json}\n{immutable_goal_json}\n{compiler_report_json}" )); } } @@ -295,10 +295,10 @@ mod tests { use super::*; #[test] - fn execution_planner_prompt_v8_pins_long_horizon_and_compiler_invariants() { + fn execution_planner_prompt_v10_pins_long_horizon_and_compiler_invariants() { // Pins: the current emitted prompt version and its compiler-facing guidance // change together so live planner provenance identifies this exact contract. - assert_eq!(EXECUTION_PLANNER_PROMPT_VERSION, "execution-planner-v8"); + assert_eq!(EXECUTION_PLANNER_PROMPT_VERSION, "execution-planner-v10"); assert_eq!( EXECUTION_PLANNER_PROMPT, concat!( @@ -309,12 +309,13 @@ mod tests { "- Choose exactly one explicit `plan.cancel_policy`: `retain_effects` or `compensate_committed`.\n", "- Treat the frozen `budget.deadline_at` as the absolute Durable-run deadline. Never emit a wait, retry window, or active task whose bound reaches or exceeds it.\n", "- Use `WaitUntil` for a calendar-time delay. Its `wake` is a tagged temporal target, either `{\"kind\":\"at\",\"at\":\"\"}` for an exact instant or `{\"kind\":\"after\",\"delay_seconds\":}` for a delay measured from the moment the node starts waiting. Emit exactly those fields for the chosen shape and nothing else. The resolved wake time must land before the run deadline. Its declared `result` is the structured value made available to downstream nodes after the timer fires.\n", - "- Give every `Review` and `WaitSignal` an explicit `wait_policy` of `{\"expiry\": , \"on_expiry\": }`, using the same two temporal-target shapes. An expiry action is either `{\"kind\":\"fail_task\"}` or `{\"kind\":\"continue_with\",\"output\":}`. Represent human and external waits only with these storage-backed wait operations; never keep an `Agent` or `Capability` active while waiting for a person, callback, schedule, or retry time.\n", - "- Always set `plan.input_wait_policy`. It is required, and it governs every task that pauses for runtime input rather than one named node, so its `on_expiry` accepts only `{\"kind\":\"fail_task\"}` — never `continue_with`.\n", + "- Give every `Review` and `WaitSignal` an explicit `wait_policy` of `{\"expiry\": , \"on_expiry\": }`, using the same two temporal-target shapes. An expiry action is either `{\"kind\":\"fail_task\"}` or `{\"kind\":\"continue_with\",\"output\":}`. Represent planned approval and external callback waits with these storage-backed wait operations; never keep an `Agent` or `Capability` active while waiting for a person, callback, schedule, or retry time.\n", + "- A runtime `NeedsInput` outcome parks the task durably and indefinitely until authorized human input arrives or the run is explicitly cancelled, without holding active compute, a worker, model call, sandbox, process, or network connection.\n", "- Decompose long work into bounded active tasks separated by durable nodes. Never plan a continuously running multi-hour or multi-day model call, tool call, shell process, network connection, or sandbox; use a registered asynchronous capability when the catalog explicitly provides one.\n", "- Set every node's `compensation` explicitly. Use `null` unless the node is a direct side-effecting `Capability` whose exact catalog entry advertises the same compensator and bounded input mapping, and that compensator has `requires_sandbox=false`. Never add compensation to reads, agents, maps, reduces, reviews, signals, or outputs, never use a sandbox-backed compensator, and never invent rollback authority.\n", "- An amendment must preserve compensation for work that is running or committed and must not weaken the run's cancellation policy.\n", "- Every goal-entry ID, completion-check ID, execution-node ID, and every ID referenced from those structures must match `[a-z][a-z0-9_-]{0,63}`.\n", + "- Every `plan.nodes[].requirement_ids` and `goal.completion_checks[].requirement_ids` entry must reference an ID from `goal.requirements`. Every `goal.completion_checks[].constraint_ids` entry must reference an ID from `goal.constraints`. Never cross those ID domains, and never put a constraint ID on a plan node. Every goal requirement must be served by at least one node.\n", "- Link every requirement and every constraint to at least one completion check via `requirement_ids` and `constraint_ids`.\n", "- Put every goal requirement ID in at least one completion check's `requirement_ids`. If the plan has only one completion check, it must list every requirement ID. For a simple `Agent`-to-`Output` plan, prefer one `OutputSchema` check listing all requirement IDs.\n", "- Use only whole-value binding objects of exactly `{\"$ref\":\"\"}` with no sibling keys or string interpolation. A reference path may select the complete `$.input` or `$.nodes..output` value, or append dot-separated object fields such as `$.input.query` and `$.nodes.lookup.output.items`; node references may read only declared dependencies. Never use bracket/index syntax.\n", diff --git a/crates/moa-brain/src/prompts/execution_planner.md b/crates/moa-brain/src/prompts/execution_planner.md index 23b214675..9610c0e14 100644 --- a/crates/moa-brain/src/prompts/execution_planner.md +++ b/crates/moa-brain/src/prompts/execution_planner.md @@ -7,12 +7,13 @@ Compiler invariants: - Choose exactly one explicit `plan.cancel_policy`: `retain_effects` or `compensate_committed`. - Treat the frozen `budget.deadline_at` as the absolute Durable-run deadline. Never emit a wait, retry window, or active task whose bound reaches or exceeds it. - Use `WaitUntil` for a calendar-time delay. Its `wake` is a tagged temporal target, either `{"kind":"at","at":""}` for an exact instant or `{"kind":"after","delay_seconds":}` for a delay measured from the moment the node starts waiting. Emit exactly those fields for the chosen shape and nothing else. The resolved wake time must land before the run deadline. Its declared `result` is the structured value made available to downstream nodes after the timer fires. -- Give every `Review` and `WaitSignal` an explicit `wait_policy` of `{"expiry": , "on_expiry": }`, using the same two temporal-target shapes. An expiry action is either `{"kind":"fail_task"}` or `{"kind":"continue_with","output":}`. Represent human and external waits only with these storage-backed wait operations; never keep an `Agent` or `Capability` active while waiting for a person, callback, schedule, or retry time. -- Always set `plan.input_wait_policy`. It is required, and it governs every task that pauses for runtime input rather than one named node, so its `on_expiry` accepts only `{"kind":"fail_task"}` — never `continue_with`. +- Give every `Review` and `WaitSignal` an explicit `wait_policy` of `{"expiry": , "on_expiry": }`, using the same two temporal-target shapes. An expiry action is either `{"kind":"fail_task"}` or `{"kind":"continue_with","output":}`. Represent planned approval and external callback waits with these storage-backed wait operations; never keep an `Agent` or `Capability` active while waiting for a person, callback, schedule, or retry time. +- A runtime `NeedsInput` outcome parks the task durably and indefinitely until authorized human input arrives or the run is explicitly cancelled, without holding active compute, a worker, model call, sandbox, process, or network connection. - Decompose long work into bounded active tasks separated by durable nodes. Never plan a continuously running multi-hour or multi-day model call, tool call, shell process, network connection, or sandbox; use a registered asynchronous capability when the catalog explicitly provides one. - Set every node's `compensation` explicitly. Use `null` unless the node is a direct side-effecting `Capability` whose exact catalog entry advertises the same compensator and bounded input mapping, and that compensator has `requires_sandbox=false`. Never add compensation to reads, agents, maps, reduces, reviews, signals, or outputs, never use a sandbox-backed compensator, and never invent rollback authority. - An amendment must preserve compensation for work that is running or committed and must not weaken the run's cancellation policy. - Every goal-entry ID, completion-check ID, execution-node ID, and every ID referenced from those structures must match `[a-z][a-z0-9_-]{0,63}`. +- Every `plan.nodes[].requirement_ids` and `goal.completion_checks[].requirement_ids` entry must reference an ID from `goal.requirements`. Every `goal.completion_checks[].constraint_ids` entry must reference an ID from `goal.constraints`. Never cross those ID domains, and never put a constraint ID on a plan node. Every goal requirement must be served by at least one node. - Link every requirement and every constraint to at least one completion check via `requirement_ids` and `constraint_ids`. - Put every goal requirement ID in at least one completion check's `requirement_ids`. If the plan has only one completion check, it must list every requirement ID. For a simple `Agent`-to-`Output` plan, prefer one `OutputSchema` check listing all requirement IDs. - Use only whole-value binding objects of exactly `{"$ref":""}` with no sibling keys or string interpolation. A reference path may select the complete `$.input` or `$.nodes..output` value, or append dot-separated object fields such as `$.input.query` and `$.nodes.lookup.output.items`; node references may read only declared dependencies. Never use bracket/index syntax. diff --git a/crates/moa-brain/tests/brain_turn_offline.rs b/crates/moa-brain/tests/brain_turn_offline.rs index 1f999740f..15edc4191 100644 --- a/crates/moa-brain/tests/brain_turn_offline.rs +++ b/crates/moa-brain/tests/brain_turn_offline.rs @@ -906,10 +906,6 @@ fn execution_planning_candidate(objective: &str, max_attempts: u32) -> String { }, "plan": { "cancel_policy": "retain_effects", - "input_wait_policy": { - "expiry": {"kind": "after", "delay_seconds": 3600}, - "on_expiry": {"kind": "fail_task"} - }, "input_schema": { "type": "object" }, "output_schema": { "type": "object" }, "nodes": [{ diff --git a/crates/moa-config/src/context.rs b/crates/moa-config/src/context.rs index 73efe1ef4..b9b3954a6 100644 --- a/crates/moa-config/src/context.rs +++ b/crates/moa-config/src/context.rs @@ -73,14 +73,6 @@ pub struct SessionLimitsConfig { pub worker_resume_max_per_window: u32, /// Rolling-window length, in milliseconds, for the guarded parent-resume budget. pub worker_resume_window_ms: u64, - /// Maximum time a child `request_input` round-trip blocks on its awakeable before - /// returning a "no input received" result so the child can proceed or abort. Kept - /// large because a human answer (audience = user) may take minutes. - pub worker_input_timeout_ms: u64, - /// Maximum time a coordinator security-input round-trip blocks before the turn - /// stops with a safe timeout result. Kept large because the owning user may take - /// minutes to answer. - pub coordinator_input_timeout_ms: u64, /// Target cadence, in milliseconds, at which an active child refreshes its /// telemetry-plane heartbeat while running. Sizes the heartbeat the watchdog /// observes; consumers treat `0` as the built-in default cadence. @@ -111,8 +103,6 @@ impl Default for SessionLimitsConfig { worker_cleanup_grace_ms: 60_000, worker_resume_max_per_window: 6, worker_resume_window_ms: 600_000, - worker_input_timeout_ms: 1_800_000, - coordinator_input_timeout_ms: 1_800_000, worker_heartbeat_interval_ms: 15_000, worker_heartbeat_stale_ms: 60_000, } @@ -468,12 +458,6 @@ mod tests { let limits = SessionLimitsConfig::default(); assert_eq!(limits.worker_resume_max_per_window, 6); assert_eq!(limits.worker_resume_window_ms, 600_000); - // The needs_input round-trip ships with a large (but finite) default so a human - // answer has time to arrive without blocking a child turn forever. - assert_eq!(limits.worker_input_timeout_ms, 1_800_000); - // The coordinator's security-input suspend is bounded by the same human-scale - // default and cannot fence a session forever. - assert_eq!(limits.coordinator_input_timeout_ms, 1_800_000); } #[test] @@ -482,8 +466,6 @@ mod tests { let overlay = EnvOverlay { session_limits_worker_resume_max_per_window: Some(3), session_limits_worker_resume_window_ms: Some(120_000), - session_limits_worker_input_timeout_ms: Some(90_000), - session_limits_coordinator_input_timeout_ms: Some(75_000), ..EnvOverlay::default() }; @@ -495,8 +477,6 @@ mod tests { let limits = &config.session_limits; assert_eq!(limits.worker_resume_max_per_window, 3); assert_eq!(limits.worker_resume_window_ms, 120_000); - assert_eq!(limits.worker_input_timeout_ms, 90_000); - assert_eq!(limits.coordinator_input_timeout_ms, 75_000); } #[test] diff --git a/crates/moa-config/src/env_overlay/mod.rs b/crates/moa-config/src/env_overlay/mod.rs index 60ddbd399..793317a94 100644 --- a/crates/moa-config/src/env_overlay/mod.rs +++ b/crates/moa-config/src/env_overlay/mod.rs @@ -628,10 +628,6 @@ pub struct EnvOverlay { pub session_limits_worker_resume_max_per_window: Option, /// `MOA_SESSION_LIMITS_WORKER_RESUME_WINDOW_MS`. pub session_limits_worker_resume_window_ms: Option, - /// `MOA_SESSION_LIMITS_WORKER_INPUT_TIMEOUT_MS`. - pub session_limits_worker_input_timeout_ms: Option, - /// `MOA_SESSION_LIMITS_COORDINATOR_INPUT_TIMEOUT_MS`. - pub session_limits_coordinator_input_timeout_ms: Option, /// `MOA_SESSION_LIMITS_WORKER_HEARTBEAT_INTERVAL_MS`. pub session_limits_worker_heartbeat_interval_ms: Option, /// `MOA_SESSION_LIMITS_WORKER_HEARTBEAT_STALE_MS`. diff --git a/crates/moa-config/src/execution.rs b/crates/moa-config/src/execution.rs index 32616a27f..c91b74b37 100644 --- a/crates/moa-config/src/execution.rs +++ b/crates/moa-config/src/execution.rs @@ -8,6 +8,7 @@ use super::require_positive_limit; /// Provisional physical execution-task window pending the measured T3.3 default. const DEFAULT_MAX_IN_FLIGHT_TASKS: usize = 64; +const MAX_TERMINAL_DRAIN_PAGE_TASKS: usize = 1_000; /// Tenant-independent defaults for execution planning and resource envelopes. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -262,6 +263,11 @@ impl ExecutionConfig { .to_string(), )); } + if self.max_in_flight_tasks > MAX_TERMINAL_DRAIN_PAGE_TASKS { + return Err(MoaError::ConfigError(format!( + "execution.max_in_flight_tasks must not exceed {MAX_TERMINAL_DRAIN_PAGE_TASKS} so one terminal drain page fences every current task owner" + ))); + } if self.dispatch_batch_size < 3 { return Err(MoaError::ConfigError( "execution.dispatch_batch_size must be at least 3 so every reconciliation lane makes progress" @@ -349,7 +355,7 @@ fn usize_as_u64(name: &str, value: usize) -> Result { #[cfg(test)] mod tests { - use super::{DEFAULT_MAX_IN_FLIGHT_TASKS, ExecutionConfig}; + use super::{DEFAULT_MAX_IN_FLIGHT_TASKS, ExecutionConfig, MAX_TERMINAL_DRAIN_PAGE_TASKS}; #[test] fn execution_config_defaults_match_the_resource_contract() { @@ -414,6 +420,19 @@ mod tests { batch.dispatch_batch_size = batch.max_in_flight_tasks + 1; assert!(batch.validate().is_err()); + let oversized_in_flight = ExecutionConfig { + max_in_flight_tasks: MAX_TERMINAL_DRAIN_PAGE_TASKS + 1, + ..ExecutionConfig::default() + }; + let error = oversized_in_flight + .validate() + .expect_err("one terminal page must cover every current task owner"); + assert!( + error + .to_string() + .contains("max_in_flight_tasks must not exceed 1000") + ); + let starving_batch = ExecutionConfig { dispatch_batch_size: 2, ..ExecutionConfig::default() diff --git a/crates/moa-eval/examples/generate_execution_corpus.rs b/crates/moa-eval/examples/generate_execution_corpus.rs index 5120ce18d..22971ea7b 100644 --- a/crates/moa-eval/examples/generate_execution_corpus.rs +++ b/crates/moa-eval/examples/generate_execution_corpus.rs @@ -473,12 +473,6 @@ fn contract_case(index: usize) -> ExecutionContractCase { }, plan: ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, - input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { - expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { - delay_seconds: 86_400, - }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, - }, input_schema: json!({ "type": "object", "properties": { diff --git a/crates/moa-eval/scenarios/execution/contract-recorded.jsonl b/crates/moa-eval/scenarios/execution/contract-recorded.jsonl index 697377061..7cbdcb5c8 100644 --- a/crates/moa-eval/scenarios/execution/contract-recorded.jsonl +++ b/crates/moa-eval/scenarios/execution/contract-recorded.jsonl @@ -1,80 +1,80 @@ -{"schema_version":1,"case_id":"contract-000","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (000).","requirements":[{"id":"req-screen-000","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-000","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-000","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-000","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-000-a","issuer-000-b","issuer-000-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-000","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-000","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-000","description":"Require complete map coverage","requirement_ids":["req-screen-000"],"constraint_ids":["constraint-exclusions-000"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-000","description":"Require citations for every issuer","requirement_ids":["req-report-000"],"constraint_ids":["constraint-definition-000"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-000"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-000-a"},{"ticker":"issuer-000-b"},{"ticker":"issuer-000-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-000"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-000"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-000-a","issuer-000-b","issuer-000-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-001","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (001).","requirements":[{"id":"req-screen-001","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-001","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-001","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-001","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-001-a","issuer-001-b","issuer-001-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-001","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-001","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-001","description":"Require complete map coverage","requirement_ids":["req-screen-001"],"constraint_ids":["constraint-exclusions-001"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-001","description":"Require citations for every issuer","requirement_ids":["req-report-001"],"constraint_ids":["constraint-definition-001"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-001"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-001-a"},{"ticker":"issuer-001-b"},{"ticker":"issuer-001-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-001"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-001"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-001-a","issuer-001-b","issuer-001-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-002","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (002).","requirements":[{"id":"req-screen-002","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-002","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-002","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-002","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-002-a","issuer-002-b","issuer-002-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-002","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-002","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-002","description":"Require complete map coverage","requirement_ids":["req-screen-002"],"constraint_ids":["constraint-exclusions-002"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-002","description":"Require citations for every issuer","requirement_ids":["req-report-002"],"constraint_ids":["constraint-definition-002"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-002"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-002-a"},{"ticker":"issuer-002-b"},{"ticker":"issuer-002-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-002"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-002"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-002-a","issuer-002-b","issuer-002-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-003","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (003).","requirements":[{"id":"req-screen-003","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-003","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-003","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-003","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-003-a","issuer-003-b","issuer-003-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-003","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-003","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-003","description":"Require complete map coverage","requirement_ids":["req-screen-003"],"constraint_ids":["constraint-exclusions-003"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-003","description":"Require citations for every issuer","requirement_ids":["req-report-003"],"constraint_ids":["constraint-definition-003"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-003"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-003-a"},{"ticker":"issuer-003-b"},{"ticker":"issuer-003-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-003"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-003"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-003-a","issuer-003-b","issuer-003-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-004","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (004).","requirements":[{"id":"req-screen-004","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-004","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-004","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-004","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-004-a","issuer-004-b","issuer-004-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-004","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-004","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-004","description":"Require complete map coverage","requirement_ids":["req-screen-004"],"constraint_ids":["constraint-exclusions-004"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-004","description":"Require citations for every issuer","requirement_ids":["req-report-004"],"constraint_ids":["constraint-definition-004"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-004"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-004-a"},{"ticker":"issuer-004-b"},{"ticker":"issuer-004-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-004"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-004"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-004-a","issuer-004-b","issuer-004-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-005","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (005).","requirements":[{"id":"req-screen-005","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-005","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-005","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-005","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-005-a","issuer-005-b","issuer-005-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-005","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-005","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-005","description":"Require complete map coverage","requirement_ids":["req-screen-005"],"constraint_ids":["constraint-exclusions-005"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-005","description":"Require citations for every issuer","requirement_ids":["req-report-005"],"constraint_ids":["constraint-definition-005"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-005"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-005-a"},{"ticker":"issuer-005-b"},{"ticker":"issuer-005-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-005"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-005"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-005-a","issuer-005-b","issuer-005-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-006","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (006).","requirements":[{"id":"req-screen-006","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-006","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-006","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-006","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-006-a","issuer-006-b","issuer-006-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-006","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-006","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-006","description":"Require complete map coverage","requirement_ids":["req-screen-006"],"constraint_ids":["constraint-exclusions-006"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-006","description":"Require citations for every issuer","requirement_ids":["req-report-006"],"constraint_ids":["constraint-definition-006"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-006"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-006-a"},{"ticker":"issuer-006-b"},{"ticker":"issuer-006-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-006"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-006"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-006-a","issuer-006-b","issuer-006-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-007","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (007).","requirements":[{"id":"req-screen-007","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-007","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-007","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-007","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-007-a","issuer-007-b","issuer-007-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-007","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-007","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-007","description":"Require complete map coverage","requirement_ids":["req-screen-007"],"constraint_ids":["constraint-exclusions-007"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-007","description":"Require citations for every issuer","requirement_ids":["req-report-007"],"constraint_ids":["constraint-definition-007"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-007"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-007-a"},{"ticker":"issuer-007-b"},{"ticker":"issuer-007-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-007"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-007"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-007-a","issuer-007-b","issuer-007-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-008","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (008).","requirements":[{"id":"req-screen-008","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-008","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-008","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-008","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-008-a","issuer-008-b","issuer-008-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-008","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-008","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-008","description":"Require complete map coverage","requirement_ids":["req-screen-008"],"constraint_ids":["constraint-exclusions-008"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-008","description":"Require citations for every issuer","requirement_ids":["req-report-008"],"constraint_ids":["constraint-definition-008"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-008"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-008-a"},{"ticker":"issuer-008-b"},{"ticker":"issuer-008-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-008"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-008"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-008-a","issuer-008-b","issuer-008-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-009","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (009).","requirements":[{"id":"req-screen-009","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-009","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-009","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-009","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-009-a","issuer-009-b","issuer-009-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-009","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-009","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-009","description":"Require complete map coverage","requirement_ids":["req-screen-009"],"constraint_ids":["constraint-exclusions-009"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-009","description":"Require citations for every issuer","requirement_ids":["req-report-009"],"constraint_ids":["constraint-definition-009"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-009"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-009-a"},{"ticker":"issuer-009-b"},{"ticker":"issuer-009-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-009"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-009"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-009-a","issuer-009-b","issuer-009-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-010","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (010).","requirements":[{"id":"req-screen-010","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-010","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-010","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-010","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-010-a","issuer-010-b","issuer-010-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-010","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-010","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-010","description":"Require complete map coverage","requirement_ids":["req-screen-010"],"constraint_ids":["constraint-exclusions-010"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-010","description":"Require citations for every issuer","requirement_ids":["req-report-010"],"constraint_ids":["constraint-definition-010"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-010"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-010-a"},{"ticker":"issuer-010-b"},{"ticker":"issuer-010-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-010"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-010"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-010-a","issuer-010-b","issuer-010-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-011","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (011).","requirements":[{"id":"req-screen-011","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-011","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-011","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-011","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-011-a","issuer-011-b","issuer-011-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-011","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-011","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-011","description":"Require complete map coverage","requirement_ids":["req-screen-011"],"constraint_ids":["constraint-exclusions-011"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-011","description":"Require citations for every issuer","requirement_ids":["req-report-011"],"constraint_ids":["constraint-definition-011"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-011"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-011-a"},{"ticker":"issuer-011-b"},{"ticker":"issuer-011-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-011"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-011"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-011-a","issuer-011-b","issuer-011-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-012","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (012).","requirements":[{"id":"req-screen-012","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-012","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-012","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-012","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-012-a","issuer-012-b","issuer-012-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-012","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-012","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-012","description":"Require complete map coverage","requirement_ids":["req-screen-012"],"constraint_ids":["constraint-exclusions-012"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-012","description":"Require citations for every issuer","requirement_ids":["req-report-012"],"constraint_ids":["constraint-definition-012"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-012"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-012-a"},{"ticker":"issuer-012-b"},{"ticker":"issuer-012-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-012"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-012"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-012-a","issuer-012-b","issuer-012-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-013","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (013).","requirements":[{"id":"req-screen-013","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-013","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-013","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-013","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-013-a","issuer-013-b","issuer-013-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-013","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-013","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-013","description":"Require complete map coverage","requirement_ids":["req-screen-013"],"constraint_ids":["constraint-exclusions-013"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-013","description":"Require citations for every issuer","requirement_ids":["req-report-013"],"constraint_ids":["constraint-definition-013"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-013"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-013-a"},{"ticker":"issuer-013-b"},{"ticker":"issuer-013-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-013"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-013"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-013-a","issuer-013-b","issuer-013-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-014","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (014).","requirements":[{"id":"req-screen-014","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-014","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-014","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-014","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-014-a","issuer-014-b","issuer-014-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-014","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-014","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-014","description":"Require complete map coverage","requirement_ids":["req-screen-014"],"constraint_ids":["constraint-exclusions-014"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-014","description":"Require citations for every issuer","requirement_ids":["req-report-014"],"constraint_ids":["constraint-definition-014"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-014"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-014-a"},{"ticker":"issuer-014-b"},{"ticker":"issuer-014-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-014"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-014"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-014-a","issuer-014-b","issuer-014-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-015","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (015).","requirements":[{"id":"req-screen-015","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-015","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-015","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-015","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-015-a","issuer-015-b","issuer-015-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-015","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-015","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-015","description":"Require complete map coverage","requirement_ids":["req-screen-015"],"constraint_ids":["constraint-exclusions-015"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-015","description":"Require citations for every issuer","requirement_ids":["req-report-015"],"constraint_ids":["constraint-definition-015"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-015"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-015-a"},{"ticker":"issuer-015-b"},{"ticker":"issuer-015-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-015"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-015"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-015-a","issuer-015-b","issuer-015-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-016","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (016).","requirements":[{"id":"req-screen-016","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-016","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-016","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-016","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-016-a","issuer-016-b","issuer-016-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-016","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-016","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-016","description":"Require complete map coverage","requirement_ids":["req-screen-016"],"constraint_ids":["constraint-exclusions-016"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-016","description":"Require citations for every issuer","requirement_ids":["req-report-016"],"constraint_ids":["constraint-definition-016"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-016"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-016-a"},{"ticker":"issuer-016-b"},{"ticker":"issuer-016-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-016"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-016"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-016-a","issuer-016-b","issuer-016-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-017","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (017).","requirements":[{"id":"req-screen-017","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-017","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-017","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-017","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-017-a","issuer-017-b","issuer-017-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-017","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-017","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-017","description":"Require complete map coverage","requirement_ids":["req-screen-017"],"constraint_ids":["constraint-exclusions-017"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-017","description":"Require citations for every issuer","requirement_ids":["req-report-017"],"constraint_ids":["constraint-definition-017"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-017"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-017-a"},{"ticker":"issuer-017-b"},{"ticker":"issuer-017-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-017"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-017"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-017-a","issuer-017-b","issuer-017-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-018","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (018).","requirements":[{"id":"req-screen-018","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-018","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-018","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-018","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-018-a","issuer-018-b","issuer-018-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-018","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-018","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-018","description":"Require complete map coverage","requirement_ids":["req-screen-018"],"constraint_ids":["constraint-exclusions-018"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-018","description":"Require citations for every issuer","requirement_ids":["req-report-018"],"constraint_ids":["constraint-definition-018"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-018"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-018-a"},{"ticker":"issuer-018-b"},{"ticker":"issuer-018-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-018"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-018"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-018-a","issuer-018-b","issuer-018-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-019","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (019).","requirements":[{"id":"req-screen-019","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-019","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-019","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-019","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-019-a","issuer-019-b","issuer-019-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-019","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-019","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-019","description":"Require complete map coverage","requirement_ids":["req-screen-019"],"constraint_ids":["constraint-exclusions-019"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-019","description":"Require citations for every issuer","requirement_ids":["req-report-019"],"constraint_ids":["constraint-definition-019"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-019"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-019-a"},{"ticker":"issuer-019-b"},{"ticker":"issuer-019-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-019"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-019"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-019-a","issuer-019-b","issuer-019-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-020","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (020).","requirements":[{"id":"req-screen-020","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-020","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-020","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-020","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-020-a","issuer-020-b","issuer-020-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-020","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-020","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-020","description":"Require complete map coverage","requirement_ids":["req-screen-020"],"constraint_ids":["constraint-exclusions-020"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-020","description":"Require citations for every issuer","requirement_ids":["req-report-020"],"constraint_ids":["constraint-definition-020"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-020"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-020-a"},{"ticker":"issuer-020-b"},{"ticker":"issuer-020-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-020"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-020"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-020-a","issuer-020-b","issuer-020-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-021","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (021).","requirements":[{"id":"req-screen-021","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-021","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-021","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-021","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-021-a","issuer-021-b","issuer-021-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-021","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-021","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-021","description":"Require complete map coverage","requirement_ids":["req-screen-021"],"constraint_ids":["constraint-exclusions-021"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-021","description":"Require citations for every issuer","requirement_ids":["req-report-021"],"constraint_ids":["constraint-definition-021"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-021"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-021-a"},{"ticker":"issuer-021-b"},{"ticker":"issuer-021-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-021"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-021"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-021-a","issuer-021-b","issuer-021-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-022","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (022).","requirements":[{"id":"req-screen-022","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-022","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-022","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-022","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-022-a","issuer-022-b","issuer-022-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-022","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-022","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-022","description":"Require complete map coverage","requirement_ids":["req-screen-022"],"constraint_ids":["constraint-exclusions-022"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-022","description":"Require citations for every issuer","requirement_ids":["req-report-022"],"constraint_ids":["constraint-definition-022"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-022"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-022-a"},{"ticker":"issuer-022-b"},{"ticker":"issuer-022-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-022"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-022"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-022-a","issuer-022-b","issuer-022-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-023","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (023).","requirements":[{"id":"req-screen-023","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-023","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-023","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-023","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-023-a","issuer-023-b","issuer-023-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-023","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-023","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-023","description":"Require complete map coverage","requirement_ids":["req-screen-023"],"constraint_ids":["constraint-exclusions-023"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-023","description":"Require citations for every issuer","requirement_ids":["req-report-023"],"constraint_ids":["constraint-definition-023"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-023"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-023-a"},{"ticker":"issuer-023-b"},{"ticker":"issuer-023-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-023"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-023"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-023-a","issuer-023-b","issuer-023-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-024","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (024).","requirements":[{"id":"req-screen-024","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-024","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-024","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-024","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-024-a","issuer-024-b","issuer-024-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-024","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-024","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-024","description":"Require complete map coverage","requirement_ids":["req-screen-024"],"constraint_ids":["constraint-exclusions-024"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-024","description":"Require citations for every issuer","requirement_ids":["req-report-024"],"constraint_ids":["constraint-definition-024"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-024"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-024-a"},{"ticker":"issuer-024-b"},{"ticker":"issuer-024-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-024"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-024"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-024-a","issuer-024-b","issuer-024-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-025","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (025).","requirements":[{"id":"req-screen-025","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-025","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-025","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-025","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-025-a","issuer-025-b","issuer-025-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-025","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-025","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-025","description":"Require complete map coverage","requirement_ids":["req-screen-025"],"constraint_ids":["constraint-exclusions-025"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-025","description":"Require citations for every issuer","requirement_ids":["req-report-025"],"constraint_ids":["constraint-definition-025"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-025"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-025-a"},{"ticker":"issuer-025-b"},{"ticker":"issuer-025-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-025"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-025"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-025-a","issuer-025-b","issuer-025-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-026","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (026).","requirements":[{"id":"req-screen-026","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-026","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-026","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-026","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-026-a","issuer-026-b","issuer-026-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-026","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-026","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-026","description":"Require complete map coverage","requirement_ids":["req-screen-026"],"constraint_ids":["constraint-exclusions-026"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-026","description":"Require citations for every issuer","requirement_ids":["req-report-026"],"constraint_ids":["constraint-definition-026"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-026"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-026-a"},{"ticker":"issuer-026-b"},{"ticker":"issuer-026-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-026"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-026"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-026-a","issuer-026-b","issuer-026-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-027","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (027).","requirements":[{"id":"req-screen-027","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-027","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-027","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-027","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-027-a","issuer-027-b","issuer-027-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-027","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-027","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-027","description":"Require complete map coverage","requirement_ids":["req-screen-027"],"constraint_ids":["constraint-exclusions-027"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-027","description":"Require citations for every issuer","requirement_ids":["req-report-027"],"constraint_ids":["constraint-definition-027"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-027"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-027-a"},{"ticker":"issuer-027-b"},{"ticker":"issuer-027-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-027"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-027"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-027-a","issuer-027-b","issuer-027-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-028","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (028).","requirements":[{"id":"req-screen-028","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-028","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-028","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-028","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-028-a","issuer-028-b","issuer-028-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-028","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-028","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-028","description":"Require complete map coverage","requirement_ids":["req-screen-028"],"constraint_ids":["constraint-exclusions-028"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-028","description":"Require citations for every issuer","requirement_ids":["req-report-028"],"constraint_ids":["constraint-definition-028"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-028"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-028-a"},{"ticker":"issuer-028-b"},{"ticker":"issuer-028-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-028"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-028"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-028-a","issuer-028-b","issuer-028-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-029","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (029).","requirements":[{"id":"req-screen-029","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-029","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-029","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-029","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-029-a","issuer-029-b","issuer-029-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-029","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-029","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-029","description":"Require complete map coverage","requirement_ids":["req-screen-029"],"constraint_ids":["constraint-exclusions-029"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-029","description":"Require citations for every issuer","requirement_ids":["req-report-029"],"constraint_ids":["constraint-definition-029"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-029"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-029-a"},{"ticker":"issuer-029-b"},{"ticker":"issuer-029-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-029"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-029"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-029-a","issuer-029-b","issuer-029-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-030","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (030).","requirements":[{"id":"req-screen-030","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-030","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-030","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-030","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-030-a","issuer-030-b","issuer-030-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-030","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-030","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-030","description":"Require complete map coverage","requirement_ids":["req-screen-030"],"constraint_ids":["constraint-exclusions-030"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-030","description":"Require citations for every issuer","requirement_ids":["req-report-030"],"constraint_ids":["constraint-definition-030"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-030"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-030-a"},{"ticker":"issuer-030-b"},{"ticker":"issuer-030-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-030"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-030"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-030-a","issuer-030-b","issuer-030-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-031","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (031).","requirements":[{"id":"req-screen-031","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-031","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-031","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-031","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-031-a","issuer-031-b","issuer-031-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-031","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-031","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-031","description":"Require complete map coverage","requirement_ids":["req-screen-031"],"constraint_ids":["constraint-exclusions-031"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-031","description":"Require citations for every issuer","requirement_ids":["req-report-031"],"constraint_ids":["constraint-definition-031"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-031"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-031-a"},{"ticker":"issuer-031-b"},{"ticker":"issuer-031-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-031"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-031"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-031-a","issuer-031-b","issuer-031-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-032","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (032).","requirements":[{"id":"req-screen-032","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-032","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-032","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-032","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-032-a","issuer-032-b","issuer-032-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-032","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-032","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-032","description":"Require complete map coverage","requirement_ids":["req-screen-032"],"constraint_ids":["constraint-exclusions-032"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-032","description":"Require citations for every issuer","requirement_ids":["req-report-032"],"constraint_ids":["constraint-definition-032"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-032"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-032-a"},{"ticker":"issuer-032-b"},{"ticker":"issuer-032-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-032"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-032"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-032-a","issuer-032-b","issuer-032-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-033","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (033).","requirements":[{"id":"req-screen-033","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-033","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-033","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-033","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-033-a","issuer-033-b","issuer-033-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-033","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-033","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-033","description":"Require complete map coverage","requirement_ids":["req-screen-033"],"constraint_ids":["constraint-exclusions-033"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-033","description":"Require citations for every issuer","requirement_ids":["req-report-033"],"constraint_ids":["constraint-definition-033"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-033"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-033-a"},{"ticker":"issuer-033-b"},{"ticker":"issuer-033-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-033"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-033"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-033-a","issuer-033-b","issuer-033-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-034","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (034).","requirements":[{"id":"req-screen-034","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-034","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-034","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-034","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-034-a","issuer-034-b","issuer-034-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-034","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-034","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-034","description":"Require complete map coverage","requirement_ids":["req-screen-034"],"constraint_ids":["constraint-exclusions-034"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-034","description":"Require citations for every issuer","requirement_ids":["req-report-034"],"constraint_ids":["constraint-definition-034"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-034"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-034-a"},{"ticker":"issuer-034-b"},{"ticker":"issuer-034-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-034"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-034"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-034-a","issuer-034-b","issuer-034-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-035","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (035).","requirements":[{"id":"req-screen-035","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-035","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-035","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-035","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-035-a","issuer-035-b","issuer-035-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-035","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-035","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-035","description":"Require complete map coverage","requirement_ids":["req-screen-035"],"constraint_ids":["constraint-exclusions-035"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-035","description":"Require citations for every issuer","requirement_ids":["req-report-035"],"constraint_ids":["constraint-definition-035"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-035"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-035-a"},{"ticker":"issuer-035-b"},{"ticker":"issuer-035-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-035"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-035"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-035-a","issuer-035-b","issuer-035-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-036","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (036).","requirements":[{"id":"req-screen-036","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-036","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-036","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-036","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-036-a","issuer-036-b","issuer-036-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-036","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-036","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-036","description":"Require complete map coverage","requirement_ids":["req-screen-036"],"constraint_ids":["constraint-exclusions-036"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-036","description":"Require citations for every issuer","requirement_ids":["req-report-036"],"constraint_ids":["constraint-definition-036"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-036"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-036-a"},{"ticker":"issuer-036-b"},{"ticker":"issuer-036-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-036"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-036"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-036-a","issuer-036-b","issuer-036-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-037","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (037).","requirements":[{"id":"req-screen-037","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-037","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-037","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-037","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-037-a","issuer-037-b","issuer-037-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-037","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-037","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-037","description":"Require complete map coverage","requirement_ids":["req-screen-037"],"constraint_ids":["constraint-exclusions-037"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-037","description":"Require citations for every issuer","requirement_ids":["req-report-037"],"constraint_ids":["constraint-definition-037"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-037"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-037-a"},{"ticker":"issuer-037-b"},{"ticker":"issuer-037-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-037"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-037"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-037-a","issuer-037-b","issuer-037-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-038","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (038).","requirements":[{"id":"req-screen-038","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-038","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-038","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-038","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-038-a","issuer-038-b","issuer-038-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-038","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-038","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-038","description":"Require complete map coverage","requirement_ids":["req-screen-038"],"constraint_ids":["constraint-exclusions-038"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-038","description":"Require citations for every issuer","requirement_ids":["req-report-038"],"constraint_ids":["constraint-definition-038"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-038"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-038-a"},{"ticker":"issuer-038-b"},{"ticker":"issuer-038-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-038"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-038"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-038-a","issuer-038-b","issuer-038-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-039","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (039).","requirements":[{"id":"req-screen-039","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-039","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-039","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-039","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-039-a","issuer-039-b","issuer-039-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-039","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-039","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-039","description":"Require complete map coverage","requirement_ids":["req-screen-039"],"constraint_ids":["constraint-exclusions-039"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-039","description":"Require citations for every issuer","requirement_ids":["req-report-039"],"constraint_ids":["constraint-definition-039"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-039"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-039-a"},{"ticker":"issuer-039-b"},{"ticker":"issuer-039-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-039"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-039"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-039-a","issuer-039-b","issuer-039-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-040","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (040).","requirements":[{"id":"req-screen-040","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-040","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-040","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-040","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-040-a","issuer-040-b","issuer-040-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-040","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-040","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-040","description":"Require complete map coverage","requirement_ids":["req-screen-040"],"constraint_ids":["constraint-exclusions-040"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-040","description":"Require citations for every issuer","requirement_ids":["req-report-040"],"constraint_ids":["constraint-definition-040"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-040"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-040-a"},{"ticker":"issuer-040-b"},{"ticker":"issuer-040-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-040"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-040"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-040-a","issuer-040-b","issuer-040-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-041","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (041).","requirements":[{"id":"req-screen-041","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-041","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-041","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-041","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-041-a","issuer-041-b","issuer-041-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-041","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-041","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-041","description":"Require complete map coverage","requirement_ids":["req-screen-041"],"constraint_ids":["constraint-exclusions-041"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-041","description":"Require citations for every issuer","requirement_ids":["req-report-041"],"constraint_ids":["constraint-definition-041"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-041"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-041-a"},{"ticker":"issuer-041-b"},{"ticker":"issuer-041-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-041"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-041"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-041-a","issuer-041-b","issuer-041-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-042","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (042).","requirements":[{"id":"req-screen-042","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-042","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-042","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-042","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-042-a","issuer-042-b","issuer-042-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-042","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-042","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-042","description":"Require complete map coverage","requirement_ids":["req-screen-042"],"constraint_ids":["constraint-exclusions-042"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-042","description":"Require citations for every issuer","requirement_ids":["req-report-042"],"constraint_ids":["constraint-definition-042"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-042"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-042-a"},{"ticker":"issuer-042-b"},{"ticker":"issuer-042-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-042"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-042"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-042-a","issuer-042-b","issuer-042-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-043","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (043).","requirements":[{"id":"req-screen-043","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-043","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-043","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-043","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-043-a","issuer-043-b","issuer-043-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-043","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-043","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-043","description":"Require complete map coverage","requirement_ids":["req-screen-043"],"constraint_ids":["constraint-exclusions-043"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-043","description":"Require citations for every issuer","requirement_ids":["req-report-043"],"constraint_ids":["constraint-definition-043"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-043"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-043-a"},{"ticker":"issuer-043-b"},{"ticker":"issuer-043-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-043"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-043"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-043-a","issuer-043-b","issuer-043-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-044","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (044).","requirements":[{"id":"req-screen-044","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-044","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-044","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-044","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-044-a","issuer-044-b","issuer-044-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-044","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-044","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-044","description":"Require complete map coverage","requirement_ids":["req-screen-044"],"constraint_ids":["constraint-exclusions-044"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-044","description":"Require citations for every issuer","requirement_ids":["req-report-044"],"constraint_ids":["constraint-definition-044"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-044"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-044-a"},{"ticker":"issuer-044-b"},{"ticker":"issuer-044-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-044"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-044"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-044-a","issuer-044-b","issuer-044-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-045","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (045).","requirements":[{"id":"req-screen-045","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-045","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-045","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-045","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-045-a","issuer-045-b","issuer-045-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-045","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-045","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-045","description":"Require complete map coverage","requirement_ids":["req-screen-045"],"constraint_ids":["constraint-exclusions-045"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-045","description":"Require citations for every issuer","requirement_ids":["req-report-045"],"constraint_ids":["constraint-definition-045"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-045"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-045-a"},{"ticker":"issuer-045-b"},{"ticker":"issuer-045-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-045"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-045"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-045-a","issuer-045-b","issuer-045-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-046","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (046).","requirements":[{"id":"req-screen-046","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-046","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-046","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-046","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-046-a","issuer-046-b","issuer-046-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-046","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-046","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-046","description":"Require complete map coverage","requirement_ids":["req-screen-046"],"constraint_ids":["constraint-exclusions-046"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-046","description":"Require citations for every issuer","requirement_ids":["req-report-046"],"constraint_ids":["constraint-definition-046"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-046"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-046-a"},{"ticker":"issuer-046-b"},{"ticker":"issuer-046-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-046"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-046"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-046-a","issuer-046-b","issuer-046-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-047","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (047).","requirements":[{"id":"req-screen-047","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-047","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-047","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-047","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-047-a","issuer-047-b","issuer-047-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-047","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-047","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-047","description":"Require complete map coverage","requirement_ids":["req-screen-047"],"constraint_ids":["constraint-exclusions-047"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-047","description":"Require citations for every issuer","requirement_ids":["req-report-047"],"constraint_ids":["constraint-definition-047"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-047"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-047-a"},{"ticker":"issuer-047-b"},{"ticker":"issuer-047-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-047"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-047"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-047-a","issuer-047-b","issuer-047-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-048","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (048).","requirements":[{"id":"req-screen-048","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-048","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-048","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-048","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-048-a","issuer-048-b","issuer-048-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-048","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-048","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-048","description":"Require complete map coverage","requirement_ids":["req-screen-048"],"constraint_ids":["constraint-exclusions-048"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-048","description":"Require citations for every issuer","requirement_ids":["req-report-048"],"constraint_ids":["constraint-definition-048"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-048"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-048-a"},{"ticker":"issuer-048-b"},{"ticker":"issuer-048-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-048"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-048"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-048-a","issuer-048-b","issuer-048-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-049","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (049).","requirements":[{"id":"req-screen-049","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-049","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-049","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-049","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-049-a","issuer-049-b","issuer-049-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-049","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-049","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-049","description":"Require complete map coverage","requirement_ids":["req-screen-049"],"constraint_ids":["constraint-exclusions-049"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-049","description":"Require citations for every issuer","requirement_ids":["req-report-049"],"constraint_ids":["constraint-definition-049"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-049"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-049-a"},{"ticker":"issuer-049-b"},{"ticker":"issuer-049-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-049"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-049"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-049-a","issuer-049-b","issuer-049-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-050","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (050).","requirements":[{"id":"req-screen-050","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-050","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-050","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-050","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-050-a","issuer-050-b","issuer-050-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-050","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-050","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-050","description":"Require complete map coverage","requirement_ids":["req-screen-050"],"constraint_ids":["constraint-exclusions-050"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-050","description":"Require citations for every issuer","requirement_ids":["req-report-050"],"constraint_ids":["constraint-definition-050"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-050"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-050-a"},{"ticker":"issuer-050-b"},{"ticker":"issuer-050-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-050"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-050"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-050-a","issuer-050-b","issuer-050-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-051","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (051).","requirements":[{"id":"req-screen-051","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-051","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-051","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-051","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-051-a","issuer-051-b","issuer-051-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-051","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-051","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-051","description":"Require complete map coverage","requirement_ids":["req-screen-051"],"constraint_ids":["constraint-exclusions-051"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-051","description":"Require citations for every issuer","requirement_ids":["req-report-051"],"constraint_ids":["constraint-definition-051"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-051"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-051-a"},{"ticker":"issuer-051-b"},{"ticker":"issuer-051-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-051"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-051"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-051-a","issuer-051-b","issuer-051-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-052","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (052).","requirements":[{"id":"req-screen-052","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-052","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-052","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-052","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-052-a","issuer-052-b","issuer-052-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-052","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-052","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-052","description":"Require complete map coverage","requirement_ids":["req-screen-052"],"constraint_ids":["constraint-exclusions-052"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-052","description":"Require citations for every issuer","requirement_ids":["req-report-052"],"constraint_ids":["constraint-definition-052"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-052"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-052-a"},{"ticker":"issuer-052-b"},{"ticker":"issuer-052-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-052"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-052"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-052-a","issuer-052-b","issuer-052-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-053","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (053).","requirements":[{"id":"req-screen-053","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-053","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-053","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-053","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-053-a","issuer-053-b","issuer-053-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-053","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-053","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-053","description":"Require complete map coverage","requirement_ids":["req-screen-053"],"constraint_ids":["constraint-exclusions-053"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-053","description":"Require citations for every issuer","requirement_ids":["req-report-053"],"constraint_ids":["constraint-definition-053"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-053"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-053-a"},{"ticker":"issuer-053-b"},{"ticker":"issuer-053-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-053"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-053"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-053-a","issuer-053-b","issuer-053-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-054","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (054).","requirements":[{"id":"req-screen-054","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-054","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-054","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-054","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-054-a","issuer-054-b","issuer-054-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-054","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-054","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-054","description":"Require complete map coverage","requirement_ids":["req-screen-054"],"constraint_ids":["constraint-exclusions-054"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-054","description":"Require citations for every issuer","requirement_ids":["req-report-054"],"constraint_ids":["constraint-definition-054"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-054"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-054-a"},{"ticker":"issuer-054-b"},{"ticker":"issuer-054-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-054"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-054"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-054-a","issuer-054-b","issuer-054-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-055","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (055).","requirements":[{"id":"req-screen-055","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-055","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-055","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-055","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-055-a","issuer-055-b","issuer-055-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-055","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-055","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-055","description":"Require complete map coverage","requirement_ids":["req-screen-055"],"constraint_ids":["constraint-exclusions-055"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-055","description":"Require citations for every issuer","requirement_ids":["req-report-055"],"constraint_ids":["constraint-definition-055"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-055"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-055-a"},{"ticker":"issuer-055-b"},{"ticker":"issuer-055-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-055"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-055"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-055-a","issuer-055-b","issuer-055-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-056","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (056).","requirements":[{"id":"req-screen-056","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-056","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-056","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-056","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-056-a","issuer-056-b","issuer-056-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-056","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-056","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-056","description":"Require complete map coverage","requirement_ids":["req-screen-056"],"constraint_ids":["constraint-exclusions-056"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-056","description":"Require citations for every issuer","requirement_ids":["req-report-056"],"constraint_ids":["constraint-definition-056"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-056"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-056-a"},{"ticker":"issuer-056-b"},{"ticker":"issuer-056-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-056"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-056"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-056-a","issuer-056-b","issuer-056-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-057","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (057).","requirements":[{"id":"req-screen-057","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-057","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-057","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-057","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-057-a","issuer-057-b","issuer-057-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-057","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-057","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-057","description":"Require complete map coverage","requirement_ids":["req-screen-057"],"constraint_ids":["constraint-exclusions-057"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-057","description":"Require citations for every issuer","requirement_ids":["req-report-057"],"constraint_ids":["constraint-definition-057"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-057"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-057-a"},{"ticker":"issuer-057-b"},{"ticker":"issuer-057-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-057"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-057"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-057-a","issuer-057-b","issuer-057-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-058","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (058).","requirements":[{"id":"req-screen-058","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-058","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-058","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-058","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-058-a","issuer-058-b","issuer-058-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-058","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-058","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-058","description":"Require complete map coverage","requirement_ids":["req-screen-058"],"constraint_ids":["constraint-exclusions-058"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-058","description":"Require citations for every issuer","requirement_ids":["req-report-058"],"constraint_ids":["constraint-definition-058"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-058"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-058-a"},{"ticker":"issuer-058-b"},{"ticker":"issuer-058-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-058"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-058"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-058-a","issuer-058-b","issuer-058-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-059","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (059).","requirements":[{"id":"req-screen-059","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-059","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-059","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-059","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-059-a","issuer-059-b","issuer-059-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-059","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-059","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-059","description":"Require complete map coverage","requirement_ids":["req-screen-059"],"constraint_ids":["constraint-exclusions-059"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-059","description":"Require citations for every issuer","requirement_ids":["req-report-059"],"constraint_ids":["constraint-definition-059"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-059"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-059-a"},{"ticker":"issuer-059-b"},{"ticker":"issuer-059-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-059"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-059"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-059-a","issuer-059-b","issuer-059-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-060","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (060).","requirements":[{"id":"req-screen-060","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-060","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-060","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-060","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-060-a","issuer-060-b","issuer-060-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-060","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-060","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-060","description":"Require complete map coverage","requirement_ids":["req-screen-060"],"constraint_ids":["constraint-exclusions-060"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-060","description":"Require citations for every issuer","requirement_ids":["req-report-060"],"constraint_ids":["constraint-definition-060"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-060"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-060-a"},{"ticker":"issuer-060-b"},{"ticker":"issuer-060-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-060"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-060"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-060-a","issuer-060-b","issuer-060-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-061","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (061).","requirements":[{"id":"req-screen-061","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-061","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-061","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-061","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-061-a","issuer-061-b","issuer-061-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-061","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-061","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-061","description":"Require complete map coverage","requirement_ids":["req-screen-061"],"constraint_ids":["constraint-exclusions-061"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-061","description":"Require citations for every issuer","requirement_ids":["req-report-061"],"constraint_ids":["constraint-definition-061"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-061"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-061-a"},{"ticker":"issuer-061-b"},{"ticker":"issuer-061-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-061"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-061"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-061-a","issuer-061-b","issuer-061-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-062","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (062).","requirements":[{"id":"req-screen-062","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-062","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-062","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-062","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-062-a","issuer-062-b","issuer-062-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-062","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-062","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-062","description":"Require complete map coverage","requirement_ids":["req-screen-062"],"constraint_ids":["constraint-exclusions-062"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-062","description":"Require citations for every issuer","requirement_ids":["req-report-062"],"constraint_ids":["constraint-definition-062"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-062"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-062-a"},{"ticker":"issuer-062-b"},{"ticker":"issuer-062-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-062"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-062"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-062-a","issuer-062-b","issuer-062-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-063","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (063).","requirements":[{"id":"req-screen-063","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-063","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-063","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-063","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-063-a","issuer-063-b","issuer-063-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-063","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-063","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-063","description":"Require complete map coverage","requirement_ids":["req-screen-063"],"constraint_ids":["constraint-exclusions-063"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-063","description":"Require citations for every issuer","requirement_ids":["req-report-063"],"constraint_ids":["constraint-definition-063"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-063"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-063-a"},{"ticker":"issuer-063-b"},{"ticker":"issuer-063-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-063"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-063"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-063-a","issuer-063-b","issuer-063-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-064","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (064).","requirements":[{"id":"req-screen-064","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-064","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-064","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-064","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-064-a","issuer-064-b","issuer-064-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-064","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-064","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-064","description":"Require complete map coverage","requirement_ids":["req-screen-064"],"constraint_ids":["constraint-exclusions-064"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-064","description":"Require citations for every issuer","requirement_ids":["req-report-064"],"constraint_ids":["constraint-definition-064"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-064"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-064-a"},{"ticker":"issuer-064-b"},{"ticker":"issuer-064-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-064"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-064"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-064-a","issuer-064-b","issuer-064-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-065","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (065).","requirements":[{"id":"req-screen-065","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-065","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-065","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-065","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-065-a","issuer-065-b","issuer-065-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-065","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-065","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-065","description":"Require complete map coverage","requirement_ids":["req-screen-065"],"constraint_ids":["constraint-exclusions-065"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-065","description":"Require citations for every issuer","requirement_ids":["req-report-065"],"constraint_ids":["constraint-definition-065"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-065"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-065-a"},{"ticker":"issuer-065-b"},{"ticker":"issuer-065-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-065"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-065"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-065-a","issuer-065-b","issuer-065-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-066","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (066).","requirements":[{"id":"req-screen-066","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-066","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-066","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-066","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-066-a","issuer-066-b","issuer-066-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-066","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-066","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-066","description":"Require complete map coverage","requirement_ids":["req-screen-066"],"constraint_ids":["constraint-exclusions-066"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-066","description":"Require citations for every issuer","requirement_ids":["req-report-066"],"constraint_ids":["constraint-definition-066"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-066"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-066-a"},{"ticker":"issuer-066-b"},{"ticker":"issuer-066-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-066"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-066"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-066-a","issuer-066-b","issuer-066-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-067","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (067).","requirements":[{"id":"req-screen-067","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-067","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-067","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-067","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-067-a","issuer-067-b","issuer-067-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-067","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-067","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-067","description":"Require complete map coverage","requirement_ids":["req-screen-067"],"constraint_ids":["constraint-exclusions-067"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-067","description":"Require citations for every issuer","requirement_ids":["req-report-067"],"constraint_ids":["constraint-definition-067"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-067"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-067-a"},{"ticker":"issuer-067-b"},{"ticker":"issuer-067-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-067"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-067"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-067-a","issuer-067-b","issuer-067-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-068","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (068).","requirements":[{"id":"req-screen-068","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-068","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-068","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-068","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-068-a","issuer-068-b","issuer-068-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-068","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-068","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-068","description":"Require complete map coverage","requirement_ids":["req-screen-068"],"constraint_ids":["constraint-exclusions-068"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-068","description":"Require citations for every issuer","requirement_ids":["req-report-068"],"constraint_ids":["constraint-definition-068"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-068"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-068-a"},{"ticker":"issuer-068-b"},{"ticker":"issuer-068-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-068"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-068"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-068-a","issuer-068-b","issuer-068-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-069","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (069).","requirements":[{"id":"req-screen-069","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-069","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-069","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-069","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-069-a","issuer-069-b","issuer-069-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-069","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-069","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-069","description":"Require complete map coverage","requirement_ids":["req-screen-069"],"constraint_ids":["constraint-exclusions-069"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-069","description":"Require citations for every issuer","requirement_ids":["req-report-069"],"constraint_ids":["constraint-definition-069"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-069"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-069-a"},{"ticker":"issuer-069-b"},{"ticker":"issuer-069-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-069"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-069"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-069-a","issuer-069-b","issuer-069-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-070","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (070).","requirements":[{"id":"req-screen-070","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-070","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-070","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-070","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-070-a","issuer-070-b","issuer-070-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-070","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-070","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-070","description":"Require complete map coverage","requirement_ids":["req-screen-070"],"constraint_ids":["constraint-exclusions-070"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-070","description":"Require citations for every issuer","requirement_ids":["req-report-070"],"constraint_ids":["constraint-definition-070"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-070"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-070-a"},{"ticker":"issuer-070-b"},{"ticker":"issuer-070-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-070"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-070"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-070-a","issuer-070-b","issuer-070-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-071","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (071).","requirements":[{"id":"req-screen-071","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-071","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-071","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-071","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-071-a","issuer-071-b","issuer-071-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-071","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-071","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-071","description":"Require complete map coverage","requirement_ids":["req-screen-071"],"constraint_ids":["constraint-exclusions-071"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-071","description":"Require citations for every issuer","requirement_ids":["req-report-071"],"constraint_ids":["constraint-definition-071"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-071"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-071-a"},{"ticker":"issuer-071-b"},{"ticker":"issuer-071-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-071"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-071"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-071-a","issuer-071-b","issuer-071-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-072","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (072).","requirements":[{"id":"req-screen-072","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-072","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-072","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-072","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-072-a","issuer-072-b","issuer-072-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-072","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-072","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-072","description":"Require complete map coverage","requirement_ids":["req-screen-072"],"constraint_ids":["constraint-exclusions-072"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-072","description":"Require citations for every issuer","requirement_ids":["req-report-072"],"constraint_ids":["constraint-definition-072"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-072"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-072-a"},{"ticker":"issuer-072-b"},{"ticker":"issuer-072-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-072"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-072"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-072-a","issuer-072-b","issuer-072-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-073","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (073).","requirements":[{"id":"req-screen-073","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-073","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-073","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-073","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-073-a","issuer-073-b","issuer-073-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-073","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-073","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-073","description":"Require complete map coverage","requirement_ids":["req-screen-073"],"constraint_ids":["constraint-exclusions-073"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-073","description":"Require citations for every issuer","requirement_ids":["req-report-073"],"constraint_ids":["constraint-definition-073"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-073"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-073-a"},{"ticker":"issuer-073-b"},{"ticker":"issuer-073-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-073"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-073"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-073-a","issuer-073-b","issuer-073-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-074","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (074).","requirements":[{"id":"req-screen-074","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-074","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-074","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-074","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-074-a","issuer-074-b","issuer-074-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-074","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-074","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-074","description":"Require complete map coverage","requirement_ids":["req-screen-074"],"constraint_ids":["constraint-exclusions-074"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-074","description":"Require citations for every issuer","requirement_ids":["req-report-074"],"constraint_ids":["constraint-definition-074"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-074"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-074-a"},{"ticker":"issuer-074-b"},{"ticker":"issuer-074-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-074"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-074"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-074-a","issuer-074-b","issuer-074-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-075","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (075).","requirements":[{"id":"req-screen-075","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-075","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-075","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-075","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-075-a","issuer-075-b","issuer-075-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-075","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-075","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-075","description":"Require complete map coverage","requirement_ids":["req-screen-075"],"constraint_ids":["constraint-exclusions-075"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-075","description":"Require citations for every issuer","requirement_ids":["req-report-075"],"constraint_ids":["constraint-definition-075"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-075"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-075-a"},{"ticker":"issuer-075-b"},{"ticker":"issuer-075-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-075"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-075"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-075-a","issuer-075-b","issuer-075-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-076","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (076).","requirements":[{"id":"req-screen-076","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-076","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-076","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-076","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-076-a","issuer-076-b","issuer-076-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-076","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-076","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-076","description":"Require complete map coverage","requirement_ids":["req-screen-076"],"constraint_ids":["constraint-exclusions-076"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-076","description":"Require citations for every issuer","requirement_ids":["req-report-076"],"constraint_ids":["constraint-definition-076"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-076"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-076-a"},{"ticker":"issuer-076-b"},{"ticker":"issuer-076-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-076"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-076"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-076-a","issuer-076-b","issuer-076-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-077","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (077).","requirements":[{"id":"req-screen-077","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-077","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-077","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-077","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-077-a","issuer-077-b","issuer-077-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-077","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-077","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-077","description":"Require complete map coverage","requirement_ids":["req-screen-077"],"constraint_ids":["constraint-exclusions-077"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-077","description":"Require citations for every issuer","requirement_ids":["req-report-077"],"constraint_ids":["constraint-definition-077"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-077"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-077-a"},{"ticker":"issuer-077-b"},{"ticker":"issuer-077-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-077"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-077"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-077-a","issuer-077-b","issuer-077-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-078","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (078).","requirements":[{"id":"req-screen-078","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-078","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-078","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-078","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-078-a","issuer-078-b","issuer-078-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-078","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-078","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-078","description":"Require complete map coverage","requirement_ids":["req-screen-078"],"constraint_ids":["constraint-exclusions-078"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-078","description":"Require citations for every issuer","requirement_ids":["req-report-078"],"constraint_ids":["constraint-definition-078"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-078"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-078-a"},{"ticker":"issuer-078-b"},{"ticker":"issuer-078-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-078"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-078"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-078-a","issuer-078-b","issuer-078-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} -{"schema_version":1,"case_id":"contract-079","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (079).","requirements":[{"id":"req-screen-079","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-079","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-079","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-079","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-079-a","issuer-079-b","issuer-079-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-079","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-079","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-079","description":"Require complete map coverage","requirement_ids":["req-screen-079"],"constraint_ids":["constraint-exclusions-079"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-079","description":"Require citations for every issuer","requirement_ids":["req-report-079"],"constraint_ids":["constraint-definition-079"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_wait_policy":{"expiry":{"kind":"after","delay_seconds":86400},"on_expiry":{"kind":"fail_task"}},"input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-079"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-079-a"},{"ticker":"issuer-079-b"},{"ticker":"issuer-079-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-079"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-079"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-079-a","issuer-079-b","issuer-079-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-000","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (000).","requirements":[{"id":"req-screen-000","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-000","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-000","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-000","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-000-a","issuer-000-b","issuer-000-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-000","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-000","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-000","description":"Require complete map coverage","requirement_ids":["req-screen-000"],"constraint_ids":["constraint-exclusions-000"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-000","description":"Require citations for every issuer","requirement_ids":["req-report-000"],"constraint_ids":["constraint-definition-000"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-000"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-000-a"},{"ticker":"issuer-000-b"},{"ticker":"issuer-000-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-000"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-000"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-000-a","issuer-000-b","issuer-000-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-001","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (001).","requirements":[{"id":"req-screen-001","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-001","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-001","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-001","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-001-a","issuer-001-b","issuer-001-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-001","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-001","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-001","description":"Require complete map coverage","requirement_ids":["req-screen-001"],"constraint_ids":["constraint-exclusions-001"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-001","description":"Require citations for every issuer","requirement_ids":["req-report-001"],"constraint_ids":["constraint-definition-001"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-001"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-001-a"},{"ticker":"issuer-001-b"},{"ticker":"issuer-001-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-001"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-001"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-001-a","issuer-001-b","issuer-001-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-002","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (002).","requirements":[{"id":"req-screen-002","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-002","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-002","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-002","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-002-a","issuer-002-b","issuer-002-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-002","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-002","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-002","description":"Require complete map coverage","requirement_ids":["req-screen-002"],"constraint_ids":["constraint-exclusions-002"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-002","description":"Require citations for every issuer","requirement_ids":["req-report-002"],"constraint_ids":["constraint-definition-002"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-002"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-002-a"},{"ticker":"issuer-002-b"},{"ticker":"issuer-002-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-002"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-002"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-002-a","issuer-002-b","issuer-002-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-003","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (003).","requirements":[{"id":"req-screen-003","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-003","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-003","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-003","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-003-a","issuer-003-b","issuer-003-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-003","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-003","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-003","description":"Require complete map coverage","requirement_ids":["req-screen-003"],"constraint_ids":["constraint-exclusions-003"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-003","description":"Require citations for every issuer","requirement_ids":["req-report-003"],"constraint_ids":["constraint-definition-003"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-003"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-003-a"},{"ticker":"issuer-003-b"},{"ticker":"issuer-003-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-003"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-003"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-003-a","issuer-003-b","issuer-003-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-004","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (004).","requirements":[{"id":"req-screen-004","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-004","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-004","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-004","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-004-a","issuer-004-b","issuer-004-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-004","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-004","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-004","description":"Require complete map coverage","requirement_ids":["req-screen-004"],"constraint_ids":["constraint-exclusions-004"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-004","description":"Require citations for every issuer","requirement_ids":["req-report-004"],"constraint_ids":["constraint-definition-004"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-004"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-004-a"},{"ticker":"issuer-004-b"},{"ticker":"issuer-004-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-004"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-004"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-004-a","issuer-004-b","issuer-004-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-005","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (005).","requirements":[{"id":"req-screen-005","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-005","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-005","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-005","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-005-a","issuer-005-b","issuer-005-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-005","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-005","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-005","description":"Require complete map coverage","requirement_ids":["req-screen-005"],"constraint_ids":["constraint-exclusions-005"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-005","description":"Require citations for every issuer","requirement_ids":["req-report-005"],"constraint_ids":["constraint-definition-005"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-005"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-005-a"},{"ticker":"issuer-005-b"},{"ticker":"issuer-005-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-005"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-005"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-005-a","issuer-005-b","issuer-005-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-006","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (006).","requirements":[{"id":"req-screen-006","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-006","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-006","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-006","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-006-a","issuer-006-b","issuer-006-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-006","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-006","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-006","description":"Require complete map coverage","requirement_ids":["req-screen-006"],"constraint_ids":["constraint-exclusions-006"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-006","description":"Require citations for every issuer","requirement_ids":["req-report-006"],"constraint_ids":["constraint-definition-006"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-006"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-006-a"},{"ticker":"issuer-006-b"},{"ticker":"issuer-006-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-006"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-006"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-006-a","issuer-006-b","issuer-006-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-007","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (007).","requirements":[{"id":"req-screen-007","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-007","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-007","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-007","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-007-a","issuer-007-b","issuer-007-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-007","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-007","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-007","description":"Require complete map coverage","requirement_ids":["req-screen-007"],"constraint_ids":["constraint-exclusions-007"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-007","description":"Require citations for every issuer","requirement_ids":["req-report-007"],"constraint_ids":["constraint-definition-007"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-007"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-007-a"},{"ticker":"issuer-007-b"},{"ticker":"issuer-007-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-007"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-007"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-007-a","issuer-007-b","issuer-007-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-008","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (008).","requirements":[{"id":"req-screen-008","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-008","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-008","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-008","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-008-a","issuer-008-b","issuer-008-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-008","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-008","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-008","description":"Require complete map coverage","requirement_ids":["req-screen-008"],"constraint_ids":["constraint-exclusions-008"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-008","description":"Require citations for every issuer","requirement_ids":["req-report-008"],"constraint_ids":["constraint-definition-008"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-008"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-008-a"},{"ticker":"issuer-008-b"},{"ticker":"issuer-008-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-008"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-008"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-008-a","issuer-008-b","issuer-008-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-009","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (009).","requirements":[{"id":"req-screen-009","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-009","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-009","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-009","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-009-a","issuer-009-b","issuer-009-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-009","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-009","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-009","description":"Require complete map coverage","requirement_ids":["req-screen-009"],"constraint_ids":["constraint-exclusions-009"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-009","description":"Require citations for every issuer","requirement_ids":["req-report-009"],"constraint_ids":["constraint-definition-009"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-009"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-009-a"},{"ticker":"issuer-009-b"},{"ticker":"issuer-009-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-009"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-009"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-009-a","issuer-009-b","issuer-009-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-010","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (010).","requirements":[{"id":"req-screen-010","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-010","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-010","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-010","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-010-a","issuer-010-b","issuer-010-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-010","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-010","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-010","description":"Require complete map coverage","requirement_ids":["req-screen-010"],"constraint_ids":["constraint-exclusions-010"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-010","description":"Require citations for every issuer","requirement_ids":["req-report-010"],"constraint_ids":["constraint-definition-010"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-010"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-010-a"},{"ticker":"issuer-010-b"},{"ticker":"issuer-010-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-010"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-010"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-010-a","issuer-010-b","issuer-010-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-011","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (011).","requirements":[{"id":"req-screen-011","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-011","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-011","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-011","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-011-a","issuer-011-b","issuer-011-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-011","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-011","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-011","description":"Require complete map coverage","requirement_ids":["req-screen-011"],"constraint_ids":["constraint-exclusions-011"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-011","description":"Require citations for every issuer","requirement_ids":["req-report-011"],"constraint_ids":["constraint-definition-011"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-011"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-011-a"},{"ticker":"issuer-011-b"},{"ticker":"issuer-011-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-011"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-011"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-011-a","issuer-011-b","issuer-011-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-012","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (012).","requirements":[{"id":"req-screen-012","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-012","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-012","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-012","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-012-a","issuer-012-b","issuer-012-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-012","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-012","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-012","description":"Require complete map coverage","requirement_ids":["req-screen-012"],"constraint_ids":["constraint-exclusions-012"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-012","description":"Require citations for every issuer","requirement_ids":["req-report-012"],"constraint_ids":["constraint-definition-012"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-012"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-012-a"},{"ticker":"issuer-012-b"},{"ticker":"issuer-012-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-012"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-012"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-012-a","issuer-012-b","issuer-012-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-013","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (013).","requirements":[{"id":"req-screen-013","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-013","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-013","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-013","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-013-a","issuer-013-b","issuer-013-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-013","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-013","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-013","description":"Require complete map coverage","requirement_ids":["req-screen-013"],"constraint_ids":["constraint-exclusions-013"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-013","description":"Require citations for every issuer","requirement_ids":["req-report-013"],"constraint_ids":["constraint-definition-013"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-013"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-013-a"},{"ticker":"issuer-013-b"},{"ticker":"issuer-013-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-013"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-013"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-013-a","issuer-013-b","issuer-013-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-014","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (014).","requirements":[{"id":"req-screen-014","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-014","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-014","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-014","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-014-a","issuer-014-b","issuer-014-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-014","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-014","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-014","description":"Require complete map coverage","requirement_ids":["req-screen-014"],"constraint_ids":["constraint-exclusions-014"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-014","description":"Require citations for every issuer","requirement_ids":["req-report-014"],"constraint_ids":["constraint-definition-014"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-014"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-014-a"},{"ticker":"issuer-014-b"},{"ticker":"issuer-014-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-014"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-014"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-014-a","issuer-014-b","issuer-014-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-015","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (015).","requirements":[{"id":"req-screen-015","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-015","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-015","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-015","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-015-a","issuer-015-b","issuer-015-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-015","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-015","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-015","description":"Require complete map coverage","requirement_ids":["req-screen-015"],"constraint_ids":["constraint-exclusions-015"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-015","description":"Require citations for every issuer","requirement_ids":["req-report-015"],"constraint_ids":["constraint-definition-015"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-015"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-015-a"},{"ticker":"issuer-015-b"},{"ticker":"issuer-015-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-015"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-015"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-015-a","issuer-015-b","issuer-015-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-016","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (016).","requirements":[{"id":"req-screen-016","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-016","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-016","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-016","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-016-a","issuer-016-b","issuer-016-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-016","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-016","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-016","description":"Require complete map coverage","requirement_ids":["req-screen-016"],"constraint_ids":["constraint-exclusions-016"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-016","description":"Require citations for every issuer","requirement_ids":["req-report-016"],"constraint_ids":["constraint-definition-016"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-016"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-016-a"},{"ticker":"issuer-016-b"},{"ticker":"issuer-016-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-016"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-016"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-016-a","issuer-016-b","issuer-016-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-017","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (017).","requirements":[{"id":"req-screen-017","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-017","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-017","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-017","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-017-a","issuer-017-b","issuer-017-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-017","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-017","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-017","description":"Require complete map coverage","requirement_ids":["req-screen-017"],"constraint_ids":["constraint-exclusions-017"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-017","description":"Require citations for every issuer","requirement_ids":["req-report-017"],"constraint_ids":["constraint-definition-017"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-017"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-017-a"},{"ticker":"issuer-017-b"},{"ticker":"issuer-017-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-017"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-017"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-017-a","issuer-017-b","issuer-017-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-018","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (018).","requirements":[{"id":"req-screen-018","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-018","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-018","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-018","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-018-a","issuer-018-b","issuer-018-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-018","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-018","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-018","description":"Require complete map coverage","requirement_ids":["req-screen-018"],"constraint_ids":["constraint-exclusions-018"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-018","description":"Require citations for every issuer","requirement_ids":["req-report-018"],"constraint_ids":["constraint-definition-018"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-018"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-018-a"},{"ticker":"issuer-018-b"},{"ticker":"issuer-018-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-018"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-018"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-018-a","issuer-018-b","issuer-018-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-019","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (019).","requirements":[{"id":"req-screen-019","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-019","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-019","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-019","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-019-a","issuer-019-b","issuer-019-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-019","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-019","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-019","description":"Require complete map coverage","requirement_ids":["req-screen-019"],"constraint_ids":["constraint-exclusions-019"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-019","description":"Require citations for every issuer","requirement_ids":["req-report-019"],"constraint_ids":["constraint-definition-019"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-019"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-019-a"},{"ticker":"issuer-019-b"},{"ticker":"issuer-019-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-019"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-019"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-019-a","issuer-019-b","issuer-019-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-020","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (020).","requirements":[{"id":"req-screen-020","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-020","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-020","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-020","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-020-a","issuer-020-b","issuer-020-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-020","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-020","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-020","description":"Require complete map coverage","requirement_ids":["req-screen-020"],"constraint_ids":["constraint-exclusions-020"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-020","description":"Require citations for every issuer","requirement_ids":["req-report-020"],"constraint_ids":["constraint-definition-020"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-020"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-020-a"},{"ticker":"issuer-020-b"},{"ticker":"issuer-020-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-020"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-020"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-020-a","issuer-020-b","issuer-020-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-021","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (021).","requirements":[{"id":"req-screen-021","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-021","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-021","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-021","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-021-a","issuer-021-b","issuer-021-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-021","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-021","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-021","description":"Require complete map coverage","requirement_ids":["req-screen-021"],"constraint_ids":["constraint-exclusions-021"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-021","description":"Require citations for every issuer","requirement_ids":["req-report-021"],"constraint_ids":["constraint-definition-021"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-021"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-021-a"},{"ticker":"issuer-021-b"},{"ticker":"issuer-021-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-021"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-021"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-021-a","issuer-021-b","issuer-021-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-022","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (022).","requirements":[{"id":"req-screen-022","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-022","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-022","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-022","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-022-a","issuer-022-b","issuer-022-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-022","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-022","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-022","description":"Require complete map coverage","requirement_ids":["req-screen-022"],"constraint_ids":["constraint-exclusions-022"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-022","description":"Require citations for every issuer","requirement_ids":["req-report-022"],"constraint_ids":["constraint-definition-022"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-022"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-022-a"},{"ticker":"issuer-022-b"},{"ticker":"issuer-022-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-022"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-022"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-022-a","issuer-022-b","issuer-022-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-023","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (023).","requirements":[{"id":"req-screen-023","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-023","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-023","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-023","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-023-a","issuer-023-b","issuer-023-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-023","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-023","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-023","description":"Require complete map coverage","requirement_ids":["req-screen-023"],"constraint_ids":["constraint-exclusions-023"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-023","description":"Require citations for every issuer","requirement_ids":["req-report-023"],"constraint_ids":["constraint-definition-023"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-023"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-023-a"},{"ticker":"issuer-023-b"},{"ticker":"issuer-023-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-023"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-023"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-023-a","issuer-023-b","issuer-023-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-024","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (024).","requirements":[{"id":"req-screen-024","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-024","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-024","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-024","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-024-a","issuer-024-b","issuer-024-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-024","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-024","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-024","description":"Require complete map coverage","requirement_ids":["req-screen-024"],"constraint_ids":["constraint-exclusions-024"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-024","description":"Require citations for every issuer","requirement_ids":["req-report-024"],"constraint_ids":["constraint-definition-024"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-024"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-024-a"},{"ticker":"issuer-024-b"},{"ticker":"issuer-024-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-024"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-024"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-024-a","issuer-024-b","issuer-024-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-025","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (025).","requirements":[{"id":"req-screen-025","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-025","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-025","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-025","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-025-a","issuer-025-b","issuer-025-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-025","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-025","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-025","description":"Require complete map coverage","requirement_ids":["req-screen-025"],"constraint_ids":["constraint-exclusions-025"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-025","description":"Require citations for every issuer","requirement_ids":["req-report-025"],"constraint_ids":["constraint-definition-025"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-025"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-025-a"},{"ticker":"issuer-025-b"},{"ticker":"issuer-025-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-025"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-025"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-025-a","issuer-025-b","issuer-025-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-026","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (026).","requirements":[{"id":"req-screen-026","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-026","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-026","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-026","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-026-a","issuer-026-b","issuer-026-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-026","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-026","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-026","description":"Require complete map coverage","requirement_ids":["req-screen-026"],"constraint_ids":["constraint-exclusions-026"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-026","description":"Require citations for every issuer","requirement_ids":["req-report-026"],"constraint_ids":["constraint-definition-026"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-026"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-026-a"},{"ticker":"issuer-026-b"},{"ticker":"issuer-026-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-026"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-026"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-026-a","issuer-026-b","issuer-026-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-027","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (027).","requirements":[{"id":"req-screen-027","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-027","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-027","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-027","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-027-a","issuer-027-b","issuer-027-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-027","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-027","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-027","description":"Require complete map coverage","requirement_ids":["req-screen-027"],"constraint_ids":["constraint-exclusions-027"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-027","description":"Require citations for every issuer","requirement_ids":["req-report-027"],"constraint_ids":["constraint-definition-027"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-027"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-027-a"},{"ticker":"issuer-027-b"},{"ticker":"issuer-027-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-027"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-027"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-027-a","issuer-027-b","issuer-027-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-028","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (028).","requirements":[{"id":"req-screen-028","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-028","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-028","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-028","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-028-a","issuer-028-b","issuer-028-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-028","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-028","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-028","description":"Require complete map coverage","requirement_ids":["req-screen-028"],"constraint_ids":["constraint-exclusions-028"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-028","description":"Require citations for every issuer","requirement_ids":["req-report-028"],"constraint_ids":["constraint-definition-028"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-028"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-028-a"},{"ticker":"issuer-028-b"},{"ticker":"issuer-028-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-028"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-028"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-028-a","issuer-028-b","issuer-028-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-029","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (029).","requirements":[{"id":"req-screen-029","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-029","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-029","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-029","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-029-a","issuer-029-b","issuer-029-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-029","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-029","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-029","description":"Require complete map coverage","requirement_ids":["req-screen-029"],"constraint_ids":["constraint-exclusions-029"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-029","description":"Require citations for every issuer","requirement_ids":["req-report-029"],"constraint_ids":["constraint-definition-029"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-029"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-029-a"},{"ticker":"issuer-029-b"},{"ticker":"issuer-029-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-029"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-029"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-029-a","issuer-029-b","issuer-029-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-030","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (030).","requirements":[{"id":"req-screen-030","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-030","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-030","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-030","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-030-a","issuer-030-b","issuer-030-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-030","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-030","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-030","description":"Require complete map coverage","requirement_ids":["req-screen-030"],"constraint_ids":["constraint-exclusions-030"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-030","description":"Require citations for every issuer","requirement_ids":["req-report-030"],"constraint_ids":["constraint-definition-030"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-030"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-030-a"},{"ticker":"issuer-030-b"},{"ticker":"issuer-030-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-030"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-030"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-030-a","issuer-030-b","issuer-030-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-031","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (031).","requirements":[{"id":"req-screen-031","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-031","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-031","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-031","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-031-a","issuer-031-b","issuer-031-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-031","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-031","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-031","description":"Require complete map coverage","requirement_ids":["req-screen-031"],"constraint_ids":["constraint-exclusions-031"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-031","description":"Require citations for every issuer","requirement_ids":["req-report-031"],"constraint_ids":["constraint-definition-031"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-031"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-031-a"},{"ticker":"issuer-031-b"},{"ticker":"issuer-031-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-031"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-031"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-031-a","issuer-031-b","issuer-031-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-032","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (032).","requirements":[{"id":"req-screen-032","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-032","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-032","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-032","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-032-a","issuer-032-b","issuer-032-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-032","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-032","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-032","description":"Require complete map coverage","requirement_ids":["req-screen-032"],"constraint_ids":["constraint-exclusions-032"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-032","description":"Require citations for every issuer","requirement_ids":["req-report-032"],"constraint_ids":["constraint-definition-032"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-032"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-032-a"},{"ticker":"issuer-032-b"},{"ticker":"issuer-032-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-032"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-032"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-032-a","issuer-032-b","issuer-032-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-033","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (033).","requirements":[{"id":"req-screen-033","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-033","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-033","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-033","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-033-a","issuer-033-b","issuer-033-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-033","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-033","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-033","description":"Require complete map coverage","requirement_ids":["req-screen-033"],"constraint_ids":["constraint-exclusions-033"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-033","description":"Require citations for every issuer","requirement_ids":["req-report-033"],"constraint_ids":["constraint-definition-033"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-033"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-033-a"},{"ticker":"issuer-033-b"},{"ticker":"issuer-033-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-033"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-033"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-033-a","issuer-033-b","issuer-033-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-034","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (034).","requirements":[{"id":"req-screen-034","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-034","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-034","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-034","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-034-a","issuer-034-b","issuer-034-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-034","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-034","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-034","description":"Require complete map coverage","requirement_ids":["req-screen-034"],"constraint_ids":["constraint-exclusions-034"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-034","description":"Require citations for every issuer","requirement_ids":["req-report-034"],"constraint_ids":["constraint-definition-034"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-034"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-034-a"},{"ticker":"issuer-034-b"},{"ticker":"issuer-034-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-034"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-034"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-034-a","issuer-034-b","issuer-034-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-035","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (035).","requirements":[{"id":"req-screen-035","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-035","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-035","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-035","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-035-a","issuer-035-b","issuer-035-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-035","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-035","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-035","description":"Require complete map coverage","requirement_ids":["req-screen-035"],"constraint_ids":["constraint-exclusions-035"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-035","description":"Require citations for every issuer","requirement_ids":["req-report-035"],"constraint_ids":["constraint-definition-035"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-035"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-035-a"},{"ticker":"issuer-035-b"},{"ticker":"issuer-035-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-035"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-035"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-035-a","issuer-035-b","issuer-035-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-036","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (036).","requirements":[{"id":"req-screen-036","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-036","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-036","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-036","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-036-a","issuer-036-b","issuer-036-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-036","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-036","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-036","description":"Require complete map coverage","requirement_ids":["req-screen-036"],"constraint_ids":["constraint-exclusions-036"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-036","description":"Require citations for every issuer","requirement_ids":["req-report-036"],"constraint_ids":["constraint-definition-036"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-036"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-036-a"},{"ticker":"issuer-036-b"},{"ticker":"issuer-036-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-036"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-036"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-036-a","issuer-036-b","issuer-036-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-037","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (037).","requirements":[{"id":"req-screen-037","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-037","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-037","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-037","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-037-a","issuer-037-b","issuer-037-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-037","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-037","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-037","description":"Require complete map coverage","requirement_ids":["req-screen-037"],"constraint_ids":["constraint-exclusions-037"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-037","description":"Require citations for every issuer","requirement_ids":["req-report-037"],"constraint_ids":["constraint-definition-037"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-037"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-037-a"},{"ticker":"issuer-037-b"},{"ticker":"issuer-037-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-037"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-037"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-037-a","issuer-037-b","issuer-037-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-038","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (038).","requirements":[{"id":"req-screen-038","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-038","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-038","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-038","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-038-a","issuer-038-b","issuer-038-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-038","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-038","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-038","description":"Require complete map coverage","requirement_ids":["req-screen-038"],"constraint_ids":["constraint-exclusions-038"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-038","description":"Require citations for every issuer","requirement_ids":["req-report-038"],"constraint_ids":["constraint-definition-038"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-038"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-038-a"},{"ticker":"issuer-038-b"},{"ticker":"issuer-038-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-038"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-038"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-038-a","issuer-038-b","issuer-038-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-039","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (039).","requirements":[{"id":"req-screen-039","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-039","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-039","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-039","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-039-a","issuer-039-b","issuer-039-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-039","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-039","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-039","description":"Require complete map coverage","requirement_ids":["req-screen-039"],"constraint_ids":["constraint-exclusions-039"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-039","description":"Require citations for every issuer","requirement_ids":["req-report-039"],"constraint_ids":["constraint-definition-039"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-039"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-039-a"},{"ticker":"issuer-039-b"},{"ticker":"issuer-039-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-039"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-039"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-039-a","issuer-039-b","issuer-039-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-040","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (040).","requirements":[{"id":"req-screen-040","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-040","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-040","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-040","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-040-a","issuer-040-b","issuer-040-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-040","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-040","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-040","description":"Require complete map coverage","requirement_ids":["req-screen-040"],"constraint_ids":["constraint-exclusions-040"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-040","description":"Require citations for every issuer","requirement_ids":["req-report-040"],"constraint_ids":["constraint-definition-040"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-040"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-040-a"},{"ticker":"issuer-040-b"},{"ticker":"issuer-040-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-040"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-040"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-040-a","issuer-040-b","issuer-040-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-041","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (041).","requirements":[{"id":"req-screen-041","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-041","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-041","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-041","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-041-a","issuer-041-b","issuer-041-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-041","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-041","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-041","description":"Require complete map coverage","requirement_ids":["req-screen-041"],"constraint_ids":["constraint-exclusions-041"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-041","description":"Require citations for every issuer","requirement_ids":["req-report-041"],"constraint_ids":["constraint-definition-041"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-041"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-041-a"},{"ticker":"issuer-041-b"},{"ticker":"issuer-041-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-041"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-041"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-041-a","issuer-041-b","issuer-041-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-042","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (042).","requirements":[{"id":"req-screen-042","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-042","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-042","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-042","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-042-a","issuer-042-b","issuer-042-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-042","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-042","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-042","description":"Require complete map coverage","requirement_ids":["req-screen-042"],"constraint_ids":["constraint-exclusions-042"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-042","description":"Require citations for every issuer","requirement_ids":["req-report-042"],"constraint_ids":["constraint-definition-042"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-042"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-042-a"},{"ticker":"issuer-042-b"},{"ticker":"issuer-042-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-042"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-042"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-042-a","issuer-042-b","issuer-042-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-043","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (043).","requirements":[{"id":"req-screen-043","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-043","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-043","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-043","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-043-a","issuer-043-b","issuer-043-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-043","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-043","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-043","description":"Require complete map coverage","requirement_ids":["req-screen-043"],"constraint_ids":["constraint-exclusions-043"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-043","description":"Require citations for every issuer","requirement_ids":["req-report-043"],"constraint_ids":["constraint-definition-043"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-043"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-043-a"},{"ticker":"issuer-043-b"},{"ticker":"issuer-043-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-043"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-043"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-043-a","issuer-043-b","issuer-043-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-044","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (044).","requirements":[{"id":"req-screen-044","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-044","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-044","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-044","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-044-a","issuer-044-b","issuer-044-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-044","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-044","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-044","description":"Require complete map coverage","requirement_ids":["req-screen-044"],"constraint_ids":["constraint-exclusions-044"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-044","description":"Require citations for every issuer","requirement_ids":["req-report-044"],"constraint_ids":["constraint-definition-044"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-044"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-044-a"},{"ticker":"issuer-044-b"},{"ticker":"issuer-044-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-044"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-044"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-044-a","issuer-044-b","issuer-044-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-045","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (045).","requirements":[{"id":"req-screen-045","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-045","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-045","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-045","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-045-a","issuer-045-b","issuer-045-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-045","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-045","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-045","description":"Require complete map coverage","requirement_ids":["req-screen-045"],"constraint_ids":["constraint-exclusions-045"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-045","description":"Require citations for every issuer","requirement_ids":["req-report-045"],"constraint_ids":["constraint-definition-045"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-045"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-045-a"},{"ticker":"issuer-045-b"},{"ticker":"issuer-045-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-045"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-045"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-045-a","issuer-045-b","issuer-045-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-046","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (046).","requirements":[{"id":"req-screen-046","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-046","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-046","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-046","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-046-a","issuer-046-b","issuer-046-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-046","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-046","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-046","description":"Require complete map coverage","requirement_ids":["req-screen-046"],"constraint_ids":["constraint-exclusions-046"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-046","description":"Require citations for every issuer","requirement_ids":["req-report-046"],"constraint_ids":["constraint-definition-046"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-046"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-046-a"},{"ticker":"issuer-046-b"},{"ticker":"issuer-046-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-046"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-046"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-046-a","issuer-046-b","issuer-046-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-047","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (047).","requirements":[{"id":"req-screen-047","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-047","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-047","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-047","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-047-a","issuer-047-b","issuer-047-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-047","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-047","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-047","description":"Require complete map coverage","requirement_ids":["req-screen-047"],"constraint_ids":["constraint-exclusions-047"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-047","description":"Require citations for every issuer","requirement_ids":["req-report-047"],"constraint_ids":["constraint-definition-047"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-047"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-047-a"},{"ticker":"issuer-047-b"},{"ticker":"issuer-047-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-047"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-047"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-047-a","issuer-047-b","issuer-047-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-048","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (048).","requirements":[{"id":"req-screen-048","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-048","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-048","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-048","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-048-a","issuer-048-b","issuer-048-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-048","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-048","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-048","description":"Require complete map coverage","requirement_ids":["req-screen-048"],"constraint_ids":["constraint-exclusions-048"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-048","description":"Require citations for every issuer","requirement_ids":["req-report-048"],"constraint_ids":["constraint-definition-048"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-048"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-048-a"},{"ticker":"issuer-048-b"},{"ticker":"issuer-048-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-048"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-048"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-048-a","issuer-048-b","issuer-048-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-049","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (049).","requirements":[{"id":"req-screen-049","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-049","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-049","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-049","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-049-a","issuer-049-b","issuer-049-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-049","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-049","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-049","description":"Require complete map coverage","requirement_ids":["req-screen-049"],"constraint_ids":["constraint-exclusions-049"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-049","description":"Require citations for every issuer","requirement_ids":["req-report-049"],"constraint_ids":["constraint-definition-049"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-049"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-049-a"},{"ticker":"issuer-049-b"},{"ticker":"issuer-049-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-049"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-049"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-049-a","issuer-049-b","issuer-049-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-050","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (050).","requirements":[{"id":"req-screen-050","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-050","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-050","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-050","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-050-a","issuer-050-b","issuer-050-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-050","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-050","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-050","description":"Require complete map coverage","requirement_ids":["req-screen-050"],"constraint_ids":["constraint-exclusions-050"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-050","description":"Require citations for every issuer","requirement_ids":["req-report-050"],"constraint_ids":["constraint-definition-050"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-050"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-050-a"},{"ticker":"issuer-050-b"},{"ticker":"issuer-050-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-050"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-050"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-050-a","issuer-050-b","issuer-050-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-051","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (051).","requirements":[{"id":"req-screen-051","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-051","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-051","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-051","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-051-a","issuer-051-b","issuer-051-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-051","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-051","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-051","description":"Require complete map coverage","requirement_ids":["req-screen-051"],"constraint_ids":["constraint-exclusions-051"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-051","description":"Require citations for every issuer","requirement_ids":["req-report-051"],"constraint_ids":["constraint-definition-051"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-051"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-051-a"},{"ticker":"issuer-051-b"},{"ticker":"issuer-051-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-051"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-051"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-051-a","issuer-051-b","issuer-051-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-052","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (052).","requirements":[{"id":"req-screen-052","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-052","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-052","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-052","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-052-a","issuer-052-b","issuer-052-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-052","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-052","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-052","description":"Require complete map coverage","requirement_ids":["req-screen-052"],"constraint_ids":["constraint-exclusions-052"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-052","description":"Require citations for every issuer","requirement_ids":["req-report-052"],"constraint_ids":["constraint-definition-052"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-052"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-052-a"},{"ticker":"issuer-052-b"},{"ticker":"issuer-052-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-052"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-052"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-052-a","issuer-052-b","issuer-052-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-053","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (053).","requirements":[{"id":"req-screen-053","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-053","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-053","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-053","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-053-a","issuer-053-b","issuer-053-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-053","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-053","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-053","description":"Require complete map coverage","requirement_ids":["req-screen-053"],"constraint_ids":["constraint-exclusions-053"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-053","description":"Require citations for every issuer","requirement_ids":["req-report-053"],"constraint_ids":["constraint-definition-053"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-053"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-053-a"},{"ticker":"issuer-053-b"},{"ticker":"issuer-053-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-053"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-053"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-053-a","issuer-053-b","issuer-053-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-054","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (054).","requirements":[{"id":"req-screen-054","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-054","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-054","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-054","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-054-a","issuer-054-b","issuer-054-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-054","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-054","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-054","description":"Require complete map coverage","requirement_ids":["req-screen-054"],"constraint_ids":["constraint-exclusions-054"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-054","description":"Require citations for every issuer","requirement_ids":["req-report-054"],"constraint_ids":["constraint-definition-054"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-054"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-054-a"},{"ticker":"issuer-054-b"},{"ticker":"issuer-054-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-054"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-054"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-054-a","issuer-054-b","issuer-054-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-055","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (055).","requirements":[{"id":"req-screen-055","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-055","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-055","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-055","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-055-a","issuer-055-b","issuer-055-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-055","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-055","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-055","description":"Require complete map coverage","requirement_ids":["req-screen-055"],"constraint_ids":["constraint-exclusions-055"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-055","description":"Require citations for every issuer","requirement_ids":["req-report-055"],"constraint_ids":["constraint-definition-055"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-055"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-055-a"},{"ticker":"issuer-055-b"},{"ticker":"issuer-055-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-055"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-055"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-055-a","issuer-055-b","issuer-055-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-056","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (056).","requirements":[{"id":"req-screen-056","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-056","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-056","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-056","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-056-a","issuer-056-b","issuer-056-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-056","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-056","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-056","description":"Require complete map coverage","requirement_ids":["req-screen-056"],"constraint_ids":["constraint-exclusions-056"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-056","description":"Require citations for every issuer","requirement_ids":["req-report-056"],"constraint_ids":["constraint-definition-056"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-056"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-056-a"},{"ticker":"issuer-056-b"},{"ticker":"issuer-056-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-056"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-056"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-056-a","issuer-056-b","issuer-056-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-057","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (057).","requirements":[{"id":"req-screen-057","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-057","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-057","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-057","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-057-a","issuer-057-b","issuer-057-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-057","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-057","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-057","description":"Require complete map coverage","requirement_ids":["req-screen-057"],"constraint_ids":["constraint-exclusions-057"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-057","description":"Require citations for every issuer","requirement_ids":["req-report-057"],"constraint_ids":["constraint-definition-057"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-057"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-057-a"},{"ticker":"issuer-057-b"},{"ticker":"issuer-057-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-057"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-057"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-057-a","issuer-057-b","issuer-057-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-058","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (058).","requirements":[{"id":"req-screen-058","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-058","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-058","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-058","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-058-a","issuer-058-b","issuer-058-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-058","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-058","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-058","description":"Require complete map coverage","requirement_ids":["req-screen-058"],"constraint_ids":["constraint-exclusions-058"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-058","description":"Require citations for every issuer","requirement_ids":["req-report-058"],"constraint_ids":["constraint-definition-058"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-058"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-058-a"},{"ticker":"issuer-058-b"},{"ticker":"issuer-058-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-058"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-058"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-058-a","issuer-058-b","issuer-058-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-059","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (059).","requirements":[{"id":"req-screen-059","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-059","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-059","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-059","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-059-a","issuer-059-b","issuer-059-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-059","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-059","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-059","description":"Require complete map coverage","requirement_ids":["req-screen-059"],"constraint_ids":["constraint-exclusions-059"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-059","description":"Require citations for every issuer","requirement_ids":["req-report-059"],"constraint_ids":["constraint-definition-059"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-059"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-059-a"},{"ticker":"issuer-059-b"},{"ticker":"issuer-059-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-059"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-059"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-059-a","issuer-059-b","issuer-059-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-060","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (060).","requirements":[{"id":"req-screen-060","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-060","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-060","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-060","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-060-a","issuer-060-b","issuer-060-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-060","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-060","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-060","description":"Require complete map coverage","requirement_ids":["req-screen-060"],"constraint_ids":["constraint-exclusions-060"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-060","description":"Require citations for every issuer","requirement_ids":["req-report-060"],"constraint_ids":["constraint-definition-060"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-060"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-060-a"},{"ticker":"issuer-060-b"},{"ticker":"issuer-060-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-060"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-060"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-060-a","issuer-060-b","issuer-060-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-061","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (061).","requirements":[{"id":"req-screen-061","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-061","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-061","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-061","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-061-a","issuer-061-b","issuer-061-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-061","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-061","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-061","description":"Require complete map coverage","requirement_ids":["req-screen-061"],"constraint_ids":["constraint-exclusions-061"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-061","description":"Require citations for every issuer","requirement_ids":["req-report-061"],"constraint_ids":["constraint-definition-061"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-061"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-061-a"},{"ticker":"issuer-061-b"},{"ticker":"issuer-061-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-061"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-061"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-061-a","issuer-061-b","issuer-061-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-062","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (062).","requirements":[{"id":"req-screen-062","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-062","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-062","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-062","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-062-a","issuer-062-b","issuer-062-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-062","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-062","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-062","description":"Require complete map coverage","requirement_ids":["req-screen-062"],"constraint_ids":["constraint-exclusions-062"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-062","description":"Require citations for every issuer","requirement_ids":["req-report-062"],"constraint_ids":["constraint-definition-062"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-062"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-062-a"},{"ticker":"issuer-062-b"},{"ticker":"issuer-062-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-062"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-062"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-062-a","issuer-062-b","issuer-062-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-063","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (063).","requirements":[{"id":"req-screen-063","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-063","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-063","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-063","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-063-a","issuer-063-b","issuer-063-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-063","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-063","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-063","description":"Require complete map coverage","requirement_ids":["req-screen-063"],"constraint_ids":["constraint-exclusions-063"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-063","description":"Require citations for every issuer","requirement_ids":["req-report-063"],"constraint_ids":["constraint-definition-063"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-063"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-063-a"},{"ticker":"issuer-063-b"},{"ticker":"issuer-063-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-063"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-063"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-063-a","issuer-063-b","issuer-063-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-064","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (064).","requirements":[{"id":"req-screen-064","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-064","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-064","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-064","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-064-a","issuer-064-b","issuer-064-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-064","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-064","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-064","description":"Require complete map coverage","requirement_ids":["req-screen-064"],"constraint_ids":["constraint-exclusions-064"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-064","description":"Require citations for every issuer","requirement_ids":["req-report-064"],"constraint_ids":["constraint-definition-064"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-064"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-064-a"},{"ticker":"issuer-064-b"},{"ticker":"issuer-064-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-064"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-064"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-064-a","issuer-064-b","issuer-064-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-065","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (065).","requirements":[{"id":"req-screen-065","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-065","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-065","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-065","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-065-a","issuer-065-b","issuer-065-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-065","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-065","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-065","description":"Require complete map coverage","requirement_ids":["req-screen-065"],"constraint_ids":["constraint-exclusions-065"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-065","description":"Require citations for every issuer","requirement_ids":["req-report-065"],"constraint_ids":["constraint-definition-065"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-065"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-065-a"},{"ticker":"issuer-065-b"},{"ticker":"issuer-065-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-065"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-065"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-065-a","issuer-065-b","issuer-065-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-066","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (066).","requirements":[{"id":"req-screen-066","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-066","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-066","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-066","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-066-a","issuer-066-b","issuer-066-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-066","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-066","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-066","description":"Require complete map coverage","requirement_ids":["req-screen-066"],"constraint_ids":["constraint-exclusions-066"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-066","description":"Require citations for every issuer","requirement_ids":["req-report-066"],"constraint_ids":["constraint-definition-066"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-066"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-066-a"},{"ticker":"issuer-066-b"},{"ticker":"issuer-066-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-066"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-066"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-066-a","issuer-066-b","issuer-066-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-067","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (067).","requirements":[{"id":"req-screen-067","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-067","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-067","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-067","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-067-a","issuer-067-b","issuer-067-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-067","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-067","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-067","description":"Require complete map coverage","requirement_ids":["req-screen-067"],"constraint_ids":["constraint-exclusions-067"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-067","description":"Require citations for every issuer","requirement_ids":["req-report-067"],"constraint_ids":["constraint-definition-067"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-067"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-067-a"},{"ticker":"issuer-067-b"},{"ticker":"issuer-067-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-067"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-067"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-067-a","issuer-067-b","issuer-067-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-068","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (068).","requirements":[{"id":"req-screen-068","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-068","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-068","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-068","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-068-a","issuer-068-b","issuer-068-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-068","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-068","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-068","description":"Require complete map coverage","requirement_ids":["req-screen-068"],"constraint_ids":["constraint-exclusions-068"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-068","description":"Require citations for every issuer","requirement_ids":["req-report-068"],"constraint_ids":["constraint-definition-068"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-068"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-068-a"},{"ticker":"issuer-068-b"},{"ticker":"issuer-068-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-068"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-068"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-068-a","issuer-068-b","issuer-068-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-069","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (069).","requirements":[{"id":"req-screen-069","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-069","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-069","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-069","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-069-a","issuer-069-b","issuer-069-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-069","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-069","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-069","description":"Require complete map coverage","requirement_ids":["req-screen-069"],"constraint_ids":["constraint-exclusions-069"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-069","description":"Require citations for every issuer","requirement_ids":["req-report-069"],"constraint_ids":["constraint-definition-069"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-069"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-069-a"},{"ticker":"issuer-069-b"},{"ticker":"issuer-069-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-069"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-069"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-069-a","issuer-069-b","issuer-069-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-070","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (070).","requirements":[{"id":"req-screen-070","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-070","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-070","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-070","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-070-a","issuer-070-b","issuer-070-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-070","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-070","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-070","description":"Require complete map coverage","requirement_ids":["req-screen-070"],"constraint_ids":["constraint-exclusions-070"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-070","description":"Require citations for every issuer","requirement_ids":["req-report-070"],"constraint_ids":["constraint-definition-070"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-070"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-070-a"},{"ticker":"issuer-070-b"},{"ticker":"issuer-070-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-070"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-070"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-070-a","issuer-070-b","issuer-070-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-071","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (071).","requirements":[{"id":"req-screen-071","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-071","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-071","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-071","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-071-a","issuer-071-b","issuer-071-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-071","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-071","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-071","description":"Require complete map coverage","requirement_ids":["req-screen-071"],"constraint_ids":["constraint-exclusions-071"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-071","description":"Require citations for every issuer","requirement_ids":["req-report-071"],"constraint_ids":["constraint-definition-071"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-071"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-071-a"},{"ticker":"issuer-071-b"},{"ticker":"issuer-071-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-071"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-071"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-071-a","issuer-071-b","issuer-071-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-072","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (072).","requirements":[{"id":"req-screen-072","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-072","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-072","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-072","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-072-a","issuer-072-b","issuer-072-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-072","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-072","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-072","description":"Require complete map coverage","requirement_ids":["req-screen-072"],"constraint_ids":["constraint-exclusions-072"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-072","description":"Require citations for every issuer","requirement_ids":["req-report-072"],"constraint_ids":["constraint-definition-072"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-072"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-072-a"},{"ticker":"issuer-072-b"},{"ticker":"issuer-072-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-072"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-072"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-072-a","issuer-072-b","issuer-072-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-073","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (073).","requirements":[{"id":"req-screen-073","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-073","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-073","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-073","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-073-a","issuer-073-b","issuer-073-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-073","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-073","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-073","description":"Require complete map coverage","requirement_ids":["req-screen-073"],"constraint_ids":["constraint-exclusions-073"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-073","description":"Require citations for every issuer","requirement_ids":["req-report-073"],"constraint_ids":["constraint-definition-073"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-073"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-073-a"},{"ticker":"issuer-073-b"},{"ticker":"issuer-073-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-073"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-073"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-073-a","issuer-073-b","issuer-073-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-074","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (074).","requirements":[{"id":"req-screen-074","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-074","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-074","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-074","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-074-a","issuer-074-b","issuer-074-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-074","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-074","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-074","description":"Require complete map coverage","requirement_ids":["req-screen-074"],"constraint_ids":["constraint-exclusions-074"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-074","description":"Require citations for every issuer","requirement_ids":["req-report-074"],"constraint_ids":["constraint-definition-074"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-074"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-074-a"},{"ticker":"issuer-074-b"},{"ticker":"issuer-074-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-074"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-074"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-074-a","issuer-074-b","issuer-074-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-075","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (075).","requirements":[{"id":"req-screen-075","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-075","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-075","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-075","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-075-a","issuer-075-b","issuer-075-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-075","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-075","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-075","description":"Require complete map coverage","requirement_ids":["req-screen-075"],"constraint_ids":["constraint-exclusions-075"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-075","description":"Require citations for every issuer","requirement_ids":["req-report-075"],"constraint_ids":["constraint-definition-075"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-075"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-075-a"},{"ticker":"issuer-075-b"},{"ticker":"issuer-075-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-075"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-075"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-075-a","issuer-075-b","issuer-075-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-076","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (076).","requirements":[{"id":"req-screen-076","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-076","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-076","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-076","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-076-a","issuer-076-b","issuer-076-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-076","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-076","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-076","description":"Require complete map coverage","requirement_ids":["req-screen-076"],"constraint_ids":["constraint-exclusions-076"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-076","description":"Require citations for every issuer","requirement_ids":["req-report-076"],"constraint_ids":["constraint-definition-076"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-076"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-076-a"},{"ticker":"issuer-076-b"},{"ticker":"issuer-076-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-076"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-076"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-076-a","issuer-076-b","issuer-076-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-077","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (077).","requirements":[{"id":"req-screen-077","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-077","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-077","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-077","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-077-a","issuer-077-b","issuer-077-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-077","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-077","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-077","description":"Require complete map coverage","requirement_ids":["req-screen-077"],"constraint_ids":["constraint-exclusions-077"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-077","description":"Require citations for every issuer","requirement_ids":["req-report-077"],"constraint_ids":["constraint-definition-077"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-077"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-077-a"},{"ticker":"issuer-077-b"},{"ticker":"issuer-077-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-077"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-077"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-077-a","issuer-077-b","issuer-077-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-078","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (078).","requirements":[{"id":"req-screen-078","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-078","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-078","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-078","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-078-a","issuer-078-b","issuer-078-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-078","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-078","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-078","description":"Require complete map coverage","requirement_ids":["req-screen-078"],"constraint_ids":["constraint-exclusions-078"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-078","description":"Require citations for every issuer","requirement_ids":["req-report-078"],"constraint_ids":["constraint-definition-078"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-078"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-078-a"},{"ticker":"issuer-078-b"},{"ticker":"issuer-078-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-078"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-078"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-078-a","issuer-078-b","issuer-078-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} +{"schema_version":1,"case_id":"contract-079","candidate":{"goal":{"objective":"Screen a complete issuer universe for five years and report defined AI mentions with citations (079).","requirements":[{"id":"req-screen-079","description":"Screen every issuer over the trailing five years for AI mentions with evidence citations."},{"id":"req-report-079","description":"Produce one structured report with issuer counts and source evidence."}],"deliverables":[{"id":"deliverable-report-079","description":"Final structured research report","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"id":"coverage-079","description":"Complete issuer universe","map_node_id":"screen","expected_items":["issuer-079-a","issuer-079-b","issuer-079-c"],"require_all":true}],"constraints":[{"id":"constraint-exclusions-079","description":"Exclude duplicate filings and preserve analyst-note provenance."},{"id":"constraint-definition-079","description":"Define AI mentions as case-insensitive artificial intelligence or AI references."}],"completion_checks":[{"id":"check-coverage-079","description":"Require complete map coverage","requirement_ids":["req-screen-079"],"constraint_ids":["constraint-exclusions-079"],"kind":{"kind":"map_coverage","map_node_id":"screen"}},{"id":"check-citations-079","description":"Require citations for every issuer","requirement_ids":["req-report-079"],"constraint_ids":["constraint-definition-079"],"kind":{"kind":"citations","node_ids":["screen"],"min_per_task":1}}]},"plan":{"cancel_policy":"retain_effects","input_schema":{"type":"object","properties":{"years":{"type":"integer"},"definition":{"type":"string"}},"required":["years","definition"],"additionalProperties":false},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"nodes":[{"id":"screen","requirement_ids":["req-screen-079"],"depends_on":[],"when":null,"input":{},"output_schema":{"type":"array","items":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false}},"operation":{"kind":"map","items":[{"ticker":"issuer-079-a"},{"ticker":"issuer-079-b"},{"ticker":"issuer-079-c"}],"item_key":"/ticker","max_items":3,"item_output_schema":{"type":"object","properties":{"count":{"type":"integer"}},"required":["count"],"additionalProperties":false},"task":{"kind":"capability","reference":{"name":"fixture.research","version":"1"}}},"compensation":null,"retry":{"max_attempts":2,"initial_backoff_ms":10,"max_backoff_ms":100},"budget":null},{"id":"report","requirement_ids":["req-report-079"],"depends_on":["screen"],"when":null,"input":{},"output_schema":{"type":"object","properties":{"report":{"type":"string","minLength":1}},"required":["report"],"additionalProperties":false},"operation":{"kind":"output","value":{"report":"report-079"}},"compensation":null,"retry":{"max_attempts":1,"initial_backoff_ms":0,"max_backoff_ms":0},"budget":null}]},"run_input":{"years":5,"definition":"artificial intelligence or AI"}},"expected":{"requirements":[{"expectation_id":"expected-screen","all_terms":["every issuer","five years","citations"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-report","all_terms":["structured report","source evidence"],"any_terms":[],"forbidden_terms":[]}],"constraints":[{"expectation_id":"expected-exclusions","all_terms":["exclude duplicate","provenance"],"any_terms":[],"forbidden_terms":[]},{"expectation_id":"expected-definition","all_terms":["define ai mentions","artificial intelligence"],"any_terms":[],"forbidden_terms":[]}],"deliverables":[{"expectation_id":"expected-deliverable","output_pointer":"/report","schema":{"type":"string","minLength":1}}],"coverage":[{"expectation_id":"expected-coverage","map_node_id":"screen","expected_keys":["issuer-079-a","issuer-079-b","issuer-079-c"],"require_all":true}],"completion_checks":[{"expectation_id":"expected-check-coverage","kind":"map_coverage","requirement_expectation_ids":["expected-screen"],"constraint_expectation_ids":["expected-exclusions"]},{"expectation_id":"expected-check-citations","kind":"citations","requirement_expectation_ids":["expected-report"],"constraint_expectation_ids":["expected-definition"]}],"run_input":[{"expectation_id":"expected-years","pointer":"/years","value":5},{"expectation_id":"expected-definition-input","pointer":"/definition","value":"artificial intelligence or AI"}]},"tags":["bulk-universe","time-range","evidence-citations","exclusions","deliverables","definitions","multi-constraint"]} diff --git a/crates/moa-eval/scenarios/execution/manifest.toml b/crates/moa-eval/scenarios/execution/manifest.toml index a4079d483..20cd75a48 100644 --- a/crates/moa-eval/scenarios/execution/manifest.toml +++ b/crates/moa-eval/scenarios/execution/manifest.toml @@ -7,7 +7,7 @@ count = 328 [contract] path = "contract-recorded.jsonl" -sha256 = "3099323222930134e78f6945fe288015c3704c3b09c2f6b7d18def0a2035de5a" +sha256 = "8703c2c3f99dfb583265a51967d1abd49c5de029d20254d90a5853285a1ad575" count = 80 [task_quality] diff --git a/crates/moa-eval/tests/eval_offline/execution_snapshot.rs b/crates/moa-eval/tests/eval_offline/execution_snapshot.rs index ee7ce2b9b..6fe171b11 100644 --- a/crates/moa-eval/tests/eval_offline/execution_snapshot.rs +++ b/crates/moa-eval/tests/eval_offline/execution_snapshot.rs @@ -264,6 +264,7 @@ fn runtime_parts( waiting_input_tenant_admin_task_count: 0, waiting_input_external_task_count: 0, approved_budget: approved_budget.clone(), + budget_deadline_suspended_at: None, reserved: ExecutionEstimate::default(), consumed: estimate, budget_overrun: false, @@ -481,12 +482,6 @@ fn canonical_plan(catalog_hash: ExecutionHash) -> CanonicalExecutionPlan { CanonicalExecutionPlan { definition: ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, - input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { - expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::At { - at: fixed_time() + chrono::TimeDelta::hours(1), - }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, - }, input_schema: json!({ "type": "object" }), output_schema: json!({ "type": "object" }), nodes: vec![moa_artifacts::execution_plan::ExecutionNode { diff --git a/crates/moa-execution/src/compiler/mod.rs b/crates/moa-execution/src/compiler/mod.rs index 58c0c7e80..4ec82aba4 100644 --- a/crates/moa-execution/src/compiler/mod.rs +++ b/crates/moa-execution/src/compiler/mod.rs @@ -746,13 +746,6 @@ fn validate_temporal_contract( ); } - validate_input_wait_policy( - &plan.input_wait_policy, - "plan.input_wait_policy", - now, - deadline_at, - report, - ); for (index, node) in plan.nodes.iter().enumerate() { let path = format!("plan.nodes[{index}].operation"); match &node.operation { @@ -819,34 +812,6 @@ fn validate_wait_policy( ); } -/// Validates the plan-level expiry policy for runtime `NeedsInput` outcomes. -/// -/// Unlike a `Review` or `WaitSignal` policy, this one policy settles whichever -/// logical task returned `NeedsInput`, so no single declared output can be checked -/// against a specific node's `output_schema` at compile time. `ContinueWith` is -/// therefore rejected outright rather than deferred to the materialization -/// transaction, where the schema check is a non-retryable failure. -fn validate_input_wait_policy( - policy: &moa_artifacts::execution_plan::ExecutionWaitPolicy, - path: &str, - now: DateTime, - deadline_at: DateTime, - report: &mut ExecutionValidationReport, -) { - validate_wait_policy(policy, path, now, deadline_at, report); - if matches!( - policy.on_expiry, - ExecutionWaitExpiryAction::ContinueWith { .. } - ) { - report.error( - "unsupported_input_wait_expiry", - format!("{path}.on_expiry"), - "input wait expiry must fail the waiting task; continue_with cannot be validated \ - against the output schema of the node that requested input", - ); - } -} - fn validate_temporal_target( target: &ExecutionTemporalTarget, path: &str, diff --git a/crates/moa-execution/src/compiler/tests.rs b/crates/moa-execution/src/compiler/tests.rs index d1cb7d38c..fd51eeb07 100644 --- a/crates/moa-execution/src/compiler/tests.rs +++ b/crates/moa-execution/src/compiler/tests.rs @@ -6,8 +6,8 @@ use chrono::Utc; use moa_artifacts::execution_plan::{ CompletionCheck, CompletionCheckKind, ExecutionBudgetLimit, ExecutionCancelPolicy, ExecutionGoalContract, ExecutionNode, ExecutionOperation, ExecutionPlanDefinition, - ExecutionRequirement, ExecutionTemporalTarget, ExecutionWaitExpiryAction, ExecutionWaitPolicy, - PlanAmendment, PlanAmendmentOperation, RetryPolicy, + ExecutionRequirement, ExecutionTemporalTarget, PlanAmendment, PlanAmendmentOperation, + RetryPolicy, }; use moa_config::ExecutionConfig; use serde_json::json; @@ -412,10 +412,6 @@ fn output_only_compile_request() -> CompileExecutionRequest { }, plan: ExecutionPlanDefinition { cancel_policy: ExecutionCancelPolicy::RetainEffects, - input_wait_policy: ExecutionWaitPolicy { - expiry: ExecutionTemporalTarget::After { delay_seconds: 60 }, - on_expiry: ExecutionWaitExpiryAction::FailTask, - }, input_schema: json!({ "type": "object" }), output_schema: json!({ "type": "object" }), nodes: vec![ExecutionNode { diff --git a/crates/moa-execution/src/compiler/validation/wait_feasibility.rs b/crates/moa-execution/src/compiler/validation/wait_feasibility.rs index 552138cf7..b116738c0 100644 --- a/crates/moa-execution/src/compiler/validation/wait_feasibility.rs +++ b/crates/moa-execution/src/compiler/validation/wait_feasibility.rs @@ -40,9 +40,8 @@ use crate::{ /// Two further contingent waits are deliberately excluded, because counting them would /// reject plans that are feasible on every execution that does not hit them: /// -/// - `plan.input_wait_policy.expiry`, which settles whichever task returned `NeedsInput`. -/// It applies to no node in particular and to every node in principle, so charging it -/// per node would inflate the path by the node count on plans that never ask for input. +/// - Runtime `NeedsInput`, which is an indefinite human wait and applies to no node in +/// particular. It pauses the active run deadline while the run is fully parked. /// - `RetryPolicy` backoff, which is millisecond-scale, contingent on failure, and /// already multiplied into the resource estimate rather than the schedule. /// diff --git a/crates/moa-execution/src/interpreter/tests.rs b/crates/moa-execution/src/interpreter/tests.rs index 6a0b38f9e..26e41b861 100644 --- a/crates/moa-execution/src/interpreter/tests.rs +++ b/crates/moa-execution/src/interpreter/tests.rs @@ -4,8 +4,7 @@ use chrono::Utc; use moa_artifacts::execution_plan::{ CapabilityReference, CompensationInputMapping, ExecutionBudgetLimit, ExecutionCancelPolicy, ExecutionCompensation, ExecutionGoalContract, ExecutionNode, ExecutionOperation, - ExecutionPlanDefinition, ExecutionTemporalTarget, ExecutionWaitExpiryAction, - ExecutionWaitPolicy, MapTask, RetryPolicy, + ExecutionPlanDefinition, MapTask, RetryPolicy, }; use super::materialize::logical_task; @@ -24,7 +23,6 @@ fn map_execution_task_validates_the_item_output_schema() { let plan = CanonicalExecutionPlan { definition: ExecutionPlanDefinition { cancel_policy: ExecutionCancelPolicy::RetainEffects, - input_wait_policy: test_input_wait_policy(), input_schema: serde_json::json!({}), output_schema: serde_json::json!({}), nodes: vec![ExecutionNode { @@ -142,7 +140,6 @@ fn only_direct_capability_task_materializes_compensation_contract() { plan: CanonicalExecutionPlan { definition: ExecutionPlanDefinition { cancel_policy: ExecutionCancelPolicy::CompensateCommitted, - input_wait_policy: test_input_wait_policy(), input_schema: serde_json::json!({}), output_schema: serde_json::json!({}), nodes: vec![direct_node.clone()], @@ -222,10 +219,3 @@ fn only_direct_capability_task_materializes_compensation_contract() { .expect("aggregate capability task should materialize"); assert_eq!(aggregate.compensation, None); } - -fn test_input_wait_policy() -> ExecutionWaitPolicy { - ExecutionWaitPolicy { - expiry: ExecutionTemporalTarget::After { delay_seconds: 60 }, - on_expiry: ExecutionWaitExpiryAction::FailTask, - } -} diff --git a/crates/moa-execution/src/repository/capacity.rs b/crates/moa-execution/src/repository/capacity.rs index bf8f54e8b..83569c10e 100644 --- a/crates/moa-execution/src/repository/capacity.rs +++ b/crates/moa-execution/src/repository/capacity.rs @@ -394,6 +394,20 @@ impl ExecutionRepository { i64::from(config.max_tenant_active_tasks), ) .await?; + // Admission already holds the fleet active-tasks bucket for the whole + // transaction. That fleet lock is the barrier that prevents another + // multi-resource task transaction from reaching a tenant active-tasks row + // while this batch is open. Lock the watchdog's scheduled-trigger buckets + // now, before `lock_oldest_ready_task` takes a run row: task settlement may + // have pre-released active-task capacity and therefore legitimately hold + // scheduled-trigger capacity before it asks for the same run row. + prelock_capacity_dimensions_in_tx( + conn.as_mut(), + config, + TenantId::from(tenant_id), + &[ExecutionCapacityDimension::ScheduledTriggers], + ) + .await?; tenant_available_by_id.insert(tenant_id, available); available } diff --git a/crates/moa-execution/src/repository/compensation.rs b/crates/moa-execution/src/repository/compensation.rs index dcd6dfa84..c07e54071 100644 --- a/crates/moa-execution/src/repository/compensation.rs +++ b/crates/moa-execution/src/repository/compensation.rs @@ -22,15 +22,16 @@ use super::{ }, outcome::record_task_outcome_in_conn, projection::budget_ledger, - ready::transition_node_counters_in_tx, + ready::{transition_node_counters_in_tx, transition_node_counters_with_input_audience_in_tx}, rows::*, - run::enqueue_run_activation_in_conn, + run::{enqueue_run_activation_in_conn, load_current_terminal_task_cancellation_dispatches}, sql::*, task::settle_external_job_terminal_in_conn as settle_task_external_job_terminal_in_conn, terminal::{ PendingTerminalAdvanceCommit, PendingTerminalAdvanceOutcome, PendingTerminalAdvanceStage, ReplanStopReceipt, drain_run_triggers_page_in_conn, }, + transition::refresh_run_after_wait_settlement_in_conn, trigger::{ ExecutionTriggerKind, ExecutionTriggerWrite, NewExecutionTrigger, create_trigger_with_dispatch_in_conn, release_trigger_capacity_in_conn, trigger_from_row, @@ -355,7 +356,10 @@ impl ExecutionRepository { conn.as_mut(), config, visible_run.tenant_id, - &[ExecutionCapacityDimension::ActiveTasks], + &[ + ExecutionCapacityDimension::ActiveTasks, + ExecutionCapacityDimension::ScheduledTriggers, + ], ) .await?; let Some(run_row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) diff --git a/crates/moa-execution/src/repository/compensation/pending_terminal.rs b/crates/moa-execution/src/repository/compensation/pending_terminal.rs index 3a76085c1..afa36e1e4 100644 --- a/crates/moa-execution/src/repository/compensation/pending_terminal.rs +++ b/crates/moa-execution/src/repository/compensation/pending_terminal.rs @@ -59,6 +59,10 @@ impl ExecutionRepository { conn.commit().await.map_err(storage_error)?; return Ok(PendingTerminalAdvanceOutcome::Conflict); } + if run.budget_deadline_suspended_at.is_some() { + conn.commit().await.map_err(storage_error)?; + return Ok(PendingTerminalAdvanceOutcome::Conflict); + } if expected_wake_epoch <= run.processed_wake_epoch { let commit = replayed_pending_terminal_commit(&mut conn, config, run).await?; conn.commit().await.map_err(storage_error)?; @@ -162,6 +166,88 @@ impl ExecutionRepository { .await } + /// Persists one externally requested cancellation and advances its first bounded drain page. + /// + /// Unlike completion-derived terminalization, a public cancellation may arrive after a + /// storage-only run has already acknowledged its current controller wake. In that case this + /// transition claims a fresh wake inside the same transaction without dispatching a controller + /// activation or moving the parked run into active capacity. + #[allow(clippy::too_many_arguments)] + pub async fn fence_cancellation_terminal_and_enqueue_settlement( + &self, + config: &ExecutionConfig, + scope: ExecutionScope, + run_uid: Uuid, + controller_generation: u64, + expected_wake_epoch: u64, + pending: PendingExecutionTerminal, + now: DateTime, + page_limit: u32, + ) -> Result { + validate_pending_terminal_page_limit(page_limit)?; + pending.validate()?; + if pending.status != ExecutionRunStatus::Cancelled + || pending.reason != ExecutionTerminalReason::Cancelled + || pending.terminal_evidence.cause != ExecutionTerminalCause::Cancellation + { + return Err(Error::InvalidRepositoryInput { + message: + "external cancellation fence requires an exact cancellation terminal intent" + .to_string(), + }); + } + let mut conn = scope.begin(&self.pool).await?; + let Some(mut run) = load_and_lock_pending_terminal_run(&mut conn, config, run_uid).await? + else { + conn.commit().await.map_err(storage_error)?; + return Ok(PendingTerminalAdvanceOutcome::NotFound); + }; + if run.status == ExecutionRunStatus::Cancelled { + let commit = replayed_pending_terminal_commit(&mut conn, config, run).await?; + conn.commit().await.map_err(storage_error)?; + return Ok(PendingTerminalAdvanceOutcome::Replayed(Box::new(commit))); + } + if run.status.is_terminal() || run.status == ExecutionRunStatus::Compensating { + conn.commit().await.map_err(storage_error)?; + return Ok(PendingTerminalAdvanceOutcome::Conflict); + } + if let Some(current) = &run.pending_terminal { + if current != &pending { + conn.commit().await.map_err(storage_error)?; + return Ok(PendingTerminalAdvanceOutcome::Conflict); + } + let commit = replayed_pending_terminal_commit(&mut conn, config, run).await?; + conn.commit().await.map_err(storage_error)?; + return Ok(PendingTerminalAdvanceOutcome::Replayed(Box::new(commit))); + } + if run.controller_generation != controller_generation + || run.wake_epoch != expected_wake_epoch + { + conn.commit().await.map_err(storage_error)?; + return Ok(PendingTerminalAdvanceOutcome::Conflict); + } + let terminal_wake_epoch = if run.processed_wake_epoch == expected_wake_epoch { + run = claim_fresh_terminal_mutation_wake_in_conn(&mut conn, &run, now).await?; + run.wake_epoch + } else if run.processed_wake_epoch < expected_wake_epoch { + expected_wake_epoch + } else { + conn.commit().await.map_err(storage_error)?; + return Ok(PendingTerminalAdvanceOutcome::Conflict); + }; + advance_pending_terminal_page_in_conn( + conn, + config, + run, + controller_generation, + terminal_wake_epoch, + Some(pending), + now, + page_limit, + ) + .await + } + /// Persists an exact replan-stop receipt and advances its first bounded terminal-drain page. #[allow(clippy::too_many_arguments)] pub async fn fence_replan_stop_and_enqueue_settlement( @@ -705,6 +791,31 @@ async fn load_and_lock_pending_terminal_run( Ok(Some(run)) } +async fn claim_fresh_terminal_mutation_wake_in_conn( + conn: &mut ScopedConn<'_>, + run: &ExecutionRunRecord, + now: DateTime, +) -> Result { + let row = sqlx::query( + "UPDATE moa.execution_run SET wake_epoch=wake_epoch+1, updated_at=$4 \ + WHERE run_uid=$1 AND controller_generation=$2 AND wake_epoch=$3 \ + AND processed_wake_epoch=wake_epoch AND pending_terminal_status IS NULL \ + AND status NOT IN ('completed','partial','blocked','unsupported','failed','cancelled', \ + 'compensating') RETURNING *", + ) + .bind(run.run_uid) + .bind(to_i64(run.controller_generation, "controller generation")?) + .bind(to_i64(run.wake_epoch, "wake epoch")?) + .bind(now) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)? + .ok_or_else(|| Error::Storage { + message: "external cancellation lost its fresh terminal-wake fence".to_string(), + })?; + run_from_row(&row) +} + async fn replayed_pending_terminal_commit( conn: &mut ScopedConn<'_>, config: &ExecutionConfig, @@ -748,6 +859,9 @@ async fn replayed_pending_terminal_commit( } else { None }; + let cancellation_dispatches = + load_current_terminal_task_cancellation_dispatches(conn, &run, config.max_in_flight_tasks) + .await?; let continuation = load_pending_terminal_continuation(conn, &run).await?; let stage = if run.status.is_terminal() { if run.manual_repair_required { @@ -767,7 +881,7 @@ async fn replayed_pending_terminal_commit( stage, settled_task_count: 0, drained_trigger_count: 0, - cancellation_dispatches: Vec::new(), + cancellation_dispatches, compensation_admission, continuation: continuation.map(Box::new), work_remaining, @@ -881,8 +995,14 @@ async fn advance_pending_terminal_page_in_conn( let task_rows = sqlx::query( "SELECT task.* FROM moa.execution_task AS task WHERE task.run_uid=$1 \ AND task.status NOT IN ('completed','skipped','failed','cancelled','unknown_outcome') \ - AND task.attempt_state <> 'cancelling' \ - ORDER BY CASE WHEN task.attempt_state IN ('dispatching','running') THEN 0 ELSE 1 END, \ + AND (task.attempt_state <> 'cancelling' OR (task.external_job_uid IS NULL AND EXISTS ( \ + SELECT 1 FROM moa.execution_capacity_reservation AS active \ + WHERE active.run_uid=task.run_uid AND active.task_id=task.task_id \ + AND active.attempt_generation=task.attempt_generation \ + AND active.resource_dimension='active_tasks' \ + AND active.state IN ('reserved','reconciling')))) \ + ORDER BY CASE WHEN task.attempt_state IN ('dispatching','running','cancelling') \ + THEN 0 ELSE 1 END, \ task.task_id LIMIT $2 FOR UPDATE", ) .bind(run.run_uid) @@ -959,7 +1079,9 @@ async fn advance_pending_terminal_page_in_conn( } if matches!( task.attempt_state, - ExecutionAttemptState::Dispatching | ExecutionAttemptState::Running + ExecutionAttemptState::Dispatching + | ExecutionAttemptState::Running + | ExecutionAttemptState::Cancelling ) { cancellation_dispatches.push( enqueue_pending_terminal_task_cancellation( @@ -975,6 +1097,21 @@ async fn advance_pending_terminal_page_in_conn( continue; } let original_status = task.status; + let input_audience = if original_status == ExecutionTaskStatus::WaitingInput { + Some( + task.current_outcome + .as_ref() + .and_then(|outcome| match &outcome.result { + ExecutionTaskResult::NeedsInput { audience, .. } => Some(audience.clone()), + _ => None, + }) + .ok_or_else(|| Error::InvalidRepositoryData { + message: "terminal drain lost the waiting-input audience".to_string(), + })?, + ) + } else { + None + }; match record_task_outcome_in_conn( &mut conn, run.run_uid, @@ -988,15 +1125,35 @@ async fn advance_pending_terminal_page_in_conn( .await? { TaskOutcomeWrite::Applied { task, .. } | TaskOutcomeWrite::Replayed { task, .. } => { - transition_node_counters_in_tx( - &mut conn, - run.run_uid, - &task.node_id, - &task.item_key, - original_status, - ExecutionTaskStatus::Cancelled, - ) - .await?; + if let Some(audience) = input_audience.as_ref() { + transition_node_counters_with_input_audience_in_tx( + &mut conn, + run.run_uid, + &task.node_id, + &task.item_key, + original_status, + ExecutionTaskStatus::Cancelled, + audience, + ) + .await?; + refresh_run_after_wait_settlement_in_conn( + &mut conn, + run.run_uid, + task.task_id, + now, + ) + .await?; + } else { + transition_node_counters_in_tx( + &mut conn, + run.run_uid, + &task.node_id, + &task.item_key, + original_status, + ExecutionTaskStatus::Cancelled, + ) + .await?; + } } TaskOutcomeWrite::Rejected { reason, .. } => { return Err(Error::InvalidRepositoryData { @@ -1393,7 +1550,8 @@ async fn enqueue_pending_terminal_task_cancellation( now: DateTime, ) -> Result { let row = sqlx::query( - "SELECT reservation.reservation_uid, trigger.trigger_uid \ + "SELECT reservation.reservation_uid, reservation.controller_generation, \ + trigger.trigger_uid \ FROM moa.execution_capacity_reservation AS reservation \ JOIN moa.execution_trigger AS trigger ON trigger.run_uid=reservation.run_uid \ AND trigger.task_id=reservation.task_id \ @@ -1402,13 +1560,12 @@ async fn enqueue_pending_terminal_task_cancellation( AND trigger.trigger_kind='task_watchdog' \ AND trigger.state = 'pending' \ WHERE reservation.run_uid=$1 AND reservation.task_id=$2 \ - AND reservation.controller_generation=$3 AND reservation.attempt_generation=$4 \ + AND reservation.attempt_generation=$3 \ AND reservation.resource_dimension='active_tasks' \ AND reservation.state IN ('reserved','reconciling') FOR UPDATE OF reservation, trigger", ) .bind(run.run_uid) .bind(task.task_id.as_uuid()) - .bind(to_i64(run.controller_generation, "controller generation")?) .bind(to_i64(task.attempt_generation, "task attempt generation")?) .fetch_optional(conn.as_mut()) .await @@ -1428,6 +1585,7 @@ async fn enqueue_pending_terminal_task_cancellation( ), })?; let capacity_reservation_uid: Uuid = row.try_get("reservation_uid").map_err(row_error)?; + let attempt_controller_generation = required_u64(&row, "controller_generation")?; let watchdog_trigger_uid: Uuid = row.try_get("trigger_uid").map_err(row_error)?; let cancellation_dispatch_uid = pending_terminal_cancel_dispatch_uid( active_dispatch_uid, @@ -1439,7 +1597,7 @@ async fn enqueue_pending_terminal_task_cancellation( last_progress_at=GREATEST(last_progress_at,$6), updated_at=NOW() \ WHERE run_uid=$1 AND task_id=$2 \ AND generation=$3 AND attempt_generation=$4 AND active_dispatch_uid=$5 \ - AND attempt_state IN ('dispatching','running')", + AND attempt_state IN ('dispatching','running','cancelling')", ) .bind(run.run_uid) .bind(task.task_id.as_uuid()) @@ -1475,7 +1633,7 @@ async fn enqueue_pending_terminal_task_cancellation( run_uid: run.run_uid, task_id: task.task_id, controller_generation: run.controller_generation, - attempt_controller_generation: run.controller_generation, + attempt_controller_generation, task_generation: task.generation, attempt_generation: task.attempt_generation, active_dispatch_uid, @@ -1791,7 +1949,7 @@ async fn checkpoint_pending_terminal_wake( last_progress_at=GREATEST(last_progress_at,$7), updated_at=NOW() \ WHERE run_uid=$1 AND controller_generation=$2 AND wake_epoch >= $3 \ AND processed_wake_epoch < $3 \ - AND activation_state IN ('queued','advancing','paused') RETURNING *", + AND activation_state IN ('idle','queued','advancing','paused') RETURNING *", ) .bind(run_uid) .bind(to_i64(controller_generation, "controller generation")?) @@ -1918,6 +2076,7 @@ async fn finalize_pending_terminal_exact( reserved_cost_microusd=0, reserved_tokens=0, reserved_tasks=0, \ reserved_tool_calls=0, reserved_retrieved_bytes=0, \ activation_state='terminal', waiting_reasons='[]'::JSONB, next_wake_at=NULL, \ + budget_deadline_suspended_at=NULL, \ waiting_task_count=0, waiting_input_task_count=0, waiting_review_task_count=0, \ waiting_signal_task_count=0, waiting_timer_task_count=0, \ waiting_external_task_count=0, waiting_replan_task_count=0, \ diff --git a/crates/moa-execution/src/repository/completion.rs b/crates/moa-execution/src/repository/completion.rs index 79811bc1a..f8a1627bf 100644 --- a/crates/moa-execution/src/repository/completion.rs +++ b/crates/moa-execution/src/repository/completion.rs @@ -1,11 +1,18 @@ //! Bounded persisted completion scanning and exact verifier materialization. +mod coverage; + use std::collections::{BTreeMap, BTreeSet}; use moa_artifacts::execution_plan::{CompletionCheckKind, ExecutionFailureClass, RetryPolicy}; use moa_config::ExecutionConfig; use serde::{Deserialize, Serialize}; +use self::coverage::{ + CoverageTaskEvidence, PersistedCoverageEvaluation, accumulate_task_coverage_evidence, + coverage_by_node, load_persisted_coverage_evaluations, prepare_coverage_evidence, + resolve_coverage_expectation, +}; use super::*; use super::{ materialize::prepare_task_materialization_batch, @@ -94,6 +101,7 @@ struct CompletionTaskEvidence { authorization_denied: bool, unsupported_by_requirement: BTreeMap, citation_failures: BTreeMap, + coverage: BTreeMap, } #[derive(Clone, Debug, Default, Deserialize, Serialize)] @@ -393,6 +401,14 @@ impl ExecutionRepository { scanned_tasks = u32::try_from(tasks.len()).map_err(|_| Error::ArithmeticOverflow { context: "completion page task count".to_string(), })?; + for coverage in &run.goal.coverage { + let expectation = + resolve_coverage_expectation(conn.as_mut(), &run, coverage).await?; + prepare_coverage_evidence(coverage, &expectation, &mut evidence)?; + for task in &tasks { + accumulate_task_coverage_evidence(coverage, task, &expectation, &mut evidence)?; + } + } for task in &tasks { accumulate_task_evidence(&run, task, &mut evidence)?; } @@ -552,10 +568,14 @@ impl ExecutionRepository { conn.commit().await.map_err(storage_error)?; return Ok(CompletionAdvanceOutcome::WaitingForVerifiers); } + let coverage_evaluations = + load_persisted_coverage_evaluations(conn.as_mut(), &run, &node_evidence, &evidence) + .await?; let (mut evaluation, terminal_output) = evaluate_persisted_completion( &run, &node_evidence, &evidence, + &coverage_evaluations, &verifier_tasks, request.now, )?; @@ -603,12 +623,16 @@ impl ExecutionRepository { }); } let typed_failure = load_earliest_typed_task_failure(conn.as_mut(), run.run_uid).await?; + let typed_failure_class = typed_failure.as_ref().map(|failure| failure.class.clone()); let terminal_projection = terminal_projection_for_evaluation(&evaluation, terminal_output, typed_failure)?; if evaluation.status != CompletionStatus::Completed { - let cause = ExecutionTerminalCause::Completion { - limit_stop: evaluation.limit_stop, - }; + let cause = typed_failure_class.map_or( + ExecutionTerminalCause::Completion { + limit_stop: evaluation.limit_stop, + }, + |class| ExecutionTerminalCause::TaskFailure { class }, + ); let terminal_evidence = terminal_evidence_from_evaluation(cause, &evaluation)?; let reason = execution_terminal_reason( &terminal_evidence.cause, @@ -836,7 +860,7 @@ async fn materialize_verifiers_in_tx( nodes: &CompletionNodeEvidence, limit: u32, ) -> Result<(Vec, bool)> { - let coverage_by_node = coverage_by_node(run, nodes); + let coverage_by_node = coverage_by_node(run, nodes, evidence)?; let unresolved = unsatisfied_requirements(run, nodes, &coverage_by_node); let terminal_output = nodes.terminal_output.clone(); let existing = sqlx::query_scalar::<_, String>( @@ -1207,6 +1231,7 @@ fn evaluate_persisted_completion( run: &ExecutionRunRecord, nodes: &CompletionNodeEvidence, evidence: &CompletionTaskEvidence, + coverage_evaluations: &[PersistedCoverageEvaluation], verifier_tasks: &[ExecutionTaskRecord], now: DateTime, ) -> Result<(CompletionEvaluation, Option)> { @@ -1215,11 +1240,10 @@ fn evaluate_persisted_completion( let mut coverage_by_node = BTreeMap::new(); let mut failed_coverage = Vec::new(); for coverage in &run.goal.coverage { - let passed = nodes - .coverage_passed - .get(&coverage.id) - .copied() - .unwrap_or(false); + let passed = coverage_evaluations + .iter() + .find(|evaluation| evaluation.coverage_id == coverage.id) + .is_some_and(|evaluation| evaluation.passed); coverage_by_node .entry(coverage.map_node_id.clone()) .and_modify(|node_passed| *node_passed &= passed) @@ -1274,18 +1298,13 @@ fn evaluate_persisted_completion( ) } CompletionCheckKind::MapCoverage { map_node_id } => { - let matching = run - .goal - .coverage + let matching = coverage_evaluations .iter() .filter(|coverage| coverage.map_node_id == *map_node_id) .collect::>(); let passed = !matching.is_empty() && coverage_by_node.get(map_node_id).copied().unwrap_or(false); - ( - passed, - json!({"map_node_id": map_node_id, "persisted_materialization_complete": passed}), - ) + (passed, serde_json::to_value(matching)?) } CompletionCheckKind::Citations { .. } => { let failed = evidence @@ -1472,25 +1491,6 @@ fn unsatisfied_requirements( partition_requirements(run, nodes, coverage).1 } -fn coverage_by_node( - run: &ExecutionRunRecord, - nodes: &CompletionNodeEvidence, -) -> BTreeMap { - let mut by_node = BTreeMap::new(); - for coverage in &run.goal.coverage { - let passed = nodes - .coverage_passed - .get(&coverage.id) - .copied() - .unwrap_or(false); - by_node - .entry(coverage.map_node_id.clone()) - .and_modify(|current| *current &= passed) - .or_insert(passed); - } - by_node -} - fn validate_completion_runtime_bounds( run: &ExecutionRunRecord, config: &ExecutionConfig, @@ -1544,7 +1544,7 @@ async fn load_earliest_typed_task_failure( run_uid: Uuid, ) -> Result> { let row = sqlx::query( - "SELECT current_outcome \ + "SELECT status,current_outcome \ FROM moa.execution_task \ WHERE run_uid = $1 \ AND status IN ('failed', 'unknown_outcome') \ @@ -1559,19 +1559,32 @@ async fn load_earliest_typed_task_failure( let Some(row) = row else { return Ok(None); }; + let status: String = row.try_get("status").map_err(row_error)?; let outcome: Value = row.try_get("current_outcome").map_err(row_error)?; let outcome: ExecutionTaskOutcome = serde_json::from_value(outcome).map_err(|error| Error::InvalidRepositoryData { message: format!("terminal failure outcome is undecodable: {error}"), })?; - Ok(match outcome.result { - ExecutionTaskResult::Failed { class, message } => Some(ExecutionTaskFailure { + let failure = match (status.as_str(), outcome.result) { + ("failed", ExecutionTaskResult::Failed { class, message }) => ExecutionTaskFailure { class, message, capability_ref: None, - }), - _ => None, - }) + }, + ("unknown_outcome", ExecutionTaskResult::UnknownOutcome { message }) => { + ExecutionTaskFailure { + class: ExecutionFailureClass::Terminal, + message, + capability_ref: None, + } + } + (status, _) => { + return Err(Error::InvalidRepositoryData { + message: format!("terminal task status `{status}` has an incompatible outcome"), + }); + } + }; + Ok(Some(failure)) } fn terminal_projection_for_evaluation( diff --git a/crates/moa-execution/src/repository/completion/coverage.rs b/crates/moa-execution/src/repository/completion/coverage.rs new file mode 100644 index 000000000..341ee81eb --- /dev/null +++ b/crates/moa-execution/src/repository/completion/coverage.rs @@ -0,0 +1,480 @@ +//! Sequential persisted map-coverage evidence resolution and evaluation. + +use std::collections::{BTreeMap, BTreeSet}; + +use moa_artifacts::execution_plan::{CoverageRequirement, ExecutionOperation}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use sqlx::{PgConnection, Row}; +use uuid::Uuid; + +use super::{CompletionNodeEvidence, CompletionTaskEvidence}; +use crate::{ + Error, Result, + bindings::{BindingContext, extract_map_key, resolve_bindings}, + capability::hash_serializable, + repository::{ExecutionRunRecord, ExecutionTaskRecord, row_error, sqlx_error}, + state::ExecutionTaskStatus, +}; + +const MAX_COVERAGE_EVIDENCE_SAMPLES: usize = 128; +const COVERAGE_EXPECTED_HASH_DOMAIN: &str = "moa.execution.coverage-expected"; +const COVERAGE_OBSERVED_HASH_DOMAIN: &str = "moa.execution.coverage-observed"; + +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +#[serde(default, deny_unknown_fields)] +pub(super) struct CoverageTaskEvidence { + expected_count: u64, + observed_count: u64, + matched_count: u64, + unexpected_count: u64, + failed_count: u64, + completed_count: u64, + completed_matched_count: u64, + expected_keys_hash: String, + observed_terminal_hash: String, +} + +#[derive(Clone, Debug)] +pub(super) struct ResolvedCoverageExpectation { + map_node_id: String, + expected_keys: BTreeSet, + expected_keys_hash: String, +} + +#[derive(Clone, Debug, Serialize)] +pub(super) struct PersistedCoverageEvaluation { + pub(super) coverage_id: String, + pub(super) map_node_id: String, + pub(super) passed: bool, + expected_count: u64, + observed_count: u64, + matched_count: u64, + missing_count: u64, + extra_count: u64, + failed_count: u64, + completed_count: u64, + expected_keys_hash: String, + observed_terminal_hash: String, + missing_keys: Vec, + extra_keys: Vec, + failed_keys: Vec, + completed_keys: Vec, + samples_truncated: bool, +} + +pub(super) async fn resolve_coverage_expectation( + conn: &mut PgConnection, + run: &ExecutionRunRecord, + coverage: &CoverageRequirement, +) -> Result { + let node = run + .active_plan + .definition + .nodes + .iter() + .find(|node| node.id == coverage.map_node_id) + .ok_or_else(|| Error::InvalidRepositoryData { + message: format!( + "coverage {} references missing map node {}", + coverage.id, coverage.map_node_id + ), + })?; + let ExecutionOperation::Map { + item_key, + max_items, + .. + } = &node.operation + else { + return Err(Error::InvalidRepositoryData { + message: format!("coverage {} does not reference a map node", coverage.id), + }); + }; + let dependencies = node.depends_on.iter().cloned().collect::>(); + let dependency_ids = dependencies.iter().cloned().collect::>(); + let mut node_outputs = BTreeMap::new(); + if !dependency_ids.is_empty() { + let rows = sqlx::query( + "SELECT node_id,aggregate_output FROM moa.execution_node_state \ + WHERE run_uid=$1 AND node_id = ANY($2::TEXT[]) ORDER BY node_id", + ) + .bind(run.run_uid) + .bind(&dependency_ids) + .fetch_all(&mut *conn) + .await + .map_err(sqlx_error)?; + for row in rows { + let node_id: String = row.try_get("node_id").map_err(row_error)?; + let aggregate_output: Option = + row.try_get("aggregate_output").map_err(row_error)?; + if let Some(output) = aggregate_output { + node_outputs.insert(node_id, output); + } + } + } + let expected = resolve_bindings( + &coverage.expected_items, + &BindingContext { + run_input: &run.input, + node_outputs: &node_outputs, + dependencies: &dependencies, + item: None, + item_key: None, + }, + )?; + let expected = expected + .as_array() + .ok_or_else(|| Error::InvalidRepositoryData { + message: format!( + "coverage {} expected_items did not resolve to an array", + coverage.id + ), + })?; + let expected_count = u64::try_from(expected.len()).map_err(|_| Error::ArithmeticOverflow { + context: format!("coverage {} expected item count", coverage.id), + })?; + if expected_count > *max_items { + return Err(Error::InvalidRepositoryData { + message: format!( + "coverage {} expected item count {expected_count} exceeds map {} max_items {max_items}", + coverage.id, coverage.map_node_id + ), + }); + } + let mut expected_keys = BTreeSet::new(); + for item in expected { + let key = extract_map_key(item, item_key)?; + if !expected_keys.insert(key) { + return Err(Error::InvalidRepositoryData { + message: format!("coverage {} contains duplicate expected keys", coverage.id), + }); + } + } + let expected_keys_hash = + hash_serializable(COVERAGE_EXPECTED_HASH_DOMAIN, &expected_keys)?.to_string(); + Ok(ResolvedCoverageExpectation { + map_node_id: coverage.map_node_id.clone(), + expected_keys, + expected_keys_hash, + }) +} + +pub(super) fn prepare_coverage_evidence( + coverage: &CoverageRequirement, + expectation: &ResolvedCoverageExpectation, + evidence: &mut CompletionTaskEvidence, +) -> Result<()> { + let expected_count = + u64::try_from(expectation.expected_keys.len()).map_err(|_| Error::ArithmeticOverflow { + context: "coverage expected item count".to_string(), + })?; + let persisted = evidence.coverage.entry(coverage.id.clone()).or_default(); + if persisted.expected_keys_hash.is_empty() { + persisted.expected_count = expected_count; + persisted.expected_keys_hash = expectation.expected_keys_hash.clone(); + } else if persisted.expected_count != expected_count + || persisted.expected_keys_hash != expectation.expected_keys_hash + { + return Err(Error::InvalidRepositoryData { + message: format!( + "coverage {} expected item universe changed during completion scan", + coverage.id + ), + }); + } + Ok(()) +} + +pub(super) fn accumulate_task_coverage_evidence( + coverage: &CoverageRequirement, + task: &ExecutionTaskRecord, + expectation: &ResolvedCoverageExpectation, + evidence: &mut CompletionTaskEvidence, +) -> Result<()> { + if expectation.map_node_id != task.node_id { + return Ok(()); + } + let is_completed = task.status == ExecutionTaskStatus::Completed; + let is_failed = matches!( + task.status, + ExecutionTaskStatus::Failed + | ExecutionTaskStatus::UnknownOutcome + | ExecutionTaskStatus::Cancelled + ); + if !is_completed && !is_failed { + return Ok(()); + } + let persisted = + evidence + .coverage + .get_mut(&coverage.id) + .ok_or_else(|| Error::InvalidRepositoryData { + message: format!("coverage {} has no initialized task evidence", coverage.id), + })?; + persisted.observed_count = + checked_evidence_increment(persisted.observed_count, "coverage observed item count")?; + let matched = expectation.expected_keys.contains(&task.item_key); + if matched { + persisted.matched_count = + checked_evidence_increment(persisted.matched_count, "coverage matched item count")?; + } else { + persisted.unexpected_count = checked_evidence_increment( + persisted.unexpected_count, + "coverage unexpected item count", + )?; + } + if is_failed { + persisted.failed_count = + checked_evidence_increment(persisted.failed_count, "coverage failed item count")?; + } else { + persisted.completed_count = + checked_evidence_increment(persisted.completed_count, "coverage completed item count")?; + if matched { + persisted.completed_matched_count = checked_evidence_increment( + persisted.completed_matched_count, + "coverage completed matched item count", + )?; + } + } + persisted.observed_terminal_hash = hash_serializable( + COVERAGE_OBSERVED_HASH_DOMAIN, + &( + &persisted.observed_terminal_hash, + &task.item_key, + task.status, + ), + )? + .to_string(); + Ok(()) +} + +fn checked_evidence_increment(value: u64, context: &str) -> Result { + value + .checked_add(1) + .ok_or_else(|| Error::ArithmeticOverflow { + context: context.to_string(), + }) +} + +fn persisted_coverage_passed( + coverage: &CoverageRequirement, + nodes: &CompletionNodeEvidence, + evidence: &CompletionTaskEvidence, +) -> Result { + let Some(task_evidence) = evidence.coverage.get(&coverage.id) else { + return Ok(false); + }; + if task_evidence.matched_count > task_evidence.expected_count + || task_evidence + .matched_count + .checked_add(task_evidence.unexpected_count) + != Some(task_evidence.observed_count) + || task_evidence + .failed_count + .checked_add(task_evidence.completed_count) + != Some(task_evidence.observed_count) + || task_evidence.completed_matched_count > task_evidence.matched_count + { + return Err(Error::InvalidRepositoryData { + message: format!("coverage {} has inconsistent persisted counts", coverage.id), + }); + } + let node_passed = nodes + .coverage_passed + .get(&coverage.id) + .copied() + .unwrap_or(false); + let expected_passed = if coverage.require_all { + task_evidence.matched_count == task_evidence.expected_count + } else { + task_evidence.expected_count == 0 || task_evidence.completed_matched_count > 0 + }; + Ok(node_passed + && task_evidence.unexpected_count == 0 + && task_evidence.failed_count == 0 + && expected_passed) +} + +pub(super) fn coverage_by_node( + run: &ExecutionRunRecord, + nodes: &CompletionNodeEvidence, + evidence: &CompletionTaskEvidence, +) -> Result> { + let mut by_node = BTreeMap::new(); + for coverage in &run.goal.coverage { + let passed = persisted_coverage_passed(coverage, nodes, evidence)?; + by_node + .entry(coverage.map_node_id.clone()) + .and_modify(|current| *current &= passed) + .or_insert(passed); + } + Ok(by_node) +} + +pub(super) async fn load_persisted_coverage_evaluations( + conn: &mut PgConnection, + run: &ExecutionRunRecord, + nodes: &CompletionNodeEvidence, + evidence: &CompletionTaskEvidence, +) -> Result> { + let mut sample_budget = MAX_COVERAGE_EVIDENCE_SAMPLES; + let mut evaluations = Vec::with_capacity(run.goal.coverage.len()); + for coverage in &run.goal.coverage { + let expectation = resolve_coverage_expectation(conn, run, coverage).await?; + let counts = + evidence + .coverage + .get(&coverage.id) + .ok_or_else(|| Error::InvalidRepositoryData { + message: format!("coverage {} has no persisted task evidence", coverage.id), + })?; + let expected_keys = expectation + .expected_keys + .iter() + .cloned() + .collect::>(); + let missing_count = counts + .expected_count + .checked_sub(counts.matched_count) + .ok_or_else(|| Error::InvalidRepositoryData { + message: format!( + "coverage {} matched count exceeds expected count", + coverage.id + ), + })?; + let missing_keys = load_coverage_key_samples( + conn, + run.run_uid, + &coverage.map_node_id, + CoverageSampleKind::Missing(&expected_keys), + &mut sample_budget, + ) + .await?; + let extra_keys = load_coverage_key_samples( + conn, + run.run_uid, + &coverage.map_node_id, + CoverageSampleKind::Extra(&expected_keys), + &mut sample_budget, + ) + .await?; + let failed_keys = load_coverage_key_samples( + conn, + run.run_uid, + &coverage.map_node_id, + CoverageSampleKind::Failed, + &mut sample_budget, + ) + .await?; + let completed_keys = load_coverage_key_samples( + conn, + run.run_uid, + &coverage.map_node_id, + CoverageSampleKind::Completed, + &mut sample_budget, + ) + .await?; + let sampled = + missing_keys.len() + extra_keys.len() + failed_keys.len() + completed_keys.len(); + let total = missing_count + .checked_add(counts.unexpected_count) + .and_then(|value| value.checked_add(counts.failed_count)) + .and_then(|value| value.checked_add(counts.completed_count)) + .ok_or_else(|| Error::ArithmeticOverflow { + context: "coverage evidence sample count".to_string(), + })?; + evaluations.push(PersistedCoverageEvaluation { + coverage_id: coverage.id.clone(), + map_node_id: coverage.map_node_id.clone(), + passed: persisted_coverage_passed(coverage, nodes, evidence)?, + expected_count: counts.expected_count, + observed_count: counts.observed_count, + matched_count: counts.matched_count, + missing_count, + extra_count: counts.unexpected_count, + failed_count: counts.failed_count, + completed_count: counts.completed_count, + expected_keys_hash: counts.expected_keys_hash.clone(), + observed_terminal_hash: counts.observed_terminal_hash.clone(), + missing_keys, + extra_keys, + failed_keys, + completed_keys, + samples_truncated: u64::try_from(sampled).map_or(true, |sampled| sampled < total), + }); + } + Ok(evaluations) +} + +enum CoverageSampleKind<'a> { + Missing(&'a [String]), + Extra(&'a [String]), + Failed, + Completed, +} + +async fn load_coverage_key_samples( + conn: &mut PgConnection, + run_uid: Uuid, + node_id: &str, + kind: CoverageSampleKind<'_>, + remaining_budget: &mut usize, +) -> Result> { + if *remaining_budget == 0 { + return Ok(Vec::new()); + } + let limit = *remaining_budget; + let limit = i64::try_from(limit).map_err(|_| Error::ArithmeticOverflow { + context: "coverage evidence sample limit".to_string(), + })?; + let rows = match kind { + CoverageSampleKind::Missing(expected) => sqlx::query_scalar::<_, String>( + "SELECT key FROM unnest($3::TEXT[]) AS expected(key) WHERE NOT EXISTS ( \ + SELECT 1 FROM moa.execution_task task WHERE task.run_uid=$1 \ + AND task.node_id=$2 AND task.item_key=expected.key \ + AND task.status IN ('completed','failed','unknown_outcome','cancelled')) \ + ORDER BY key LIMIT $4", + ) + .bind(run_uid) + .bind(node_id) + .bind(expected) + .bind(limit) + .fetch_all(&mut *conn) + .await + .map_err(sqlx_error)?, + CoverageSampleKind::Extra(expected) => sqlx::query_scalar::<_, String>( + "SELECT item_key FROM moa.execution_task WHERE run_uid=$1 AND node_id=$2 \ + AND status IN ('completed','failed','unknown_outcome','cancelled') \ + AND NOT (item_key = ANY($3::TEXT[])) ORDER BY item_key LIMIT $4", + ) + .bind(run_uid) + .bind(node_id) + .bind(expected) + .bind(limit) + .fetch_all(&mut *conn) + .await + .map_err(sqlx_error)?, + CoverageSampleKind::Failed => sqlx::query_scalar::<_, String>( + "SELECT item_key FROM moa.execution_task WHERE run_uid=$1 AND node_id=$2 \ + AND status IN ('failed','unknown_outcome','cancelled') ORDER BY item_key LIMIT $3", + ) + .bind(run_uid) + .bind(node_id) + .bind(limit) + .fetch_all(&mut *conn) + .await + .map_err(sqlx_error)?, + CoverageSampleKind::Completed => sqlx::query_scalar::<_, String>( + "SELECT item_key FROM moa.execution_task WHERE run_uid=$1 AND node_id=$2 \ + AND status='completed' ORDER BY item_key LIMIT $3", + ) + .bind(run_uid) + .bind(node_id) + .bind(limit) + .fetch_all(&mut *conn) + .await + .map_err(sqlx_error)?, + }; + *remaining_budget = remaining_budget.saturating_sub(rows.len()); + Ok(rows) +} diff --git a/crates/moa-execution/src/repository/mod.rs b/crates/moa-execution/src/repository/mod.rs index 46fb1595f..e39e7ea8b 100644 --- a/crates/moa-execution/src/repository/mod.rs +++ b/crates/moa-execution/src/repository/mod.rs @@ -382,6 +382,8 @@ pub struct ExecutionRunRecord { pub waiting_input_external_task_count: u64, /// Approved resource limits. pub approved_budget: ExecutionBudgetLimit, + /// Start of the current human-input interval excluded from wall-clock deadline accounting. + pub budget_deadline_suspended_at: Option>, /// Resources held by nonterminal tasks. pub reserved: ExecutionEstimate, /// Reconciled actual resources and terminal logical tasks. diff --git a/crates/moa-execution/src/repository/outbox.rs b/crates/moa-execution/src/repository/outbox.rs index c0a9a182a..a2000ec38 100644 --- a/crates/moa-execution/src/repository/outbox.rs +++ b/crates/moa-execution/src/repository/outbox.rs @@ -746,6 +746,7 @@ impl ExecutionRepository { 'pausing', 'paused', 'compensating' ) AND budget_deadline_at IS NOT NULL + AND budget_deadline_suspended_at IS NULL AND budget_deadline_at <= now() LIMIT $1 ) AS overdue diff --git a/crates/moa-execution/src/repository/outcome.rs b/crates/moa-execution/src/repository/outcome.rs index 85e1c5360..5e1f1678f 100644 --- a/crates/moa-execution/src/repository/outcome.rs +++ b/crates/moa-execution/src/repository/outcome.rs @@ -11,6 +11,9 @@ use super::{ sql::*, transition::task_outcome_is_exact_replay, }; +use amendment_reconciliation::reconcile_amendment_node_state_in_conn; + +mod amendment_reconciliation; impl ExecutionRepository { /// Records one cumulative task outcome under the current generation fence. @@ -530,6 +533,16 @@ impl ExecutionRepository { .ok_or_else(|| Error::InvalidRepositoryInput { message: "execution plan revision overflow".to_string(), })?; + reconcile_amendment_node_state_in_conn(conn.as_mut(), &run, &task, &validated.active_plan) + .await?; + sqlx::query(SUPERSEDE_REPLAN_TASK_SQL) + .bind(run_uid) + .bind(task.task_id.as_uuid()) + .bind(task_audit) + .bind(superseded_outcome) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; let history = json!({ "base_plan_revision": expected_revision, "plan_revision": next_revision, @@ -577,6 +590,7 @@ impl ExecutionRepository { "run consumed tasks", )?) .bind(reconciliation.budget_overrun) + .bind(task.task_id.as_uuid()) .fetch_optional(conn.as_mut()) .await .map_err(sqlx_error)?; @@ -585,14 +599,6 @@ impl ExecutionRepository { return Ok(AmendmentWrite::Conflict); }; let run = run_from_row(&row)?; - sqlx::query(SUPERSEDE_REPLAN_TASK_SQL) - .bind(run_uid) - .bind(task.task_id.as_uuid()) - .bind(task_audit) - .bind(superseded_outcome) - .fetch_one(conn.as_mut()) - .await - .map_err(sqlx_error)?; sqlx::query( "INSERT INTO moa.execution_amendment_receipt (tenant_id,run_uid, \ base_plan_revision,amendment_hash,receipt_kind,superseded_task_id, \ diff --git a/crates/moa-execution/src/repository/outcome/amendment_reconciliation.rs b/crates/moa-execution/src/repository/outcome/amendment_reconciliation.rs new file mode 100644 index 000000000..bbb922e53 --- /dev/null +++ b/crates/moa-execution/src/repository/outcome/amendment_reconciliation.rs @@ -0,0 +1,196 @@ +//! Atomic node-state reconciliation for compiler-validated plan amendments. + +use std::collections::{BTreeMap, BTreeSet}; + +use sqlx::{PgConnection, Row}; +use uuid::Uuid; + +use crate::{CanonicalExecutionPlan, Error, Result}; + +use super::super::{ + ExecutionRunRecord, ExecutionTaskRecord, row_error, rows::required_u64, sqlx_error, to_i64, +}; + +/// Reconciles persisted node state with one validated replacement plan. +pub(super) async fn reconcile_amendment_node_state_in_conn( + conn: &mut PgConnection, + run: &ExecutionRunRecord, + superseded_task: &ExecutionTaskRecord, + active_plan: &CanonicalExecutionPlan, +) -> Result<()> { + let rows = sqlx::query( + "SELECT node_id,node_status,materialization_cursor,materialization_complete, \ + total_task_count FROM moa.execution_node_state \ + WHERE run_uid=$1 ORDER BY node_order FOR UPDATE", + ) + .bind(run.run_uid) + .fetch_all(&mut *conn) + .await + .map_err(sqlx_error)?; + let current_nodes = run + .active_plan + .definition + .nodes + .iter() + .map(|node| (node.id.as_str(), node)) + .collect::>(); + if rows.len() != current_nodes.len() { + return Err(Error::InvalidRepositoryData { + message: "persisted node state does not exactly cover the active plan before amendment" + .to_string(), + }); + } + let replacement_nodes = active_plan + .definition + .nodes + .iter() + .map(|node| (node.id.as_str(), node)) + .collect::>(); + let mut preserved = BTreeMap::new(); + for row in &rows { + let node_id: String = row.try_get("node_id").map_err(row_error)?; + let status: String = row.try_get("node_status").map_err(row_error)?; + let cursor = required_u64(row, "materialization_cursor")?; + let materialization_complete: bool = + row.try_get("materialization_complete").map_err(row_error)?; + let total_tasks = required_u64(row, "total_task_count")?; + let current = + current_nodes + .get(node_id.as_str()) + .ok_or_else(|| Error::InvalidRepositoryData { + message: format!( + "persisted amendment node `{node_id}` is absent from active plan" + ), + })?; + let unstarted = + status == "pending" && cursor == 0 && !materialization_complete && total_tasks == 0; + let superseded_wait = + node_id == superseded_task.node_id && status == "waiting" && total_tasks == 1; + if unstarted || superseded_wait { + continue; + } + let replacement = replacement_nodes.get(node_id.as_str()).ok_or_else(|| { + Error::InvalidRepositoryInput { + message: format!("amendment removes started node `{node_id}`"), + } + })?; + if *current != *replacement { + return Err(Error::InvalidRepositoryInput { + message: format!("amendment rewrites started node `{node_id}`"), + }); + } + preserved.insert(node_id, status); + } + if replacement_nodes.contains_key(superseded_task.node_id.as_str()) { + return Err(Error::InvalidRepositoryInput { + message: "amendment retains the superseded WaitingReplan node identity".to_string(), + }); + } + + let shift = i64::try_from( + run.active_plan + .definition + .nodes + .len() + .saturating_add(active_plan.definition.nodes.len()) + .saturating_add(1), + ) + .map_err(|_| Error::ArithmeticOverflow { + context: "amendment node-order reconciliation shift".to_string(), + })?; + if !preserved.is_empty() { + let preserved_ids = preserved.keys().cloned().collect::>(); + sqlx::query( + "UPDATE moa.execution_node_state SET node_order=node_order+$3,updated_at=NOW() \ + WHERE run_uid=$1 AND node_id=ANY($2::TEXT[])", + ) + .bind(run.run_uid) + .bind(&preserved_ids) + .bind(shift) + .execute(&mut *conn) + .await + .map_err(sqlx_error)?; + } + let preserved_ids = preserved.keys().cloned().collect::>(); + sqlx::query( + "DELETE FROM moa.execution_node_state WHERE run_uid=$1 \ + AND NOT (node_id=ANY($2::TEXT[]))", + ) + .bind(run.run_uid) + .bind(&preserved_ids) + .execute(&mut *conn) + .await + .map_err(sqlx_error)?; + + let resolved_dependencies = preserved + .iter() + .filter(|(_, status)| matches!(status.as_str(), "completed" | "skipped")) + .map(|(node_id, _)| node_id.as_str()) + .collect::>(); + for (node_order, node) in active_plan.definition.nodes.iter().enumerate() { + let node_order = i64::try_from(node_order).map_err(|_| Error::ArithmeticOverflow { + context: "amendment node order".to_string(), + })?; + if preserved.contains_key(&node.id) { + let updated = sqlx::query( + "UPDATE moa.execution_node_state SET node_order=$3,updated_at=NOW() \ + WHERE run_uid=$1 AND node_id=$2", + ) + .bind(run.run_uid) + .bind(&node.id) + .bind(node_order) + .execute(&mut *conn) + .await + .map_err(sqlx_error)?; + if updated.rows_affected() != 1 { + return Err(Error::InvalidRepositoryData { + message: format!("preserved amendment node `{}` disappeared", node.id), + }); + } + continue; + } + if node.depends_on.iter().any(|dependency| { + preserved + .get(dependency) + .is_some_and(|status| matches!(status.as_str(), "failed" | "cancelled")) + }) { + return Err(Error::InvalidRepositoryInput { + message: format!("amendment node `{}` depends on a failed node", node.id), + }); + } + let dependency_count = + u64::try_from(node.depends_on.len()).map_err(|_| Error::ArithmeticOverflow { + context: "amendment node dependency count".to_string(), + })?; + let resolved_count = u64::try_from( + node.depends_on + .iter() + .filter(|dependency| resolved_dependencies.contains(dependency.as_str())) + .count(), + ) + .map_err(|_| Error::ArithmeticOverflow { + context: "amendment resolved dependency count".to_string(), + })?; + let remaining = dependency_count + .checked_sub(resolved_count) + .ok_or_else(|| Error::InvalidRepositoryData { + message: format!("amendment node `{}` has invalid dependency counts", node.id), + })?; + sqlx::query( + "INSERT INTO moa.execution_node_state (node_state_uid,tenant_id,run_uid,node_id, \ + node_order,dependency_count,remaining_dependency_count) \ + VALUES ($1,$2,$3,$4,$5,$6,$7)", + ) + .bind(Uuid::new_v5(&run.run_uid, node.id.as_bytes())) + .bind(run.tenant_id.0) + .bind(run.run_uid) + .bind(&node.id) + .bind(node_order) + .bind(to_i64(dependency_count, "amendment node dependency count")?) + .bind(to_i64(remaining, "amendment remaining dependency count")?) + .execute(&mut *conn) + .await + .map_err(sqlx_error)?; + } + Ok(()) +} diff --git a/crates/moa-execution/src/repository/ready.rs b/crates/moa-execution/src/repository/ready.rs index fb68eb0c4..9765785d0 100644 --- a/crates/moa-execution/src/repository/ready.rs +++ b/crates/moa-execution/src/repository/ready.rs @@ -15,6 +15,7 @@ use crate::schema::validate_instance; use super::*; use super::{ + capacity::{ExecutionCapacityDimension, prelock_capacity_dimensions_in_tx}, materialize::{ensure_materialization_replay_matches, prepare_task_materialization_batch}, outcome_support::outcome_projection_fields, rows::*, @@ -965,7 +966,39 @@ impl ExecutionRepository { message: "ready materialization cursor overflow".to_string(), } })?; + let may_enter_storage_wait = tasks.first().is_some_and(|task| { + matches!( + &task.kind, + LogicalTaskKind::Review { .. } + | LogicalTaskKind::WaitSignal { .. } + | LogicalTaskKind::WaitUntil { .. } + ) + }); let mut conn = scope.begin(&self.pool).await?; + let visible_tenant_id = if may_enter_storage_wait { + let tenant_id = sqlx::query_scalar::<_, Uuid>( + "SELECT tenant_id FROM moa.execution_run WHERE run_uid=$1", + ) + .bind(run_uid) + .fetch_optional(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let Some(tenant_id) = tenant_id else { + conn.commit().await.map_err(storage_error)?; + return Ok(ReadyMaterializationOutcome::Conflict); + }; + let tenant_id = TenantId(tenant_id); + prelock_capacity_dimensions_in_tx( + conn.as_mut(), + config, + tenant_id, + &[ExecutionCapacityDimension::ScheduledTriggers], + ) + .await?; + Some(tenant_id) + } else { + None + }; let Some(run_row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) .bind(run_uid) .fetch_optional(conn.as_mut()) @@ -976,6 +1009,12 @@ impl ExecutionRepository { return Ok(ReadyMaterializationOutcome::Conflict); }; let run = run_from_row(&run_row)?; + if visible_tenant_id.is_some_and(|tenant_id| tenant_id != run.tenant_id) { + return Err(Error::InvalidRepositoryData { + message: "execution run tenant changed while acquiring storage-wait capacity locks" + .to_string(), + }); + } if run.plan_revision != plan_revision || !matches!( run.status, @@ -1637,14 +1676,14 @@ impl ExecutionRepository { } })?; let citation_ids = sqlx::query_scalar::<_, String>( - "SELECT citation.value ->> 'source_id' \ - FROM moa.execution_task task \ - CROSS JOIN LATERAL jsonb_array_elements(task.citations) \ - WITH ORDINALITY AS citation(value, position) \ - WHERE task.run_uid = $1 \ - AND NULLIF(btrim(citation.value ->> 'source_id'), '') IS NOT NULL \ - ORDER BY task.node_id, task.item_key, task.task_id, citation.position \ - LIMIT $2", + "SELECT source_id FROM ( \ + SELECT DISTINCT citation.value ->> 'source_id' AS source_id \ + FROM moa.execution_task AS task \ + CROSS JOIN LATERAL jsonb_array_elements(task.citations) AS citation(value) \ + WHERE task.run_uid = $1 \ + AND NULLIF(btrim(citation.value ->> 'source_id'), '') IS NOT NULL \ + ) AS citation_ids \ + ORDER BY source_id LIMIT $2", ) .bind(run_uid) .bind(citation_limit) @@ -1652,10 +1691,10 @@ impl ExecutionRepository { .await .map_err(sqlx_error)?; let failures = sqlx::query_scalar::<_, String>( - "SELECT COALESCE(error, current_outcome #>> '{result,message}') \ + "SELECT COALESCE(error ->> 'message', current_outcome #>> '{result,message}') \ FROM moa.execution_task WHERE run_uid = $1 \ AND status IN ('failed', 'unknown_outcome') \ - AND COALESCE(error, current_outcome #>> '{result,message}') IS NOT NULL \ + AND COALESCE(error ->> 'message', current_outcome #>> '{result,message}') IS NOT NULL \ ORDER BY node_id, item_key, task_id LIMIT $2", ) .bind(run_uid) @@ -2579,7 +2618,7 @@ async fn transition_node_counters_inner( Ok(()) } -async fn cancel_unmaterialized_dependents_in_tx( +pub(super) async fn cancel_unmaterialized_dependents_in_tx( conn: &mut PgConnection, run: &ExecutionRunRecord, failed_node_id: &str, diff --git a/crates/moa-execution/src/repository/replan_stop.rs b/crates/moa-execution/src/repository/replan_stop.rs index 10c965b82..ee845c86a 100644 --- a/crates/moa-execution/src/repository/replan_stop.rs +++ b/crates/moa-execution/src/repository/replan_stop.rs @@ -5,6 +5,7 @@ use moa_core::types::identifiers::SessionId; use super::*; use super::{ capacity::{ExecutionCapacityDimension, prelock_capacity_dimensions_in_tx}, + ready::cancel_unmaterialized_dependents_in_tx, rows::{required_u64, run_from_row}, run::enqueue_run_activation_in_conn, sql::LOAD_RUN_SQL, @@ -170,7 +171,7 @@ impl ExecutionRepository { return Ok(ReplanStopIntentWriteOutcome::Conflict); } let task = sqlx::query( - "SELECT generation,status,current_outcome FROM moa.execution_task \ + "SELECT node_id,generation,status,current_outcome FROM moa.execution_task \ WHERE run_uid=$1 AND task_id=$2 FOR UPDATE", ) .bind(run.run_uid) @@ -183,6 +184,7 @@ impl ExecutionRepository { return Ok(ReplanStopIntentWriteOutcome::Conflict); }; let task_generation = required_u64(&task, "generation")?; + let origin_node_id: String = task.try_get("node_id").map_err(row_error)?; let task_status: String = task.try_get("status").map_err(row_error)?; let current_outcome: Option = task .try_get::, _>("current_outcome") @@ -199,6 +201,7 @@ impl ExecutionRepository { conn.commit().await.map_err(storage_error)?; return Ok(ReplanStopIntentWriteOutcome::Conflict); } + cancel_unmaterialized_dependents_in_tx(conn.as_mut(), &run, &origin_node_id).await?; let dispatch = enqueue_run_activation_in_conn( conn.as_mut(), diff --git a/crates/moa-execution/src/repository/rows.rs b/crates/moa-execution/src/repository/rows.rs index 088a805a1..30b4e815d 100644 --- a/crates/moa-execution/src/repository/rows.rs +++ b/crates/moa-execution/src/repository/rows.rs @@ -250,6 +250,9 @@ pub(super) fn run_from_row(row: &PgRow) -> Result { max_retrieved_bytes: optional_u64(row, "budget_max_retrieved_bytes")?, deadline_at: row.try_get("budget_deadline_at").map_err(row_error)?, }, + budget_deadline_suspended_at: row + .try_get("budget_deadline_suspended_at") + .map_err(row_error)?, reserved: estimate_from_row(row, "reserved")?, consumed: estimate_from_row(row, "consumed")?, budget_overrun: row.try_get("budget_overrun").map_err(row_error)?, diff --git a/crates/moa-execution/src/repository/run.rs b/crates/moa-execution/src/repository/run.rs index dcd544b2a..96b7df865 100644 --- a/crates/moa-execution/src/repository/run.rs +++ b/crates/moa-execution/src/repository/run.rs @@ -10,7 +10,7 @@ use super::{ }, outbox::{ ExecutionDispatchKind, ExecutionDispatchRecord, NewExecutionDispatch, - enqueue_dispatch_in_conn, + dispatch_from_row_for_repository, enqueue_dispatch_in_conn, }, rows::*, sql::*, @@ -40,6 +40,8 @@ pub struct ExecutionCancellationProjection { pub run: ExecutionRunRecord, /// Completed plan-node identities, bounded by the compiler-capped active plan. pub completed_node_ids: Vec, + /// Exact terminal-cancellation receipts still joined to a current cancelling task attempt. + pub task_cancellation_dispatches: Vec, } const LOAD_RUN_BY_IDEMPOTENCY_FOR_SESSION_SQL: &str = r#" SELECT * @@ -350,6 +352,7 @@ impl ExecutionRepository { scope: ExecutionScope, run_uid: Uuid, expected_session_id: SessionId, + max_current_task_owners: usize, ) -> Result> { let mut conn = scope.begin(&self.pool).await?; let row = sqlx::query(LOAD_RUN_FOR_SESSION_SQL) @@ -394,10 +397,26 @@ impl ExecutionRepository { message: "cancellation projection exceeded its active-plan bound".to_string(), }); } + let task_cancellation_dispatches = if run.status == ExecutionRunStatus::Cancelled + || run + .pending_terminal + .as_ref() + .is_some_and(|pending| pending.status == ExecutionRunStatus::Cancelled) + { + load_current_terminal_task_cancellation_dispatches( + &mut conn, + &run, + max_current_task_owners, + ) + .await? + } else { + Vec::new() + }; conn.commit().await.map_err(storage_error)?; Ok(Some(ExecutionCancellationProjection { run, completed_node_ids, + task_cancellation_dispatches, })) } @@ -967,6 +986,52 @@ impl ExecutionRepository { } } +pub(super) async fn load_current_terminal_task_cancellation_dispatches( + conn: &mut ScopedConn<'_>, + run: &ExecutionRunRecord, + max_current_task_owners: usize, +) -> Result> { + let fetch_limit = i64::try_from(max_current_task_owners) + .map_err(|_| Error::InvalidRepositoryInput { + message: "cancellation owner bound exceeds PostgreSQL BIGINT".to_string(), + })? + .checked_add(1) + .ok_or_else(|| Error::ArithmeticOverflow { + context: "cancellation owner projection limit".to_string(), + })?; + let rows = sqlx::query( + "SELECT dispatch.* FROM moa.execution_dispatch_outbox AS dispatch \ + JOIN moa.execution_task AS task ON task.run_uid=dispatch.run_uid \ + AND task.task_id=dispatch.task_id \ + WHERE dispatch.run_uid=$1 AND dispatch.tenant_id=$2 \ + AND dispatch.dispatch_kind='task_attempt_cancel' \ + AND dispatch.controller_generation=$3 \ + AND dispatch.attempt_generation=task.attempt_generation \ + AND task.attempt_state='cancelling' \ + AND task.active_dispatch_uid IS NOT NULL \ + AND dispatch.payload->>'active_dispatch_uid'=task.active_dispatch_uid::TEXT \ + AND dispatch.payload->>'task_generation'=task.generation::TEXT \ + AND dispatch.payload->>'attempt_generation'=task.attempt_generation::TEXT \ + AND dispatch.payload->>'controller_generation'=$3::TEXT \ + AND dispatch.payload->>'reason'='run_terminal' \ + ORDER BY dispatch.task_id, dispatch.dispatch_uid LIMIT $4", + ) + .bind(run.run_uid) + .bind(run.tenant_id.0) + .bind(to_i64(run.controller_generation, "controller generation")?) + .bind(fetch_limit) + .fetch_all(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if rows.len() > max_current_task_owners { + return Err(Error::InvalidRepositoryData { + message: "cancellation owner projection exceeded its configured in-flight bound" + .to_string(), + }); + } + rows.iter().map(dispatch_from_row_for_repository).collect() +} + /// Arms the exact immutable run-deadline trigger inside the caller's transaction. /// /// Multi-resource admission callers must prelock `ActiveRuns` then `ScheduledTriggers` before @@ -980,6 +1045,9 @@ pub(super) async fn arm_run_deadline_in_conn( if run.status.is_terminal() { return Ok(RunDeadlineArmOutcome::Terminal); } + if run.budget_deadline_suspended_at.is_some() { + return Ok(RunDeadlineArmOutcome::NoDeadline); + } let Some(deadline_at) = run.approved_budget.deadline_at else { return Ok(RunDeadlineArmOutcome::NoDeadline); }; diff --git a/crates/moa-execution/src/repository/sql.rs b/crates/moa-execution/src/repository/sql.rs index ed82fcc22..2591d177f 100644 --- a/crates/moa-execution/src/repository/sql.rs +++ b/crates/moa-execution/src/repository/sql.rs @@ -778,12 +778,33 @@ pub(super) const SUPERSEDE_REPLAN_TASK_SQL: &str = r#" "#; pub(super) const APPEND_AMENDMENT_SQL: &str = r#" + WITH remaining_wait AS ( + SELECT MIN(waiting_since) AS waiting_since + FROM moa.execution_task + WHERE run_uid=$1 AND task_id<>$14 + AND status IN ('waiting_input','waiting_review','waiting_signal', + 'waiting_timer','waiting_external','waiting_replan') + ), remaining_reasons AS ( + SELECT COALESCE(jsonb_agg(reason ORDER BY ordinal), '[]'::JSONB) AS reasons + FROM moa.execution_run AS source, + jsonb_array_elements(source.waiting_reasons) + WITH ORDINALITY AS item(reason, ordinal) + WHERE source.run_uid=$1 AND COALESCE(reason->>'task_id','')<>$14::TEXT + ) UPDATE moa.execution_run SET active_plan = $4, active_plan_hash = $5, plan_revision = $3, plan_history = plan_history || jsonb_build_array($6::JSONB), - status = 'running', + status = CASE + WHEN waiting_input_task_count > 0 THEN 'waiting_input' + WHEN waiting_review_task_count > 0 THEN 'waiting_review' + WHEN waiting_signal_task_count > 0 THEN 'waiting_signal' + WHEN waiting_timer_task_count > 0 THEN 'waiting_timer' + WHEN waiting_external_task_count > 0 THEN 'waiting_external' + WHEN waiting_replan_task_count - 1 > 0 THEN 'waiting_replan' + ELSE 'running' + END, reserved_cost_microusd = $7, reserved_tokens = $8, reserved_tasks = $9, @@ -792,8 +813,16 @@ pub(super) const APPEND_AMENDMENT_SQL: &str = r#" consumed_tasks = $12, budget_overrun = $13, progress_cancelled_tasks = progress_cancelled_tasks + 1, + waiting_task_count = waiting_task_count - 1, + waiting_replan_task_count = waiting_replan_task_count - 1, + waiting_reasons = remaining_reasons.reasons, + waiting_reasons_truncated = jsonb_array_length(remaining_reasons.reasons) + < waiting_task_count - 1, + waiting_since = remaining_wait.waiting_since, updated_at = NOW() + FROM remaining_wait, remaining_reasons WHERE run_uid = $1 AND plan_revision = $2 AND status = 'waiting_replan' + AND waiting_task_count > 0 AND waiting_replan_task_count = 1 RETURNING * "#; diff --git a/crates/moa-execution/src/repository/task.rs b/crates/moa-execution/src/repository/task.rs index f06f3beb9..5bcf4f2fb 100644 --- a/crates/moa-execution/src/repository/task.rs +++ b/crates/moa-execution/src/repository/task.rs @@ -29,18 +29,12 @@ use super::{ transition_node_counters_with_input_audience_in_tx, }, rows::*, - run::enqueue_run_activation_in_conn, + run::{arm_run_deadline_in_conn, enqueue_run_activation_in_conn}, sql::*, transition::{refresh_run_after_wait_settlement_in_conn, task_outcome_is_exact_replay}, - trigger::{ - ExecutionTriggerKind, ExecutionTriggerSupersedeOutcome, NewExecutionTrigger, - create_trigger_with_dispatch_in_conn, supersede_trigger_in_conn, - }, + trigger::{ExecutionTriggerKind, ExecutionTriggerSupersedeOutcome, supersede_trigger_in_conn}, }; -const TASK_INPUT_WAIT_TRIGGER_NAMESPACE: Uuid = - Uuid::from_u128(0x9a2e_18f4_1c5e_57c4_8bf5_ea73_52e9_4a11); - /// Immutable identity of one admitted bounded task-attempt slice. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct TaskAttemptFence { @@ -555,8 +549,7 @@ pub struct ResolveTaskAttemptReviewRequest { pub resolved_at: DateTime, } -struct SettleTaskAttemptRequest<'a> { - config: &'a ExecutionConfig, +struct SettleTaskAttemptRequest { fence: TaskAttemptFence, outcome: ExecutionTaskOutcome, retry_at: Option>, @@ -1412,7 +1405,8 @@ impl ExecutionRepository { "UPDATE moa.execution_task \ SET status = 'running', attempt_state = 'running', \ attempt_started_at = COALESCE(attempt_started_at, NOW()), \ - started_at = COALESCE(started_at, NOW()), last_progress_at = NOW(), \ + started_at = COALESCE(started_at, NOW()), \ + last_progress_at = GREATEST(last_progress_at, clock_timestamp()), \ progress_step_bound_seconds = NULL, \ updated_at = NOW() \ WHERE run_uid = $1 AND task_id = $2 AND status = 'dispatching' \ @@ -1429,7 +1423,8 @@ impl ExecutionRepository { sqlx::query( "UPDATE moa.execution_run \ SET status = CASE WHEN status = 'queued' THEN 'running' ELSE status END, \ - started_at = COALESCE(started_at, NOW()), last_progress_at = NOW(), \ + started_at = COALESCE(started_at, NOW()), \ + last_progress_at = GREATEST(last_progress_at, clock_timestamp()), \ updated_at = NOW() WHERE run_uid = $1", ) .bind(fence.run_uid) @@ -1830,14 +1825,13 @@ impl ExecutionRepository { /// can admit the next slice. pub async fn settle_task_attempt( &self, - config: &ExecutionConfig, + _config: &ExecutionConfig, fence: TaskAttemptFence, outcome: ExecutionTaskOutcome, retry_at: Option>, settled_at: DateTime, ) -> Result { self.settle_task_attempt_inner(SettleTaskAttemptRequest { - config, fence, outcome, retry_at, @@ -1852,7 +1846,7 @@ impl ExecutionRepository { /// Finalizes a cancelling attempt only after exact sandbox release proof is available. pub async fn settle_released_task_attempt( &self, - config: &ExecutionConfig, + _config: &ExecutionConfig, fence: TaskAttemptFence, outcome: ExecutionTaskOutcome, retry_at: Option>, @@ -1872,7 +1866,6 @@ impl ExecutionRepository { return Ok(TaskAttemptSettlementOutcome::Stale); } self.settle_task_attempt_inner(SettleTaskAttemptRequest { - config, fence, outcome, retry_at, @@ -1997,7 +1990,7 @@ impl ExecutionRepository { /// Settles an input wait while preserving the exact resumable agent continuation. pub async fn settle_released_task_attempt_with_checkpoint( &self, - config: &ExecutionConfig, + _config: &ExecutionConfig, fence: TaskAttemptFence, outcome: ExecutionTaskOutcome, settled_at: DateTime, @@ -2011,7 +2004,6 @@ impl ExecutionRepository { return Ok(TaskAttemptSettlementOutcome::InvalidState); } self.settle_task_attempt_inner(SettleTaskAttemptRequest { - config, fence, outcome, retry_at: None, @@ -2198,10 +2190,9 @@ impl ExecutionRepository { async fn settle_task_attempt_inner( &self, - request: SettleTaskAttemptRequest<'_>, + request: SettleTaskAttemptRequest, ) -> Result { let SettleTaskAttemptRequest { - config, fence, outcome, retry_at, @@ -2263,6 +2254,16 @@ impl ExecutionRepository { conn.rollback().await.map_err(storage_error)?; return Ok(TaskAttemptSettlementOutcome::Stale); } + // Preserve the global capacity lock order before locking the run: task settlement may + // make the run fully input-parked and supersede its deadline trigger. Deadline delivery + // takes ScheduledTriggers before the run, so taking that bucket here prevents a + // run-to-trigger lock inversion without retaining active task capacity. + prelock_existing_capacity_dimensions_in_tx( + conn.as_mut(), + fence.tenant_id, + &[ExecutionCapacityDimension::ScheduledTriggers], + ) + .await?; let Some(run_row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) .bind(fence.run_uid) .fetch_optional(conn.as_mut()) @@ -2284,35 +2285,6 @@ impl ExecutionRepository { return Ok(TaskAttemptSettlementOutcome::NotFound); }; let task = task_from_row(&task_row)?; - // A relative input-wait expiry resolves against wait entry, not compile time, so a delay - // that was legal when the plan compiled can land past the run deadline here. That is a - // product outcome for a long-horizon run, so the task fails terminally with a typed - // deadline failure instead of parking on a wait that can never settle in time. - let needs_input = matches!(outcome.result, ExecutionTaskResult::NeedsInput { .. }); - let outcome = match run.approved_budget.deadline_at { - Some(run_deadline_at) if needs_input => { - match crate::interpreter::resolve_temporal_target_within_deadline( - &run.active_plan.definition.input_wait_policy.expiry, - settled_at, - run_deadline_at, - )? { - crate::interpreter::TemporalTargetResolution::Due(_) => outcome, - crate::interpreter::TemporalTargetResolution::DeadlineExceeded { - due_at, - run_deadline_at, - } => failed_task_outcome( - moa_artifacts::execution_plan::ExecutionFailureClass::DeadlineExceeded, - format!( - "input wait on node `{}` entered at {settled_at} resolves at \ - {due_at}, at or after the run deadline {run_deadline_at}", - task.node_id - ), - outcome.usage.clone(), - ), - } - } - _ => outcome, - }; if capacity == CapacityReleaseOutcome::AlreadyReleased { let replay = task_attempt_settlement_replayed(&task, &fence, &outcome); if replay { @@ -2503,21 +2475,6 @@ impl ExecutionRepository { .await .map_err(sqlx_error)?; let task = task_from_row(&row)?; - let input_wait_due_at = if task.status == ExecutionTaskStatus::WaitingInput { - let run_deadline_at = - run.approved_budget - .deadline_at - .ok_or_else(|| Error::InvalidRepositoryInput { - message: "input waits require an absolute run deadline".to_string(), - })?; - Some(crate::interpreter::resolve_temporal_target( - &run.active_plan.definition.input_wait_policy.expiry, - settled_at, - run_deadline_at, - )?) - } else { - None - }; if task.status == ExecutionTaskStatus::WaitingInput { let input_audience = task .current_outcome @@ -2559,20 +2516,6 @@ impl ExecutionRepository { task_id: task.task_id, audience, question, - wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { - expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::At { - at: input_wait_due_at.ok_or_else(|| Error::InvalidRepositoryData { - message: "waiting-input task is missing its resolved expiry" - .to_string(), - })?, - }, - on_expiry: run - .active_plan - .definition - .input_wait_policy - .on_expiry - .clone(), - }, }, settled_at, ) @@ -2588,45 +2531,8 @@ impl ExecutionRepository { ) .await?; } - if task.status == ExecutionTaskStatus::WaitingInput { - let due_at = input_wait_due_at.ok_or_else(|| Error::InvalidRepositoryData { - message: "waiting-input task is missing its resolved expiry".to_string(), - })?; - let trigger_uid = Uuid::new_v5( - &TASK_INPUT_WAIT_TRIGGER_NAMESPACE, - format!( - "{}:{}:{}:{}", - fence.run_uid, task.task_id, task.generation, settled_at - ) - .as_bytes(), - ); - create_trigger_with_dispatch_in_conn( - conn.as_mut(), - config, - &NewExecutionTrigger { - trigger_uid, - tenant_id: fence.tenant_id, - run_uid: Some(fence.run_uid), - task_id: Some(fence.task_id.as_uuid()), - compensation_id: None, - schedule_uid: None, - schedule_incarnation: None, - kind: ExecutionTriggerKind::WaitExpiry, - controller_generation: Some(fence.controller_generation), - attempt_generation: Some(task.generation), - compensation_generation: None, - compensation_attempt_generation: None, - occurrence_sequence: None, - due_at, - payload: json!({ - "task_generation": task.generation, - "waiting_since": settled_at, - "source": "active_task_input_wait", - }), - }, - ) + suspend_run_deadline_if_fully_waiting_for_input_in_tx(&mut conn, fence.run_uid, settled_at) .await?; - } let activation_at = retry_at.unwrap_or(settled_at); enqueue_run_activation_in_conn( conn.as_mut(), @@ -4744,7 +4650,7 @@ impl ExecutionRepository { resume_input, } = request; let mut conn = scope.begin(&self.pool).await?; - let locked_wait_trigger_uid = if kind == ResumeKind::Input { + if kind == ResumeKind::Input { let config = config.ok_or_else(|| Error::InvalidRepositoryInput { message: "input resume requires validated execution capacity configuration" .to_string(), @@ -4772,26 +4678,7 @@ impl ExecutionRepository { ], ) .await?; - let trigger_uids = sqlx::query_scalar::<_, Uuid>( - "SELECT trigger_uid FROM moa.execution_trigger \ - WHERE run_uid=$1 AND task_id=$2 AND trigger_kind='wait_expiry' \ - AND state = 'pending' \ - ORDER BY trigger_uid LIMIT 2 FOR UPDATE", - ) - .bind(run_uid) - .bind(task_id.as_uuid()) - .fetch_all(conn.as_mut()) - .await - .map_err(sqlx_error)?; - if trigger_uids.len() > 1 { - return Err(Error::InvalidRepositoryData { - message: "waiting-input task owns multiple active expiry triggers".to_string(), - }); - } - trigger_uids.into_iter().next() - } else { - None - }; + } let Some(run_row) = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) .bind(run_uid) .fetch_optional(conn.as_mut()) @@ -4875,10 +4762,18 @@ impl ExecutionRepository { TransitionRejection::InvalidRunStatus, )); } - let admission_rejection = if run - .approved_budget - .deadline_at - .is_some_and(|deadline| Utc::now() > deadline) + let resumed_at = Utc::now(); + let deadline_was_resumed = if kind == ResumeKind::Input { + resume_run_deadline_after_input_wait_in_tx(&mut conn, run_uid, task_id, resumed_at) + .await? + } else { + false + }; + let admission_rejection = if kind == ResumeKind::Retry + && run + .approved_budget + .deadline_at + .is_some_and(|deadline| Utc::now() > deadline) { Some(( moa_artifacts::execution_plan::ExecutionFailureClass::DeadlineExceeded, @@ -4904,7 +4799,6 @@ impl ExecutionRepository { TransitionRejection::CounterOverflow, )); }; - let resumed_at = Utc::now(); let input_audience = if kind == ResumeKind::Input { Some( task.current_outcome @@ -4922,33 +4816,6 @@ impl ExecutionRepository { None }; if kind == ResumeKind::Input { - let Some(trigger_uid) = locked_wait_trigger_uid else { - return Err(Error::InvalidRepositoryData { - message: "waiting-input task is missing its active expiry trigger".to_string(), - }); - }; - match supersede_trigger_in_conn( - conn.as_mut(), - trigger_uid, - ExecutionTriggerKind::WaitExpiry, - Some(run.controller_generation), - Some(task.generation), - None, - None, - ) - .await? - { - ExecutionTriggerSupersedeOutcome::Superseded - | ExecutionTriggerSupersedeOutcome::AlreadySuperseded - | ExecutionTriggerSupersedeOutcome::AlreadyInactive => {} - ExecutionTriggerSupersedeOutcome::StaleOrMissing => { - return Err(Error::InvalidRepositoryData { - message: "waiting-input expiry trigger lost its generation fence" - .to_string(), - }); - } - } - let current_checkpoint = sqlx::query( "SELECT * FROM moa.execution_task_checkpoint WHERE tenant_id=$1 AND run_uid=$2 \ AND task_id=$3 AND superseded_at IS NULL FOR UPDATE", @@ -5060,6 +4927,23 @@ impl ExecutionRepository { ) .await?; } + if deadline_was_resumed { + let row = sqlx::query(LOAD_RUN_FOR_UPDATE_SQL) + .bind(run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let resumed_run = run_from_row(&row)?; + let _ = arm_run_deadline_in_conn( + conn.as_mut(), + config.ok_or_else(|| Error::InvalidRepositoryInput { + message: "input resume requires validated execution capacity configuration" + .to_string(), + })?, + &resumed_run, + ) + .await?; + } if !matches!( run.status, ExecutionRunStatus::PauseRequested @@ -5308,6 +5192,180 @@ impl ExecutionRepository { } } +/// Suspends wall-clock run deadline accounting once no executable work remains and every +/// durable wait is human input. Active attempt watchdogs have already been settled before this +/// helper runs; review, signal, timer, and provider waits keep their ordinary deadlines. +async fn suspend_run_deadline_if_fully_waiting_for_input_in_tx( + conn: &mut ScopedConn<'_>, + run_uid: Uuid, + suspended_at: DateTime, +) -> Result { + let row = sqlx::query( + "SELECT budget_deadline_at, budget_deadline_suspended_at, ready_task_count, \ + active_task_count, waiting_task_count, waiting_input_task_count \ + FROM moa.execution_run WHERE run_uid=$1 FOR UPDATE", + ) + .bind(run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + let ready_task_count = required_u64(&row, "ready_task_count")?; + let active_task_count = required_u64(&row, "active_task_count")?; + let waiting_task_count = required_u64(&row, "waiting_task_count")?; + let waiting_input_task_count = required_u64(&row, "waiting_input_task_count")?; + let deadline_at: Option> = + row.try_get("budget_deadline_at").map_err(row_error)?; + let deadline_suspended_at: Option> = row + .try_get("budget_deadline_suspended_at") + .map_err(row_error)?; + if ready_task_count != 0 + || active_task_count != 0 + || waiting_task_count == 0 + || waiting_task_count != waiting_input_task_count + || deadline_at.is_none() + || deadline_suspended_at.is_some() + { + return Ok(false); + } + let deadline_at = deadline_at.ok_or_else(|| Error::InvalidRepositoryData { + message: "fully input-waiting run lost its deadline before suspension".to_string(), + })?; + if deadline_at <= suspended_at { + return Err(Error::InvalidRepositoryData { + message: "active attempt settled NeedsInput after the run deadline".to_string(), + }); + } + let deadline_triggers = sqlx::query( + "SELECT trigger_uid, controller_generation FROM moa.execution_trigger \ + WHERE run_uid=$1 AND trigger_kind='run_deadline' AND state='pending' \ + ORDER BY trigger_uid LIMIT 2 FOR UPDATE", + ) + .bind(run_uid) + .fetch_all(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if deadline_triggers.len() > 1 { + return Err(Error::InvalidRepositoryData { + message: "input-waiting run owns multiple active deadline triggers".to_string(), + }); + } + for trigger in deadline_triggers { + let trigger_uid: Uuid = trigger.try_get("trigger_uid").map_err(row_error)?; + let trigger_generation = required_u64(&trigger, "controller_generation")?; + if supersede_trigger_in_conn( + conn.as_mut(), + trigger_uid, + ExecutionTriggerKind::RunDeadline, + Some(trigger_generation), + None, + None, + None, + ) + .await? + == ExecutionTriggerSupersedeOutcome::StaleOrMissing + { + return Err(Error::InvalidRepositoryData { + message: "input-waiting run deadline lost its exact trigger fence".to_string(), + }); + } + } + let run_updated = sqlx::query( + "UPDATE moa.execution_run SET budget_deadline_suspended_at=$2, waiting_since=$2, \ + next_wake_at=(SELECT min(due_at) FROM moa.execution_trigger \ + WHERE run_uid=$1 AND state='pending'), \ + last_progress_at=GREATEST(last_progress_at,$2), updated_at=NOW() \ + WHERE run_uid=$1 AND budget_deadline_at=$3 \ + AND budget_deadline_suspended_at IS NULL \ + AND EXISTS (SELECT 1 FROM moa.execution_task WHERE run_uid=$1 \ + AND status='waiting_input')", + ) + .bind(run_uid) + .bind(suspended_at) + .bind(deadline_at) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if run_updated.rows_affected() != 1 { + return Err(Error::InvalidRepositoryData { + message: "input-wait deadline suspension lost its run fence".to_string(), + }); + } + Ok(true) +} + +/// Shifts the run deadline by exactly the human-wait interval when input wakes a suspended run. +async fn resume_run_deadline_after_input_wait_in_tx( + conn: &mut ScopedConn<'_>, + run_uid: Uuid, + task_id: ExecutionTaskId, + resumed_at: DateTime, +) -> Result { + let (deadline_at, suspended_at) = + sqlx::query_as::<_, (Option>, Option>)>( + "SELECT budget_deadline_at,budget_deadline_suspended_at \ + FROM moa.execution_run WHERE run_uid=$1 FOR UPDATE", + ) + .bind(run_uid) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if suspended_at.is_none() { + return Ok(false); + } + let deadline_at = deadline_at.ok_or_else(|| Error::InvalidRepositoryData { + message: "suspended input wait is missing its original run deadline".to_string(), + })?; + let suspended_at = suspended_at.ok_or_else(|| Error::InvalidRepositoryData { + message: "input wait is missing its deadline suspension timestamp".to_string(), + })?; + let wait_duration = resumed_at.signed_duration_since(suspended_at); + if wait_duration < chrono::TimeDelta::zero() { + return Err(Error::InvalidRepositoryData { + message: "input resume predates its deadline suspension".to_string(), + }); + } + let restored_deadline = deadline_at + .checked_add_signed(wait_duration) + .ok_or_else(|| Error::ArithmeticOverflow { + context: "resumed input-wait run deadline".to_string(), + })?; + let run_updated = sqlx::query( + "UPDATE moa.execution_run SET budget_deadline_at=$2, \ + budget_deadline_suspended_at=NULL, \ + last_progress_at=GREATEST(last_progress_at,$3), updated_at=NOW() \ + WHERE run_uid=$1 AND budget_deadline_at=$4 \ + AND budget_deadline_suspended_at=$5", + ) + .bind(run_uid) + .bind(restored_deadline) + .bind(resumed_at) + .bind(deadline_at) + .bind(suspended_at) + .execute(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if run_updated.rows_affected() != 1 { + return Err(Error::InvalidRepositoryData { + message: "input resume lost its suspended run deadline fence".to_string(), + }); + } + let task_exists: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM moa.execution_task \ + WHERE run_uid=$1 AND task_id=$2 AND status='waiting_input')", + ) + .bind(run_uid) + .bind(task_id.as_uuid()) + .fetch_one(conn.as_mut()) + .await + .map_err(sqlx_error)?; + if !task_exists { + return Err(Error::InvalidRepositoryData { + message: "input resume lost its waiting-task fence".to_string(), + }); + } + Ok(true) +} + #[cfg(test)] mod tests { use super::{ diff --git a/crates/moa-execution/src/repository/transition.rs b/crates/moa-execution/src/repository/transition.rs index bf7d90f42..81d6037cf 100644 --- a/crates/moa-execution/src/repository/transition.rs +++ b/crates/moa-execution/src/repository/transition.rs @@ -420,7 +420,7 @@ impl ExecutionRepository { .ok_or_else(|| Error::InvalidRepositoryData { message: "current wait trigger task has no wait-entry timestamp".to_string(), })?; - let settlement = settlement_for_delivered_trigger(&run, &task, trigger.kind)?; + let settlement = settlement_for_delivered_trigger(&task, trigger.kind)?; let outcome = settle_wait_locked_in_conn( &mut conn, &run, @@ -489,7 +489,7 @@ async fn settle_wait_locked_in_conn( TransitionRejection::GenerationMismatch, )); } - let outcome = wait_settlement_outcome(run, task, &settlement)?; + let outcome = wait_settlement_outcome(task, &settlement)?; if run.status.is_terminal() { return Ok(TransitionOutcome::Rejected( TransitionRejection::InvalidRunStatus, @@ -512,13 +512,8 @@ async fn settle_wait_locked_in_conn( TransitionRejection::DeadlineElapsed, )); } - let due_at = wait_settlement_due_at( - run, - task, - &settlement, - expected_waiting_since, - run_deadline_at, - )?; + let due_at = + wait_settlement_due_at(task, &settlement, expected_waiting_since, run_deadline_at)?; if settled_at < due_at { return Ok(TransitionOutcome::Rejected( TransitionRejection::InvalidTaskStatus, @@ -690,7 +685,6 @@ pub(super) async fn refresh_run_after_wait_settlement_in_conn( } fn settlement_for_delivered_trigger( - run: &ExecutionRunRecord, task: &ExecutionTaskRecord, kind: ExecutionTriggerKind, ) -> Result { @@ -706,12 +700,6 @@ fn settlement_for_delivered_trigger( }, ExecutionTriggerKind::WaitExpiry => { let action = match (&task.status, &task.kind) { - (ExecutionTaskStatus::WaitingInput, _) => run - .active_plan - .definition - .input_wait_policy - .on_expiry - .clone(), ( ExecutionTaskStatus::WaitingReview, LogicalTaskKind::Review { wait_policy, .. }, @@ -772,7 +760,6 @@ fn wait_settlement_is_exact_replay( } fn wait_settlement_outcome( - run: &ExecutionRunRecord, task: &ExecutionTaskRecord, settlement: &WaitSettlement, ) -> Result { @@ -790,9 +777,6 @@ fn wait_settlement_outcome( }, WaitSettlement::WaitExpired { action, .. } => { let persisted_action = match (&task.status, &task.kind) { - (ExecutionTaskStatus::WaitingInput, _) => { - &run.active_plan.definition.input_wait_policy.on_expiry - } ( ExecutionTaskStatus::WaitingReview, LogicalTaskKind::Review { wait_policy, .. }, @@ -829,7 +813,6 @@ fn wait_settlement_outcome( } fn wait_settlement_due_at( - run: &ExecutionRunRecord, task: &ExecutionTaskRecord, settlement: &WaitSettlement, waiting_since: DateTime, @@ -845,9 +828,6 @@ fn wait_settlement_due_at( } }, WaitSettlement::WaitExpired { .. } => match (&task.status, &task.kind) { - (ExecutionTaskStatus::WaitingInput, _) => { - &run.active_plan.definition.input_wait_policy.expiry - } (ExecutionTaskStatus::WaitingReview, LogicalTaskKind::Review { wait_policy, .. }) | ( ExecutionTaskStatus::WaitingSignal, diff --git a/crates/moa-execution/src/repository/trigger.rs b/crates/moa-execution/src/repository/trigger.rs index 358e97d1e..180243fc5 100644 --- a/crates/moa-execution/src/repository/trigger.rs +++ b/crates/moa-execution/src/repository/trigger.rs @@ -652,8 +652,18 @@ impl ExecutionRepository { .ok_or_else(|| Error::InvalidRepositoryData { message: "run deadline trigger is missing run identity".to_string(), })?; - let run = sqlx::query_as::<_, (i64, i64, String, Option>)>( - "SELECT controller_generation, wake_epoch, status, budget_deadline_at \ + let run = sqlx::query_as::< + _, + ( + i64, + i64, + String, + Option>, + Option>, + ), + >( + "SELECT controller_generation, wake_epoch, status, budget_deadline_at, \ + budget_deadline_suspended_at \ FROM moa.execution_run \ WHERE tenant_id=$1 AND run_uid=$2 FOR UPDATE", ) @@ -662,12 +672,30 @@ impl ExecutionRepository { .fetch_optional(conn.as_mut()) .await .map_err(sqlx_error)?; - let Some((controller_generation, wake_epoch, status, approved_deadline_at)) = run else { + let Some((controller_generation, wake_epoch, status, approved_deadline_at, suspended_at)) = + run + else { conn.commit().await.map_err(storage_error)?; return Ok(ExecutionRunDeadlineTriggerOutcome::NoOp( ExecutionTriggerNoOp::NotFound, )); }; + if suspended_at.is_some() { + supersede_trigger_in_conn( + conn.as_mut(), + trigger_uid, + ExecutionTriggerKind::RunDeadline, + trigger.controller_generation, + None, + None, + None, + ) + .await?; + conn.commit().await.map_err(storage_error)?; + return Ok(ExecutionRunDeadlineTriggerOutcome::NoOp( + ExecutionTriggerNoOp::StaleGeneration, + )); + } if matches!( status.as_str(), "completed" | "partial" | "blocked" | "unsupported" | "failed" | "cancelled" @@ -2337,6 +2365,7 @@ async fn run_deadline_is_current( SELECT EXISTS ( SELECT 1 FROM moa.execution_run WHERE run_uid = $1 AND tenant_id = $2 AND budget_deadline_at = $3 + AND budget_deadline_suspended_at IS NULL AND status NOT IN ( 'completed', 'partial', 'blocked', 'unsupported', 'failed', 'cancelled' ) diff --git a/crates/moa-execution/src/state.rs b/crates/moa-execution/src/state.rs index eed42a9ef..ed6340051 100644 --- a/crates/moa-execution/src/state.rs +++ b/crates/moa-execution/src/state.rs @@ -951,8 +951,6 @@ pub enum WaitingReason { audience: InputAudience, /// Exact task question. question: String, - /// Absolute expiry and deterministic expiry action. - wait_policy: ExecutionWaitPolicy, }, /// One task needs a tenant review decision. Review { diff --git a/crates/moa-execution/tests/compiler.rs b/crates/moa-execution/tests/compiler.rs index 89f9a2f14..572c75b9c 100644 --- a/crates/moa-execution/tests/compiler.rs +++ b/crates/moa-execution/tests/compiler.rs @@ -238,12 +238,6 @@ fn compile_accepts_wait_until_strictly_between_validation_time_and_run_deadline( fn compile_accepts_wait_entry_relative_targets_inside_the_remaining_horizon() { // Pins: reusable relative waits remain relative until their task enters storage-only waiting. let mut request = valid_request(); - request.plan.input_wait_policy = ExecutionWaitPolicy { - expiry: ExecutionTemporalTarget::After { - delay_seconds: 7_200, - }, - on_expiry: ExecutionWaitExpiryAction::FailTask, - }; request.plan.nodes[0].operation = ExecutionOperation::WaitUntil { wake: ExecutionTemporalTarget::After { delay_seconds: 3_600, @@ -2307,7 +2301,6 @@ fn valid_request() -> CompileExecutionRequest { }, plan: ExecutionPlanDefinition { cancel_policy: ExecutionCancelPolicy::RetainEffects, - input_wait_policy: default_wait_policy(), input_schema: json!({ "type": "object", "required": ["order_id"], @@ -2809,18 +2802,6 @@ fn generous_budget() -> ExecutionBudgetLimit { } } -fn default_wait_policy() -> ExecutionWaitPolicy { - ExecutionWaitPolicy { - expiry: ExecutionTemporalTarget::At { - at: Utc - .with_ymd_and_hms(2026, 7, 20, 0, 0, 0) - .single() - .expect("wait expiry"), - }, - on_expiry: ExecutionWaitExpiryAction::FailTask, - } -} - fn now() -> chrono::DateTime { Utc.with_ymd_and_hms(2026, 7, 13, 12, 0, 0) .single() diff --git a/crates/moa-execution/tests/completion.rs b/crates/moa-execution/tests/completion.rs index b4e24a1a3..fc108a8a3 100644 --- a/crates/moa-execution/tests/completion.rs +++ b/crates/moa-execution/tests/completion.rs @@ -5,8 +5,8 @@ use moa_artifacts::execution_plan::{ CapabilityReference, CompletionCheck, CompletionCheckKind, CoverageRequirement, ExecutionBudgetLimit, ExecutionCancelPolicy, ExecutionCitation, ExecutionDeliverable, ExecutionGoalContract, ExecutionNode, ExecutionOperation, ExecutionPlanDefinition, - ExecutionRequirement, ExecutionTaskOutcome, ExecutionTaskResult, ExecutionTemporalTarget, - ExecutionUsage, ExecutionWaitExpiryAction, ExecutionWaitPolicy, MapTask, RetryPolicy, + ExecutionRequirement, ExecutionTaskOutcome, ExecutionTaskResult, ExecutionUsage, MapTask, + RetryPolicy, }; use moa_execution::{ budget::BudgetLedger, @@ -594,12 +594,6 @@ fn canonical(nodes: Vec) -> CanonicalExecutionPlan { CanonicalExecutionPlan { definition: ExecutionPlanDefinition { cancel_policy: ExecutionCancelPolicy::RetainEffects, - input_wait_policy: ExecutionWaitPolicy { - expiry: ExecutionTemporalTarget::After { - delay_seconds: 3_600, - }, - on_expiry: ExecutionWaitExpiryAction::FailTask, - }, input_schema: json!({ "type": "object" }), output_schema: json!({ "type": "object" }), nodes, diff --git a/crates/moa-execution/tests/execution_db/compensation_attempts_db.rs b/crates/moa-execution/tests/execution_db/compensation_attempts_db.rs index 93208cf1d..ef169a5f9 100644 --- a/crates/moa-execution/tests/execution_db/compensation_attempts_db.rs +++ b/crates/moa-execution/tests/execution_db/compensation_attempts_db.rs @@ -39,6 +39,65 @@ use moa_execution::{ use super::support::*; use std::time::Duration as StdDuration; +#[tokio::test] +async fn compensation_admission_locks_watchdog_capacity_before_run_db() -> TestResult { + // Pins: compensation admission reserves both active-task and watchdog capacity before it + // locks the run. A settlement that already owns ScheduledTriggers can therefore retain the + // run lock without forming a run-to-capacity cycle with compensation admission. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let (run, _) = compensating_run(&repository, scope, tenant_id, &["lock-order"]).await?; + let run_uid = run.run_uid; + let config = ExecutionConfig::default(); + + let mut capacity_holder = pool.begin().await?; + let locked: Vec = sqlx::query_scalar( + "SELECT capacity_bucket_uid FROM moa.execution_capacity_bucket \ + WHERE resource_dimension='scheduled_triggers' \ + AND ((scope_kind='fleet' AND tenant_id IS NULL) \ + OR (scope_kind='tenant' AND tenant_id=$1)) \ + ORDER BY CASE scope_kind WHEN 'fleet' THEN 0 ELSE 1 END FOR UPDATE", + ) + .bind(tenant_id.0) + .fetch_all(&mut *capacity_holder) + .await?; + assert_eq!(locked.len(), 2); + + let admission_repository = repository.clone(); + let admission_config = config.clone(); + let mut admission = tokio::spawn(async move { + admission_repository + .admit_next_compensation_attempt( + scope, + &admission_config, + run_uid, + moa_test_support::fixtures::pg_now(), + ) + .await + }); + assert!( + tokio::time::timeout(StdDuration::from_millis(100), &mut admission) + .await + .is_err(), + "admission must wait for ScheduledTriggers capacity" + ); + sqlx::query("SELECT run_uid FROM moa.execution_run WHERE run_uid=$1 FOR UPDATE NOWAIT") + .bind(run_uid) + .fetch_one(&mut *capacity_holder) + .await?; + capacity_holder.commit().await?; + + assert!(matches!( + tokio::time::timeout(StdDuration::from_secs(5), admission).await???, + CompensationAttemptAdmissionOutcome::Admitted(_) + | CompensationAttemptAdmissionOutcome::Replayed(_) + )); + Ok(()) +} + #[tokio::test] async fn nonterminal_guard_uses_partial_index_for_more_than_2500_tasks_db() -> TestResult { // Pins: compensation admission's forward-work guard remains an indexed existence probe even diff --git a/crates/moa-execution/tests/execution_db/completion_projection_db.rs b/crates/moa-execution/tests/execution_db/completion_projection_db.rs index 7c8088517..6ca7d1231 100644 --- a/crates/moa-execution/tests/execution_db/completion_projection_db.rs +++ b/crates/moa-execution/tests/execution_db/completion_projection_db.rs @@ -1,7 +1,8 @@ //! Bounded persisted completion projection and terminal-evidence contracts. use moa_artifacts::execution_plan::{ - CompletionCheck, CompletionCheckKind, ExecutionNode, ExecutionOperation, + CompletionCheck, CompletionCheckKind, CoverageRequirement, ExecutionNode, ExecutionOperation, + MapTask, }; use moa_execution::{ capability::node_output_hash, @@ -15,7 +16,10 @@ use moa_execution::{ }, terminal::{PendingTerminalAdvanceOutcome, PendingTerminalAdvanceStage}, }, - state::{ExecutionLimitStop, ExecutionTerminalEvidence}, + state::{ + ExecutionLimitStop, ExecutionTerminalCause, ExecutionTerminalEvidence, + ExecutionTerminalReason, + }, }; use super::support::*; @@ -46,6 +50,105 @@ fn output_node_with_dependencies(id: &str, depends_on: &[&str]) -> ExecutionNode } } +fn map_node_with_max_items(id: &str, max_items: u64) -> ExecutionNode { + let mut node = output_node_with_dependencies(id, &[]); + node.operation = ExecutionOperation::Map { + items: json!([]), + item_key: "/id".to_string(), + max_items, + item_output_schema: json!({ "type": "object" }), + task: MapTask::Agent { + instructions: "process one bounded item".to_string(), + skill_refs: Vec::new(), + capability_refs: Vec::new(), + max_turns: 1, + }, + }; + node +} + +#[tokio::test] +async fn completion_rejects_resolved_coverage_above_map_max_items_db() -> TestResult { + // Pins: a dynamic coverage universe cannot allocate or hash more expected keys than the + // referenced map was allowed to materialize, even when persisted state reaches completion. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let mut candidate = new_run( + tenant_id, + None, + "coverage-max-items", + ExecutionRunStatus::Queued, + budget(1), + ); + candidate.input = json!({ + "expected": [ + { "id": "company-a" }, + { "id": "company-b" } + ] + }); + candidate.plan.definition.nodes = vec![map_node_with_max_items("companies", 1)]; + candidate.goal.coverage = vec![CoverageRequirement { + id: "all_companies".to_string(), + description: "Cover every requested company".to_string(), + map_node_id: "companies".to_string(), + expected_items: json!({ "$ref": "$.input.expected" }), + require_all: true, + }]; + let run = create_run(&repository, scope, candidate).await?; + + sqlx::query( + "UPDATE moa.execution_node_state SET node_status='completed', \ + materialization_complete=TRUE,aggregate_complete=TRUE,aggregate_output='{}'::JSONB, \ + updated_at=NOW() WHERE run_uid=$1 AND node_id='companies'", + ) + .bind(run.run_uid) + .execute(&pool) + .await?; + set_run_status_path(&pool, run.run_uid, &["running"]).await?; + let current = repository + .load_run(scope, run.run_uid) + .await? + .expect("coverage run remains visible"); + let RunControllerClaimOutcome::Claimed(current) = repository + .claim_controller_wake( + scope, + run.run_uid, + current.controller_generation, + current.wake_epoch, + ) + .await? + else { + panic!("coverage completion wake must be claimable"); + }; + + let error = repository + .advance_completion_projection( + scope, + &ExecutionConfig::default(), + CompletionAdvanceRequest { + run_uid: run.run_uid, + controller_generation: current.controller_generation, + wake_epoch: current.wake_epoch, + page_size: 1, + now: Utc::now(), + }, + ) + .await + .expect_err("coverage above map max_items must fail before evidence is built"); + let message = match error { + moa_execution::Error::InvalidRepositoryData { message } => message, + other => panic!("unexpected coverage bound error: {other:?}"), + }; + assert_eq!( + message, + "coverage all_companies expected item count 2 exceeds map companies max_items 1" + ); + Ok(()) +} + #[tokio::test] async fn failed_and_unknown_outcome_tasks_cancel_transitive_unmaterialized_dependents_db() -> TestResult { @@ -58,18 +161,19 @@ async fn failed_and_unknown_outcome_tasks_cancel_transitive_unmaterialized_depen let scope = ExecutionScope::Tenant { tenant_id }; let config = ExecutionConfig::default(); - for (case, outcome, expected_task_status) in [ + for (case, outcome, expected_task_status, expected_failure_class) in [ ( - "failed", + "invalid-output", ExecutionTaskOutcome { schema_version: 1, usage: usage(1), result: ExecutionTaskResult::Failed { - class: ExecutionFailureClass::Terminal, - message: "source failed".to_string(), + class: ExecutionFailureClass::InvalidOutput, + message: "source output did not match its schema".to_string(), }, }, ExecutionTaskStatus::Failed, + ExecutionFailureClass::InvalidOutput, ), ( "unknown-outcome", @@ -81,6 +185,7 @@ async fn failed_and_unknown_outcome_tasks_cancel_transitive_unmaterialized_depen }, }, ExecutionTaskStatus::UnknownOutcome, + ExecutionFailureClass::Terminal, ), ] { let mut candidate = new_run( @@ -98,11 +203,6 @@ async fn failed_and_unknown_outcome_tasks_cancel_transitive_unmaterialized_depen candidate.plan.estimate.tasks = 3; let run = create_run(&repository, scope, candidate).await?; let source = logical_task(run.run_uid, "source", case, estimate(1)); - assert!( - repository - .initialize_scheduler_state(scope, run.run_uid) - .await? - ); assert!(matches!( repository .materialize_ready_page( @@ -231,6 +331,22 @@ async fn failed_and_unknown_outcome_tasks_cancel_transitive_unmaterialized_depen ExecutionRunStatus::Failed, "{case}" ); + assert_eq!( + pending_terminal.reason, + ExecutionTerminalReason::TaskFailure, + "{case}" + ); + assert_eq!( + pending_terminal.terminal_evidence, + ExecutionTerminalEvidence { + cause: ExecutionTerminalCause::TaskFailure { + class: expected_failure_class.clone(), + }, + satisfied_requirement_count: 0, + requirement_count: 0, + }, + "{case}" + ); break; } other => panic!("{case} completion projection did not advance: {other:?}"), @@ -464,7 +580,10 @@ async fn replan_stop_completion_pages_rebind_exact_wake_without_duplicate_verifi ExecutionRunStatus::Queued, budget(20), ); - candidate.plan.definition.nodes = vec![output_node()]; + candidate.plan.definition.nodes = vec![ + output_node(), + output_node_with_dependencies("unmaterialized_descendant", &["output"]), + ]; candidate.goal.completion_checks = vec![CompletionCheck { id: "semantic".to_string(), description: "Verifier must not be materialized after ReplanStop".to_string(), @@ -561,25 +680,44 @@ async fn replan_stop_completion_pages_rebind_exact_wake_without_duplicate_verifi .execute(&pool) .await?; let amendment_hash: ExecutionHash = "a".repeat(64).parse()?; + let stop_request = NewExecutionReplanStopIntent { + run_uid: run.run_uid, + session_id: run.session_id, + base_plan_revision: run.plan_revision, + origin_task_id: origin, + task_generation: 1, + amendment_hash, + stop_reason: ReplanStopReason::RepeatedFailure, + detail: Some("same failure exhausted replan policy".to_string()), + }; let ReplanStopIntentWriteOutcome::Applied(queued) = repository - .request_replan_stop( - scope, - &ExecutionConfig::default(), - NewExecutionReplanStopIntent { - run_uid: run.run_uid, - session_id: run.session_id, - base_plan_revision: run.plan_revision, - origin_task_id: origin, - task_generation: 1, - amendment_hash, - stop_reason: ReplanStopReason::RepeatedFailure, - detail: Some("same failure exhausted replan policy".to_string()), - }, - ) - .await? + .request_replan_stop(scope, &ExecutionConfig::default(), stop_request.clone()) + .await + .expect("fresh ReplanStop request must remain repository-valid") else { panic!("fresh ReplanStop intent must persist with one activation"); }; + let descendant: (String, bool, bool, i64, i64) = sqlx::query_as( + "SELECT node_status,materialization_complete,aggregate_complete, \ + remaining_dependency_count,total_task_count \ + FROM moa.execution_node_state WHERE run_uid=$1 AND node_id='unmaterialized_descendant'", + ) + .bind(run.run_uid) + .fetch_one(&pool) + .await + .expect("unmaterialized descendant node state must remain present"); + assert_eq!( + descendant, + ("cancelled".to_string(), true, true, 0, 0), + "ReplanStop must close every unmaterialized descendant before its controller wake" + ); + let ReplanStopIntentWriteOutcome::Replayed(replayed) = repository + .request_replan_stop(scope, &ExecutionConfig::default(), stop_request) + .await? + else { + panic!("exact ReplanStop intent replay must remain idempotent"); + }; + assert_eq!(replayed.wake_epoch, queued.wake_epoch); let mut wake_epoch = queued.wake_epoch; let mut source_progress_at = None; let mut page_count = 0_u32; @@ -684,9 +822,9 @@ async fn replan_stop_completion_pages_rebind_exact_wake_without_duplicate_verifi } other => panic!("unexpected ReplanStop completion outcome: {other:?}"), } - assert!(page_count <= 4, "ReplanStop cursor failed to make progress"); + assert!(page_count <= 5, "ReplanStop cursor failed to make progress"); } - assert_eq!(page_count, 4, "three task pages plus one node page"); + assert_eq!(page_count, 5, "three task pages plus two node pages"); let verifier_count = sqlx::query_scalar::<_, i64>( "SELECT COUNT(*) FROM moa.execution_task WHERE run_uid=$1 AND node_id LIKE '@check/%'", ) diff --git a/crates/moa-execution/tests/execution_db/execution_capacity_db.rs b/crates/moa-execution/tests/execution_db/execution_capacity_db.rs index 504f9a08a..816519ed3 100644 --- a/crates/moa-execution/tests/execution_db/execution_capacity_db.rs +++ b/crates/moa-execution/tests/execution_db/execution_capacity_db.rs @@ -1,5 +1,7 @@ //! Fleet-owned weighted-fair task admission contracts. +use std::time::Duration as StdDuration; + use chrono::DateTime; use moa_artifacts::execution_plan::{ExecutionNode, ExecutionOperation}; use moa_config::ExecutionConfig; @@ -529,3 +531,97 @@ async fn no_work_admission_does_not_rewrite_unchanged_capacity_bucket_db() -> Te ); Ok(()) } + +#[tokio::test] +async fn ready_admission_waits_for_scheduled_capacity_before_locking_run_db() -> TestResult { + // Pins: task admission waits for ScheduledTriggers capacity before locking the ready run, so + // a pre-released task settlement that already owns ScheduledTriggers can acquire that run + // without forming a ScheduledTriggers-to-run versus run-to-ScheduledTriggers deadlock. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let run_uid = ready_run(&repository, tenant_id, "admission-scheduled-before-run", 1).await?; + let config = ExecutionConfig::default(); + + let mut scheduled_holder = ScopedConn::begin_control_plane(&pool).await?; + scheduled_holder.assume_app_role().await?; + let holder_pid: i32 = sqlx::query_scalar("SELECT pg_backend_pid()") + .fetch_one(scheduled_holder.as_mut()) + .await?; + let locked_buckets: Vec<(String, Option)> = sqlx::query_as( + "SELECT scope_kind,tenant_id FROM moa.execution_capacity_bucket \ + WHERE resource_dimension='scheduled_triggers' \ + AND (scope_kind='fleet' OR (scope_kind='tenant' AND tenant_id=$1)) \ + ORDER BY CASE scope_kind WHEN 'fleet' THEN 0 ELSE 1 END FOR UPDATE", + ) + .bind(tenant_id.0) + .fetch_all(scheduled_holder.as_mut()) + .await?; + assert_eq!( + locked_buckets, + vec![ + ("fleet".to_string(), None), + ("tenant".to_string(), Some(tenant_id.0)), + ], + "the fixture must hold both watchdog-capacity rows" + ); + + let admission_repository = repository.clone(); + let mut admission = tokio::spawn(async move { + admission_repository + .admit_ready_attempts(&config, 1, Utc::now()) + .await + }); + let blocked_pid = tokio::time::timeout(StdDuration::from_secs(5), async { + loop { + let blocked_pid: Option = sqlx::query_scalar( + "SELECT min(activity.pid) FROM pg_stat_activity AS activity \ + WHERE activity.datname=current_database() \ + AND $1 = ANY(pg_blocking_pids(activity.pid))", + ) + .bind(holder_pid) + .fetch_one(&pool) + .await?; + if let Some(blocked_pid) = blocked_pid { + return Ok::(blocked_pid); + } + tokio::task::yield_now().await; + } + }) + .await + .expect("admission must reach the held ScheduledTriggers lock")?; + assert_ne!(blocked_pid, holder_pid); + assert!( + tokio::time::timeout(StdDuration::from_millis(100), &mut admission) + .await + .is_err(), + "admission must remain blocked while ScheduledTriggers capacity is held" + ); + + let locked_run_uid = match sqlx::query_scalar::<_, Uuid>( + "SELECT run_uid FROM moa.execution_run WHERE run_uid=$1 FOR UPDATE NOWAIT", + ) + .bind(run_uid) + .fetch_one(scheduled_holder.as_mut()) + .await + { + Ok(locked_run_uid) => locked_run_uid, + Err(error) => { + scheduled_holder.rollback().await?; + let _ = tokio::time::timeout(StdDuration::from_secs(5), &mut admission).await; + panic!( + "admission locked the run before ScheduledTriggers capacity; NOWAIT failed: {error}" + ); + } + }; + assert_eq!(locked_run_uid, run_uid); + scheduled_holder.commit().await?; + + let admitted = tokio::time::timeout(StdDuration::from_secs(5), admission) + .await + .expect("admission must complete after ScheduledTriggers capacity is released")??; + assert_eq!(admitted.admitted.len(), 1); + assert_eq!(admitted.admitted[0].run_uid, run_uid); + Ok(()) +} diff --git a/crates/moa-execution/tests/execution_db/long_horizon_state_db.rs b/crates/moa-execution/tests/execution_db/long_horizon_state_db.rs index 9abf93cff..a324e4926 100644 --- a/crates/moa-execution/tests/execution_db/long_horizon_state_db.rs +++ b/crates/moa-execution/tests/execution_db/long_horizon_state_db.rs @@ -1,15 +1,22 @@ //! Long-horizon execution state, identity, RLS, and generation-fence contracts. use moa_artifacts::execution_plan::{ExecutionNode, ExecutionOperation}; +use moa_execution::repository::outbox::ExecutionDispatchKind; use moa_execution::repository::ready::{ReadyMaterializationOutcome, ReadyMaterializationRequest}; use moa_execution::repository::task::{ ActiveAttemptLiveness, NewTaskAttemptCheckpoint, TaskAttemptCheckpointKind, TaskAttemptContinuationYieldOutcome, TaskAttemptFence, TaskAttemptProgressOutcome, - TaskAttemptReleaseClaimOutcome, TaskAttemptStartOutcome, classify_active_attempt_liveness, + TaskAttemptReleaseClaimOutcome, TaskAttemptSettlementOutcome, TaskAttemptStartOutcome, + classify_active_attempt_liveness, +}; +use moa_execution::repository::terminal::{ + PendingTerminalAdvanceOutcome, PendingTerminalAdvanceStage, }; use moa_execution::repository::trigger::{ ExecutionTriggerNoOp, ExecutionWatchdogDeferOutcome, ExecutionWatchdogTriggerOutcome, }; +use moa_execution::state::ExecutionTerminalEvidence; +use moa_execution::wire::{ExecutionAttemptCancelReason, ExecutionTaskAttemptCancelRequest}; use super::support::*; @@ -360,6 +367,431 @@ async fn start_admitted_attempts( Ok((run.run_uid, started)) } +#[tokio::test] +async fn terminal_drain_recovers_orphaned_cancelling_task_once_db() -> TestResult { + // Pins: if an attempt workflow fails after claiming its release boundary but before it can + // settle, explicit terminal drain must create the exact cancellation dispatch. Replaying the + // same terminal fence must not duplicate either that outbox row or active-capacity ownership. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let config = ExecutionConfig::default(); + let (run_uid, started) = start_admitted_attempts( + &repository, + tenant_id, + "orphaned-cancelling-terminal-drain", + &config, + &["orphaned"], + ) + .await?; + let [(fence, _)] = started[..] else { + panic!("one exact task attempt must start"); + }; + let active = listed_task(&repository, scope, run_uid, fence.task_id).await?; + assert!(matches!( + repository + .begin_task_attempt_release( + fence, + active.generation, + "task_outcome", + moa_test_support::fixtures::pg_now(), + ) + .await?, + TaskAttemptReleaseClaimOutcome::Applied(_) + )); + let orphaned = listed_task(&repository, scope, run_uid, fence.task_id).await?; + assert_eq!(orphaned.attempt_state, ExecutionAttemptState::Cancelling); + + let current = repository + .load_run(scope, run_uid) + .await? + .expect("orphaned attempt run remains visible"); + let pending = PendingExecutionTerminal { + status: ExecutionRunStatus::Cancelled, + reason: ExecutionTerminalReason::Cancelled, + terminal_evidence: ExecutionTerminalEvidence { + cause: ExecutionTerminalCause::Cancellation, + satisfied_requirement_count: 0, + requirement_count: 1, + }, + completion_check_results: Vec::new(), + terminal_gaps: Vec::new(), + output: None, + cancellation_reason: Some("operator cancelled orphaned attempt".to_string()), + }; + let PendingTerminalAdvanceOutcome::Applied(first) = repository + .fence_completion_terminal_and_enqueue_settlement( + &config, + scope, + run_uid, + current.controller_generation, + current.wake_epoch, + pending.clone(), + moa_test_support::fixtures::pg_now(), + 1, + ) + .await? + else { + panic!("terminal drain must recover the orphaned cancelling attempt"); + }; + assert_eq!(first.stage, PendingTerminalAdvanceStage::Draining); + assert_eq!(first.cancellation_dispatches.len(), 1); + let cancellation_dispatch = &first.cancellation_dispatches[0]; + assert_eq!( + cancellation_dispatch.kind, + ExecutionDispatchKind::TaskAttemptCancel + ); + let cancellation: ExecutionTaskAttemptCancelRequest = + serde_json::from_value(cancellation_dispatch.payload.clone())?; + assert_eq!(cancellation.active_dispatch_uid, fence.dispatch_uid); + assert_eq!( + cancellation.capacity_reservation_uid, + fence.capacity_reservation_uid + ); + assert_eq!( + cancellation.watchdog_trigger_uid, + fence.watchdog_trigger_uid + ); + assert_eq!( + cancellation.reason, + ExecutionAttemptCancelReason::RunTerminal + ); + let cancellation_projection = repository + .load_cancellation_projection_for_session( + scope, + run_uid, + current.session_id, + config.max_in_flight_tasks, + ) + .await? + .expect("the exact session-owned cancellation projection remains visible"); + assert_eq!( + cancellation_projection.task_cancellation_dispatches, + vec![cancellation_dispatch.clone()], + "crash replay must reconstruct only the current cancelling attempt receipt" + ); + + let PendingTerminalAdvanceOutcome::Replayed(replayed) = repository + .fence_completion_terminal_and_enqueue_settlement( + &config, + scope, + run_uid, + current.controller_generation, + current.wake_epoch, + pending.clone(), + moa_test_support::fixtures::pg_now(), + 1, + ) + .await? + else { + panic!("the exact terminal fence must replay idempotently"); + }; + assert!(replayed.work_remaining); + assert_eq!( + replayed.cancellation_dispatches, + vec![cancellation_dispatch.clone()], + "same-wake replay must carry the current owner fence without duplicating it" + ); + + let cancellation_rows: i64 = sqlx::query_scalar( + "SELECT count(*) FROM moa.execution_dispatch_outbox \ + WHERE run_uid=$1 AND task_id=$2 AND dispatch_kind='task_attempt_cancel'", + ) + .bind(run_uid) + .bind(fence.task_id.as_uuid()) + .fetch_one(&pool) + .await?; + assert_eq!(cancellation_rows, 1); + let capacity_rows: Vec<(Uuid, String)> = sqlx::query_as( + "SELECT reservation_uid,state FROM moa.execution_capacity_reservation \ + WHERE run_uid=$1 AND task_id=$2 AND resource_dimension='active_tasks'", + ) + .bind(run_uid) + .bind(fence.task_id.as_uuid()) + .fetch_all(&pool) + .await?; + assert_eq!( + capacity_rows, + vec![(fence.capacity_reservation_uid, "reconciling".to_string())] + ); + Ok(()) +} + +#[tokio::test] +async fn external_cancellation_claims_fresh_wake_for_parked_replan_once_db() -> TestResult { + // Pins: cancelling a storage-only WaitingReplan run whose current wake is already + // acknowledged must claim one internal terminal-drain wake, install the cancellation, and + // replay without dispatching a redundant controller activation. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let config = ExecutionConfig::default(); + let (run_uid, started) = start_admitted_attempts( + &repository, + tenant_id, + "parked-replan-external-cancellation", + &config, + &["replan"], + ) + .await?; + let [(fence, _)] = started[..] else { + panic!("one exact task attempt must start"); + }; + let TaskAttemptSettlementOutcome::Applied { run: waiting, task } = repository + .settle_task_attempt(&config, fence, needs_replan(1), None, Utc::now()) + .await? + else { + panic!("task must enter its durable replan wait"); + }; + assert_eq!(waiting.status, ExecutionRunStatus::WaitingReplan); + assert_eq!(task.status, ExecutionTaskStatus::WaitingReplan); + + let RunControllerClaimOutcome::Claimed(claimed) = repository + .claim_controller_wake( + scope, + run_uid, + waiting.controller_generation, + waiting.wake_epoch, + ) + .await? + else { + panic!("the replan wait wake must be claimable"); + }; + let RunControllerCompletionOutcome::Applied { + run: parked, + continuation, + } = repository + .complete_controller_wake( + scope, + &config, + run_uid, + RunControllerCompletionRequest { + controller_generation: claimed.controller_generation, + wake_epoch: claimed.wake_epoch, + checkpoint: ExecutionRunActivationCheckpoint { + status: ExecutionRunStatus::WaitingReplan, + activation_state: ExecutionActivationState::Idle, + next_wake_at: claimed.next_wake_at, + waiting_since: claimed.waiting_since, + ready_task_count: claimed.ready_task_count, + active_task_count: claimed.active_task_count, + }, + continuation_payload: None, + continuation_not_before_at: Utc::now(), + }, + ) + .await? + else { + panic!("the replan wait must become storage-only"); + }; + assert!(continuation.is_none()); + assert_eq!(parked.status, ExecutionRunStatus::WaitingReplan); + assert_eq!(parked.activation_state, ExecutionActivationState::Idle); + assert_eq!(parked.wake_epoch, parked.processed_wake_epoch); + let parked_wake_epoch = parked.wake_epoch; + + let pending = PendingExecutionTerminal { + status: ExecutionRunStatus::Cancelled, + reason: ExecutionTerminalReason::Cancelled, + terminal_evidence: ExecutionTerminalEvidence { + cause: ExecutionTerminalCause::Cancellation, + satisfied_requirement_count: 0, + requirement_count: 0, + }, + completion_check_results: Vec::new(), + terminal_gaps: Vec::new(), + output: None, + cancellation_reason: Some("operator cancelled parked replan".to_string()), + }; + let PendingTerminalAdvanceOutcome::Applied(first) = repository + .fence_cancellation_terminal_and_enqueue_settlement( + &config, + scope, + run_uid, + parked.controller_generation, + parked_wake_epoch, + pending.clone(), + moa_test_support::fixtures::pg_now(), + 1, + ) + .await? + else { + panic!("first external cancellation must install its terminal fence"); + }; + assert_eq!(first.run.status, ExecutionRunStatus::WaitingReplan); + assert_eq!(first.run.processed_wake_epoch, parked_wake_epoch + 1); + assert_eq!(first.run.pending_terminal, Some(pending.clone())); + assert_eq!(first.settled_task_count, 1); + let continuation = first + .continuation + .as_ref() + .expect("bounded trigger cleanup must own one continuation"); + assert_eq!(continuation.kind, ExecutionDispatchKind::RunActivation); + assert_eq!(continuation.wake_epoch, Some(first.run.wake_epoch)); + assert!(first.run.wake_epoch > first.run.processed_wake_epoch); + assert_eq!( + listed_task(&repository, scope, run_uid, fence.task_id) + .await? + .status, + ExecutionTaskStatus::Cancelled + ); + + let PendingTerminalAdvanceOutcome::Replayed(replayed) = repository + .fence_cancellation_terminal_and_enqueue_settlement( + &config, + scope, + run_uid, + parked.controller_generation, + parked_wake_epoch, + pending.clone(), + moa_test_support::fixtures::pg_now(), + 1, + ) + .await? + else { + panic!("the exact external cancellation must replay"); + }; + assert_eq!(replayed.run.status, ExecutionRunStatus::WaitingReplan); + assert_eq!(replayed.run.pending_terminal, Some(pending.clone())); + assert_eq!(replayed.run.wake_epoch, first.run.wake_epoch); + assert_eq!(replayed.settled_task_count, 0); + let internal_wake_activation_count: i64 = sqlx::query_scalar( + "SELECT count(*) FROM moa.execution_dispatch_outbox \ + WHERE run_uid=$1 AND dispatch_kind='run_activation' AND wake_epoch=$2", + ) + .bind(run_uid) + .bind(i64::try_from(parked_wake_epoch + 1)?) + .fetch_one(&pool) + .await?; + assert_eq!(internal_wake_activation_count, 0); + + let PendingTerminalAdvanceOutcome::Applied(terminal) = repository + .advance_pending_terminal_settlement( + &config, + scope, + run_uid, + parked.controller_generation, + first.run.wake_epoch, + moa_test_support::fixtures::pg_now(), + 1, + ) + .await? + else { + panic!("the bounded cleanup continuation must finalize cancellation"); + }; + assert_eq!(terminal.run.status, ExecutionRunStatus::Cancelled); + assert_eq!( + terminal.run.terminal_evidence, + Some(pending.terminal_evidence) + ); + assert_eq!( + terminal.run.cancellation_reason, + pending.cancellation_reason + ); + assert!(terminal.run.pending_terminal.is_none()); + Ok(()) +} + +#[tokio::test] +async fn terminal_drain_prioritizes_a_current_cancelling_owner_over_storage_work_db() -> TestResult +{ + // Pins: a bounded first terminal page must fence every current provider owner before it + // spends page capacity settling an idle waiting task. The lower task ID is deliberately + // parked so task-ID ordering alone would strand the higher cancelling owner. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let repository = ExecutionRepository::new(test_db.store().pool().clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let config = ExecutionConfig { + max_in_flight_tasks: 3, + maximum_activation_steps: 1, + dispatch_batch_size: 3, + ..ExecutionConfig::default() + }; + let (run_uid, mut started) = start_admitted_attempts( + &repository, + tenant_id, + "current-owner-before-storage-terminal-drain", + &config, + &["first", "second"], + ) + .await?; + started.sort_by_key(|(fence, _)| fence.task_id); + let storage_fence = started[0].0; + let cancelling_fence = started[1].0; + + let TaskAttemptSettlementOutcome::Applied { task, .. } = repository + .settle_task_attempt(&config, storage_fence, needs_input(1), None, Utc::now()) + .await? + else { + panic!("lower-ID task must park on human input"); + }; + assert_eq!(task.status, ExecutionTaskStatus::WaitingInput); + let active = listed_task(&repository, scope, run_uid, cancelling_fence.task_id).await?; + assert!(matches!( + repository + .begin_task_attempt_release( + cancelling_fence, + active.generation, + "task_outcome", + moa_test_support::fixtures::pg_now(), + ) + .await?, + TaskAttemptReleaseClaimOutcome::Applied(_) + )); + + let current = repository + .load_run(scope, run_uid) + .await? + .expect("terminal-drain run remains visible"); + let pending = PendingExecutionTerminal { + status: ExecutionRunStatus::Cancelled, + reason: ExecutionTerminalReason::Cancelled, + terminal_evidence: ExecutionTerminalEvidence { + cause: ExecutionTerminalCause::Cancellation, + satisfied_requirement_count: 0, + requirement_count: 1, + }, + completion_check_results: Vec::new(), + terminal_gaps: Vec::new(), + output: None, + cancellation_reason: Some("operator cancelled both tasks".to_string()), + }; + let PendingTerminalAdvanceOutcome::Applied(commit) = repository + .fence_completion_terminal_and_enqueue_settlement( + &config, + scope, + run_uid, + current.controller_generation, + current.wake_epoch, + pending, + moa_test_support::fixtures::pg_now(), + 1, + ) + .await? + else { + panic!("first terminal page must apply"); + }; + assert_eq!(commit.settled_task_count, 0); + assert_eq!(commit.cancellation_dispatches.len(), 1); + let cancellation: ExecutionTaskAttemptCancelRequest = + serde_json::from_value(commit.cancellation_dispatches[0].payload.clone())?; + assert_eq!(cancellation.task_id, cancelling_fence.task_id); + assert_eq!( + cancellation.active_dispatch_uid, + cancelling_fence.dispatch_uid + ); + assert_eq!( + cancellation.reason, + ExecutionAttemptCancelReason::RunTerminal + ); + Ok(()) +} + #[tokio::test] async fn attempt_heartbeat_keeps_a_progressing_attempt_live_while_a_wedged_one_stalls_db() -> TestResult { diff --git a/crates/moa-execution/tests/execution_db/planning_and_audit_db.rs b/crates/moa-execution/tests/execution_db/planning_and_audit_db.rs index c1d65e241..1ab2b843d 100644 --- a/crates/moa-execution/tests/execution_db/planning_and_audit_db.rs +++ b/crates/moa-execution/tests/execution_db/planning_and_audit_db.rs @@ -1,6 +1,8 @@ //! Planning-context, normalized-audit, confirmation, and amendment persistence contracts. use super::support::*; +use moa_artifacts::execution_plan::{ExecutionNode, ExecutionOperation}; +use moa_execution::capability::node_output_hash; use moa_execution::repository::planning_budget::{ AmendmentPlanningCallReconcileOutcome, AmendmentPlanningCallReconcileRequest, AmendmentPlanningCallReservation, AmendmentPlanningCallReservationOutcome, @@ -487,6 +489,34 @@ async fn normalized_planning_audits_return_first_measurements_and_conflict_db() assert_eq!(planner_evidence.usage.input_tokens_cache_read, 5); assert_eq!(planner_evidence.usage.output_tokens, 8); assert_eq!(planner_evidence.cost_microusd, 29); + let reconstructed = moa_test_support::execution_audits::load_execution_planning_audits( + test_db.database_url(), + session_id, + ) + .await?; + assert_eq!( + reconstructed.len(), + 2, + "route and planner audit must both reconstruct" + ); + let ExecutionPlanningAuditPayload::PlannerCall { + usage, + cost_microusd, + .. + } = &reconstructed[1].payload + else { + panic!("second reconstructed audit must be the planner call"); + }; + assert_eq!( + *usage, + ExecutionRouteUsage { + input_tokens_uncached: 21, + input_tokens_cache_write: 3, + input_tokens_cache_read: 5, + output_tokens: 8, + } + ); + assert_eq!(*cost_microusd, 29); let mut planner_retry = planner.clone(); let ExecutionPlanningAuditPayload::PlannerCall { duration_micros, @@ -871,23 +901,63 @@ async fn confirmation_is_plan_hash_bound_and_exact_replay_only_db() -> TestResul #[tokio::test] async fn amendment_append_is_revision_fenced_and_preserves_initial_plan_db() -> TestResult { - // Pins: accepted replans preserve confirmation identity, append history, and supersede one waiting task. + // Pins: accepted replans atomically supersede their wait and replace only mutable node state; + // completed dependencies survive while replacement nodes become immediately actionable. let test_db = moa_test_support::postgres::bootstrap_test_db().await?; let repository = ExecutionRepository::new(test_db.store().pool().clone()); let tenant_id = TenantId::new(); let scope = ExecutionScope::Tenant { tenant_id }; - let created = create_run( - &repository, - scope, - new_run( - tenant_id, - None, - "amendment", - ExecutionRunStatus::AwaitingConfirmation, - budget(10), - ), - ) - .await?; + let retry = RetryPolicy { + max_attempts: 1, + initial_backoff_ms: 0, + max_backoff_ms: 0, + }; + let node = |id: &str, depends_on: Vec| ExecutionNode { + id: id.to_string(), + requirement_ids: vec!["req".to_string()], + depends_on, + when: None, + input: json!({}), + output_schema: json!({"type": "object"}), + operation: ExecutionOperation::Output { value: json!({}) }, + compensation: None, + retry: retry.clone(), + budget: None, + }; + let preserved = node("preserved", Vec::new()); + let review_wait_policy = ExecutionWaitPolicy { + expiry: ExecutionTemporalTarget::After { delay_seconds: 600 }, + on_expiry: ExecutionWaitExpiryAction::FailTask, + }; + let review = ExecutionNode { + id: "preserved_review".to_string(), + requirement_ids: vec!["req".to_string()], + depends_on: Vec::new(), + when: None, + input: json!({}), + output_schema: json!({"type": "object"}), + operation: ExecutionOperation::Review { + prompt: "approve preserved work".to_string(), + wait_policy: review_wait_policy.clone(), + }, + compensation: None, + retry: retry.clone(), + budget: None, + }; + let replan = node("replan", vec!["preserved".to_string()]); + let old_terminal = node("old_terminal", vec!["replan".to_string()]); + let replacement = node("replacement", vec!["preserved".to_string()]); + let new_terminal = node("new_terminal", vec!["replacement".to_string()]); + let mut candidate = new_run( + tenant_id, + None, + "amendment", + ExecutionRunStatus::AwaitingConfirmation, + budget(10), + ); + candidate.plan.definition.nodes = vec![preserved.clone(), review.clone(), replan, old_terminal]; + candidate.plan.estimate.tasks = 4; + let created = create_run(&repository, scope, candidate).await?; let ConfirmationOutcome::Confirmed(run) = repository .confirm_run( scope, @@ -899,9 +969,19 @@ async fn amendment_append_is_revision_fenced_and_preserves_initial_plan_db() -> else { panic!("amendment fixture must begin from a confirmed plan"); }; + let mut review_task = logical_task(run.run_uid, "preserved_review", "", estimate(1)); + review_task.kind = LogicalTaskKind::Review { + prompt: "approve preserved work".to_string(), + wait_policy: review_wait_policy, + }; let task = logical_task(run.run_uid, "replan", "", estimate(1)); repository - .materialize_tasks(scope, run.run_uid, 1, vec![task.clone()]) + .materialize_tasks( + scope, + run.run_uid, + 1, + vec![review_task.clone(), task.clone()], + ) .await?; reserve_and_start(&repository, scope, run.run_uid, task.task_id).await?; assert!(matches!( @@ -910,6 +990,70 @@ async fn amendment_append_is_revision_fenced_and_preserves_initial_plan_db() -> .await?, TaskOutcomeWrite::Applied { .. } )); + set_task_status_path( + test_db.store().pool(), + review_task.task_id, + task_setup_path("waiting_review"), + ) + .await?; + let review_waiting_since = moa_test_support::fixtures::pg_now() - Duration::minutes(1); + let review_expiry_at = review_waiting_since + Duration::minutes(10); + sqlx::query( + "UPDATE moa.execution_task SET attempt_state='waiting',waiting_since=$2,updated_at=NOW() \ + WHERE task_id=$1", + ) + .bind(review_task.task_id.as_uuid()) + .bind(review_waiting_since) + .execute(test_db.store().pool()) + .await?; + let preserved_output = json!({"result": "kept"}); + sqlx::query( + "UPDATE moa.execution_node_state SET node_status='completed', \ + materialization_complete=TRUE,aggregate_complete=TRUE,aggregate_output=$3, \ + aggregate_output_hash=$4 WHERE run_uid=$1 AND node_id=$2", + ) + .bind(run.run_uid) + .bind("preserved") + .bind(&preserved_output) + .bind(node_output_hash(&preserved_output)?.to_string()) + .execute(test_db.store().pool()) + .await?; + sqlx::query( + "UPDATE moa.execution_node_state SET node_status='waiting', \ + materialization_complete=TRUE,total_task_count=1,waiting_task_count=1 \ + WHERE run_uid=$1 AND node_id='replan'", + ) + .bind(run.run_uid) + .execute(test_db.store().pool()) + .await?; + sqlx::query( + "UPDATE moa.execution_node_state SET node_status='waiting', \ + materialization_complete=TRUE,total_task_count=1,waiting_task_count=1 \ + WHERE run_uid=$1 AND node_id='preserved_review'", + ) + .bind(run.run_uid) + .execute(test_db.store().pool()) + .await?; + let preserved_review_reason = moa_execution::state::WaitingReason::Review { + task_id: review_task.task_id, + prompt: "approve preserved work".to_string(), + wait_policy: ExecutionWaitPolicy { + expiry: ExecutionTemporalTarget::At { + at: review_expiry_at, + }, + on_expiry: ExecutionWaitExpiryAction::FailTask, + }, + }; + sqlx::query( + "UPDATE moa.execution_run SET waiting_task_count=2,waiting_review_task_count=1, \ + waiting_replan_task_count=1,waiting_since=$2,waiting_reasons=$3, \ + waiting_reasons_truncated=TRUE WHERE run_uid=$1", + ) + .bind(run.run_uid) + .bind(review_waiting_since) + .bind(serde_json::to_value([preserved_review_reason.clone()])?) + .execute(test_db.store().pool()) + .await?; let amendment = PlanAmendment { base_plan_revision: 1, @@ -917,10 +1061,13 @@ async fn amendment_append_is_revision_fenced_and_preserves_initial_plan_db() -> evidence: json!({ "source": "unavailable" }), operations: Vec::new(), }; + let mut replacement_plan = canonical_plan(2); + replacement_plan.definition.nodes = vec![preserved, review, replacement, new_terminal]; + replacement_plan.estimate.tasks = 3; let validated = ValidatedAmendment { amendment_hash: amendment_hash(&amendment)?, amendment, - active_plan: canonical_plan(2), + active_plan: replacement_plan.clone(), requirement_mapping: [("replacement".to_string(), vec!["req".to_string()])] .into_iter() .collect(), @@ -939,6 +1086,40 @@ async fn amendment_append_is_revision_fenced_and_preserves_initial_plan_db() -> .await?, AmendmentWrite::Conflict ); + sqlx::query( + "UPDATE moa.execution_run SET waiting_task_count=3,waiting_replan_task_count=2 \ + WHERE run_uid=$1", + ) + .bind(run.run_uid) + .execute(test_db.store().pool()) + .await?; + assert_eq!( + repository + .append_amendment( + scope, + &ExecutionConfig::default(), + run.run_uid, + 1, + validated.clone(), + ) + .await?, + AmendmentWrite::Conflict, + "one planner amendment must supersede exactly one WaitingReplan task" + ); + assert_eq!( + listed_task(&repository, scope, run.run_uid, task.task_id) + .await? + .status, + ExecutionTaskStatus::WaitingReplan, + "the exact-count conflict must roll back task supersession" + ); + sqlx::query( + "UPDATE moa.execution_run SET waiting_task_count=2,waiting_replan_task_count=1 \ + WHERE run_uid=$1", + ) + .bind(run.run_uid) + .execute(test_db.store().pool()) + .await?; let AmendmentWrite::Applied(amended) = repository .append_amendment( scope, @@ -951,7 +1132,20 @@ async fn amendment_append_is_revision_fenced_and_preserves_initial_plan_db() -> else { panic!("expected applied amendment"); }; + let assert_preserved_review_wait = |projection: &ExecutionRunRecord| { + assert_eq!(projection.status, ExecutionRunStatus::WaitingReview); + assert_eq!(projection.waiting_task_count, 1); + assert_eq!(projection.waiting_review_task_count, 1); + assert_eq!(projection.waiting_replan_task_count, 0); + assert_eq!( + projection.waiting_reasons, + vec![preserved_review_reason.clone()] + ); + assert!(!projection.waiting_reasons_truncated); + assert_eq!(projection.waiting_since, Some(review_waiting_since)); + }; assert_eq!(amended.task_ids_to_release, vec![task.task_id]); + assert_preserved_review_wait(&amended.run); let applied_wake_epoch = amended.run.wake_epoch; let amendment_dispatch: (i64, String) = sqlx::query_as( "SELECT wake_epoch, payload->>'reason' FROM moa.execution_dispatch_outbox \ @@ -979,6 +1173,7 @@ async fn amendment_append_is_revision_fenced_and_preserves_initial_plan_db() -> }; assert_eq!(replayed.run.wake_epoch, applied_wake_epoch); assert_eq!(replayed.task_ids_to_release, vec![task.task_id]); + assert_preserved_review_wait(&replayed.run); let AmendmentReplayOutcome::Replayed(recovered) = repository .recover_amendment_handoff(scope, run.run_uid, run.session_id, 1, &amendment_digest) .await? @@ -989,6 +1184,7 @@ async fn amendment_append_is_revision_fenced_and_preserves_initial_plan_db() -> }; assert_eq!(recovered.run.wake_epoch, applied_wake_epoch); assert_eq!(recovered.task_ids_to_release, vec![task.task_id]); + assert_preserved_review_wait(&recovered.run); assert_eq!( repository .recover_amendment_handoff( @@ -1003,9 +1199,9 @@ async fn amendment_append_is_revision_fenced_and_preserves_initial_plan_db() -> ); let amended = amended.run; assert_eq!(amended.plan_revision, 2); - assert_eq!(amended.status, ExecutionRunStatus::Running); + assert_preserved_review_wait(&amended); assert_eq!(amended.initial_plan_hash, run.initial_plan_hash); - assert_eq!(amended.active_plan_hash, canonical_plan(2).plan_hash); + assert_eq!(amended.active_plan_hash, replacement_plan.plan_hash); assert_eq!(amended.confirmed_plan_hash, Some(run.active_plan_hash)); assert_ne!(amended.confirmed_plan_hash, Some(amended.active_plan_hash)); assert_eq!(amended.plan_history.len(), 1); @@ -1030,6 +1226,34 @@ async fn amendment_append_is_revision_fenced_and_preserves_initial_plan_db() -> ); assert_eq!(amended.reserved.tasks, 0); assert_eq!(amended.consumed.tasks, 1); + let node_states: Vec<(String, String, i64, i64)> = sqlx::query_as( + "SELECT node_id,node_status,dependency_count,remaining_dependency_count \ + FROM moa.execution_node_state WHERE run_uid=$1 ORDER BY node_order", + ) + .bind(run.run_uid) + .fetch_all(test_db.store().pool()) + .await?; + assert_eq!( + node_states, + vec![ + ("preserved".to_string(), "completed".to_string(), 0, 0), + ("preserved_review".to_string(), "waiting".to_string(), 0, 0,), + ("replacement".to_string(), "pending".to_string(), 1, 0), + ("new_terminal".to_string(), "pending".to_string(), 1, 1), + ] + ); + let activation = repository + .load_activation_projection(scope, run.run_uid, 10) + .await? + .expect("amended run remains schedulable"); + assert_eq!( + activation + .nodes + .iter() + .map(|node| node.node_id.as_str()) + .collect::>(), + vec!["replacement"] + ); assert_eq!( repository .confirm_run( @@ -1044,10 +1268,16 @@ async fn amendment_append_is_revision_fenced_and_preserves_initial_plan_db() -> let page = repository .list_tasks(scope, run.run_uid, ExecutionTaskPageRequest::default()) .await?; - assert_eq!(page.tasks[0].status, ExecutionTaskStatus::Cancelled); - assert_eq!(page.tasks[0].actual_tasks, 1); + let persisted_task = page + .tasks + .iter() + .find(|persisted| persisted.task_id == task.task_id) + .expect("superseded replan task must remain in the run task projection") + .clone(); + assert_eq!(persisted_task.status, ExecutionTaskStatus::Cancelled); + assert_eq!(persisted_task.actual_tasks, 1); assert_eq!( - page.tasks[0].current_outcome, + persisted_task.current_outcome, Some(ExecutionTaskOutcome { schema_version: 1, usage: usage(1), @@ -1060,7 +1290,6 @@ async fn amendment_append_is_revision_fenced_and_preserves_initial_plan_db() -> .load_run(scope, run.run_uid) .await? .expect("amended run remains visible"); - let persisted_task = page.tasks[0].clone(); assert!(!persisted_task.outcome_audit.is_empty()); for replacement in [json!([]), json!([{ "replacement": true }])] { assert_db_error_contains( diff --git a/crates/moa-execution/tests/execution_db/support.rs b/crates/moa-execution/tests/execution_db/support.rs index 64d6451d3..8616e8501 100644 --- a/crates/moa-execution/tests/execution_db/support.rs +++ b/crates/moa-execution/tests/execution_db/support.rs @@ -593,12 +593,6 @@ pub(crate) fn canonical_plan(seed: u8) -> CanonicalExecutionPlan { CanonicalExecutionPlan { definition: moa_artifacts::execution_plan::ExecutionPlanDefinition { cancel_policy: ExecutionCancelPolicy::RetainEffects, - input_wait_policy: ExecutionWaitPolicy { - expiry: ExecutionTemporalTarget::After { - delay_seconds: 3_600, - }, - on_expiry: ExecutionWaitExpiryAction::FailTask, - }, input_schema: json!({ "type": "object" }), output_schema: json!({ "type": "object" }), nodes: Vec::new(), diff --git a/crates/moa-execution/tests/execution_db/trigger_outbox_db.rs b/crates/moa-execution/tests/execution_db/trigger_outbox_db.rs index 3a0f44a82..c2eae77fa 100644 --- a/crates/moa-execution/tests/execution_db/trigger_outbox_db.rs +++ b/crates/moa-execution/tests/execution_db/trigger_outbox_db.rs @@ -71,6 +71,113 @@ fn watchdog_output_node() -> ExecutionNode { } } +#[tokio::test] +async fn task_start_uses_post_lock_progress_time_db() -> TestResult { + // Pins: a task-start transaction whose PostgreSQL NOW() predates a contended run lock still + // advances both task and run progress monotonically after the lock owner commits newer times. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let config = execution_capacity_config(); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let mut candidate = new_run( + tenant_id, + None, + "task-start-post-lock-time", + ExecutionRunStatus::Queued, + budget(10), + ); + candidate.plan.definition.nodes = vec![watchdog_output_node()]; + let run = create_run(&repository, scope, candidate).await?; + repository + .initialize_scheduler_state(scope, run.run_uid) + .await?; + assert!(matches!( + repository + .materialize_ready_page( + scope, + &config, + ReadyMaterializationRequest { + run_uid: run.run_uid, + plan_revision: 1, + node_id: "watchdog-work".to_string(), + expected_cursor: 0, + reduce_cursor: None, + source_exhausted: true, + terminal_output: None, + condition_skipped: false, + tasks: vec![logical_task( + run.run_uid, + "watchdog-work", + "one", + estimate(1), + )], + }, + ) + .await?, + ReadyMaterializationOutcome::Applied { .. } + )); + let admitted = repository + .admit_ready_attempts(&config, 1, Utc::now()) + .await? + .admitted + .into_iter() + .next() + .expect("one task must be admitted"); + let fence = TaskAttemptFence { + tenant_id: admitted.tenant_id, + run_uid: admitted.run_uid, + task_id: admitted.task_id, + controller_generation: admitted.controller_generation, + attempt_generation: admitted.attempt_generation, + dispatch_uid: admitted.dispatch_uid, + capacity_reservation_uid: admitted.capacity_reservation_uid, + watchdog_trigger_uid: admitted.watchdog_trigger_uid, + attempt_deadline_at: admitted.attempt_deadline_at, + }; + + let mut lock_owner = pool.begin().await?; + sqlx::query("SELECT run_uid FROM moa.execution_run WHERE run_uid=$1 FOR UPDATE") + .bind(run.run_uid) + .fetch_one(&mut *lock_owner) + .await?; + let start_repository = repository.clone(); + let start = tokio::spawn(async move { start_repository.start_task_attempt(fence).await }); + tokio::time::sleep(StdDuration::from_millis(100)).await; + let newer_progress: DateTime = sqlx::query_scalar( + "UPDATE moa.execution_task SET last_progress_at=clock_timestamp() \ + WHERE run_uid=$1 AND task_id=$2 RETURNING last_progress_at", + ) + .bind(run.run_uid) + .bind(admitted.task_id.as_uuid()) + .fetch_one(&mut *lock_owner) + .await?; + sqlx::query("UPDATE moa.execution_run SET last_progress_at=$2 WHERE run_uid=$1") + .bind(run.run_uid) + .bind(newer_progress) + .execute(&mut *lock_owner) + .await?; + lock_owner.commit().await?; + + assert!(matches!( + tokio::time::timeout(StdDuration::from_secs(5), start).await???, + TaskAttemptStartOutcome::Started(_) + )); + let persisted: (DateTime, DateTime) = sqlx::query_as( + "SELECT run.last_progress_at, task.last_progress_at \ + FROM moa.execution_run AS run JOIN moa.execution_task AS task USING (run_uid) \ + WHERE run.run_uid=$1 AND task.task_id=$2", + ) + .bind(run.run_uid) + .bind(admitted.task_id.as_uuid()) + .fetch_one(&pool) + .await?; + assert!(persisted.0 >= newer_progress); + assert!(persisted.1 >= newer_progress); + Ok(()) +} + #[tokio::test] async fn trigger_creation_is_atomic_and_firing_is_due_generation_fenced_db() -> TestResult { // Pins: a trigger and its fallback delivery commit together; early delivery does not diff --git a/crates/moa-execution/tests/execution_db/wait_entry_deadline_db.rs b/crates/moa-execution/tests/execution_db/wait_entry_deadline_db.rs index 0a24b7c0a..52a5d02cc 100644 --- a/crates/moa-execution/tests/execution_db/wait_entry_deadline_db.rs +++ b/crates/moa-execution/tests/execution_db/wait_entry_deadline_db.rs @@ -1,5 +1,7 @@ //! Wait entry that cannot finish before the run deadline projects a typed task failure. +use std::time::Duration as StdDuration; + use moa_artifacts::execution_plan::{ExecutionNode, ExecutionOperation}; use moa_execution::repository::ready::{ReadyMaterializationOutcome, ReadyMaterializationRequest}; use moa_execution::repository::task::{ @@ -8,10 +10,96 @@ use moa_execution::repository::task::{ use moa_execution::repository::terminal::{ PendingTerminalAdvanceOutcome, PendingTerminalAdvanceStage, }; +use moa_execution::repository::trigger::{ + ExecutionRunDeadlineTriggerOutcome, ExecutionTriggerNoOp, +}; use moa_execution::state::ExecutionTerminalEvidence; use super::support::*; +#[tokio::test] +async fn storage_wait_materialization_locks_trigger_capacity_before_run_db() -> TestResult { + // Pins: a storage-only task waits for ScheduledTriggers capacity before taking its run row, + // so concurrent trigger settlement cannot deadlock with wait materialization. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let config = ExecutionConfig::default(); + let mut candidate = new_run( + tenant_id, + None, + "storage-wait-lock-order", + ExecutionRunStatus::Queued, + budget(10), + ); + candidate.plan.definition.nodes = vec![output_node("wait", &[])]; + let run = create_run(&repository, scope, candidate).await?; + let run_uid = run.run_uid; + repository + .initialize_scheduler_state(scope, run_uid) + .await?; + let mut task = logical_task(run_uid, "wait", "", estimate(1)); + task.kind = LogicalTaskKind::WaitUntil { + wake: ExecutionTemporalTarget::After { delay_seconds: 60 }, + result: json!({"elapsed": true}), + }; + + let mut capacity_holder = pool.begin().await?; + let locked: Vec = sqlx::query_scalar( + "SELECT capacity_bucket_uid FROM moa.execution_capacity_bucket \ + WHERE resource_dimension='scheduled_triggers' \ + AND ((scope_kind='fleet' AND tenant_id IS NULL) \ + OR (scope_kind='tenant' AND tenant_id=$1)) \ + ORDER BY CASE scope_kind WHEN 'fleet' THEN 0 ELSE 1 END FOR UPDATE", + ) + .bind(tenant_id.0) + .fetch_all(&mut *capacity_holder) + .await?; + assert_eq!(locked.len(), 2); + + let materialization_repository = repository.clone(); + let materialization_config = config.clone(); + let mut materialization = tokio::spawn(async move { + materialization_repository + .materialize_ready_page( + scope, + &materialization_config, + ReadyMaterializationRequest { + run_uid, + plan_revision: 1, + node_id: "wait".to_string(), + expected_cursor: 0, + reduce_cursor: None, + source_exhausted: true, + terminal_output: None, + condition_skipped: false, + tasks: vec![task], + }, + ) + .await + }); + assert!( + tokio::time::timeout(StdDuration::from_millis(100), &mut materialization) + .await + .is_err(), + "wait materialization must block on ScheduledTriggers capacity" + ); + sqlx::query("SELECT run_uid FROM moa.execution_run WHERE run_uid=$1 FOR UPDATE NOWAIT") + .bind(run_uid) + .fetch_one(&mut *capacity_holder) + .await?; + capacity_holder.commit().await?; + + let outcome = tokio::time::timeout(StdDuration::from_secs(5), materialization).await???; + assert!(matches!( + outcome, + ReadyMaterializationOutcome::Applied { ref triggers, .. } if triggers.len() == 1 + )); + Ok(()) +} + fn output_node(id: &str, depends_on: &[&str]) -> ExecutionNode { ExecutionNode { id: id.to_string(), @@ -278,10 +366,11 @@ async fn storage_wait_past_run_deadline_fails_its_node_instead_of_erroring_db() } #[tokio::test] -async fn input_wait_past_run_deadline_fails_its_task_instead_of_erroring_db() -> TestResult { - // Pins: settling a NeedsInput attempt whose plan-level input-wait expiry resolves at or after - // the run deadline terminates the task with a typed DeadlineExceeded failure naming its node - // rather than aborting settlement; an expiry that still fits parks the ordinary input wait. +async fn input_wait_suspends_run_deadline_without_trigger_or_active_capacity_and_resumes_exactly_db() +-> TestResult { + // Pins: a fully human-input-parked run has no input-expiry trigger or active-task capacity, + // an already-dispatched deadline cannot terminalize it, and exact input shifts then rearms + // the run deadline without weakening per-attempt watchdogs. let test_db = moa_test_support::postgres::bootstrap_test_db().await?; let pool = test_db.store().pool().clone(); let repository = ExecutionRepository::new(pool.clone()); @@ -289,30 +378,271 @@ async fn input_wait_past_run_deadline_fails_its_task_instead_of_erroring_db() -> let scope = ExecutionScope::Tenant { tenant_id }; let config = ExecutionConfig::default(); - for (key, expiry_seconds, expected_status) in [ - ( - "input-wait-past-deadline", - 86_400_u64, - ExecutionTaskStatus::Failed, - ), - ( - "input-wait-inside-deadline", - 60, - ExecutionTaskStatus::WaitingInput, + let mut candidate = new_run( + tenant_id, + None, + "input-wait-indefinite", + ExecutionRunStatus::Queued, + budget(10), + ); + candidate.plan.definition.nodes = vec![output_node("ask", &[])]; + let run = create_run(&repository, scope, candidate).await?; + let original_deadline = run.approved_budget.deadline_at.expect("fixture deadline"); + let deadline_trigger_uid: Uuid = sqlx::query_scalar( + "SELECT trigger_uid FROM moa.execution_trigger \ + WHERE run_uid=$1 AND trigger_kind='run_deadline' AND state='pending'", + ) + .bind(run.run_uid) + .fetch_one(&pool) + .await?; + repository + .initialize_scheduler_state(scope, run.run_uid) + .await?; + assert!(matches!( + repository + .materialize_ready_page( + scope, + &config, + ReadyMaterializationRequest { + run_uid: run.run_uid, + plan_revision: 1, + node_id: "ask".to_string(), + expected_cursor: 0, + reduce_cursor: None, + source_exhausted: true, + terminal_output: None, + condition_skipped: false, + tasks: vec![logical_task(run.run_uid, "ask", "", estimate(1))], + }, + ) + .await?, + ReadyMaterializationOutcome::Applied { .. } + )); + let admission = repository + .admit_ready_attempts(&config, 1, Utc::now()) + .await?; + let admitted = admission + .admitted + .into_iter() + .find(|item| item.run_uid == run.run_uid) + .expect("the only ready task must be admitted"); + let fence = TaskAttemptFence { + tenant_id: admitted.tenant_id, + run_uid: admitted.run_uid, + task_id: admitted.task_id, + controller_generation: admitted.controller_generation, + attempt_generation: admitted.attempt_generation, + dispatch_uid: admitted.dispatch_uid, + capacity_reservation_uid: admitted.capacity_reservation_uid, + watchdog_trigger_uid: admitted.watchdog_trigger_uid, + attempt_deadline_at: admitted.attempt_deadline_at, + }; + assert!(matches!( + repository.start_task_attempt(fence).await?, + TaskAttemptStartOutcome::Started(_) + )); + let TaskAttemptSettlementOutcome::Applied { task, .. } = repository + .settle_task_attempt(&config, fence, needs_input(1), None, Utc::now()) + .await? + else { + panic!("NeedsInput settlement must apply"); + }; + assert_eq!(task.status, ExecutionTaskStatus::WaitingInput); + let parked = repository + .load_run(scope, run.run_uid) + .await? + .expect("parked run remains visible"); + assert_eq!(parked.approved_budget.deadline_at, Some(original_deadline)); + let suspended_at = parked + .budget_deadline_suspended_at + .expect("human input wait suspends wall-clock deadline accounting"); + assert_eq!(parked.active_task_count, 0); + assert_eq!(parked.waiting_task_count, 1); + assert_eq!(parked.waiting_input_task_count, 1); + let active_task_receipts: i64 = sqlx::query_scalar( + "SELECT count(*) FROM moa.execution_capacity_reservation \ + WHERE run_uid=$1 AND resource_dimension='active_tasks' \ + AND state IN ('reserved','reconciling')", + ) + .bind(run.run_uid) + .fetch_one(&pool) + .await?; + assert_eq!(active_task_receipts, 0); + let input_expiry_rows: i64 = sqlx::query_scalar( + "SELECT count(*) FROM moa.execution_trigger \ + WHERE run_uid=$1 AND task_id=$2 AND trigger_kind='wait_expiry'", + ) + .bind(run.run_uid) + .bind(task.task_id.as_uuid()) + .fetch_one(&pool) + .await?; + assert_eq!(input_expiry_rows, 0); + + assert_eq!( + repository + .prepare_run_deadline_trigger(scope, deadline_trigger_uid) + .await?, + ExecutionRunDeadlineTriggerOutcome::NoOp(ExecutionTriggerNoOp::Inactive) + ); + assert!(matches!( + repository + .fence_deadline_and_enqueue_settlement( + &config, + scope, + run.run_uid, + parked.controller_generation, + parked.wake_epoch, + original_deadline + Duration::hours(1), + 1, + ) + .await?, + PendingTerminalAdvanceOutcome::Conflict + )); + + let TransitionOutcome::Applied(resumed) = repository + .resume_task_with_input( + scope, + &config, + run.run_uid, + task.task_id, + task.generation, + json!({"answer": "continue"}), + ) + .await? + else { + panic!("exact input must resume the parked generation"); + }; + assert_eq!(resumed.status, ExecutionTaskStatus::Ready); + let resumed_run = repository + .load_run(scope, run.run_uid) + .await? + .expect("resumed run remains visible"); + assert!(resumed_run.budget_deadline_suspended_at.is_none()); + let shifted_deadline = resumed_run + .approved_budget + .deadline_at + .expect("resume restores the deadline"); + assert!(shifted_deadline > original_deadline); + assert_eq!( + shifted_deadline - original_deadline, + resumed + .ready_at + .expect("resumed task has an exact ready time") + - suspended_at + ); + let current_deadlines: Vec> = sqlx::query_scalar( + "SELECT due_at FROM moa.execution_trigger \ + WHERE run_uid=$1 AND trigger_kind='run_deadline' AND state='pending'", + ) + .bind(run.run_uid) + .fetch_all(&pool) + .await?; + assert_eq!(current_deadlines, vec![shifted_deadline]); + + let readmission = repository + .admit_ready_attempts(&config, 1, Utc::now()) + .await? + .admitted + .into_iter() + .find(|item| item.run_uid == run.run_uid) + .expect("resumed task is admitted under its new attempt generation"); + let second_fence = TaskAttemptFence { + tenant_id: readmission.tenant_id, + run_uid: readmission.run_uid, + task_id: readmission.task_id, + controller_generation: readmission.controller_generation, + attempt_generation: readmission.attempt_generation, + dispatch_uid: readmission.dispatch_uid, + capacity_reservation_uid: readmission.capacity_reservation_uid, + watchdog_trigger_uid: readmission.watchdog_trigger_uid, + attempt_deadline_at: readmission.attempt_deadline_at, + }; + assert!(matches!( + repository.start_task_attempt(second_fence).await?, + TaskAttemptStartOutcome::Started(_) + )); + let second_settlement = repository + .settle_task_attempt(&config, second_fence, needs_input(2), None, Utc::now()) + .await?; + assert!( + matches!( + second_settlement, + TaskAttemptSettlementOutcome::Applied { .. } ), - ] { - let mut candidate = new_run(tenant_id, None, key, ExecutionRunStatus::Queued, budget(10)); - candidate.plan.definition.nodes = vec![output_node("ask", &[])]; - candidate.plan.definition.input_wait_policy = ExecutionWaitPolicy { - expiry: ExecutionTemporalTarget::After { - delay_seconds: expiry_seconds, + "second NeedsInput settlement must apply, got {second_settlement:?}" + ); + let suspended_again = repository + .load_run(scope, run.run_uid) + .await? + .expect("second input wait remains visible"); + assert!(suspended_again.budget_deadline_suspended_at.is_some()); + let terminal_evidence = + moa_execution::completion::cancellation_terminal_evidence_from_completed_nodes( + &suspended_again.goal, + &suspended_again.active_plan, + &std::collections::BTreeSet::::new(), + )?; + let PendingTerminalAdvanceOutcome::Applied(cancelled) = repository + .fence_completion_terminal_and_enqueue_settlement( + &config, + scope, + run.run_uid, + suspended_again.controller_generation, + suspended_again.wake_epoch, + PendingExecutionTerminal { + status: ExecutionRunStatus::Cancelled, + reason: ExecutionTerminalReason::Cancelled, + terminal_evidence, + completion_check_results: Vec::new(), + terminal_gaps: Vec::new(), + output: None, + cancellation_reason: Some("operator cancelled during input wait".to_string()), }, - on_expiry: ExecutionWaitExpiryAction::FailTask, - }; - let run = create_run(&repository, scope, candidate).await?; - repository - .initialize_scheduler_state(scope, run.run_uid) - .await?; + Utc::now(), + 8, + ) + .await? + else { + panic!("explicit cancellation must drain a deadline-suspended input wait"); + }; + assert_eq!(cancelled.run.status, ExecutionRunStatus::Cancelled); + assert!(cancelled.run.budget_deadline_suspended_at.is_none()); + Ok(()) +} + +#[tokio::test] +async fn last_active_completion_suspends_deadline_when_sibling_waits_for_input_db() -> TestResult { + // Pins: deadline suspension is fenced by the locked run counters, not by the task whose + // settlement happens to make the run fully input-parked. A sibling may already be waiting. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let config = ExecutionConfig::default(); + let mut candidate = new_run( + tenant_id, + None, + "input-wait-last-active", + ExecutionRunStatus::Queued, + budget(10), + ); + candidate.plan.definition.nodes = vec![ + output_node("ask-a", &[]), + output_node("ask-b", &[]), + output_node("finish", &[]), + ]; + let run = create_run(&repository, scope, candidate).await?; + repository + .initialize_scheduler_state(scope, run.run_uid) + .await?; + + let tasks = [ + logical_task(run.run_uid, "ask-a", "", estimate(1)), + logical_task(run.run_uid, "ask-b", "", estimate(1)), + logical_task(run.run_uid, "finish", "", estimate(1)), + ]; + for task in &tasks { assert!(matches!( repository .materialize_ready_page( @@ -321,63 +651,142 @@ async fn input_wait_past_run_deadline_fails_its_task_instead_of_erroring_db() -> ReadyMaterializationRequest { run_uid: run.run_uid, plan_revision: 1, - node_id: "ask".to_string(), + node_id: task.node_id.clone(), expected_cursor: 0, reduce_cursor: None, source_exhausted: true, terminal_output: None, condition_skipped: false, - tasks: vec![logical_task(run.run_uid, "ask", "", estimate(1))], + tasks: vec![task.clone()], }, ) .await?, - ReadyMaterializationOutcome::Applied { .. } + ReadyMaterializationOutcome::Applied { next_cursor: 1, .. } )); - let admission = repository - .admit_ready_attempts(&config, 1, Utc::now()) - .await?; - let admitted = admission - .admitted - .into_iter() - .find(|item| item.run_uid == run.run_uid) - .expect("the only ready task must be admitted"); - let fence = TaskAttemptFence { - tenant_id: admitted.tenant_id, - run_uid: admitted.run_uid, - task_id: admitted.task_id, - controller_generation: admitted.controller_generation, - attempt_generation: admitted.attempt_generation, - dispatch_uid: admitted.dispatch_uid, - capacity_reservation_uid: admitted.capacity_reservation_uid, - watchdog_trigger_uid: admitted.watchdog_trigger_uid, - attempt_deadline_at: admitted.attempt_deadline_at, - }; + } + let admitted = repository + .admit_ready_attempts(&config, 3, Utc::now()) + .await? + .admitted; + assert_eq!(admitted.len(), 3); + let fences = admitted + .iter() + .map(|item| TaskAttemptFence { + tenant_id: item.tenant_id, + run_uid: item.run_uid, + task_id: item.task_id, + controller_generation: item.controller_generation, + attempt_generation: item.attempt_generation, + dispatch_uid: item.dispatch_uid, + capacity_reservation_uid: item.capacity_reservation_uid, + watchdog_trigger_uid: item.watchdog_trigger_uid, + attempt_deadline_at: item.attempt_deadline_at, + }) + .collect::>(); + for fence in &fences { assert!(matches!( - repository.start_task_attempt(fence).await?, + repository.start_task_attempt(*fence).await?, TaskAttemptStartOutcome::Started(_) )); - let TaskAttemptSettlementOutcome::Applied { task, .. } = repository - .settle_task_attempt(&config, fence, needs_input(1), None, Utc::now()) - .await? - else { - panic!("{key} settlement must apply"); - }; - assert_eq!(task.status, expected_status, "{key}"); - if expected_status == ExecutionTaskStatus::Failed { - let (class, message) = - failure_class(&task).expect("the failed input wait must carry a typed outcome"); - assert_eq!(class, ExecutionFailureClass::DeadlineExceeded); - assert!( - message.contains("`ask`"), - "the failure must name its node, got `{message}`" - ); - let settled_run = repository - .load_run(scope, run.run_uid) - .await? - .expect("run must remain visible"); - assert_eq!(settled_run.waiting_input_task_count, 0); - assert_eq!(settled_run.progress_failed_tasks, 1); - } } + + assert!(matches!( + repository + .settle_task_attempt(&config, fences[0], needs_input(1), None, Utc::now()) + .await?, + TaskAttemptSettlementOutcome::Applied { .. } + )); + let partly_active = repository + .load_run(scope, run.run_uid) + .await? + .expect("partly active run remains visible"); + assert_eq!(partly_active.active_task_count, 2); + assert_eq!(partly_active.waiting_input_task_count, 1); + assert!(partly_active.budget_deadline_suspended_at.is_none()); + + assert!(matches!( + repository + .settle_task_attempt(&config, fences[1], needs_input(1), None, Utc::now()) + .await?, + TaskAttemptSettlementOutcome::Applied { .. } + )); + assert!(matches!( + repository + .settle_task_attempt(&config, fences[2], completed(1), None, Utc::now()) + .await?, + TaskAttemptSettlementOutcome::Applied { .. } + )); + let fully_parked = repository + .load_run(scope, run.run_uid) + .await? + .expect("fully parked run remains visible"); + assert_eq!(fully_parked.active_task_count, 0); + assert_eq!(fully_parked.waiting_task_count, 2); + assert_eq!(fully_parked.waiting_input_task_count, 2); + assert!(fully_parked.budget_deadline_suspended_at.is_some()); + let active_task_receipts: i64 = sqlx::query_scalar( + "SELECT count(*) FROM moa.execution_capacity_reservation \ + WHERE run_uid=$1 AND resource_dimension='active_tasks' \ + AND state IN ('reserved','reconciling')", + ) + .bind(run.run_uid) + .fetch_one(&pool) + .await?; + assert_eq!(active_task_receipts, 0); + let input_expiry_rows: i64 = sqlx::query_scalar( + "SELECT count(*) FROM moa.execution_trigger \ + WHERE run_uid=$1 AND trigger_kind='wait_expiry'", + ) + .bind(run.run_uid) + .fetch_one(&pool) + .await?; + assert_eq!(input_expiry_rows, 0); + + sqlx::query("UPDATE moa.execution_run SET activation_state='idle' WHERE run_uid=$1") + .bind(run.run_uid) + .execute(&pool) + .await?; + let idle_parked = repository + .load_run(scope, run.run_uid) + .await? + .expect("fully parked run remains visible after idle checkpoint fixture"); + assert_eq!(idle_parked.activation_state, ExecutionActivationState::Idle); + + let terminal_evidence = + moa_execution::completion::cancellation_terminal_evidence_from_completed_nodes( + &idle_parked.goal, + &idle_parked.active_plan, + &std::collections::BTreeSet::from(["finish".to_string()]), + )?; + let PendingTerminalAdvanceOutcome::Applied(cancelled_page) = repository + .fence_completion_terminal_and_enqueue_settlement( + &config, + scope, + run.run_uid, + idle_parked.controller_generation, + idle_parked.wake_epoch, + PendingExecutionTerminal { + status: ExecutionRunStatus::Cancelled, + reason: ExecutionTerminalReason::Cancelled, + terminal_evidence, + completion_check_results: Vec::new(), + terminal_gaps: Vec::new(), + output: None, + cancellation_reason: Some("operator cancelled fully parked run".to_string()), + }, + Utc::now(), + 1, + ) + .await? + else { + panic!("idle external cancellation must checkpoint its bounded terminal page"); + }; + assert!(cancelled_page.work_remaining); + assert_eq!( + cancelled_page.run.activation_state, + ExecutionActivationState::Queued + ); + assert!(cancelled_page.continuation.is_some()); + assert!(cancelled_page.run.budget_deadline_suspended_at.is_some()); Ok(()) } diff --git a/crates/moa-execution/tests/interpreter.rs b/crates/moa-execution/tests/interpreter.rs index f664da22d..59ae86d62 100644 --- a/crates/moa-execution/tests/interpreter.rs +++ b/crates/moa-execution/tests/interpreter.rs @@ -9,7 +9,7 @@ use moa_artifacts::execution_plan::{ ExecutionCancelPolicy, ExecutionCondition, ExecutionGoalContract, ExecutionNode, ExecutionOperation, ExecutionPlanDefinition, ExecutionReducer, ExecutionReference, ExecutionRequirement, ExecutionTaskOutcome, ExecutionTaskResult, ExecutionTemporalTarget, - ExecutionUsage, ExecutionWaitExpiryAction, ExecutionWaitPolicy, MapTask, RetryPolicy, + ExecutionUsage, MapTask, RetryPolicy, }; use moa_config::ExecutionConfig; use moa_core::types::{ @@ -830,15 +830,6 @@ fn canonical(nodes: Vec) -> CanonicalExecutionPlan { CanonicalExecutionPlan { definition: ExecutionPlanDefinition { cancel_policy: ExecutionCancelPolicy::RetainEffects, - input_wait_policy: ExecutionWaitPolicy { - expiry: ExecutionTemporalTarget::At { - at: Utc - .with_ymd_and_hms(2026, 7, 13, 12, 0, 0) - .single() - .expect("input wait expiry"), - }, - on_expiry: ExecutionWaitExpiryAction::FailTask, - }, input_schema: json!({ "type": "object" }), output_schema: json!({ "type": "object" }), nodes, diff --git a/crates/moa-hands/src/core/dispatch.rs b/crates/moa-hands/src/core/dispatch.rs index 735f4fd27..35d72a98b 100644 --- a/crates/moa-hands/src/core/dispatch.rs +++ b/crates/moa-hands/src/core/dispatch.rs @@ -13,7 +13,9 @@ use moa_core::{ types::completion::ToolInvocation, types::hands::HandHandle, types::hands::HandStatus, - types::identifiers::{ExecutionRunScopeId, ToolCallId}, + types::identifiers::{ + ExecutionRunScopeId, HandProvisioningOperationId, SandboxWorkspaceId, ToolCallId, + }, types::resource::DeadlineGuard, types::sandbox_workspace::{ExecutionHandReleaseOwner, SandboxWorkspaceScope, WorkspaceEffect}, types::security::ToolCapabilityId, @@ -21,6 +23,7 @@ use moa_core::{ types::tools::SecuredToolOutput, types::tools::ToolDefinition, types::tools::ToolOutput, + types::worker::state::WorkerInputTarget, }; use moa_observability::current_turn_root_span; use moa_security::{OutputClassification, classify_tool_output}; @@ -127,6 +130,39 @@ pub struct ExecutionHandReleaseRequest<'a> { pub scope: ToolCallScope<'a>, } +/// One idempotent request to park a worker's sandbox for a human-input wait. +#[derive(Clone, Copy)] +pub struct WorkerHandReleaseRequest<'a> { + /// Session whose tenant owns the worker workspace and hand lease. + pub session: &'a SessionMeta, + /// Exact worker scope entering the durable wait. + pub worker_id: &'a str, + /// Turn, admission generation, and input request that own this wait. + pub input_target: &'a WorkerInputTarget, + /// Exact compute attachment captured before the input wait was registered. + pub expected: Option<&'a WorkerHandReleaseFence>, + /// Fresh bounded budget for checkpoint publication and verified destroy. + pub scope: ToolCallScope<'a>, +} + +/// Exact durable workspace and lease generation a worker is allowed to park. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct WorkerHandReleaseFence { + /// Workspace attached to the worker when the wait began. + pub workspace_id: SandboxWorkspaceId, + /// Single-writer epoch that owned the working copy. + pub writer_epoch: u64, + /// Compute instance generation that owned the working copy. + pub instance_generation: u64, + /// Provider route pinned on the workspace. + pub provider: String, + /// Provisioning operation that created the exact compute instance. + pub provisioning_operation_id: HandProvisioningOperationId, + /// Durable hand-lease generation attached to the workspace. + pub hand_lease_generation: u64, +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(super) enum WorkspaceCommitMode { Inline, diff --git a/crates/moa-hands/src/core/mod.rs b/crates/moa-hands/src/core/mod.rs index 0d07c551c..acad9f8b0 100644 --- a/crates/moa-hands/src/core/mod.rs +++ b/crates/moa-hands/src/core/mod.rs @@ -49,7 +49,8 @@ use crate::adapters::local::LocalHandProvider; pub use dispatch::{ AuthorizedToolCall, DeferredWorkspaceToolOutput, ExecutionHandReleaseRequest, - JournaledWorkspaceCommit, PendingConnectorToolOutput, + JournaledWorkspaceCommit, PendingConnectorToolOutput, WorkerHandReleaseFence, + WorkerHandReleaseRequest, }; use leases::{HAND_LEASE_SESSION_PAGE_SIZE, HandLeaseStore}; pub use maintenance_provider_inventory::SandboxProviderInventory; diff --git a/crates/moa-hands/src/core/sandbox_workspace/lifecycle.rs b/crates/moa-hands/src/core/sandbox_workspace/lifecycle.rs index 7218ca3c0..86269e718 100644 --- a/crates/moa-hands/src/core/sandbox_workspace/lifecycle.rs +++ b/crates/moa-hands/src/core/sandbox_workspace/lifecycle.rs @@ -4,6 +4,7 @@ mod commit; mod execution_release; mod management; mod materialization; +mod worker_release; use chrono::{Duration as ChronoDuration, Utc}; use moa_core::{ @@ -35,8 +36,9 @@ use super::{ }, failpoints, model::{ - AbsentTaskHandReleaseIntent, CompensationHandReleaseIntent, SandboxWorkspace, - TaskHandReleaseIntent, WorkspaceTransition, WorkspaceWriterClaim, + AbsentTaskHandReleaseIntent, CompensationHandReleaseClaimIntent, + CompensationHandReleaseIntent, SandboxWorkspace, TaskHandReleaseIntent, + WorkspaceTransition, WorkspaceWriterClaim, }, operations::WorkspaceOperationIntent, }; diff --git a/crates/moa-hands/src/core/sandbox_workspace/lifecycle/execution_release.rs b/crates/moa-hands/src/core/sandbox_workspace/lifecycle/execution_release.rs index 767dee99d..0b50ef666 100644 --- a/crates/moa-hands/src/core/sandbox_workspace/lifecycle/execution_release.rs +++ b/crates/moa-hands/src/core/sandbox_workspace/lifecycle/execution_release.rs @@ -79,9 +79,15 @@ impl ToolRouter { self.hands.workspace_repository.as_ref().ok_or_else(|| { MoaError::StorageError("workspace repository missing".to_string()) })?; + let contact_id = request + .session + .contact + .as_ref() + .map(|contact| contact.contact_id); if let Some(receipt) = repository .get_task_execution_hand_release_receipt( request.session.tenant_id, + contact_id, request.run_id, task_id, logical_generation, @@ -104,6 +110,7 @@ impl ToolRouter { .record_absent_task_execution_hand_release_receipt(AbsentTaskHandReleaseIntent { receipt_id: absence_receipt_id, tenant_id: request.session.tenant_id, + contact_id, run_id: request.run_id, task_id, logical_generation, @@ -159,6 +166,7 @@ impl ToolRouter { let (receipt_id, release_claim_token, requested_at) = repository .begin_task_execution_hand_release(TaskHandReleaseIntent { receipt_id: candidate_receipt_id, + contact_id, run_id: request.run_id, task_id, logical_generation, @@ -283,7 +291,7 @@ impl ToolRouter { } })?; if !repository - .finalize_task_yield_destroy(&final_workspace.binding()?, &final_lease) + .finalize_checkpointed_hand_destroy(&final_workspace.binding()?, &final_lease) .await? { // The compute is gone but the durable release did not commit, so the @@ -400,7 +408,7 @@ impl ToolRouter { released_at: Utc::now(), }; repository - .record_task_execution_hand_release_receipt(&receipt, release_claim_token) + .record_task_execution_hand_release_receipt(&receipt, release_claim_token, contact_id) .await } @@ -419,6 +427,11 @@ impl ToolRouter { self.hands.workspace_repository.as_ref().ok_or_else(|| { MoaError::StorageError("workspace repository missing".to_string()) })?; + let contact_id = request + .session + .contact + .as_ref() + .map(|contact| contact.contact_id); if let Some(receipt) = repository .get_compensation_execution_hand_release_receipt( request.session.tenant_id, @@ -440,14 +453,15 @@ impl ToolRouter { MoaError::StorageError("durable hand lease store missing".to_string()) })?; if let Some(claim) = repository - .claim_pending_compensation_execution_hand_release( - request.session.tenant_id, - request.run_id, + .claim_pending_compensation_execution_hand_release(CompensationHandReleaseClaimIntent { + tenant_id: request.session.tenant_id, + contact_id, + run_id: request.run_id, compensation_id, logical_generation, - request.attempt_generation, - Utc::now() + ChronoDuration::minutes(5), - ) + attempt_generation: request.attempt_generation, + recovery_claim_expires_at: Utc::now() + ChronoDuration::minutes(5), + }) .await? { let persisted_identity = match ( @@ -571,6 +585,7 @@ impl ToolRouter { request.session.id, &hand_scope, claim.claim_token, + contact_id, ) .await; } @@ -596,6 +611,7 @@ impl ToolRouter { .begin_compensation_execution_hand_release(CompensationHandReleaseIntent { receipt_id, tenant_id: request.session.tenant_id, + contact_id, session_id: request.session.id, run_id: request.run_id, compensation_id, @@ -690,6 +706,7 @@ impl ToolRouter { request.session.id, &hand_scope, claim_token, + contact_id, ) .await } diff --git a/crates/moa-hands/src/core/sandbox_workspace/lifecycle/worker_release.rs b/crates/moa-hands/src/core/sandbox_workspace/lifecycle/worker_release.rs new file mode 100644 index 000000000..1817f419e --- /dev/null +++ b/crates/moa-hands/src/core/sandbox_workspace/lifecycle/worker_release.rs @@ -0,0 +1,490 @@ +//! Worker sandbox checkpoint and compute-release boundary. + +use super::*; +use moa_core::types::worker::state::WorkerInputTarget; + +use crate::core::{ + WorkerHandReleaseFence, WorkerHandReleaseRequest, lifecycle::workspace_lease_scope, +}; + +impl ToolRouter { + /// Captures the exact live worker attachment allowed to enter an input wait. + /// + /// Callers must journal this result before asking to park the hand. `None` + /// proves there was no live hand at capture time; a later hand therefore cannot + /// be released by replaying that request. + pub async fn capture_worker_hand_release_fence( + &self, + session: &SessionMeta, + worker_id: &str, + ) -> Result> { + if worker_id.trim().is_empty() { + return Err(MoaError::ValidationError( + "worker hand release fence requires a worker".to_string(), + )); + } + let workspace_scope = SandboxWorkspaceScope::Worker { + session_id: session.id, + worker_id: worker_id.to_string(), + }; + let lease_scope = workspace_lease_scope(&workspace_scope); + let scope_key = crate::core::HandScopeKey::new(session.tenant_id, session.id, &lease_scope); + let Some(repository) = self.hands.workspace_repository.as_ref() else { + self.verify_worker_hand_absent( + session, + &lease_scope, + &scope_key, + "worker-hand-release-fence", + ) + .await?; + return Ok(None); + }; + let Some(workspace) = repository + .get_by_scope(session.tenant_id, &workspace_scope) + .await? + else { + self.verify_worker_hand_absent( + session, + &lease_scope, + &scope_key, + "worker-hand-release-fence", + ) + .await?; + return Ok(None); + }; + let lease_store = self.hands.hand_leases.as_ref().ok_or_else(|| { + MoaError::StorageError("durable hand lease store missing".to_string()) + })?; + let lease = lease_store + .get( + session.tenant_id, + session.id, + &lease_scope, + &workspace.provider, + ) + .await?; + if workspace.state == SandboxWorkspaceState::Ready { + self.verify_worker_hand_absent( + session, + &lease_scope, + &scope_key, + "worker-hand-release-fence", + ) + .await?; + return Ok(None); + } + let lease = lease.ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { + operation_id: "worker-hand-release-fence".to_string(), + })?; + let binding = workspace.binding()?; + if workspace.state != SandboxWorkspaceState::Active + || lease.status != HandLeaseStatus::Active + || lease.handle.is_none() + || lease.attachment != Some(lease_attachment(&binding)?) + { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: "worker-hand-release-fence".to_string(), + }); + } + Ok(Some(WorkerHandReleaseFence { + workspace_id: workspace.workspace_id, + writer_epoch: binding.writer_epoch, + instance_generation: binding.instance_generation, + provider: workspace.provider, + provisioning_operation_id: lease.provisioning_operation_id, + hand_lease_generation: u64::try_from(lease.generation).map_err(|_| { + MoaError::StorageError("hand lease generation is invalid".to_string()) + })?, + })) + } + + /// Checkpoints and releases one worker's sandbox before a durable human-input wait. + /// + /// The wait target supplies the replay identity. A successful return proves that + /// the worker has no active durable hand lease. Its retained workspace remains + /// `Ready`; the next sandbox dispatch provisions fresh compute and restores the + /// exact committed portable checkpoint. + pub async fn checkpoint_and_release_worker_hand( + &self, + request: WorkerHandReleaseRequest<'_>, + ) -> Result<()> { + validate_worker_input_target(request.worker_id, request.input_target)?; + let operation_key = worker_wait_operation_key(request.input_target); + let workspace_scope = SandboxWorkspaceScope::Worker { + session_id: request.session.id, + worker_id: request.worker_id.to_string(), + }; + let lease_scope = workspace_lease_scope(&workspace_scope); + let scope_key = crate::core::HandScopeKey::new( + request.session.tenant_id, + request.session.id, + &lease_scope, + ); + let repository = match self.hands.workspace_repository.as_ref() { + Some(repository) => repository, + None => { + if request.expected.is_some() { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_key, + }); + } + self.verify_worker_hand_absent( + request.session, + &lease_scope, + &scope_key, + &operation_key, + ) + .await?; + return Ok(()); + } + }; + let Some(initial_workspace) = repository + .get_by_scope(request.session.tenant_id, &workspace_scope) + .await? + else { + if request.expected.is_some() { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_key, + }); + } + self.verify_worker_hand_absent( + request.session, + &lease_scope, + &scope_key, + &operation_key, + ) + .await?; + return Ok(()); + }; + let lease_store = self.hands.hand_leases.as_ref().ok_or_else(|| { + MoaError::StorageError("durable hand lease store missing".to_string()) + })?; + let initial_lease = lease_store + .get( + request.session.tenant_id, + request.session.id, + &lease_scope, + &initial_workspace.provider, + ) + .await?; + if initial_workspace.state == SandboxWorkspaceState::Ready { + verify_released_worker_fence( + request.expected, + &initial_workspace, + initial_lease.as_ref(), + &operation_key, + )?; + self.verify_worker_hand_absent( + request.session, + &lease_scope, + &scope_key, + &operation_key, + ) + .await?; + return Ok(()); + } + let initial_lease = + initial_lease.ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_key.clone(), + })?; + if !worker_fence_matches(request.expected, &initial_workspace, &initial_lease)? + || initial_lease.status != HandLeaseStatus::Active + || initial_lease.handle.is_none() + || initial_lease.attachment != Some(lease_attachment(&initial_workspace.binding()?)?) + { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_key, + }); + } + let hand = initial_lease + .handle + .as_ref() + .map(|handle| handle.handle.clone()) + .ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_key.clone(), + })?; + let tool_call_id = + worker_wait_tool_call_id(initial_workspace.workspace_id, request.input_target); + self.commit_workspace_after_tool(WorkspaceCommitExecution { + session: request.session, + workspace_scope: &workspace_scope, + tool_call_id, + provider_name: &initial_workspace.provider, + hand: &hand, + call_scope: request.scope, + release_compute: true, + }) + .await?; + + let mut workspace = repository + .get_by_scope(request.session.tenant_id, &workspace_scope) + .await? + .ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_key.clone(), + })?; + let mut lease = lease_store + .get( + request.session.tenant_id, + request.session.id, + &lease_scope, + &initial_workspace.provider, + ) + .await? + .ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_key.clone(), + })?; + + // Reconciliation can prove the checkpoint bytes while retaining compute. + // Finish that exact attachment as a separate fenced destroy step. + if lease.status == HandLeaseStatus::Active { + if !self + .confirmed_workspace_commit_replay(&workspace, tool_call_id) + .await? + { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_key, + }); + } + let current_hand = lease + .handle + .as_ref() + .map(|handle| handle.handle.clone()) + .ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_key.clone(), + })?; + let provider = self + .hands + .providers + .get(&initial_workspace.provider) + .ok_or_else(|| { + MoaError::ProviderError(format!( + "hand provider {} is not registered", + initial_workspace.provider + )) + })?; + self.run_within_scope(request.scope, provider.destroy(¤t_hand)) + .await + .map_err(|error| { + tracing::warn!( + operation_id = %operation_key, + error = %error, + "worker hand destroy outcome is ambiguous" + ); + MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_key.clone(), + } + })?; + if !repository + .finalize_checkpointed_hand_destroy(&workspace.binding()?, &lease) + .await? + { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_key, + }); + } + let key = crate::core::lifecycle::session_provider_key( + request.session, + Some(&lease_scope), + &initial_workspace.provider, + ); + self.remove_cached_binding_if_matches( + &key, + ¤t_hand, + Some(initial_lease.generation), + ) + .await; + self.remove_installed_marker( + crate::core::lifecycle::manifest_scope_key(request.session, Some(&lease_scope)), + &initial_workspace.provider, + ) + .await; + workspace = repository + .get_by_scope(request.session.tenant_id, &workspace_scope) + .await? + .ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_key.clone(), + })?; + lease = lease_store + .get( + request.session.tenant_id, + request.session.id, + &lease_scope, + &initial_workspace.provider, + ) + .await? + .ok_or_else(|| MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_key.clone(), + })?; + } + + let exact_generation = lease.generation == initial_lease.generation + && lease.provisioning_operation_id == initial_lease.provisioning_operation_id; + if workspace.state != SandboxWorkspaceState::Ready + || workspace.checkpoint_id.is_none() + || workspace.checkpoint_generation <= initial_workspace.checkpoint_generation + || !exact_generation + || lease.status != HandLeaseStatus::Destroyed + || lease.handle.is_some() + || lease.attachment.is_some() + { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_key, + }); + } + self.verify_worker_hand_absent(request.session, &lease_scope, &scope_key, &operation_key) + .await + } + + async fn verify_worker_hand_absent( + &self, + session: &SessionMeta, + lease_scope: &str, + scope_key: &crate::core::HandScopeKey, + operation_key: &str, + ) -> Result<()> { + let durably_live = match self.hands.hand_leases.as_ref() { + Some(leases) => { + leases + .has_live_owner(session.tenant_id, session.id, lease_scope) + .await? + } + None => false, + }; + let cached = self + .hands + .active_hands + .read() + .await + .keys() + .any(|key| &key.scope == scope_key); + if durably_live || cached { + return Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_key.to_string(), + }); + } + Ok(()) + } +} + +fn validate_worker_input_target(worker_id: &str, target: &WorkerInputTarget) -> Result<()> { + if worker_id.trim().is_empty() + || target.turn_id.trim().is_empty() + || target.generation == 0 + || target.input_request_id.trim().is_empty() + { + return Err(MoaError::ValidationError( + "worker hand park requires a worker, turn, positive generation, and input request" + .to_string(), + )); + } + Ok(()) +} + +fn worker_fence_matches( + expected: Option<&WorkerHandReleaseFence>, + workspace: &SandboxWorkspace, + lease: &HandLease, +) -> Result { + let Some(expected) = expected else { + return Ok(false); + }; + Ok(expected.workspace_id == workspace.workspace_id + && expected.writer_epoch + == u64::try_from(workspace.writer_epoch).map_err(|_| { + MoaError::StorageError("workspace writer epoch is invalid".to_string()) + })? + && expected.instance_generation + == u64::try_from(workspace.instance_generation).map_err(|_| { + MoaError::StorageError("workspace instance generation is invalid".to_string()) + })? + && expected.provider == workspace.provider + && expected.provisioning_operation_id == lease.provisioning_operation_id + && expected.hand_lease_generation + == u64::try_from(lease.generation).map_err(|_| { + MoaError::StorageError("hand lease generation is invalid".to_string()) + })?) +} + +fn verify_released_worker_fence( + expected: Option<&WorkerHandReleaseFence>, + workspace: &SandboxWorkspace, + lease: Option<&HandLease>, + operation_key: &str, +) -> Result<()> { + match (expected, lease) { + (None, None) => Ok(()), + (None, Some(lease)) + if lease.status == HandLeaseStatus::Destroyed && lease.handle.is_none() => + { + Ok(()) + } + (Some(_), Some(lease)) + if worker_fence_matches(expected, workspace, lease)? + && lease.status == HandLeaseStatus::Destroyed + && lease.handle.is_none() + && lease.attachment.is_none() => + { + Ok(()) + } + _ => Err(MoaError::ExternalEffectUnknownOutcome { + operation_id: operation_key.to_string(), + }), + } +} + +fn worker_wait_tool_call_id( + workspace_id: SandboxWorkspaceId, + target: &WorkerInputTarget, +) -> ToolCallId { + ToolCallId(Uuid::new_v5( + &workspace_id.0, + worker_wait_operation_key(target).as_bytes(), + )) +} + +fn worker_wait_operation_key(target: &WorkerInputTarget) -> String { + format!( + "worker-input-wait-v1:{}:{}:{}:{}:{}", + target.turn_id.len(), + target.turn_id, + target.generation, + target.input_request_id.len(), + target.input_request_id + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn worker_wait_identity_is_generation_fenced_offline() { + // Pins: reusing one input request id from a different worker generation cannot + // replay the prior checkpoint-and-release operation. + let workspace_id = SandboxWorkspaceId::new(); + let first = WorkerInputTarget { + turn_id: "turn-1".to_string(), + generation: 3, + input_request_id: "input-7".to_string(), + }; + let superseding = WorkerInputTarget { + generation: 4, + ..first.clone() + }; + assert_ne!( + worker_wait_tool_call_id(workspace_id, &first), + worker_wait_tool_call_id(workspace_id, &superseding) + ); + } + + #[test] + fn worker_wait_identity_rejects_unfenced_targets_offline() { + // Pins: an unversioned input request cannot own sandbox release. + let target = WorkerInputTarget { + turn_id: "turn-1".to_string(), + generation: 0, + input_request_id: "input-7".to_string(), + }; + assert!(validate_worker_input_target("worker-1", &target).is_err()); + } +} diff --git a/crates/moa-hands/src/core/sandbox_workspace/model.rs b/crates/moa-hands/src/core/sandbox_workspace/model.rs index 815768a90..e57bfffab 100644 --- a/crates/moa-hands/src/core/sandbox_workspace/model.rs +++ b/crates/moa-hands/src/core/sandbox_workspace/model.rs @@ -4,6 +4,7 @@ use chrono::{DateTime, Utc}; use moa_core::{ error::{MoaError, Result}, types::{ + contact::ContactId, identifiers::{ HandProvisioningOperationId, ProviderAccountId, SandboxWorkspaceId, TenantId, WorkspaceCheckpointId, @@ -22,6 +23,8 @@ use crate::core::leases::{HandLease, LeaseHandle}; pub struct TaskHandReleaseIntent<'a> { /// Deterministic receipt identity. pub receipt_id: Uuid, + /// Contact whose RLS scope owns the execution, when contact-scoped. + pub contact_id: Option, /// Owning execution run. pub run_id: moa_core::types::identifiers::ExecutionRunScopeId, /// Stable task identity within the run. @@ -46,6 +49,8 @@ pub struct AbsentTaskHandReleaseIntent { pub receipt_id: Uuid, /// Tenant owning the execution. pub tenant_id: TenantId, + /// Contact whose RLS scope owns the execution, when contact-scoped. + pub contact_id: Option, /// Owning execution run. pub run_id: moa_core::types::identifiers::ExecutionRunScopeId, /// Stable task identity. @@ -64,6 +69,8 @@ pub struct CompensationHandReleaseIntent<'a> { pub receipt_id: Uuid, /// Tenant owning the execution. pub tenant_id: TenantId, + /// Contact whose RLS scope owns the execution, when contact-scoped. + pub contact_id: Option, /// Parent session whose hand scope is inspected. pub session_id: moa_core::types::identifiers::SessionId, /// Owning execution run. @@ -84,6 +91,24 @@ pub struct CompensationHandReleaseIntent<'a> { pub recovery_claim_expires_at: DateTime, } +/// Exact pending compensation release whose expired storage claim is renewed. +pub struct CompensationHandReleaseClaimIntent { + /// Tenant owning the execution. + pub tenant_id: TenantId, + /// Contact whose RLS scope owns the execution, when contact-scoped. + pub contact_id: Option, + /// Owning execution run. + pub run_id: moa_core::types::identifiers::ExecutionRunScopeId, + /// Stable compensation identity. + pub compensation_id: moa_core::types::identifiers::ExecutionCompensationScopeId, + /// Exact logical compensation generation. + pub logical_generation: u64, + /// Exact bounded attempt generation. + pub attempt_generation: u64, + /// Short database claim expiry used for storage-only finalization retries. + pub recovery_claim_expires_at: DateTime, +} + /// Renewed recovery authority for one already-persisted compensation release. #[derive(Clone, Debug, PartialEq, Eq)] pub struct CompensationHandReleaseClaim { diff --git a/crates/moa-hands/src/core/sandbox_workspace/repository/lifecycle.rs b/crates/moa-hands/src/core/sandbox_workspace/repository/lifecycle.rs index 2052cd9df..b209b9e76 100644 --- a/crates/moa-hands/src/core/sandbox_workspace/repository/lifecycle.rs +++ b/crates/moa-hands/src/core/sandbox_workspace/repository/lifecycle.rs @@ -15,6 +15,7 @@ impl PostgresWorkspaceRepository { ) -> Result<(Uuid, Uuid, chrono::DateTime)> { let TaskHandReleaseIntent { receipt_id, + contact_id, run_id, task_id, logical_generation, @@ -60,7 +61,9 @@ impl PostgresWorkspaceRepository { ) })?; let claim_token = Uuid::now_v7(); - let mut conn = self.begin(workspace.tenant_id).await?; + let mut conn = self + .begin_with_contact(workspace.tenant_id, contact_id) + .await?; let row = sqlx::query( r#" INSERT INTO moa.sandbox_execution_hand_release_receipts ( @@ -170,6 +173,7 @@ impl PostgresWorkspaceRepository { pub async fn get_task_execution_hand_release_receipt( &self, tenant_id: TenantId, + contact_id: Option, run_id: ExecutionRunScopeId, task_id: ExecutionTaskScopeId, logical_generation: u64, @@ -185,7 +189,7 @@ impl PostgresWorkspaceRepository { "execution task attempt generation overflows Postgres bigint".to_string(), ) })?; - let mut conn = self.begin(tenant_id).await?; + let mut conn = self.begin_with_contact(tenant_id, contact_id).await?; let row = sqlx::query( r#" SELECT receipt_id, tenant_id, run_uid, owner_kind, task_id, compensation_id, @@ -238,7 +242,9 @@ impl PostgresWorkspaceRepository { "execution task attempt generation overflows Postgres bigint".to_string(), ) })?; - let mut conn = self.begin(intent.tenant_id).await?; + let mut conn = self + .begin_with_contact(intent.tenant_id, intent.contact_id) + .await?; sqlx::query( r#" WITH locked_task AS MATERIALIZED ( @@ -350,6 +356,7 @@ impl PostgresWorkspaceRepository { let CompensationHandReleaseIntent { receipt_id, tenant_id, + contact_id, session_id, run_id, compensation_id, @@ -381,7 +388,7 @@ impl PostgresWorkspaceRepository { ) })?; let claim_token = Uuid::now_v7(); - let mut conn = self.begin(tenant_id).await?; + let mut conn = self.begin_with_contact(tenant_id, contact_id).await?; let row = sqlx::query( r#" INSERT INTO moa.sandbox_execution_hand_release_receipts ( @@ -509,13 +516,17 @@ impl PostgresWorkspaceRepository { /// finalize the persisted receipt using its original provisioning identity. pub async fn claim_pending_compensation_execution_hand_release( &self, - tenant_id: TenantId, - run_id: ExecutionRunScopeId, - compensation_id: ExecutionCompensationScopeId, - logical_generation: u64, - attempt_generation: u64, - recovery_claim_expires_at: chrono::DateTime, + intent: CompensationHandReleaseClaimIntent, ) -> Result> { + let CompensationHandReleaseClaimIntent { + tenant_id, + contact_id, + run_id, + compensation_id, + logical_generation, + attempt_generation, + recovery_claim_expires_at, + } = intent; let logical_generation = i64::try_from(logical_generation).map_err(|_| { MoaError::ValidationError( "compensation logical generation overflows Postgres bigint".to_string(), @@ -527,7 +538,7 @@ impl PostgresWorkspaceRepository { ) })?; let claim_token = Uuid::now_v7(); - let mut conn = self.begin(tenant_id).await?; + let mut conn = self.begin_with_contact(tenant_id, contact_id).await?; let row = sqlx::query( r#" UPDATE moa.sandbox_execution_hand_release_receipts AS receipt @@ -593,6 +604,7 @@ impl PostgresWorkspaceRepository { session_id: SessionId, hand_scope: &str, claim_token: Uuid, + contact_id: Option, ) -> Result { let (compensation_id, logical_generation) = match receipt.owner { ExecutionHandReleaseOwner::Compensation { @@ -632,7 +644,9 @@ impl PostgresWorkspaceRepository { "compensation hand release identity must be wholly present or absent".to_string(), )); } - let mut conn = self.begin(receipt.tenant_id).await?; + let mut conn = self + .begin_with_contact(receipt.tenant_id, contact_id) + .await?; let row = sqlx::query( r#" UPDATE moa.sandbox_execution_hand_release_receipts AS receipt @@ -716,6 +730,7 @@ impl PostgresWorkspaceRepository { &self, receipt: &ExecutionHandReleaseReceipt, claim_token: Uuid, + contact_id: Option, ) -> Result { let (task_id, logical_generation) = match receipt.owner { ExecutionHandReleaseOwner::Task { @@ -803,7 +818,9 @@ impl PostgresWorkspaceRepository { "checkpoint logical bytes overflow Postgres bigint".to_string(), ) })?; - let mut conn = self.begin(receipt.tenant_id).await?; + let mut conn = self + .begin_with_contact(receipt.tenant_id, contact_id) + .await?; let row = sqlx::query( r#" UPDATE moa.sandbox_execution_hand_release_receipts AS receipt @@ -946,10 +963,11 @@ impl PostgresWorkspaceRepository { /// Finalizes verified compute destruction after a checkpoint was already committed. /// /// This is the recovery seam for an ambiguous checkpoint attempt that was later - /// reconciled with its attachment retained. Provider destruction happens before + /// reconciled with its attachment retained. Execution tasks and conversational + /// workers share this exact atomic boundary. Provider destruction happens before /// this call; the lease, capacity charge, and workspace state then advance under /// the exact hand and workspace generations in one transaction. - pub async fn finalize_task_yield_destroy( + pub async fn finalize_checkpointed_hand_destroy( &self, binding: &WorkspaceBinding, lease: &HandLease, diff --git a/crates/moa-hands/src/core/sandbox_workspace/repository/mod.rs b/crates/moa-hands/src/core/sandbox_workspace/repository/mod.rs index f9d486b03..e182a3049 100644 --- a/crates/moa-hands/src/core/sandbox_workspace/repository/mod.rs +++ b/crates/moa-hands/src/core/sandbox_workspace/repository/mod.rs @@ -8,6 +8,7 @@ mod lifecycle; use moa_core::{ error::{MoaError, Result}, types::{ + contact::ContactId, identifiers::{ ExecutionCompensationScopeId, ExecutionRunScopeId, ExecutionTaskScopeId, HandProvisioningOperationId, ProviderAccountId, SandboxWorkspaceId, SessionId, @@ -34,10 +35,10 @@ use super::{ failpoints, model::{ AbsentTaskHandReleaseIntent, ActivateHydratedWorkspaceRequest, - CompensationHandReleaseClaim, CompensationHandReleaseIntent, CreateWorkspaceRequest, - SandboxWorkspace, TaskHandReleaseIntent, WorkspaceGrant, WorkspaceGrantRelation, - WorkspaceGrantSubjectType, WorkspaceProviderAccount, WorkspaceTransition, - WorkspaceWriterClaim, + CompensationHandReleaseClaim, CompensationHandReleaseClaimIntent, + CompensationHandReleaseIntent, CreateWorkspaceRequest, SandboxWorkspace, + TaskHandReleaseIntent, WorkspaceGrant, WorkspaceGrantRelation, WorkspaceGrantSubjectType, + WorkspaceProviderAccount, WorkspaceTransition, WorkspaceWriterClaim, }, operations::ClaimedWorkspaceOperation, }; @@ -70,6 +71,14 @@ impl PostgresWorkspaceRepository { } async fn begin(&self, tenant_id: TenantId) -> Result> { + self.begin_with_contact(tenant_id, None).await + } + + async fn begin_with_contact( + &self, + tenant_id: TenantId, + contact_id: Option, + ) -> Result> { if self.assume_workspace_maintenance_role { let mut conn = ScopedConn::begin_control_plane(&self.pool).await?; sqlx::query("SET LOCAL ROLE moa_workspace_maintenance") @@ -78,7 +87,11 @@ impl PostgresWorkspaceRepository { .map_err(map_sqlx_error)?; Ok(conn) } else { - ScopedConn::begin_as_app(&self.pool, &RlsContext::tenant(tenant_id), true).await + let context = contact_id.map_or_else( + || RlsContext::tenant(tenant_id), + |contact_id| RlsContext::contact(tenant_id, contact_id), + ); + ScopedConn::begin_as_app(&self.pool, &context, true).await } } diff --git a/crates/moa-hands/src/lib.rs b/crates/moa-hands/src/lib.rs index 909571dc5..e8e3cf48d 100644 --- a/crates/moa-hands/src/lib.rs +++ b/crates/moa-hands/src/lib.rs @@ -18,7 +18,7 @@ pub use core::{ ProviderCredentialSource, ProviderEndpoint, ProviderHttpAttempt, ProviderSandboxAttempt, SandboxProviderInventory, SessionHandReleasePageOutcome, TenantSandboxPolicyStore, ToolCallScope, ToolCatalogDrift, ToolCatalogPin, ToolCatalogSnapshot, ToolExecution, - ToolRegistry, ToolRouter, deployment_sandbox_policy, governed_tool_contract_revision, - local_development_sandbox_policy, mcp_tool_reference, route_sandbox_policy, - spawn_mcp_catalog_refresh, truncate_tool_span_text, + ToolRegistry, ToolRouter, WorkerHandReleaseFence, WorkerHandReleaseRequest, + deployment_sandbox_policy, governed_tool_contract_revision, local_development_sandbox_policy, + mcp_tool_reference, route_sandbox_policy, spawn_mcp_catalog_refresh, truncate_tool_span_text, }; diff --git a/crates/moa-hands/tests/hands_db/sandbox_workspace/dispatch_db.rs b/crates/moa-hands/tests/hands_db/sandbox_workspace/dispatch_db.rs index 1b242b4a7..4d435b18c 100644 --- a/crates/moa-hands/tests/hands_db/sandbox_workspace/dispatch_db.rs +++ b/crates/moa-hands/tests/hands_db/sandbox_workspace/dispatch_db.rs @@ -40,11 +40,12 @@ use moa_core::{ }, session::SessionMeta, tools::{IdempotencyClass, ToolDiffStrategy, ToolInputShape, ToolOutput, ToolPolicySpec}, + worker::state::WorkerInputTarget, }, }; use moa_hands::{ AuthorizedToolCall, HandRoute, JournaledWorkspaceCommit, ToolCallScope, ToolRegistry, - ToolRouter, + ToolRouter, WorkerHandReleaseRequest, core::{ leases::{ HandLeasePolicy, HandLeaseProvisionRequest, HandLeaseStatus, HandLeaseStore, @@ -935,6 +936,7 @@ struct GatedWorkspaceProvider { restore_calls: AtomicUsize, reconcile_calls: AtomicUsize, checkpoint_post_commit_state: WorkspacePostCommitState, + checkpoint_capacity: Option, } impl GatedWorkspaceProvider { @@ -955,6 +957,7 @@ impl GatedWorkspaceProvider { restore_calls: AtomicUsize::new(0), reconcile_calls: AtomicUsize::new(0), checkpoint_post_commit_state: WorkspacePostCommitState::AttachmentRetained, + checkpoint_capacity: None, }, started_rx, release_tx, @@ -962,17 +965,23 @@ impl GatedWorkspaceProvider { } fn management() -> Self { - let (started_tx, _started_rx) = oneshot::channel(); - let (_release_tx, release_rx) = oneshot::channel(); Self { - commit_started: Mutex::new(Some(started_tx)), - commit_release: Mutex::new(Some(release_rx)), + commit_started: Mutex::new(None), + commit_release: Mutex::new(None), commit_calls: AtomicUsize::new(0), attach_calls: AtomicUsize::new(0), checkpoint_calls: AtomicUsize::new(0), restore_calls: AtomicUsize::new(0), reconcile_calls: AtomicUsize::new(0), checkpoint_post_commit_state: WorkspacePostCommitState::ComputeDestroyed, + checkpoint_capacity: None, + } + } + + fn parking(pool: PgPool) -> Self { + Self { + checkpoint_capacity: Some(PostgresWorkspaceCapacityRepository::new(pool)), + ..Self::management() } } @@ -1144,24 +1153,32 @@ impl SandboxStorageProvider for GatedWorkspaceProvider { match request.operation.kind { WorkspaceOperationKind::Commit => { self.commit_calls.fetch_add(1, Ordering::SeqCst); - self.commit_started + let started = self + .commit_started .lock() .expect("commit gate mutex should not be poisoned") - .take() - .expect("the router must publish exactly one commit request") - .send(request.clone()) - .map_err(|_| { - MoaError::StorageError("commit observer disappeared".to_string()) - })?; + .take(); let release = self .commit_release .lock() .expect("commit release mutex should not be poisoned") - .take() - .expect("the router must await exactly one commit release"); - release.await.map_err(|_| { - MoaError::StorageError("commit release disappeared".to_string()) - })?; + .take(); + match (started, release) { + (Some(started), Some(release)) => { + started.send(request.clone()).map_err(|_| { + MoaError::StorageError("commit observer disappeared".to_string()) + })?; + release.await.map_err(|_| { + MoaError::StorageError("commit release disappeared".to_string()) + })?; + } + (None, None) => {} + _ => { + return Err(MoaError::StorageError( + "commit gate is only partially configured".to_string(), + )); + } + } } WorkspaceOperationKind::Checkpoint => { self.checkpoint_calls.fetch_add(1, Ordering::SeqCst); @@ -1172,7 +1189,16 @@ impl SandboxStorageProvider for GatedWorkspaceProvider { )); } } - Ok(self.committed_result(&request.operation)) + let result = self.committed_result(&request.operation); + if let (Some(capacity), Some(publication)) = ( + self.checkpoint_capacity.as_ref(), + result.checkpoint_publication.as_ref(), + ) { + capacity + .reserve_checkpoint_publication(&request.operation, publication.logical_bytes) + .await?; + } + Ok(result) } async fn restore_workspace( @@ -1372,6 +1398,172 @@ async fn public_management_attach_checkpoint_and_exact_restore_are_durable_db() pool.close().await; } +#[tokio::test] +#[ignore = "requires a fresh V58 compose Postgres via MOA_DATABASE_URL"] +async fn worker_input_wait_checkpoints_releases_and_rejects_stale_replay_db() { + // Pins: a worker parks only the exact hand generation captured before its input + // registration. The portable checkpoint survives fresh provision/restore, and a + // delayed replay cannot checkpoint or destroy that replacement hand. + let pool = pool().await; + let tenant_id = TenantId::new(); + let session_id = SessionId::new(); + let account_id = ProviderAccountId::new(); + let workspace_id = SandboxWorkspaceId::new(); + let worker_id = format!("worker-input-wait-{workspace_id}"); + let scope = SandboxWorkspaceScope::Worker { + session_id, + worker_id: worker_id.clone(), + }; + seed_session(&pool, session_id, tenant_id).await; + seed_named_account(&pool, account_id, GATED_PROVIDER).await; + let workspaces = PostgresWorkspaceRepository::new(pool.clone()); + workspaces + .create(&CreateWorkspaceRequest { + workspace_id, + tenant_id, + scope: scope.clone(), + provider: GATED_PROVIDER.to_string(), + provider_account_id: account_id, + provider_account_generation: 1, + durability_class: DurabilityClass::PortableFilesystem, + retention_deadline_at: None, + }) + .await + .expect("create worker input-wait workspace"); + + let provider = Arc::new(GatedWorkspaceProvider::parking(pool.clone())); + let mut registry = ToolRegistry::new(); + registry.register_hand( + "worker_input_wait_route_anchor", + "exposes the worker input-wait provider route", + serde_json::json!({ "type": "object", "additionalProperties": false }), + ToolPolicySpec { + risk_level: RiskLevel::Low, + default_effect: ActionPolicyEffect::Allow, + action_class: ActionClass::Read, + input_shape: ToolInputShape::Json, + diff_strategy: ToolDiffStrategy::None, + }, + IdempotencyClass::Idempotent, + ); + registry.retarget_hand_tools(vec![HandRoute { + provider: GATED_PROVIDER.to_string(), + tier: SandboxTier::Container, + policy: SandboxPolicySnapshot::builtin(BuiltinPolicyRevision::RouteUnset), + }]); + let mut hand_providers = HashMap::new(); + hand_providers.insert( + GATED_PROVIDER.to_string(), + Arc::clone(&provider) as Arc, + ); + let router = ToolRouter::new(registry, hand_providers, local_development_sandbox_policy()) + .with_sandbox_storage_provider(Arc::clone(&provider) as Arc) + .expect("register input-wait storage provider") + .with_workspace_repositories(pool.clone()) + .with_hand_lease_store(Arc::new(PostgresHandLeaseStore::new(pool.clone()))) + .with_hand_lease_reaper(); + let session = SessionMeta { + id: session_id, + tenant_id, + model: ModelId::new("worker-input-wait-model"), + ..SessionMeta::default() + }; + router + .attach_managed_workspace(&session, &scope, workspace_id) + .await + .expect("materialize the worker hand"); + let first_fence = router + .capture_worker_hand_release_fence(&session, &worker_id) + .await + .expect("capture exact first hand") + .expect("active worker has a hand fence"); + let target = WorkerInputTarget { + turn_id: "turn-1".to_string(), + generation: 1, + input_request_id: "input-1".to_string(), + }; + let park = WorkerHandReleaseRequest { + session: &session, + worker_id: &worker_id, + input_target: &target, + expected: Some(&first_fence), + scope: ToolCallScope::unbounded(), + }; + router + .checkpoint_and_release_worker_hand(park) + .await + .expect("checkpoint and release the exact worker hand"); + assert_eq!(provider.commit_calls.load(Ordering::SeqCst), 1); + let parked = workspaces + .get(tenant_id, workspace_id) + .await + .expect("load parked workspace") + .expect("parked workspace exists"); + assert_eq!(parked.state, SandboxWorkspaceState::Ready); + let checkpoint_id = parked + .checkpoint_id + .expect("parked workspace has a verified checkpoint"); + let parked_lease = PostgresHandLeaseStore::new(pool.clone()) + .get(tenant_id, session_id, &worker_id, GATED_PROVIDER) + .await + .expect("load parked hand lease") + .expect("parked hand lease exists"); + assert_eq!(parked_lease.status, HandLeaseStatus::Destroyed); + assert!(parked_lease.handle.is_none()); + assert_eq!( + sqlx::query_scalar::<_, i64>( + "SELECT count(*) FROM moa.sandbox_capacity_reservations \ + WHERE tenant_id = $1 AND workspace_id = $2 \ + AND resource_dimension = 'active_hands' AND reservation_state != 'released'", + ) + .bind(tenant_id) + .bind(workspace_id) + .fetch_one(&pool) + .await + .expect("count live active-hand charges"), + 0 + ); + + router + .checkpoint_and_release_worker_hand(park) + .await + .expect("exact release replay is a no-op"); + assert_eq!(provider.commit_calls.load(Ordering::SeqCst), 1); + router + .restore_managed_workspace(&session, &scope, workspace_id, checkpoint_id) + .await + .expect("fresh compute restores the parked checkpoint"); + let replacement_fence = router + .capture_worker_hand_release_fence(&session, &worker_id) + .await + .expect("capture replacement hand") + .expect("restored worker has a replacement hand"); + assert_ne!(replacement_fence, first_fence); + + let delayed_target = WorkerInputTarget { + turn_id: "turn-delayed".to_string(), + generation: 1, + input_request_id: "input-delayed".to_string(), + }; + router + .checkpoint_and_release_worker_hand(WorkerHandReleaseRequest { + input_target: &delayed_target, + ..park + }) + .await + .expect_err("unused stale release fence cannot touch replacement compute"); + assert_eq!(provider.commit_calls.load(Ordering::SeqCst), 1); + let restored = workspaces + .get(tenant_id, workspace_id) + .await + .expect("load restored workspace") + .expect("restored workspace exists"); + assert_eq!(restored.state, SandboxWorkspaceState::Active); + + cleanup(&pool, session_id, workspace_id, account_id).await; + pool.close().await; +} + #[tokio::test] #[ignore = "requires a fresh V58 compose Postgres via MOA_DATABASE_URL"] async fn may_write_result_waits_for_atomic_checkpoint_publication_db() { diff --git a/crates/moa-hands/tests/hands_db/sandbox_workspace/lifecycle_db.rs b/crates/moa-hands/tests/hands_db/sandbox_workspace/lifecycle_db.rs index 6b5467624..7532f22a5 100644 --- a/crates/moa-hands/tests/hands_db/sandbox_workspace/lifecycle_db.rs +++ b/crates/moa-hands/tests/hands_db/sandbox_workspace/lifecycle_db.rs @@ -6,6 +6,7 @@ use chrono::{Duration as ChronoDuration, Utc}; use moa_core::error::MoaError; use moa_core::types::{ action_policy::CallOrigin, + contact::ContactId, hands::{ BuiltinPolicyRevision, CpuLimit, DiskLimit, EgressPolicy, HandHandle, LifetimeLimit, MemoryLimit, SandboxPolicySnapshot, SandboxProfile, SandboxTier, @@ -33,8 +34,9 @@ use moa_hands::core::{ checkpoint::model::{CreateCheckpointRequest, PublishCheckpointCommitRequest}, model::{ AbsentTaskHandReleaseIntent, ActivateHydratedWorkspaceRequest, - CompensationHandReleaseIntent, CreateWorkspaceRequest, SandboxWorkspace, - TaskHandReleaseIntent, WorkspaceTransition, WorkspaceWriterClaim, + CompensationHandReleaseClaimIntent, CompensationHandReleaseIntent, + CreateWorkspaceRequest, SandboxWorkspace, TaskHandReleaseIntent, WorkspaceTransition, + WorkspaceWriterClaim, }, operations::{ AbsenceObservation, PostgresWorkspaceOperationRepository, WorkspaceOperationIntent, @@ -67,6 +69,7 @@ async fn seed_cancelling_compensation( pool: &sqlx::PgPool, tenant_id: TenantId, session_id: SessionId, + contact_id: Option, ) -> ( ExecutionRunScopeId, ExecutionTaskScopeId, @@ -83,10 +86,6 @@ async fn seed_cancelling_compensation( "cancel_policy": "retain_effects", "input_schema": {}, "output_schema": {}, - "input_wait_policy": { - "expiry": {"kind": "after", "delay_seconds": 1}, - "on_expiry": {"kind": "fail_task"} - }, "nodes": [{ "id": "output", "requirement_ids": [], "depends_on": [], "when": null, "input": {}, "output_schema": {}, @@ -105,13 +104,15 @@ async fn seed_cancelling_compensation( sqlx::query( "INSERT INTO moa.execution_planning_context (\ planning_context_uid, tenant_id, session_id, originating_user_sequence_num,\ - originating_user_event_hash, owner_user_id, planning_context_hash, snapshot\ - ) VALUES ($1, $2, $3, 0, $4, 'hands-release-test', $4, '{}'::JSONB)", + originating_user_event_hash, owner_user_id, planning_context_hash, snapshot,\ + contact_id\ + ) VALUES ($1, $2, $3, 0, $4, 'hands-release-test', $4, '{}'::JSONB, $5)", ) .bind(planning_context_uid) .bind(tenant_id) .bind(session_id) .bind(&context_hash) + .bind(contact_id) .execute(pool) .await .expect("seed execution planning context"); @@ -121,9 +122,9 @@ async fn seed_cancelling_compensation( planning_context_uid, planning_context_hash, owner_user_id, goal_contract,\ initial_plan, active_plan, initial_plan_hash, active_plan_hash,\ capability_catalog, authorization_envelope, source_provenance, source_kind,\ - input, admitted_identity, status\ + input, admitted_identity, status, contact_id\ ) VALUES ($1, $2, $3, 0, $4, $5, 'hands-release-test', $6, $7, $7, $8, $8,\ - $9, $10, $11, 'generated_plan', '{}'::JSONB, $12, 'queued')", + $9, $10, $11, 'generated_plan', '{}'::JSONB, $12, 'queued', $13)", ) .bind(run_id) .bind(tenant_id) @@ -148,6 +149,7 @@ async fn seed_cancelling_compensation( "identity_type": "operator", "id": run_id, "tenant_id": tenant_id, "api_key_id": null, "acting_on_behalf_of": null })) + .bind(contact_id) .execute(pool) .await .expect("seed execution run"); @@ -155,15 +157,16 @@ async fn seed_cancelling_compensation( "INSERT INTO moa.execution_task (\ task_id, run_uid, tenant_id, node_id, item_key, plan_revision, status, input,\ task_kind, retry_policy, estimate_cost_microusd, estimate_tokens, estimate_tasks,\ - estimate_tool_calls, estimate_retrieved_bytes\ + estimate_tool_calls, estimate_retrieved_bytes, contact_id\ ) VALUES ($1, $2, $3, 'forward', 'forward', 1, 'completed', '{}',\ '{\"kind\":\"output\",\"value\":null}',\ '{\"max_attempts\":2,\"initial_backoff_ms\":1,\"max_backoff_ms\":1}',\ - 0, 0, 1, 0, 0)", + 0, 0, 1, 0, 0, $4)", ) .bind(task_id) .bind(run_id) .bind(tenant_id) + .bind(contact_id) .execute(pool) .await .expect("seed forward task"); @@ -171,9 +174,9 @@ async fn seed_cancelling_compensation( "INSERT INTO moa.execution_compensation (\ compensation_id, run_uid, forward_task_id, tenant_id, registered_sequence,\ forward_generation, compensator, mapped_input, status, started_at,\ - attempt_state, attempt_started_at, attempt_deadline_at, release_intent\ + attempt_state, attempt_started_at, attempt_deadline_at, release_intent, contact_id\ ) VALUES ($1, $2, $3, $4, 1, 1, $5, '{}', 'running', now(),\ - 'cancelling', now(), now() + interval '10 minutes', 'pause')", + 'cancelling', now(), now() + interval '10 minutes', 'pause', $6)", ) .bind(compensation_id) .bind(run_id) @@ -183,6 +186,7 @@ async fn seed_cancelling_compensation( "compensator": {"name": "test.undo", "version": "contract"}, "input_mapping": {"bindings": []} })) + .bind(contact_id) .execute(pool) .await .expect("seed cancelling compensation"); @@ -194,6 +198,7 @@ async fn seed_cancelling_task( tenant_id: TenantId, run_id: ExecutionRunScopeId, node_id: &str, + contact_id: Option, ) -> ExecutionTaskScopeId { let task_id = ExecutionTaskScopeId::new(); sqlx::query( @@ -201,17 +206,18 @@ async fn seed_cancelling_task( task_id, run_uid, tenant_id, node_id, item_key, plan_revision, status, input,\ task_kind, retry_policy, estimate_cost_microusd, estimate_tokens, estimate_tasks,\ estimate_tool_calls, estimate_retrieved_bytes, reserved_tasks, reserved_at,\ - started_at, attempt_state, attempt_started_at, attempt_deadline_at\ + started_at, attempt_state, attempt_started_at, attempt_deadline_at, contact_id\ ) VALUES ($1, $2, $3, $4, $4, 1, 'running', '{}',\ '{\"kind\":\"output\",\"value\":null}',\ '{\"max_attempts\":2,\"initial_backoff_ms\":1,\"max_backoff_ms\":1}',\ 0, 0, 1, 0, 0, 1, now(), now(), 'cancelling', now(),\ - now() + interval '10 minutes')", + now() + interval '10 minutes', $5)", ) .bind(task_id) .bind(run_id) .bind(tenant_id) .bind(node_id) + .bind(contact_id) .execute(pool) .await .expect("seed cancelling execution task"); @@ -361,25 +367,34 @@ fn create_request( } #[tokio::test] -#[ignore = "requires a fresh V60 compose Postgres via MOA_DATABASE_URL"] +#[ignore = "requires Postgres for an isolated current-schema test database"] async fn cancelling_task_without_owned_compute_gets_exact_absence_receipt_db() { // Pins: a sandbox-capable task denied before provisioning still obtains a // durable exact-attempt absence receipt, while a live lease cannot be hidden // behind that no-workspace path. - let pool = PgPoolOptions::new() - .max_connections(4) - .connect(&database_url()) + let test_db = moa_test_support::postgres::bootstrap_test_db() .await - .expect("test Postgres should be reachable"); + .expect("bootstrap isolated current-schema Postgres"); + let pool = test_db.store().pool().clone(); let tenant_id = TenantId::new(); let session_id = SessionId::new(); + let contact_id = ContactId::new(); seed_session(&pool, session_id, tenant_id).await; - let (run_id, _, _) = seed_cancelling_compensation(&pool, tenant_id, session_id).await; - let task_id = seed_cancelling_task(&pool, tenant_id, run_id, "never-provisioned").await; + let (run_id, _, _) = + seed_cancelling_compensation(&pool, tenant_id, session_id, Some(contact_id)).await; + let task_id = seed_cancelling_task( + &pool, + tenant_id, + run_id, + "never-provisioned", + Some(contact_id), + ) + .await; let repository = PostgresWorkspaceRepository::new(pool.clone()); let intent = AbsentTaskHandReleaseIntent { receipt_id: uuid::Uuid::now_v7(), tenant_id, + contact_id: Some(contact_id), run_id, task_id, logical_generation: 1, @@ -409,13 +424,21 @@ async fn cancelling_task_without_owned_compute_gets_exact_absence_receipt_db() { ); assert_eq!( repository - .get_task_execution_hand_release_receipt(tenant_id, run_id, task_id, 1, 1) + .get_task_execution_hand_release_receipt( + tenant_id, + Some(contact_id), + run_id, + task_id, + 1, + 1, + ) .await .expect("replay exact absence receipt"), Some(receipt) ); - let live_task_id = seed_cancelling_task(&pool, tenant_id, run_id, "live-owner").await; + let live_task_id = + seed_cancelling_task(&pool, tenant_id, run_id, "live-owner", Some(contact_id)).await; let live_scope = format!("execution:{run_id}:{live_task_id}"); sqlx::query( "INSERT INTO moa.hand_leases (\ @@ -435,6 +458,7 @@ async fn cancelling_task_without_owned_compute_gets_exact_absence_receipt_db() { .record_absent_task_execution_hand_release_receipt(AbsentTaskHandReleaseIntent { receipt_id: uuid::Uuid::now_v7(), tenant_id, + contact_id: Some(contact_id), run_id, task_id: live_task_id, logical_generation: 1, @@ -460,12 +484,21 @@ async fn checkpointed_task_destroy_records_exact_release_receipt_db() { let pool = test_db.store().pool().clone(); let tenant_id = TenantId::new(); let session_id = SessionId::new(); + let contact_id = ContactId::new(); let account_id = ProviderAccountId::new(); let workspace_id = SandboxWorkspaceId::new(); seed_session(&pool, session_id, tenant_id).await; seed_account(&pool, account_id).await; - let (run_id, _, _) = seed_cancelling_compensation(&pool, tenant_id, session_id).await; - let task_id = seed_cancelling_task(&pool, tenant_id, run_id, "checkpointed-release").await; + let (run_id, _, _) = + seed_cancelling_compensation(&pool, tenant_id, session_id, Some(contact_id)).await; + let task_id = seed_cancelling_task( + &pool, + tenant_id, + run_id, + "checkpointed-release", + Some(contact_id), + ) + .await; let worker_id = format!("execution:{run_id}:{task_id}"); let workspace_scope = SandboxWorkspaceScope::ExecutionTask { run_id, task_id }; let workspaces = PostgresWorkspaceRepository::new(pool.clone()); @@ -523,6 +556,7 @@ async fn checkpointed_task_destroy_records_exact_release_receipt_db() { let (persisted_receipt_id, claim_token, requested_at) = workspaces .begin_task_execution_hand_release(TaskHandReleaseIntent { receipt_id, + contact_id: Some(contact_id), run_id, task_id, logical_generation: 1, @@ -660,13 +694,20 @@ async fn checkpointed_task_destroy_records_exact_release_receipt_db() { released_at: Utc::now(), }; let finalized = workspaces - .record_task_execution_hand_release_receipt(&receipt, claim_token) + .record_task_execution_hand_release_receipt(&receipt, claim_token, Some(contact_id)) .await .expect("available checkpoint must finalize the task release receipt"); assert_eq!(finalized, receipt); assert_eq!( workspaces - .get_task_execution_hand_release_receipt(tenant_id, run_id, task_id, 1, 1) + .get_task_execution_hand_release_receipt( + tenant_id, + Some(contact_id), + run_id, + task_id, + 1, + 1, + ) .await .expect("replay finalized task release receipt"), Some(receipt) @@ -674,21 +715,106 @@ async fn checkpointed_task_destroy_records_exact_release_receipt_db() { } #[tokio::test] -#[ignore = "requires a fresh V60 compose Postgres via MOA_DATABASE_URL"] +#[ignore = "requires Postgres for an isolated current-schema test database"] +async fn contact_scoped_compensation_without_compute_gets_exact_release_receipt_db() { + // Pins: a contact-scoped compensation that never provisioned compute still crosses the + // exact cancelling-owner CAS and persists its replayable verified-absence receipt. + let test_db = moa_test_support::postgres::bootstrap_test_db() + .await + .expect("bootstrap isolated current-schema Postgres"); + let pool = test_db.store().pool().clone(); + let tenant_id = TenantId::new(); + let contact_id = ContactId::new(); + let session_id = SessionId::new(); + seed_session(&pool, session_id, tenant_id).await; + let (run_id, _task_id, compensation_id) = + seed_cancelling_compensation(&pool, tenant_id, session_id, Some(contact_id)).await; + let hand_scope = format!("execution_compensation:{run_id}:{compensation_id}"); + let repository = PostgresWorkspaceRepository::new(pool.clone()); + let receipt_id = uuid::Uuid::now_v7(); + let deadline_at = Utc::now() + ChronoDuration::minutes(1); + let (persisted_receipt_id, claim_token, requested_at) = repository + .begin_compensation_execution_hand_release(CompensationHandReleaseIntent { + receipt_id, + tenant_id, + contact_id: Some(contact_id), + session_id, + run_id, + compensation_id, + logical_generation: 1, + attempt_generation: 1, + hand_scope: &hand_scope, + lease: None, + deadline_at, + recovery_claim_expires_at: deadline_at, + }) + .await + .expect("contact-scoped compensation should persist its absence intent"); + assert_eq!(persisted_receipt_id, receipt_id); + + let receipt = ExecutionHandReleaseReceipt { + receipt_id, + tenant_id, + run_id, + owner: ExecutionHandReleaseOwner::Compensation { + compensation_id, + logical_generation: 1, + }, + attempt_generation: 1, + workspace_id: None, + writer_epoch: None, + instance_generation: None, + hand_provisioning_operation_id: None, + hand_lease_generation: None, + checkpoint_id: None, + checkpoint_generation: None, + checkpoint_manifest_digest: None, + checkpoint_logical_bytes: None, + requested_at, + released_at: Utc::now(), + }; + let finalized = repository + .record_compensation_execution_hand_release_receipt( + &receipt, + session_id, + &hand_scope, + claim_token, + Some(contact_id), + ) + .await + .expect("contact-scoped compensation should finalize verified absence"); + assert_eq!(finalized, receipt); + assert_eq!( + repository + .get_compensation_execution_hand_release_receipt( + tenant_id, + run_id, + compensation_id, + 1, + 1, + ) + .await + .expect("replay contact-scoped compensation receipt"), + Some(receipt) + ); +} + +#[tokio::test] +#[ignore = "requires Postgres for an isolated current-schema test database"] async fn compensation_release_recovers_persisted_destroyed_identity_after_deadline_db() { // Pins: a crash after provider teardown but before receipt finalization reuses // the pending receipt's exact lease identity after the provider-I/O deadline; // deleting that exact destroyed row remains fail-closed. - let pool = PgPoolOptions::new() - .max_connections(4) - .connect(&database_url()) + let test_db = moa_test_support::postgres::bootstrap_test_db() .await - .expect("test Postgres should be reachable"); + .expect("bootstrap isolated current-schema Postgres"); + let pool = test_db.store().pool().clone(); let tenant_id = TenantId::new(); + let contact_id = ContactId::new(); let session_id = SessionId::new(); seed_session(&pool, session_id, tenant_id).await; let (run_id, _task_id, compensation_id) = - seed_cancelling_compensation(&pool, tenant_id, session_id).await; + seed_cancelling_compensation(&pool, tenant_id, session_id, Some(contact_id)).await; let hand_scope = format!("execution_compensation:{run_id}:{compensation_id}"); let provisioning_operation_id = HandProvisioningOperationId::new(); let generation = 7_i64; @@ -724,6 +850,7 @@ async fn compensation_release_recovers_persisted_destroyed_identity_after_deadli .begin_compensation_execution_hand_release(CompensationHandReleaseIntent { receipt_id, tenant_id, + contact_id: Some(contact_id), session_id, run_id, compensation_id, @@ -748,10 +875,10 @@ async fn compensation_release_recovers_persisted_destroyed_identity_after_deadli .expect("finalize exact lease destroy") ); sqlx::query( - "UPDATE moa.sandbox_execution_hand_release_receipts\ - SET requested_at = now() - interval '10 minutes',\ - deadline_at = now() - interval '5 minutes',\ - claim_expires_at = now() - interval '6 minutes'\ + "UPDATE moa.sandbox_execution_hand_release_receipts \ + SET requested_at = now() - interval '10 minutes', \ + deadline_at = now() - interval '5 minutes', \ + claim_expires_at = now() - interval '6 minutes' \ WHERE receipt_id = $1", ) .bind(receipt_id) @@ -759,14 +886,15 @@ async fn compensation_release_recovers_persisted_destroyed_identity_after_deadli .await .expect("advance pending release beyond its provider deadline"); let claim = repository - .claim_pending_compensation_execution_hand_release( + .claim_pending_compensation_execution_hand_release(CompensationHandReleaseClaimIntent { tenant_id, + contact_id: Some(contact_id), run_id, compensation_id, - 1, - 1, - Utc::now() + ChronoDuration::minutes(5), - ) + logical_generation: 1, + attempt_generation: 1, + recovery_claim_expires_at: Utc::now() + ChronoDuration::minutes(5), + }) .await .expect("renew storage-only recovery claim") .expect("expired pending release is reclaimable"); @@ -791,7 +919,7 @@ async fn compensation_release_recovers_persisted_destroyed_identity_after_deadli assert!(destroyed.handle.is_none()); sqlx::query( - "DELETE FROM moa.hand_leases WHERE tenant_id = $1 AND session_id = $2\ + "DELETE FROM moa.hand_leases WHERE tenant_id = $1 AND session_id = $2 \ AND worker_id = $3 AND provisioning_operation_id = $4 AND generation = $5", ) .bind(tenant_id) @@ -830,6 +958,7 @@ async fn compensation_release_recovers_persisted_destroyed_identity_after_deadli session_id, &hand_scope, claim.claim_token, + Some(contact_id), ) .await .is_err(), @@ -855,11 +984,11 @@ async fn compensation_release_recovers_persisted_destroyed_identity_after_deadli session_id, &hand_scope, claim.claim_token, + Some(contact_id), ) .await .expect("finalize recovered exact receipt"); assert_eq!(finalized, release_receipt); - pool.close().await; } #[tokio::test] diff --git a/crates/moa-migrations/migrations/postgres/V000059__long_horizon_execution.sql b/crates/moa-migrations/migrations/postgres/V000059__long_horizon_execution.sql index 27010a023..145214da3 100644 --- a/crates/moa-migrations/migrations/postgres/V000059__long_horizon_execution.sql +++ b/crates/moa-migrations/migrations/postgres/V000059__long_horizon_execution.sql @@ -183,9 +183,8 @@ AS $$ AND moa.execution_wait_expiry_action_is_valid(candidate -> 'on_expiry') $$; --- The old four-key definition remains valid only so retained terminal audit --- rows can continue to satisfy their original check constraint. Every new run --- and serving skill template is required to use the current five-key shape. +-- The current plan contract has exactly four top-level keys. Runtime input has +-- no plan-level expiry policy; input waits remain durable until input arrives. CREATE OR REPLACE FUNCTION moa.execution_plan_definition_is_current(candidate JSONB) RETURNS BOOLEAN LANGUAGE plpgsql @@ -198,14 +197,12 @@ BEGIN IF NOT moa.execution_json_object_has_exact_keys( candidate, ARRAY[ - 'cancel_policy', 'input_schema', 'output_schema', 'input_wait_policy', - 'nodes' + 'cancel_policy', 'input_schema', 'output_schema', 'nodes' ] ) OR candidate ->> 'cancel_policy' NOT IN ( 'retain_effects', 'compensate_committed' ) - OR NOT moa.execution_wait_policy_is_valid(candidate -> 'input_wait_policy') OR jsonb_typeof(candidate -> 'nodes') <> 'array' THEN RETURN FALSE; END IF; @@ -254,26 +251,16 @@ AS $$ AND candidate ->> 'plan_hash' ~ '^[0-9a-f]{64}$' $$; --- V55 bound both run snapshots to the old four-key plan validator. Replace --- those constraints at the cutover boundary so every nonterminal/current run --- uses the five-key long-horizon contract. Pre-cutover terminal rows remain --- immutable audit evidence and are the only permitted legacy shape. +-- Bind every stored run snapshot to the current plan contract. This is a hard +-- break: terminal rows do not retain an alternate legacy shape. ALTER TABLE moa.execution_run DROP CONSTRAINT execution_run_initial_plan_check, DROP CONSTRAINT execution_run_active_plan_check, ADD CONSTRAINT execution_run_initial_plan_check CHECK ( - status IN ( - 'completed', 'partial', 'blocked', 'unsupported', - 'failed', 'cancelled' - ) - OR moa.execution_plan_snapshot_is_current(initial_plan) + moa.execution_plan_snapshot_is_current(initial_plan) ), ADD CONSTRAINT execution_run_active_plan_check CHECK ( - status IN ( - 'completed', 'partial', 'blocked', 'unsupported', - 'failed', 'cancelled' - ) - OR moa.execution_plan_snapshot_is_current(active_plan) + moa.execution_plan_snapshot_is_current(active_plan) ); CREATE OR REPLACE FUNCTION moa.skill_execution_template_is_valid(candidate JSONB) @@ -305,6 +292,7 @@ ALTER TABLE moa.execution_run CHECK (activation_state IN ('idle', 'queued', 'advancing', 'paused', 'terminal')), ADD COLUMN next_wake_at TIMESTAMPTZ, ADD COLUMN waiting_since TIMESTAMPTZ, + ADD COLUMN budget_deadline_suspended_at TIMESTAMPTZ, ADD COLUMN last_progress_at TIMESTAMPTZ NOT NULL DEFAULT now(), ADD COLUMN pause_requested_at TIMESTAMPTZ, ADD COLUMN paused_at TIMESTAMPTZ, @@ -356,6 +344,10 @@ ALTER TABLE moa.execution_run ADD CONSTRAINT execution_run_pause_timestamp_order_check CHECK ( paused_at IS NULL OR (pause_requested_at IS NOT NULL AND paused_at >= pause_requested_at) + ), + ADD CONSTRAINT execution_run_budget_deadline_state_check CHECK ( + budget_deadline_suspended_at IS NULL + OR budget_deadline_at IS NOT NULL ); ALTER TABLE moa.execution_run @@ -553,7 +545,9 @@ CREATE INDEX execution_run_nonterminal_idx -- leading on the deadline keeps the guard an index-only scan. CREATE INDEX execution_run_overdue_deadline_idx ON moa.execution_run (budget_deadline_at, run_uid) - WHERE budget_deadline_at IS NOT NULL AND status IN ( + WHERE budget_deadline_at IS NOT NULL + AND budget_deadline_suspended_at IS NULL + AND status IN ( 'awaiting_confirmation', 'queued', 'running', 'waiting_input', 'waiting_review', 'waiting_signal', 'waiting_timer', 'waiting_external', 'waiting_replan', 'pause_requested', 'pausing', 'paused', 'compensating' diff --git a/crates/moa-migrations/tests/run_idempotency_db/execution_and_security_catalog.rs b/crates/moa-migrations/tests/run_idempotency_db/execution_and_security_catalog.rs index 2b5cbb506..8f34c61e4 100644 --- a/crates/moa-migrations/tests/run_idempotency_db/execution_and_security_catalog.rs +++ b/crates/moa-migrations/tests/run_idempotency_db/execution_and_security_catalog.rs @@ -838,12 +838,6 @@ async fn execution_analytics_fresh_cutover_and_exact_contract_db() { 'definition',jsonb_build_object( 'cancel_policy','retain_effects','input_schema','{}'::JSONB, 'output_schema','{}'::JSONB, - 'input_wait_policy',jsonb_build_object( - 'expiry',jsonb_build_object( - 'kind','after','delay_seconds',1 - ), - 'on_expiry',jsonb_build_object('kind','fail_task') - ), 'nodes','[]'::JSONB ), 'plan_hash',repeat('3',64),'catalog_hash',repeat('0',64), @@ -853,12 +847,6 @@ async fn execution_analytics_fresh_cutover_and_exact_contract_db() { 'definition',jsonb_build_object( 'cancel_policy','retain_effects','input_schema','{}'::JSONB, 'output_schema','{}'::JSONB, - 'input_wait_policy',jsonb_build_object( - 'expiry',jsonb_build_object( - 'kind','after','delay_seconds',1 - ), - 'on_expiry',jsonb_build_object('kind','fail_task') - ), 'nodes','[]'::JSONB ), 'plan_hash',repeat('3',64),'catalog_hash',repeat('0',64), @@ -982,12 +970,6 @@ async fn execution_analytics_fresh_cutover_and_exact_contract_db() { 'definition',jsonb_build_object(\ 'cancel_policy','retain_effects',\ 'input_schema','{}'::JSONB,'output_schema','{}'::JSONB,\ - 'input_wait_policy',jsonb_build_object(\ - 'expiry',jsonb_build_object(\ - 'kind','after','delay_seconds',1\ - ),\ - 'on_expiry',jsonb_build_object('kind','fail_task')\ - ),\ 'nodes','[]'::JSONB\ ),\ 'plan_hash',repeat('3',64),'catalog_hash',repeat('0',64),\ @@ -997,12 +979,6 @@ async fn execution_analytics_fresh_cutover_and_exact_contract_db() { 'definition',jsonb_build_object(\ 'cancel_policy','retain_effects',\ 'input_schema','{}'::JSONB,'output_schema','{}'::JSONB,\ - 'input_wait_policy',jsonb_build_object(\ - 'expiry',jsonb_build_object(\ - 'kind','after','delay_seconds',1\ - ),\ - 'on_expiry',jsonb_build_object('kind','fail_task')\ - ),\ 'nodes','[]'::JSONB\ ),\ 'plan_hash',repeat('3',64),'catalog_hash',repeat('0',64),\ @@ -1703,16 +1679,17 @@ async fn long_horizon_execution_cutover_rejects_live_runs_and_installs_fenced_ca .await .is_err(); - let catalog_shape: (bool, bool, bool, bool, bool, bool) = sqlx::query_as( + let catalog_shape: (bool, bool, bool, bool, bool, bool, bool) = sqlx::query_as( r#" SELECT - (SELECT count(*) = 169 + (SELECT count(*) = 170 FROM information_schema.columns WHERE table_schema = 'moa' AND ( (table_name = 'execution_run' AND column_name IN ( 'admitted_identity', 'controller_generation', 'activation_state', 'next_wake_at', 'waiting_since', 'last_progress_at', + 'budget_deadline_suspended_at', 'pause_requested_at', 'paused_at', 'activation_failure_count', 'ready_task_count', 'active_task_count', 'waiting_task_count', @@ -1941,6 +1918,10 @@ async fn long_horizon_execution_cutover_rejects_live_runs_and_installs_fenced_ca ,'execution_amendment_planning_settlement_pkey' ,'execution_amendment_planning_settlement_reservation_uid_key' )), + (SELECT indexdef LIKE '%budget_deadline_suspended_at IS NULL%' + FROM pg_indexes + WHERE schemaname = 'moa' + AND indexname = 'execution_run_overdue_deadline_idx'), moa.execution_admitted_identity_is_valid(admitted_identity, tenant_id) AND activation_state = 'terminal' AND status = 'cancelled', @@ -1986,7 +1967,7 @@ async fn long_horizon_execution_cutover_rejects_live_runs_and_installs_fenced_ca AND table_name = 'execution_maintenance_checkpoint' AND grantee = 'moa_app') AND - (SELECT count(*) = 10 + (SELECT count(*) = 11 FROM pg_constraint WHERE conname IN ( 'execution_compensation_release_intent_shape_check', @@ -1998,7 +1979,8 @@ async fn long_horizon_execution_cutover_rejects_live_runs_and_installs_fenced_ca 'execution_amendment_receipt_release_shape_check', 'execution_task_output_inline_size_check', 'execution_trigger_start_recovery_shape_check', - 'execution_completion_scan_kind_shape_check' + 'execution_completion_scan_kind_shape_check', + 'execution_run_budget_deadline_state_check' )) AND (SELECT count(*) = 2 @@ -2006,8 +1988,8 @@ async fn long_horizon_execution_cutover_rejects_live_runs_and_installs_fenced_ca AND bool_and( pg_get_constraintdef(oid) LIKE '%execution_plan_snapshot_is_current%' - AND pg_get_constraintdef(oid) LIKE '%completed%' - AND pg_get_constraintdef(oid) LIKE '%cancelled%' + AND pg_get_constraintdef(oid) NOT LIKE '%completed%' + AND pg_get_constraintdef(oid) NOT LIKE '%cancelled%' ) FROM pg_constraint WHERE conrelid = 'moa.execution_run'::regclass @@ -2264,7 +2246,7 @@ async fn long_horizon_execution_cutover_rejects_live_runs_and_installs_fenced_ca invalid_attempt_generation_rejected, "attempt generation may only advance one fence at a time" ); - assert_eq!(catalog_shape, (true, true, true, true, true, true)); + assert_eq!(catalog_shape, (true, true, true, true, true, true, true)); let (purge_catalog_count, purge_batch_constant, purge_catalog_drift, receipts_drain_first) = purge_catalog; assert_eq!( @@ -2300,10 +2282,6 @@ async fn seed_queued_execution_run( "cancel_policy": "retain_effects", "input_schema": {}, "output_schema": {}, - "input_wait_policy": { - "expiry": {"kind": "after", "delay_seconds": 3600}, - "on_expiry": {"kind": "fail_task"} - }, "nodes": [{ "id": "output", "requirement_ids": [], diff --git a/crates/moa-migrations/tests/run_idempotency_db/execution_compensation.rs b/crates/moa-migrations/tests/run_idempotency_db/execution_compensation.rs index b9afc43b6..4a3acf855 100644 --- a/crates/moa-migrations/tests/run_idempotency_db/execution_compensation.rs +++ b/crates/moa-migrations/tests/run_idempotency_db/execution_compensation.rs @@ -56,9 +56,18 @@ async fn execution_compensation_schema_and_transitions_are_strict_db() { ) .fetch_one(&target) .await?; - let version_neutral_shapes: (bool, bool, bool) = sqlx::query_as( - "SELECT moa.execution_plan_definition_is_valid(initial_plan -> 'definition'), \ - moa.execution_plan_definition_is_current(initial_plan -> 'definition'), \ + let plan_shape_checks: (bool, bool, bool) = sqlx::query_as( + "SELECT moa.execution_plan_definition_is_current(initial_plan -> 'definition'), \ + NOT moa.execution_plan_definition_is_current( \ + (initial_plan -> 'definition') || jsonb_build_object( \ + 'input_wait_policy', jsonb_build_object( \ + 'expiry', jsonb_build_object( \ + 'kind', 'after', 'delay_seconds', 1 \ + ), \ + 'on_expiry', jsonb_build_object('kind', 'fail_task') \ + ) \ + ) \ + ), \ NOT capability_catalog ? 'schema_version' \ FROM moa.execution_run WHERE run_uid = $1", ) @@ -98,7 +107,7 @@ async fn execution_compensation_schema_and_transitions_are_strict_db() { second, compensation_schema, action_review_owner_schema, - version_neutral_shapes, + plan_shape_checks, compensation_reason, )) } @@ -109,7 +118,7 @@ async fn execution_compensation_schema_and_transitions_are_strict_db() { second, compensation_schema, action_review_owner_schema, - version_neutral_shapes, + plan_shape_checks, compensation_reason, ) = database .finish(outcome) @@ -122,7 +131,7 @@ async fn execution_compensation_schema_and_transitions_are_strict_db() { ); assert!(compensation_schema); assert!(action_review_owner_schema); - assert_eq!(version_neutral_shapes, (false, true, true)); + assert_eq!(plan_shape_checks, (true, true, true)); assert_eq!(compensation_reason.as_deref(), Some("compensation_failed")); } @@ -142,10 +151,6 @@ async fn seed_execution_run(target: &PgPool) -> TestResult { "cancel_policy": "retain_effects", "input_schema": {}, "output_schema": {}, - "input_wait_policy": { - "expiry": {"kind": "after", "delay_seconds": 1}, - "on_expiry": {"kind": "fail_task"} - }, "nodes": [{ "id": "output", "requirement_ids": [], diff --git a/crates/moa-orchestrator/src/objects/session/handlers/progress.rs b/crates/moa-orchestrator/src/objects/session/handlers/progress.rs index 2366c53e3..2d66028c0 100644 --- a/crates/moa-orchestrator/src/objects/session/handlers/progress.rs +++ b/crates/moa-orchestrator/src/objects/session/handlers/progress.rs @@ -62,7 +62,7 @@ impl SessionImpl { annotate_restate_handler_span("Session", "turn_admission_heartbeat"); let req = req.into_inner(); let pending_state = load_pending_state(&ctx).await?; - if pending_state.active_turn_id.is_none() + if !pending_state.turn_admission_is_live() || pending_state.admission_heartbeat_generation != req.generation { return Ok(()); diff --git a/crates/moa-orchestrator/src/objects/session/handlers/turns.rs b/crates/moa-orchestrator/src/objects/session/handlers/turns.rs index a2396c08a..d39db3183 100644 --- a/crates/moa-orchestrator/src/objects/session/handlers/turns.rs +++ b/crates/moa-orchestrator/src/objects/session/handlers/turns.rs @@ -2,6 +2,7 @@ use super::*; +mod coordinator_input; mod outcomes; mod replies; mod resume; @@ -148,6 +149,17 @@ impl SessionImpl { .take_next(pending_state.turn_generation) { pending_state.active_turn_id = Some(queued.turn_id.clone()); + reacquire_parked_turn_admission_for_dispatch( + &ctx, + &mut pending_state, + &self.turn_admission, + session_id, + state + .ensure_initialized() + .map_err(moa_error_to_handler_error)? + .tenant_id, + ) + .await?; activate_coordinator_security_owner(&mut state, &queued.turn_id, queued.generation); let now = durable_utc_now(&ctx).await?; state.set_status(SessionStatus::Running, now); @@ -176,6 +188,17 @@ impl SessionImpl { if dispatch_next && let Some(next) = pending_state.pending_messages.pop_front() { let next_turn_id = generate_turn_id(&mut ctx); pending_state.active_turn_id = Some(next_turn_id.clone()); + reacquire_parked_turn_admission_for_dispatch( + &ctx, + &mut pending_state, + &self.turn_admission, + session_id, + state + .ensure_initialized() + .map_err(moa_error_to_handler_error)? + .tenant_id, + ) + .await?; activate_coordinator_security_owner(&mut state, &next_turn_id, next.generation); let now = durable_utc_now(&ctx).await?; state.set_status(SessionStatus::Running, now); @@ -221,6 +244,19 @@ impl SessionImpl { { let now = durable_utc_now(&ctx).await?; + if matches!(outcome.kind, ExecutionTurnOutcomeKind::Completed) { + reacquire_parked_turn_admission_for_dispatch( + &ctx, + &mut pending_state, + &self.turn_admission, + session_id, + state + .ensure_initialized() + .map_err(moa_error_to_handler_error)? + .tenant_id, + ) + .await?; + } let resumed = if matches!(outcome.kind, ExecutionTurnOutcomeKind::Completed) { dispatch_queued_parent_resume_if_idle( &mut ctx, @@ -262,84 +298,22 @@ impl SessionImpl { sync_status(&ctx, session_id, &state).await?; } if pending_state.active_turn_id.is_none() { + let admission_was_parked = pending_state.turn_admission_parked.take().is_some(); let tenant_id = state .ensure_initialized() .map_err(moa_error_to_handler_error)? .tenant_id; - self.turn_admission - .release(&ctx, session_id, tenant_id) - .await?; + if !admission_was_parked { + self.turn_admission + .release(&ctx, session_id, tenant_id) + .await?; + } } persist_pending_state(&ctx, &pending_state); resolve_turn_waiters(&ctx, turn_waiters, &outcome)?; Ok(()) } - pub(super) async fn handle_register_coordinator_input( - &self, - ctx: ObjectContext<'_>, - request: Json, - ) -> Result<(), HandlerError> { - annotate_restate_handler_span("Session", "register_coordinator_input"); - let request = request.into_inner(); - let session_id = parse_session_key(ctx.key())?; - let mut state = Tracked::::load(&ctx).await?; - - // Delivery history is the terminal fence for this request identity. A - // replay after the awakeable was resolved must not advertise the target - // again or emit a new question for work that has already continued. - if state.coordinator_input_already_delivered(&request.input_request_id) { - return Ok(()); - } - - if !state.register_coordinator_input(CoordinatorPendingInput { - turn_id: request.turn_id.clone(), - generation: request.generation, - input_request_id: request.input_request_id.clone(), - awakeable_id: request.awakeable_id, - waiting_workflow_id: request.waiting_workflow_id, - }) { - return Ok(()); - } - // Advertising the pending target is what lets an unaddressed plain reply be - // routed here instead of starting an ordinary turn behind the blocked one. - state.upsert_pending_user_reply_target(PendingUserReplyTarget::CoordinatorInput { - turn_id: request.turn_id, - generation: request.generation, - input_request_id: request.input_request_id.clone(), - }); - append_session_event_deduped( - &ctx, - session_id, - Event::Warning { - message: request.question, - }, - format!("coordinator_input_request:{}", request.input_request_id), - ) - .await?; - state.persist(&ctx); - Ok(()) - } - - pub(super) async fn handle_clear_coordinator_input( - &self, - ctx: ObjectContext<'_>, - request: Json, - ) -> Result<(), HandlerError> { - annotate_restate_handler_span("Session", "clear_coordinator_input"); - let request = request.into_inner(); - let mut state = Tracked::::load(&ctx).await?; - if state.clear_coordinator_input( - &request.turn_id, - request.generation, - &request.input_request_id, - &request.waiting_workflow_id, - ) { - state.persist(&ctx); - } - Ok(()) - } - pub(super) async fn handle_apply_security_assessment( &self, ctx: ObjectContext<'_>, @@ -462,3 +436,22 @@ impl SessionImpl { Ok(()) } } + +/// Reacquires shared admission before dispatching work after a human-input park. +async fn reacquire_parked_turn_admission_for_dispatch( + ctx: &ObjectContext<'_>, + pending_state: &mut SessionPendingState, + turn_admission: &crate::objects::session::admission::TurnAdmission, + session_id: SessionId, + tenant_id: moa_core::types::identifiers::TenantId, +) -> Result<(), HandlerError> { + if pending_state.turn_admission_parked.is_none() { + return Ok(()); + } + turn_admission + .acquire(ctx, session_id, tenant_id, "turn_admission_parked_dispatch") + .await?; + pending_state.turn_admission_parked = None; + arm_turn_admission_heartbeat(ctx, pending_state, turn_admission); + Ok(()) +} diff --git a/crates/moa-orchestrator/src/objects/session/handlers/turns/coordinator_input.rs b/crates/moa-orchestrator/src/objects/session/handlers/turns/coordinator_input.rs new file mode 100644 index 000000000..da105846b --- /dev/null +++ b/crates/moa-orchestrator/src/objects/session/handlers/turns/coordinator_input.rs @@ -0,0 +1,113 @@ +//! Durable coordinator human-input registration and admission parking. + +use super::*; + +impl SessionImpl { + pub(in crate::objects::session::handlers) async fn handle_register_coordinator_input( + &self, + ctx: ObjectContext<'_>, + request: Json, + ) -> Result<(), HandlerError> { + annotate_restate_handler_span("Session", "register_coordinator_input"); + let request = request.into_inner(); + let session_id = parse_session_key(ctx.key())?; + let mut state = Tracked::::load(&ctx).await?; + + // Delivery history is the terminal fence for this request identity. A + // replay after the awakeable was resolved must not advertise the target + // again or emit a new question for work that has already continued. + if state.coordinator_input_already_delivered(&request.input_request_id) { + return Ok(()); + } + if state.security_circuit.owner.as_ref() + != Some( + &moa_core::types::security::SecurityCircuitOwner::Coordinator { + turn_id: request.turn_id.clone(), + generation: request.generation, + }, + ) + { + return Err(TerminalError::new_with_code( + 409, + "coordinator input registration owner is no longer active", + ) + .into()); + } + let mut pending_state = load_pending_state(&ctx).await?; + let expected_park = ParkedCoordinatorAdmission { + turn_id: request.turn_id.clone(), + generation: request.generation, + }; + if pending_state.active_turn_id.as_deref() != Some(request.turn_id.as_str()) + || pending_state + .turn_admission_parked + .as_ref() + .is_some_and(|parked| parked != &expected_park) + { + return Err(TerminalError::new_with_code( + 409, + "coordinator input registration does not match the active turn admission", + ) + .into()); + } + + let registered = state.register_coordinator_input(CoordinatorPendingInput { + turn_id: request.turn_id.clone(), + generation: request.generation, + input_request_id: request.input_request_id.clone(), + awakeable_id: request.awakeable_id.clone(), + waiting_workflow_id: request.waiting_workflow_id.clone(), + }); + if registered { + state.upsert_pending_user_reply_target(PendingUserReplyTarget::CoordinatorInput { + turn_id: request.turn_id.clone(), + generation: request.generation, + input_request_id: request.input_request_id.clone(), + }); + append_session_event_deduped( + &ctx, + session_id, + Event::Warning { + message: request.question, + }, + format!("coordinator_input_request:{}", request.input_request_id), + ) + .await?; + state.persist(&ctx); + } + + // Persist the awakeable, reply target, and park fence before releasing the + // shared admission lease. Duplicate registration repeats the idempotent + // release, closing the crash window between those two durable systems. + if pending_state.park_turn_admission(&request.turn_id, request.generation) { + persist_pending_state(&ctx, &pending_state); + } + let tenant_id = state + .ensure_initialized() + .map_err(moa_error_to_handler_error)? + .tenant_id; + self.turn_admission + .release(&ctx, session_id, tenant_id) + .await?; + Ok(()) + } + + pub(in crate::objects::session::handlers) async fn handle_clear_coordinator_input( + &self, + ctx: ObjectContext<'_>, + request: Json, + ) -> Result<(), HandlerError> { + annotate_restate_handler_span("Session", "clear_coordinator_input"); + let request = request.into_inner(); + let mut state = Tracked::::load(&ctx).await?; + if state.clear_coordinator_input( + &request.turn_id, + request.generation, + &request.input_request_id, + &request.waiting_workflow_id, + ) { + state.persist(&ctx); + } + Ok(()) + } +} diff --git a/crates/moa-orchestrator/src/objects/session/handlers/turns/replies.rs b/crates/moa-orchestrator/src/objects/session/handlers/turns/replies.rs index 48e3f5bdc..2c949decd 100644 --- a/crates/moa-orchestrator/src/objects/session/handlers/turns/replies.rs +++ b/crates/moa-orchestrator/src/objects/session/handlers/turns/replies.rs @@ -140,6 +140,69 @@ pub(in crate::objects::session::handlers) async fn forward_user_input_reply( Ok(()) } +/// Reacquires shared coordinator admission under the exact pending-input fence. +/// +/// This runs before the matching awakeable is resolved so resumed work can never +/// perform provider or tool I/O while its fleet and tenant admission is parked. +pub(in crate::objects::session::handlers) async fn reacquire_coordinator_reply_admission( + ctx: &ObjectContext<'_>, + state: &SessionVoState, + pending_state: &mut SessionPendingState, + turn_admission: &crate::objects::session::admission::TurnAdmission, + session_id: SessionId, + target: &PendingUserReplyTarget, +) -> Result<(), HandlerError> { + let PendingUserReplyTarget::CoordinatorInput { + turn_id, + generation, + input_request_id, + } = target + else { + return Ok(()); + }; + let exact_input_is_pending = state.pending_coordinator_inputs.iter().any(|pending| { + pending.turn_id == *turn_id + && pending.generation == *generation + && pending.input_request_id == *input_request_id + }); + if !exact_input_is_pending { + return Ok(()); + } + if pending_state.active_turn_id.as_deref() != Some(turn_id) + || pending_state.turn_admission_parked.as_ref() + != Some(&ParkedCoordinatorAdmission { + turn_id: turn_id.clone(), + generation: *generation, + }) + { + return Err(TerminalError::new_with_code( + 409, + "coordinator input target does not match the parked turn admission", + ) + .into()); + } + + let tenant_id = state + .ensure_initialized() + .map_err(moa_error_to_handler_error)? + .tenant_id; + turn_admission + .acquire(ctx, session_id, tenant_id, "turn_admission_human_resume") + .await?; + // The exact predicate above is evaluated inside this single-writer virtual + // object handler, so the matching parked fence cannot change before clearing. + if !pending_state.resume_turn_admission(turn_id, *generation) { + return Err(TerminalError::new_with_code( + 409, + "coordinator turn admission changed while resuming human input", + ) + .into()); + } + arm_turn_admission_heartbeat(ctx, pending_state, turn_admission); + persist_pending_state(ctx, pending_state); + Ok(()) +} + /// Builds the exact worker input payload for a routed user reply. pub(in crate::objects::session::handlers) fn worker_provide_input_request( parent_session: SessionId, diff --git a/crates/moa-orchestrator/src/objects/session/handlers/turns/turn_start.rs b/crates/moa-orchestrator/src/objects/session/handlers/turns/turn_start.rs index 4a2bf9fb5..9870d71f4 100644 --- a/crates/moa-orchestrator/src/objects/session/handlers/turns/turn_start.rs +++ b/crates/moa-orchestrator/src/objects/session/handlers/turns/turn_start.rs @@ -103,6 +103,15 @@ pub(in crate::objects::session::handlers) async fn start_turn_inner( .into()); } MessageRouting::Reply(target) => { + reacquire_coordinator_reply_admission( + ctx, + &state, + &mut pending_state, + turn_admission, + session_id, + &target, + ) + .await?; forward_user_input_reply( ctx, &mut state, diff --git a/crates/moa-orchestrator/src/objects/session/mod.rs b/crates/moa-orchestrator/src/objects/session/mod.rs index 066a1e02a..8b5786e27 100644 --- a/crates/moa-orchestrator/src/objects/session/mod.rs +++ b/crates/moa-orchestrator/src/objects/session/mod.rs @@ -132,6 +132,10 @@ struct SessionPendingState { turn_waiters: Vec, #[serde(default)] admission_heartbeat_generation: u64, + /// Exact coordinator turn durably parked for human input and therefore owning + /// no shared fleet or tenant admission lease. + #[serde(default)] + turn_admission_parked: Option, /// Cancellation requested for a turn that has not yet reported its outcome. /// /// The scope decides queue disposition when the matching `Cancelled` callback @@ -159,7 +163,49 @@ struct PendingCancellation { scope: CancelScope, } +/// Exact active coordinator whose shared admission was released for human input. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +struct ParkedCoordinatorAdmission { + turn_id: String, + generation: u64, +} + impl SessionPendingState { + /// Returns whether the active coordinator currently owns shared turn admission. + fn turn_admission_is_live(&self) -> bool { + self.active_turn_id.is_some() && self.turn_admission_parked.is_none() + } + + /// Parks shared admission for one exact active coordinator generation. + fn park_turn_admission(&mut self, turn_id: &str, generation: u64) -> bool { + if self.active_turn_id.as_deref() != Some(turn_id) || self.turn_admission_parked.is_some() { + return false; + } + self.turn_admission_parked = Some(ParkedCoordinatorAdmission { + turn_id: turn_id.to_string(), + generation, + }); + // Invalidate the already-scheduled heartbeat. Its handler observes this + // generation before it can renew the released lease. + self.admission_heartbeat_generation = self.admission_heartbeat_generation.saturating_add(1); + true + } + + /// Marks shared admission live again for one exact parked coordinator generation. + fn resume_turn_admission(&mut self, turn_id: &str, generation: u64) -> bool { + if self.active_turn_id.as_deref() != Some(turn_id) + || self.turn_admission_parked.as_ref() + != Some(&ParkedCoordinatorAdmission { + turn_id: turn_id.to_string(), + generation, + }) + { + return false; + } + self.turn_admission_parked = None; + true + } + /// Returns whether a new turn may be admitted right now. /// /// A whole-task-tree cancellation fences admission for the window between the @@ -715,6 +761,31 @@ mod tests { assert!(!task_tree.dispatches_next_after(&cancelled)); } + #[test] + fn coordinator_human_wait_releases_and_exact_resume_reacquires_turn_admission() { + // Pins: a coordinator parked on durable human input owns no shared active-turn + // lease, stale registrations cannot change that state, and only the exact active + // turn can make admission live again before its awakeable is resolved. + let mut state = SessionPendingState { + active_turn_id: Some("turn-1".to_string()), + admission_heartbeat_generation: 7, + ..SessionPendingState::default() + }; + + state.turn_generation = 3; + assert!(!state.park_turn_admission("stale-turn", 3)); + assert!(state.turn_admission_is_live()); + assert!(state.park_turn_admission("turn-1", 3)); + assert!(!state.turn_admission_is_live()); + assert_eq!(state.admission_heartbeat_generation, 8); + assert!(!state.park_turn_admission("turn-1", 3)); + assert!(!state.resume_turn_admission("stale-turn", 3)); + assert!(!state.resume_turn_admission("turn-1", 2)); + assert!(state.resume_turn_admission("turn-1", 3)); + assert!(state.turn_admission_is_live()); + assert!(!state.resume_turn_admission("turn-1", 3)); + } + #[test] fn cancellation_without_a_matching_request_dispatches_nothing() { // Pins: an externally cancelled invocation, or a cancellation recorded for a diff --git a/crates/moa-orchestrator/src/services/execution/handlers.rs b/crates/moa-orchestrator/src/services/execution/handlers.rs index 99e06dd1e..d13608bd1 100644 --- a/crates/moa-orchestrator/src/services/execution/handlers.rs +++ b/crates/moa-orchestrator/src/services/execution/handlers.rs @@ -5,6 +5,7 @@ use super::planning_context::{PlanningContextInput, planning_context_inner}; use super::start::start_inner; use super::support::*; use super::*; +use crate::services::llm_gateway::{LLMCompletionOwner, LLMGatewayClient}; impl Execution for ExecutionImpl { #[tracing::instrument(skip(self, ctx, request))] @@ -313,10 +314,24 @@ impl Execution for ExecutionImpl { let pool = self.pool.clone(); let config = self.config.clone(); let accepted = ctx - .run(|| async move { cancel_inner(pool, config, request).await.map(Json::from) }) + .run(|| async move { + let accepted = cancel_inner(pool, config, request).await?; + pause_execution_cancel_db_handoff_for_test().await; + Ok::<_, HandlerError>(Json::from(accepted)) + }) .name("execution_cancel") .await? .into_inner(); + for dispatch_uid in accepted.llm_owner_dispatch_uids() { + crate::restate_identity::replay_safe_request( + ctx.service_client::() + .cancel_owner(Json::from(LLMCompletionOwner::execution_task_attempt( + *dispatch_uid, + ))), + ) + .call() + .await?; + } if let Some(wake_epoch) = accepted.wake_epoch() { pause_execution_mutation_handoff_for_test().await; kick_execution_dispatcher(&ctx, run_request.run_uid, wake_epoch, "cancel").await?; @@ -754,6 +769,7 @@ pub(super) async fn cancel_inner( scope, request.run.run_uid, request.run.session_id, + config.max_in_flight_tasks, ) .await .map_err(execution_error)? @@ -761,12 +777,16 @@ pub(super) async fn cancel_inner( return Ok(not_found_mutation()); }; verify_run_request(&cancellation.run, &request.run)?; + let replay_owner_dispatch_uids = + terminal_task_llm_owner_dispatch_uids(&cancellation.task_cancellation_dispatches)?; if cancellation.run.status == ExecutionRunStatus::Cancelled { - return Ok(replayed_mutation(&cancellation.run)); + return Ok(replayed_mutation(&cancellation.run) + .with_llm_owner_dispatch_uids(replay_owner_dispatch_uids)); } if let Some(pending) = &cancellation.run.pending_terminal { return Ok(if pending.status == ExecutionRunStatus::Cancelled { replayed_mutation(&cancellation.run) + .with_llm_owner_dispatch_uids(replay_owner_dispatch_uids) } else { conflict_mutation(ExecutionConflictReason::AlreadyTerminal) }); @@ -792,7 +812,7 @@ pub(super) async fn cancel_inner( }; Ok( match repository - .fence_completion_terminal_and_enqueue_settlement( + .fence_cancellation_terminal_and_enqueue_settlement( &config, scope, cancellation.run.run_uid, @@ -800,19 +820,22 @@ pub(super) async fn cancel_inner( cancellation.run.wake_epoch, pending_terminal, chrono::Utc::now(), - u32::try_from( - config - .maximum_activation_steps - .min(config.max_in_flight_tasks) - .min(1_000), - ) - .map_err(|_| invalid_execution_request("terminal page limit exceeds u32"))?, + u32::try_from(config.max_in_flight_tasks) + .map_err(|_| invalid_execution_request("terminal page limit exceeds u32"))?, ) .await .map_err(execution_error)? { - PendingTerminalAdvanceOutcome::Applied(commit) => applied_mutation(&commit.run), - PendingTerminalAdvanceOutcome::Replayed(commit) => replayed_mutation(&commit.run), + PendingTerminalAdvanceOutcome::Applied(commit) => { + let owner_dispatch_uids = + terminal_task_llm_owner_dispatch_uids(&commit.cancellation_dispatches)?; + applied_mutation(&commit.run).with_llm_owner_dispatch_uids(owner_dispatch_uids) + } + PendingTerminalAdvanceOutcome::Replayed(commit) => { + let owner_dispatch_uids = + terminal_task_llm_owner_dispatch_uids(&commit.cancellation_dispatches)?; + replayed_mutation(&commit.run).with_llm_owner_dispatch_uids(owner_dispatch_uids) + } PendingTerminalAdvanceOutcome::NotFound => not_found_mutation(), PendingTerminalAdvanceOutcome::Conflict => { conflict_mutation(ExecutionConflictReason::AlreadyTerminal) diff --git a/crates/moa-orchestrator/src/services/execution/support.rs b/crates/moa-orchestrator/src/services/execution/support.rs index 3cdf76255..de3bfa760 100644 --- a/crates/moa-orchestrator/src/services/execution/support.rs +++ b/crates/moa-orchestrator/src/services/execution/support.rs @@ -1,6 +1,11 @@ //! Shared execution-service mutation handoff types and conversion helpers. use super::*; +use crate::runtime::execution_dispatch::{ExecutionDispatchTarget, JournaledExecutionDispatch}; +use moa_execution::{ + repository::outbox::{ExecutionDispatchKind, ExecutionDispatchRecord}, + wire::ExecutionAttemptCancelReason, +}; pub(super) fn scoped_catalog_error( error: crate::connector_catalog::ScopedConnectorCatalogError, @@ -16,6 +21,7 @@ pub(super) fn scoped_catalog_error( pub(crate) struct ExecutionMutationHandoff { wake_epoch: u64, task_ids_to_release: Vec, + llm_owner_dispatch_uids: Vec, } #[derive(Clone, Debug, Deserialize, Serialize)] @@ -47,6 +53,23 @@ impl ExecutionMutationAccepted { self } + pub(crate) fn llm_owner_dispatch_uids(&self) -> &[uuid::Uuid] { + match self { + Self::Accepted { handoff, .. } => &handoff.llm_owner_dispatch_uids, + Self::Rejected { .. } => &[], + } + } + + pub(crate) fn with_llm_owner_dispatch_uids( + mut self, + llm_owner_dispatch_uids: Vec, + ) -> Self { + if let Self::Accepted { handoff, .. } = &mut self { + handoff.llm_owner_dispatch_uids = llm_owner_dispatch_uids; + } + self + } + pub(crate) fn into_response(self) -> ExecutionMutationResponse { match self { Self::Accepted { response, .. } | Self::Rejected { response } => response, @@ -376,6 +399,7 @@ pub(super) fn applied_mutation(run: &ExecutionRunRecord) -> ExecutionMutationAcc handoff: ExecutionMutationHandoff { wake_epoch: run.wake_epoch, task_ids_to_release: Vec::new(), + llm_owner_dispatch_uids: Vec::new(), }, } } @@ -388,10 +412,41 @@ pub(super) fn replayed_mutation(run: &ExecutionRunRecord) -> ExecutionMutationAc handoff: ExecutionMutationHandoff { wake_epoch: run.wake_epoch, task_ids_to_release: Vec::new(), + llm_owner_dispatch_uids: Vec::new(), }, } } +pub(super) fn terminal_task_llm_owner_dispatch_uids( + cancellation_dispatches: &[ExecutionDispatchRecord], +) -> Result, HandlerError> { + let mut owner_dispatch_uids = Vec::new(); + for dispatch in cancellation_dispatches { + if dispatch.kind != ExecutionDispatchKind::TaskAttemptCancel { + continue; + } + let target = JournaledExecutionDispatch::from(dispatch.clone()) + .target() + .map_err(execution_error)?; + let ExecutionDispatchTarget::TaskAttemptCancel(request) = target else { + return Err(TerminalError::new( + "task cancellation dispatch decoded to a different execution target", + ) + .into()); + }; + if request.reason != ExecutionAttemptCancelReason::RunTerminal { + return Err(TerminalError::new( + "terminal cancellation handoff contained a non-terminal task cancellation", + ) + .into()); + } + owner_dispatch_uids.push(request.active_dispatch_uid); + } + owner_dispatch_uids.sort_unstable(); + owner_dispatch_uids.dedup(); + Ok(owner_dispatch_uids) +} + pub(super) fn conflict_mutation(reason: ExecutionConflictReason) -> ExecutionMutationAccepted { ExecutionMutationAccepted::Rejected { response: ExecutionMutationResponse::Conflict { reason }, @@ -504,6 +559,18 @@ pub(super) async fn pause_execution_mutation_handoff_for_test() { #[cfg(not(feature = "integration"))] pub(super) async fn pause_execution_mutation_handoff_for_test() {} +#[cfg(feature = "integration")] +pub(super) async fn pause_execution_cancel_db_handoff_for_test() { + if std::env::var("MOA_EXECUTION_TEST_PAUSE_CANCEL_DB_HANDOFF").as_deref() == Ok("true") { + // Deliberately inside the journaled closure but after the database commit: recovery must + // rebuild the exact current owner fence from Postgres when this result was not journaled. + tokio::time::sleep(std::time::Duration::from_secs(30)).await; + } +} + +#[cfg(not(feature = "integration"))] +pub(super) async fn pause_execution_cancel_db_handoff_for_test() {} + pub(super) fn invalid_execution_request(message: impl Into) -> HandlerError { TerminalError::new_with_code(400, message.into()).into() } diff --git a/crates/moa-orchestrator/src/services/execution/tests.rs b/crates/moa-orchestrator/src/services/execution/tests.rs index cabdf6846..612556cd5 100644 --- a/crates/moa-orchestrator/src/services/execution/tests.rs +++ b/crates/moa-orchestrator/src/services/execution/tests.rs @@ -260,13 +260,6 @@ fn skill_revision(name: &str, revision_uid: u128) -> StoredArtifactRevision { plan: ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, - input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { - expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { - delay_seconds: 86_400, - }, - on_expiry: - moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, - }, input_schema: json!({"type": "object"}), output_schema: json!({"type": "object"}), nodes: Vec::new(), @@ -634,12 +627,6 @@ fn accepted_turn_requires_skill_template_provenance_from_planning_snapshot() { }, plan: ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, - input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { - expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { - delay_seconds: 86_400, - }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, - }, input_schema: json!({"type": "object"}), output_schema: json!({"type": "object"}), nodes: Vec::new(), @@ -715,12 +702,6 @@ fn pinned_execution_template( }, plan: ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, - input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { - expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { - delay_seconds: 86_400, - }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, - }, input_schema: json!({"type": "object"}), output_schema: json!({"type": "object"}), nodes: Vec::new(), @@ -1164,7 +1145,14 @@ fn execution_external_wait_payload_is_validated_against_node_schema() { "required": ["approved"], "properties": {"approved": {"type": "boolean"}} }, - "operation": {"kind": "review", "prompt": "Approve?"}, + "operation": { + "kind": "review", + "prompt": "Approve?", + "wait_policy": { + "expiry": {"kind": "after", "delay_seconds": 3600}, + "on_expiry": {"kind": "fail_task"} + } + }, "compensation": null, "retry": {"max_attempts": 1, "initial_backoff_ms": 1, "max_backoff_ms": 1}, "budget": null diff --git a/crates/moa-orchestrator/src/services/execution_amendment_planner/tests.rs b/crates/moa-orchestrator/src/services/execution_amendment_planner/tests.rs index 8560873da..effb50d5d 100644 --- a/crates/moa-orchestrator/src/services/execution_amendment_planner/tests.rs +++ b/crates/moa-orchestrator/src/services/execution_amendment_planner/tests.rs @@ -290,8 +290,7 @@ async fn waiting_replan_uses_confirmed_budget_for_planning_apply_and_replay_db() use moa_artifacts::execution_plan::{ CompletionCheck, CompletionCheckKind, ExecutionBudgetLimit, ExecutionCancelPolicy, ExecutionGoalContract, ExecutionNode, ExecutionOperation, ExecutionPlanDefinition, - ExecutionRequirement, ExecutionTaskOutcome, ExecutionTaskResult, ExecutionTemporalTarget, - ExecutionUsage, ExecutionWaitExpiryAction, ExecutionWaitPolicy, + ExecutionRequirement, ExecutionTaskOutcome, ExecutionTaskResult, ExecutionUsage, GeneratedAmendmentCandidate, RetryPolicy, }; use moa_core::types::execution_planning::{ @@ -383,12 +382,6 @@ async fn waiting_replan_uses_confirmed_budget_for_planning_apply_and_replay_db() fn replan_plan() -> ExecutionPlanDefinition { ExecutionPlanDefinition { cancel_policy: ExecutionCancelPolicy::RetainEffects, - input_wait_policy: ExecutionWaitPolicy { - expiry: ExecutionTemporalTarget::At { - at: chrono::Utc::now() + chrono::TimeDelta::minutes(30), - }, - on_expiry: ExecutionWaitExpiryAction::FailTask, - }, input_schema: json!({"type": "object"}), output_schema: json!({"type": "object"}), nodes: vec![ diff --git a/crates/moa-orchestrator/src/services/skill_regression/compilation.rs b/crates/moa-orchestrator/src/services/skill_regression/compilation.rs index 092a6a548..78cba12b3 100644 --- a/crates/moa-orchestrator/src/services/skill_regression/compilation.rs +++ b/crates/moa-orchestrator/src/services/skill_regression/compilation.rs @@ -96,15 +96,23 @@ pub(super) fn compile_skill_execution_template( .map_err(|error| MoaError::SerializationError(error.to_string()))?; let candidate_hash = execution_planning_hash("moa.execution.compile-candidate", &candidate_bytes); + let created_at = Utc::now(); + let horizon_seconds = i64::try_from(request.config.execution.maximum_horizon_seconds) + .map_err(|_| MoaError::ConfigError("execution maximum horizon does not fit i64".into()))?; + let horizon = chrono::TimeDelta::try_seconds(horizon_seconds).ok_or_else(|| { + MoaError::ConfigError("execution maximum horizon does not fit chrono".into()) + })?; + let deadline_at = created_at.checked_add_signed(horizon).ok_or_else(|| { + MoaError::ConfigError("execution maximum horizon exceeds timestamp range".into()) + })?; let approved_budget = ExecutionBudgetLimit { max_cost_microusd: Some(request.config.execution.max_cost_microusd), max_tokens: Some(request.config.execution.max_tokens), max_tasks: Some(request.config.execution.max_tasks), max_tool_calls: Some(request.config.execution.max_tool_calls), max_retrieved_bytes: Some(request.config.execution.max_retrieved_bytes), - deadline_at: None, + deadline_at: Some(deadline_at), }; - let created_at = Utc::now(); let started = Instant::now(); let mut outcome = if matches!(request.run_input, RegressionExecutionInput::Ambiguous) { CompileExecutionOutcome { diff --git a/crates/moa-orchestrator/src/services/tool_executor.rs b/crates/moa-orchestrator/src/services/tool_executor.rs index 1a5241b1a..da2221786 100644 --- a/crates/moa-orchestrator/src/services/tool_executor.rs +++ b/crates/moa-orchestrator/src/services/tool_executor.rs @@ -69,7 +69,8 @@ use moa_execution::wire::{ use moa_hands::{ DeferredWorkspaceToolOutput, ExecutionHandReleaseRequest, JournaledWorkspaceCommit, PendingConnectorToolOutput, SessionHandReleasePageOutcome, ToolCallScope, ToolCatalogPin, - ToolCatalogSnapshot, ToolExecution, ToolRouter, + ToolCatalogSnapshot, ToolExecution, ToolRouter, WorkerHandReleaseFence, + WorkerHandReleaseRequest, }; use moa_security::{ OutputClassification, ToolInputCanaryScreening, classify_tool_output, @@ -540,6 +541,16 @@ pub trait ToolExecutor { request: Json, ) -> Result<(), HandlerError>; + /// Captures the exact worker hand generation allowed to enter a human-input wait. + async fn capture_worker_hand_release_fence( + request: Json, + ) -> Result>, HandlerError>; + + /// Checkpoints and releases one exact worker hand before a human-input wait. + async fn checkpoint_and_release_worker_hand( + request: Json, + ) -> Result<(), HandlerError>; + /// Releases the generation-independent hand scope owned by one execution task. async fn release_execution_task_hands( request: Json, @@ -650,6 +661,31 @@ pub struct ReleaseWorkerHandsRequest { pub worker_id: String, } +/// Request to capture the exact live worker hand before registering an input wait. +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CaptureWorkerHandReleaseFenceRequest { + /// Admitted session metadata owning the worker sandbox. + pub session: SessionMeta, + /// Worker scope about to enter the durable wait. + pub worker_id: String, +} + +/// Request to park the exact captured worker hand after input registration is durable. +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CheckpointAndReleaseWorkerHandRequest { + /// Admitted session metadata owning the worker sandbox. + pub session: SessionMeta, + /// Worker scope entering the durable wait. + pub worker_id: String, + /// Exact turn, generation, and request identity that owns the wait. + pub input_target: moa_core::types::worker::state::WorkerInputTarget, + /// Exact workspace and hand generation captured before registration, or an + /// absence proof that forbids releasing a hand created later. + pub expected: Option, +} + /// Request to release one terminal or cancelled execution task's scoped hands. #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct ReleaseExecutionTaskHandsRequest { @@ -2451,6 +2487,52 @@ impl ToolExecutor for ToolExecutorImpl { Ok(()) } + #[tracing::instrument(skip(self, ctx, request))] + // SAFETY: internal worker-turn pre-park read. The admitted SessionMeta and worker + // id come from the already-authorized worker dispatch and expose no data publicly. + async fn capture_worker_hand_release_fence( + &self, + ctx: Context<'_>, + request: Json, + ) -> Result>, HandlerError> { + crate::ctx::adopt_incoming_trace_parent(&ctx); + annotate_restate_handler_span("ToolExecutor", "capture_worker_hand_release_fence"); + let request = request.into_inner(); + self.router + .capture_worker_hand_release_fence(&request.session, &request.worker_id) + .await + .map(Json::from) + .map_err(moa_error_to_handler_error) + } + + #[tracing::instrument(skip(self, ctx, request))] + // SAFETY: internal worker-turn park after its exact pending-input registration is + // durable. The captured workspace/instance/lease fence prevents a replay from + // touching compute provisioned by a later resume. + async fn checkpoint_and_release_worker_hand( + &self, + ctx: Context<'_>, + request: Json, + ) -> Result<(), HandlerError> { + crate::ctx::adopt_incoming_trace_parent(&ctx); + annotate_restate_handler_span("ToolExecutor", "checkpoint_and_release_worker_hand"); + let request = request.into_inner(); + self.router + .checkpoint_and_release_worker_hand(WorkerHandReleaseRequest { + session: &request.session, + worker_id: &request.worker_id, + input_target: &request.input_target, + expected: request.expected.as_ref(), + scope: ToolCallScope::unbounded().with_budget( + moa_core::types::resource::ResourceBudget::until( + chrono::Utc::now() + chrono::Duration::minutes(5), + ), + ), + }) + .await + .map_err(moa_error_to_handler_error) + } + #[tracing::instrument(skip(self, ctx, request))] // SAFETY: internal bounded-attempt yield; the authoritative Session is loaded and its exact // tenant/run/owner generation is fenced again by the durable release repository. diff --git a/crates/moa-orchestrator/src/workflows/execution_task_attempt/active/agent.rs b/crates/moa-orchestrator/src/workflows/execution_task_attempt/active/agent.rs index f7ae10975..9eaab4a3f 100644 --- a/crates/moa-orchestrator/src/workflows/execution_task_attempt/active/agent.rs +++ b/crates/moa-orchestrator/src/workflows/execution_task_attempt/active/agent.rs @@ -74,6 +74,20 @@ struct AgentPending { external: Option, } +fn resolve_declared_capability<'a>( + capabilities: &'a BTreeMap, + invocation: &ToolInvocation, + usage: &ExecutionUsage, +) -> Result<&'a ExecutionCapability, Box> { + capabilities.get(&invocation.name).copied().ok_or_else(|| { + Box::new(failed_task_outcome( + ExecutionFailureClass::Terminal, + format!("agent emitted undeclared capability `{}`", invocation.name), + usage.clone(), + )) + }) +} + /// Executes one bounded task-local agent model or tool boundary. pub(super) async fn execute_agent_turn( workflow: &ExecutionTaskAttemptImpl, @@ -436,12 +450,10 @@ pub(super) async fn execute_agent_turn( } let invocation = pending_tool_calls.remove(0); - let capability = capabilities.get(&invocation.name).copied().ok_or_else(|| { - TerminalError::new(format!( - "agent emitted undeclared capability `{}`", - invocation.name - )) - })?; + let capability = match resolve_declared_capability(&capabilities, &invocation, &usage) { + Ok(capability) => capability, + Err(outcome) => return Ok(ActiveTaskAttemptExit::Outcome(*outcome)), + }; if disabled_capabilities.contains_key(&invocation.name) { let tool_use_id = invocation .id @@ -814,6 +826,38 @@ mod tests { use super::*; + // Pins: a model-authored call outside the task's declared capability envelope + // becomes a typed terminal task outcome, preserving accumulated usage so the + // normal attempt-settlement path owns cleanup and controller progress. + #[test] + fn undeclared_agent_capability_becomes_terminal_task_outcome_offline() { + let invocation = ToolInvocation { + id: Some("undeclared-call".to_string()), + name: "fixture-capability__forbidden_execution_eval_action".to_string(), + input: json!({"case": "escape"}), + }; + let usage = ExecutionUsage { + cost_microusd: 17, + tokens: 23, + tool_calls: 2, + retrieved_bytes: 31, + }; + let capabilities = BTreeMap::::new(); + + let outcome = resolve_declared_capability(&capabilities, &invocation, &usage) + .expect_err("an undeclared capability must stop through task settlement"); + + assert_eq!( + *outcome, + failed_task_outcome( + ExecutionFailureClass::Terminal, + "agent emitted undeclared capability `fixture-capability__forbidden_execution_eval_action`" + .to_string(), + usage, + ) + ); + } + // Pins: once an asynchronous provider start commits, the durable checkpoint // retains the exact model invocation, effect semantics, and MOA job identity; // decoding the checkpoint must not reconstruct or resend that effect. diff --git a/crates/moa-orchestrator/src/workflows/turn_events.rs b/crates/moa-orchestrator/src/workflows/turn_events.rs index fd963972a..ac15cbe1e 100644 --- a/crates/moa-orchestrator/src/workflows/turn_events.rs +++ b/crates/moa-orchestrator/src/workflows/turn_events.rs @@ -396,13 +396,10 @@ pub(super) async fn append_turn_failed( /// erasing it would collapse a typed contract into an indistinguishable /// generic failure. Only codes on this closed list survive sanitization; /// everything else keeps the fixed class sentence. -pub(super) const COORDINATOR_SECURITY_INPUT_TIMEOUT_MESSAGE: &str = "The turn stopped safely because user direction was not received before the security-input timeout."; - /// Repository-authored failure text that may survive catch-all sanitization. pub(super) const SAFE_TERMINAL_REJECTION_CODES: &[&str] = &[ "durable_execution_requires_user_message_origin", "run_requires_user_message_origin", - COORDINATOR_SECURITY_INPUT_TIMEOUT_MESSAGE, ]; /// Returns the stable rejection code carried by a hand-authored terminal diff --git a/crates/moa-orchestrator/src/workflows/turn_execution/mod.rs b/crates/moa-orchestrator/src/workflows/turn_execution/mod.rs index 5004ef3e7..486056f35 100644 --- a/crates/moa-orchestrator/src/workflows/turn_execution/mod.rs +++ b/crates/moa-orchestrator/src/workflows/turn_execution/mod.rs @@ -284,8 +284,6 @@ enum TurnIterationOutcome { ToolBudgetExceeded(ToolBudgetExhausted), /// The prompt-injection circuit halted this coordinator turn. SecurityHalt, - /// The coordinator's bounded security-input wait expired. - SecurityInputTimedOut, } struct DurableUpgradeGuard { @@ -802,24 +800,6 @@ async fn execute_turn_inside_workflow( post_outcome_assessment, }); } - TurnIterationOutcome::SecurityInputTimedOut => { - let post_outcome_assessment = capture_current_active_segment_assessment( - workflow, - ctx, - session_id, - AssessmentPhase::Final, - &[], - last_response_cutoff_before_seq(ctx).await?, - ) - .await?; - return Ok(BodyOutcome { - kind: TurnOutcomeKind::Failed, - message: last_summary.take().unwrap_or_else(|| { - "The turn stopped safely because required user input timed out.".to_string() - }), - post_outcome_assessment, - }); - } TurnIterationOutcome::ToolBudgetExceeded(exhaustion) => { emit_tool_budget_exceeded(appender, ctx, session_id, &exhaustion).await?; let (message, sequence_num) = append_zero_cost_assistant_response_with_sequence( @@ -1504,9 +1484,6 @@ async fn run_once_inside_workflow( ToolDispatchOutcome::SecurityHalt => { return Ok(TurnIterationOutcome::SecurityHalt); } - ToolDispatchOutcome::SecurityInputTimedOut => { - return Ok(TurnIterationOutcome::SecurityInputTimedOut); - } } Ok(TurnIterationOutcome::Core(turn_outcome_for_response( diff --git a/crates/moa-orchestrator/src/workflows/turn_execution/tools.rs b/crates/moa-orchestrator/src/workflows/turn_execution/tools.rs index 50dac70bb..56dc2a3d3 100644 --- a/crates/moa-orchestrator/src/workflows/turn_execution/tools.rs +++ b/crates/moa-orchestrator/src/workflows/turn_execution/tools.rs @@ -32,8 +32,7 @@ use crate::turn::util::{ use crate::turn_driver::progress as driver_progress; use crate::workflows::errors::moa_error_to_handler_error; use crate::workflows::turn_events::{ - COORDINATOR_SECURITY_INPUT_TIMEOUT_MESSAGE, append_tool_call_event, append_tool_result_event, - record_segment_tool_use, + append_tool_call_event, append_tool_result_event, record_segment_tool_use, }; use crate::workflows::turn_progress; use crate::workflows::turn_responsiveness::{ @@ -148,8 +147,6 @@ pub(super) enum ToolDispatchOutcome { ToolBudgetExceeded(ToolBudgetExhausted), /// The prompt-injection circuit reached its halt threshold for this owner. SecurityHalt, - /// The coordinator's bounded security-input wait expired without an answer. - SecurityInputTimedOut, } #[derive(Debug, Deserialize)] @@ -480,10 +477,6 @@ pub(super) async fn dispatch_response_tool_calls( *last_summary = Some(reason); return Ok(ToolDispatchOutcome::Cancelled); } - ToolCallDisposition::SecurityInputTimedOut => { - *last_summary = Some(COORDINATOR_SECURITY_INPUT_TIMEOUT_MESSAGE.to_string()); - return Ok(ToolDispatchOutcome::SecurityInputTimedOut); - } // `handle_tool_call` already parked on the user's reply before // returning, so by the time control reaches here the suspend has been // answered and the loop may continue with the capability disabled. @@ -678,7 +671,6 @@ async fn handle_tool_call( tool_context.turn_id, tool_context.generation, suspend_tool_id, - workflow.session_limits().coordinator_input_timeout_ms, ) .await?; disposition = match input_outcome { @@ -686,7 +678,6 @@ async fn handle_tool_call( CoordinatorSecurityInputOutcome::Cancelled(reason) => { ToolCallDisposition::Cancelled(reason) } - CoordinatorSecurityInputOutcome::TimedOut => ToolCallDisposition::SecurityInputTimedOut, }; } Ok(disposition) @@ -703,8 +694,6 @@ enum ToolCallDisposition { SecurityNeedsInput, /// Cancellation won while the coordinator was parked for user input. Cancelled(String), - /// The bounded coordinator input wait expired. - SecurityInputTimedOut, } /// Records the refusal of a tool whose capability the circuit already disabled. @@ -767,7 +756,6 @@ async fn await_coordinator_security_input( turn_id: &str, generation: u64, tool_id: ToolCallId, - timeout_ms: u64, ) -> Result { let input_request_id = format!("security:{turn_id}:{generation}:{tool_id}"); let awakeable = ctx.awakeable::(); @@ -797,9 +785,6 @@ async fn await_coordinator_security_input( }, reason = ctx.promise::(driver_progress::TurnStateKey::CANCEL_REASON_PROMISE) => { CoordinatorSecurityInputOutcome::Cancelled(reason?) - }, - _ = ctx.sleep(std::time::Duration::from_millis(timeout_ms)) => { - CoordinatorSecurityInputOutcome::TimedOut } }; @@ -826,7 +811,6 @@ async fn await_coordinator_security_input( enum CoordinatorSecurityInputOutcome { Answered, Cancelled(String), - TimedOut, } /// Fixed question asked when the circuit suspends a coordinator turn. diff --git a/crates/moa-orchestrator/src/workflows/worker_turn_execution.rs b/crates/moa-orchestrator/src/workflows/worker_turn_execution.rs index cc84d2ab2..2d353bab6 100644 --- a/crates/moa-orchestrator/src/workflows/worker_turn_execution.rs +++ b/crates/moa-orchestrator/src/workflows/worker_turn_execution.rs @@ -55,6 +55,10 @@ use crate::services::{ cancel_completion_owner, completion_idempotency_key, }, session_store::RestateSessionStoreClient, + tool_executor::{ + CaptureWorkerHandReleaseFenceRequest, CheckpointAndReleaseWorkerHandRequest, + ToolExecutorClient, + }, }; use crate::tool_invocation::governed::completion_tool_catalog_pin; use crate::tool_invocation::governed::{ @@ -721,6 +725,7 @@ async fn handle_tool_call( turn_id: tool_context.turn_id, generation: tool_context.generation, worker_id, + meta, parent_session: session_id, tool_id, tool_call, @@ -850,15 +855,15 @@ async fn handle_tool_call( // Reuse the existing request_input round-trip rather than inventing a // second suspension mechanism: it already emits one `NeedsInput` signal, // registers the awakeable on the Worker VO before emitting so a reply can - // never race ahead of it, and clears the mapping on timeout. + // never race ahead of it, and clears the mapping on cancellation. if let ChildInputWaitOutcome::Cancelled(reason) = request_input_from_parent( - workflow, ctx, ChildInputRequestOwner { worker_id, turn_id: tool_context.turn_id, generation: tool_context.generation, parent_session: session_id, + meta, }, &moa_core::types::worker::commands::RequestInputInput { question: WORKER_SECURITY_INPUT_QUESTION.to_string(), @@ -1024,6 +1029,7 @@ struct ChildReportToolRequest<'a> { /// it raises so a reply or clear can name exactly this owner. generation: u64, worker_id: &'a str, + meta: &'a SessionMeta, parent_session: SessionId, tool_id: ToolCallId, tool_call: &'a ToolCallContent, @@ -1036,8 +1042,7 @@ struct ChildReportToolRequest<'a> { /// tool result, evidence) so the child's conversation stays consistent, but the work is a /// control-plane emit to the owning coordinator rather than a managed-child operation. /// `report_to_parent` returns immediately; `request_input` blocks the child turn on a -/// Restate awakeable until the coordinator answers (`ProvideInput`) or the long timeout -/// elapses. +/// Restate awakeable until the coordinator answers (`ProvideInput`) or cancels the turn. async fn handle_child_report_tool( workflow: &WorkerTurnExecutionImpl, ctx: &WorkflowContext<'_>, @@ -1048,6 +1053,7 @@ async fn handle_child_report_tool( turn_id, generation, worker_id, + meta, parent_session, tool_id, tool_call, @@ -1069,19 +1075,19 @@ async fn handle_child_report_tool( } ChildReportTool::RequestInput(input) => { match request_input_from_parent( - workflow, ctx, ChildInputRequestOwner { worker_id, turn_id, generation, parent_session, + meta, }, &input, ) .await? { - ChildInputWaitOutcome::Output { output, .. } => *output, + ChildInputWaitOutcome::Output(output) => *output, ChildInputWaitOutcome::Cancelled(reason) => { return Ok(WorkerToolCallDisposition::Cancelled(reason)); } @@ -1142,18 +1148,16 @@ async fn report_to_parent( )) } -/// Runs the child `request_input` awakeable round-trip and returns the answer (or a -/// timeout result). +/// Runs the child `request_input` awakeable round-trip and returns the answer. /// /// Mirrors the `wait_worker` awakeable pattern with the roles reversed: the child turn /// workflow registers an awakeable, stores `(input_request_id → awakeable_id)` on its own /// `Worker` VO, emits a `NeedsInput` signal (which arms an idle-coordinator resume), then -/// `select!`s the awakeable against a long timeout. A later +/// awaits the awakeable without an expiry. A later /// `Worker::post_message(ProvideInput)` resolves the awakeable from the coordinator's -/// answer. On timeout the mapping is cleared so a late `ProvideInput` is an idempotent -/// no-op, and the child receives a "no input" result so it can proceed or report blocked. +/// answer. Cancellation clears the exact mapping so a late `ProvideInput` is an +/// idempotent no-op. async fn request_input_from_parent( - workflow: &WorkerTurnExecutionImpl, ctx: &WorkflowContext<'_>, owner: ChildInputRequestOwner<'_>, input: &RequestInputInput, @@ -1163,6 +1167,7 @@ async fn request_input_from_parent( turn_id, generation, parent_session, + meta, } = owner; let input_request_id = ctx .run(|| async { Ok::<_, HandlerError>(Json::from(uuid::Uuid::now_v7().to_string())) }) @@ -1188,6 +1193,19 @@ async fn request_input_from_parent( generation, input_request_id: input_request_id.clone(), }; + // Capture the exact workspace, instance, and lease generation before publishing + // the pending-input registration. Restate journals this read at the workflow call + // position, so replay after a later resume keeps the original fence. + let expected_hand = crate::restate_identity::replay_safe_request( + ctx.service_client::() + .capture_worker_hand_release_fence(Json::from(CaptureWorkerHandReleaseFenceRequest { + session: meta.clone(), + worker_id: worker_id.to_string(), + })), + ) + .call() + .await? + .into_inner(); moa_core::coordination_counters::record_worker_vo_call(); crate::restate_identity::replay_safe_request( ctx.object_client::(worker_id.to_string()) @@ -1202,6 +1220,23 @@ async fn request_input_from_parent( .call() .await?; + // Only after the exact input mapping is durable may the hand stop. The captured + // fence makes this idempotent without allowing a replay to destroy replacement + // compute provisioned after the human answered. + crate::restate_identity::replay_safe_request( + ctx.service_client::() + .checkpoint_and_release_worker_hand(Json::from( + CheckpointAndReleaseWorkerHandRequest { + session: meta.clone(), + worker_id: worker_id.to_string(), + input_target: target.clone(), + expected: expected_hand, + }, + )), + ) + .call() + .await?; + // Emit the NeedsInput signal to the owning coordinator (arms a guarded resume if the // coordinator is idle) before waiting, so cancellation cannot leave an unrecorded target. let signal = build_needs_input_signal( @@ -1221,38 +1256,18 @@ async fn request_input_from_parent( .call() .await?; - let timeout_ms = workflow.session_limits.worker_input_timeout_ms; let output = restate_sdk::select! { answer = answer_future => { - ChildInputWaitOutcome::Output { - output: Box::new(ToolOutput::text( + ChildInputWaitOutcome::Output(Box::new(ToolOutput::text( format!("Input received: {}", answer?), Duration::ZERO, - )), - clear_registration: false, - } + ))) }, reason = ctx.promise::(driver_progress::TurnStateKey::CANCEL_REASON_PROMISE) => { ChildInputWaitOutcome::Cancelled(reason?) - }, - _ = ctx.sleep(Duration::from_millis(timeout_ms)) => { - ChildInputWaitOutcome::Output { - output: Box::new(ToolOutput::text( - "No input was received in time. Proceed with your best judgment or report that you are blocked." - .to_string(), - Duration::ZERO, - )), - clear_registration: true, - } } }; - let clear_registration = match &output { - ChildInputWaitOutcome::Output { - clear_registration, .. - } => *clear_registration, - ChildInputWaitOutcome::Cancelled(_) => true, - }; - if clear_registration { + if matches!(output, ChildInputWaitOutcome::Cancelled(_)) { moa_core::coordination_counters::record_worker_vo_call(); crate::restate_identity::replay_safe_request( ctx.object_client::(worker_id.to_string()) @@ -1268,10 +1283,7 @@ async fn request_input_from_parent( } enum ChildInputWaitOutcome { - Output { - output: Box, - clear_registration: bool, - }, + Output(Box), Cancelled(String), } @@ -1323,6 +1335,7 @@ struct ChildInputRequestOwner<'a> { turn_id: &'a str, generation: u64, parent_session: SessionId, + meta: &'a SessionMeta, } /// Builds the `NeedsInput` control-plane signal for a child `request_input` round-trip. diff --git a/crates/moa-orchestrator/tests/analytics_parity_docker.rs b/crates/moa-orchestrator/tests/analytics_parity_docker.rs index baf131e82..2d295d73c 100644 --- a/crates/moa-orchestrator/tests/analytics_parity_docker.rs +++ b/crates/moa-orchestrator/tests/analytics_parity_docker.rs @@ -1585,12 +1585,16 @@ async fn seed_execution_run_and_tasks( .await?; sqlx::query( "INSERT INTO moa.execution_run \ - (run_uid, tenant_id, session_id, originating_user_sequence_num, planning_context_uid, \ - planning_context_hash, owner_user_id, goal_contract, initial_plan, active_plan, \ + (run_uid, tenant_id, session_id, originating_user_sequence_num, planning_context_uid, \ + planning_context_hash, owner_user_id, admitted_identity, goal_contract, initial_plan, active_plan, \ initial_plan_hash, active_plan_hash, capability_catalog, authorization_envelope, \ source_provenance, source_kind, skill_template_ref, \ skill_template_revision_uid, input, status, progress_total_tasks, started_at) \ VALUES ($1, $2, $3, 1, $4, $5, 'user-1', \ + jsonb_build_object( \ + 'identity_type', 'service', 'id', $2::TEXT, \ + 'tenant_id', $2::TEXT, 'api_key_id', NULL, \ + 'acting_on_behalf_of', NULL), \ '{\"requirements\":[{\"id\":\"r1\"}]}'::JSONB, '{}'::JSONB, '{}'::JSONB, \ $6, $6, '{}'::JSONB, '{}'::JSONB, \ jsonb_build_object('kind', 'skill_template', \ diff --git a/crates/moa-orchestrator/tests/coordinator_worker_behavior_provider_e2e.rs b/crates/moa-orchestrator/tests/coordinator_worker_behavior_provider_e2e.rs index ca3a76445..0034064c8 100644 --- a/crates/moa-orchestrator/tests/coordinator_worker_behavior_provider_e2e.rs +++ b/crates/moa-orchestrator/tests/coordinator_worker_behavior_provider_e2e.rs @@ -50,8 +50,8 @@ use moa_execution::state::{ }; use moa_execution::wire::{ ExecutionCancelRequest, ExecutionConflictReason, ExecutionMutationResponse, - ExecutionPlanningContextRequest, ExecutionPlanningContextResponse, ExecutionRunRequest, - ExecutionStatusResponse, ExecutionTaskListRequest, ExecutionTaskListResponse, + ExecutionRunRequest, ExecutionStatusResponse, ExecutionTaskListRequest, + ExecutionTaskListResponse, }; use moa_wire::turn::{StartTurnRequest, StartTurnResponse, TurnOutcomeKind}; use serde::Serialize; @@ -1205,7 +1205,7 @@ impl SupplementaryLiveHarness { response, ExecutionMutationResponse::Applied { ref run } | ExecutionMutationResponse::Replayed { ref run } - if run.status == ExecutionRunStatus::Cancelled + if run.status == ExecutionRunStatus::Cancelled || !run.status.is_terminal() ) || matches!( response, ExecutionMutationResponse::Conflict { @@ -1743,6 +1743,7 @@ fn generated_plan_hash_chain_rejects_cross_surface_drift() { async fn assert_generated_plan_audits_and_authorization( harness: &SupplementaryLiveHarness, events: &[EventRecord], + planning_context_uid: uuid::Uuid, ) -> Result { let audits = supplementary_planning_audits(harness).await?; ensure!( @@ -2029,23 +2030,20 @@ async fn assert_generated_plan_audits_and_authorization( ); let expected_compiler_candidate_hash = supplementary_compile_candidate_hash(&candidate)?; - let planning_context: ExecutionPlanningContextResponse = harness - .execution_call( - "planning_context", - &ExecutionPlanningContextRequest { - tenant_id: harness.session.identity.tenant_id, - contact_id: None, - session_id: harness.session.session_id, - originating_user_sequence_num: originating_sequence, - deadline_at: chrono::Utc::now() + chrono::TimeDelta::days(1), - requested_template: None, - }, - ) - .await?; - ensure!( - !planning_context.created, - "post-admission planning context read must replay the frozen authority snapshot" - ); + let planning_context = ExecutionRepository::new( + sqlx::PgPool::connect(&test_database_url()) + .await + .context("connect supplementary planning-context repository")?, + ) + .load_planning_context_for_session( + ExecutionScope::Tenant { + tenant_id: harness.session.identity.tenant_id, + }, + planning_context_uid, + harness.session.session_id, + ) + .await? + .context("admitted run must retain its immutable planning context")?; planning_context .snapshot .validate() @@ -2521,18 +2519,30 @@ async fn recovery_matrix_assert_child_joined( parent_id: &str, child_id: &str, ) -> Result<()> { - let rows = recovery_matrix_restate_rows( - fixture, - format!("SELECT id, invoked_by_id, status FROM sys_invocation WHERE id = '{child_id}'"), - ) - .await?; - ensure!( - rows.len() == 1 - && rows[0].get("invoked_by_id").and_then(Value::as_str) == Some(parent_id) - && rows[0].get("status").and_then(Value::as_str) == Some("completed"), - "cancelled LLM child {child_id} was not joined in parent {parent_id}: {rows:?}" - ); - Ok(()) + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + let mut poll = tokio::time::interval(Duration::from_millis(25)); + poll.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + loop { + poll.tick().await; + let rows = recovery_matrix_restate_rows( + fixture, + format!("SELECT id, invoked_by_id, status FROM sys_invocation WHERE id = '{child_id}'"), + ) + .await?; + ensure!( + rows.len() == 1 + && rows[0].get("id").and_then(Value::as_str) == Some(child_id) + && rows[0].get("invoked_by_id").and_then(Value::as_str) == Some(parent_id), + "cancelled LLM child {child_id} did not retain parent {parent_id}: {rows:?}" + ); + if rows[0].get("status").and_then(Value::as_str) == Some("completed") { + return Ok(()); + } + ensure!( + tokio::time::Instant::now() < deadline, + "cancelled LLM child {child_id} was not joined in parent {parent_id}: {rows:?}" + ); + } } #[tokio::test] @@ -3078,12 +3088,6 @@ fn recovery_matrix_execution_candidate( }, plan: ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, - input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { - expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { - delay_seconds: 86_400, - }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, - }, input_schema: json!({"type": "object", "additionalProperties": false}), output_schema: output_schema.clone(), nodes: vec![ @@ -3513,12 +3517,6 @@ async fn coordinator_generated_plan_is_strict_authorized_and_terminal_provider_e started .validate() .context("ExecutionRunStarted must satisfy the strict admission contract")?; - let audit_evidence = - assert_generated_plan_audits_and_authorization(&harness, &admission_events).await?; - ensure!( - started.originating_user_sequence_num == audit_evidence.originating_sequence, - "admission origin must equal planning-audit origin" - ); ensure!( started.plan_revision == 1, "admission must start at revision one" @@ -3547,6 +3545,16 @@ async fn coordinator_generated_plan_is_strict_authorized_and_terminal_provider_e ) .await? .context("admitted generated run must be persisted")?; + let audit_evidence = assert_generated_plan_audits_and_authorization( + &harness, + &admission_events, + persisted.planning_context_uid, + ) + .await?; + ensure!( + started.originating_user_sequence_num == audit_evidence.originating_sequence, + "admission origin must equal planning-audit origin" + ); ensure!( persisted.run_uid == started.run_uid, "event run UID must equal persisted run UID" diff --git a/crates/moa-orchestrator/tests/execution_run_service_e2e.rs b/crates/moa-orchestrator/tests/execution_run_service_e2e.rs index 524322d1f..37e6c415f 100644 --- a/crates/moa-orchestrator/tests/execution_run_service_e2e.rs +++ b/crates/moa-orchestrator/tests/execution_run_service_e2e.rs @@ -132,12 +132,6 @@ async fn output_only_run_is_durable_detached_and_reaches_terminal_state() -> Res }, plan: ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, - input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { - expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { - delay_seconds: 86_400, - }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, - }, input_schema: json!({"type": "object", "additionalProperties": false}), output_schema: json!({ "type": "object", @@ -351,12 +345,6 @@ async fn cancellation_preserves_preconfirmation_null_and_postqueue_timestamp() - }, plan: ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, - input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { - expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { - delay_seconds: 86_400, - }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, - }, input_schema: json!({"type": "object"}), output_schema: json!({"type": "object"}), nodes: vec![ExecutionNode { diff --git a/crates/moa-orchestrator/tests/execution_run_service_e2e/admission_replay.rs b/crates/moa-orchestrator/tests/execution_run_service_e2e/admission_replay.rs index 7facb524a..9f75886ac 100644 --- a/crates/moa-orchestrator/tests/execution_run_service_e2e/admission_replay.rs +++ b/crates/moa-orchestrator/tests/execution_run_service_e2e/admission_replay.rs @@ -354,12 +354,6 @@ fn template_skill_source() -> String { }, plan: ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, - input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { - expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { - delay_seconds: 86_400, - }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, - }, input_schema: io_schema.clone(), output_schema: io_schema.clone(), nodes: vec![ExecutionNode { diff --git a/crates/moa-orchestrator/tests/execution_run_service_e2e/bulk_and_recovery.rs b/crates/moa-orchestrator/tests/execution_run_service_e2e/bulk_and_recovery.rs index 8582a114e..8bc338ed4 100644 --- a/crates/moa-orchestrator/tests/execution_run_service_e2e/bulk_and_recovery.rs +++ b/crates/moa-orchestrator/tests/execution_run_service_e2e/bulk_and_recovery.rs @@ -810,12 +810,6 @@ fn bulk_candidate( }, plan: ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, - input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { - expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { - delay_seconds: 86_400, - }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, - }, input_schema: json!({"type": "object", "additionalProperties": false}), output_schema: report_schema.clone(), nodes: vec![ diff --git a/crates/moa-orchestrator/tests/execution_run_service_e2e/compensation_recovery.rs b/crates/moa-orchestrator/tests/execution_run_service_e2e/compensation_recovery.rs index d5fd401b4..0864a8e89 100644 --- a/crates/moa-orchestrator/tests/execution_run_service_e2e/compensation_recovery.rs +++ b/crates/moa-orchestrator/tests/execution_run_service_e2e/compensation_recovery.rs @@ -627,12 +627,6 @@ fn compensated_plan( }); ExecutionPlanDefinition { cancel_policy, - input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { - expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { - delay_seconds: 60, - }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, - }, input_schema: json!({ "type": "object", "additionalProperties": false diff --git a/crates/moa-orchestrator/tests/execution_run_service_e2e/evaluation.rs b/crates/moa-orchestrator/tests/execution_run_service_e2e/evaluation.rs index 5f36073c5..f7fe4c684 100644 --- a/crates/moa-orchestrator/tests/execution_run_service_e2e/evaluation.rs +++ b/crates/moa-orchestrator/tests/execution_run_service_e2e/evaluation.rs @@ -14,7 +14,7 @@ use moa_test_support::{FixtureCapabilityController, OrchestratorTestFixture, Tes use crate::execution_execution_support::{ assertions::{assert_initial_route, assert_no_execution_lifecycle_events}, - evaluation::{collect_execution_eval_snapshot, collect_repository_execution_eval_snapshot}, + evaluation::collect_execution_eval_snapshot, }; /// Collects one service snapshot, evaluates typed invariants, and hard-fails any violation. @@ -56,30 +56,6 @@ pub(crate) async fn assert_execution_eval_case( Ok(result) } -/// Evaluates typed repository invariants for a service test that intentionally bypasses Session. -pub(crate) async fn assert_repository_execution_eval_case( - fixture: &OrchestratorTestFixture, - repository: &ExecutionRepository, - scope: ExecutionScope, - request: &ExecutionRunRequest, - case_id: &str, - specs: &[ExecutionInvariantSpec], -) -> Result { - let snapshot = collect_repository_execution_eval_snapshot( - repository, - scope, - &fixture.postgres_url, - request.session_id, - request.run_uid, - ) - .await?; - let result = ExecutionEvalCaseResult::evaluate(case_id, &snapshot, specs, 0)?; - if !result.passed { - anyhow::bail!("repository execution eval case `{case_id}` failed: {result:#?}"); - } - Ok(result) -} - /// Pins a non-Durable route to one typed route audit and zero execution lifecycle events. pub(crate) fn assert_non_durable_eval( audits: &[moa_core::types::execution_planning::ExecutionPlanningAuditEnvelope], diff --git a/crates/moa-orchestrator/tests/execution_run_service_e2e/observability.rs b/crates/moa-orchestrator/tests/execution_run_service_e2e/observability.rs index 38465849e..df499199b 100644 --- a/crates/moa-orchestrator/tests/execution_run_service_e2e/observability.rs +++ b/crates/moa-orchestrator/tests/execution_run_service_e2e/observability.rs @@ -92,12 +92,6 @@ async fn execution_observability_exports_stable_identity_and_replay_safe_service }, plan: ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, - input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { - expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { - delay_seconds: 86_400, - }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, - }, input_schema: json!({"type": "object", "additionalProperties": false}), output_schema: json!({ "type": "object", diff --git a/crates/moa-orchestrator/tests/execution_run_service_e2e/replan_and_completion.rs b/crates/moa-orchestrator/tests/execution_run_service_e2e/replan_and_completion.rs index 0d66c2d8c..0a015cf7b 100644 --- a/crates/moa-orchestrator/tests/execution_run_service_e2e/replan_and_completion.rs +++ b/crates/moa-orchestrator/tests/execution_run_service_e2e/replan_and_completion.rs @@ -23,7 +23,7 @@ use moa_execution::repository::{ }; use moa_execution::state::{ ExecutionRunStatus, ExecutionTaskProjection, ExecutionTaskStatus, ExecutionTerminalCause, - ExecutionTerminalEvidence, + ExecutionTerminalEvidence, ExecutionTerminalReason, }; use moa_execution::wire::{ ExecutionAmendmentRequest, ExecutionCancelRequest, ExecutionConflictReason, @@ -53,6 +53,8 @@ const USEFUL_OUTPUT_INSTRUCTION: &str = "USEFUL_REPLAN_OUTPUT_AGENT"; const USEFUL_OUTPUT_REQUIREMENT: &str = "useful_result"; const REPAIR_REQUIREMENT: &str = "repair_result"; const USEFUL_OUTPUT: &str = "preserved-useful-output"; +const FIXTURE_RELEASE_BATCH_SIZE: usize = 1; +const FIXTURE_EXECUTION_WINDOW: usize = 8; struct StartedExecution { originating_user_sequence_num: u64, @@ -770,15 +772,21 @@ async fn completion_gate_missing_company_service_e2e() -> Result<()> { .map(|item| extract_map_key(item, "/ticker")) .collect::, _>>()?; let missing_key = extract_map_key(&json!({"ticker": MISSING_COMPANY}), "/ticker")?; - let max_in_flight_tasks = ExecutionConfig::default().max_in_flight_tasks; + let max_in_flight_tasks = FIXTURE_EXECUTION_WINDOW; let fixture = replan_fixture( default_script(), FixtureCapabilityOptions { tools: vec![map_tool(TOOL, "/ticker")], - orchestrator_env: vec![( - "MOA_EXECUTION_MAX_IN_FLIGHT_TASKS".to_string(), - max_in_flight_tasks.to_string(), - )], + orchestrator_env: vec![ + ( + "MOA_EXECUTION_MAX_IN_FLIGHT_TASKS".to_string(), + max_in_flight_tasks.to_string(), + ), + ( + "MOA_EXECUTION_DISPATCH_BATCH_SIZE".to_string(), + max_in_flight_tasks.to_string(), + ), + ], }, ) .await?; @@ -898,15 +906,21 @@ async fn run_silent_incomplete_universe(universe_size: usize, tool_name: &str) - .map(|item| extract_map_key(item, "/ticker")) .collect::, _>>()?; let missing_keys = expected_keys[returned_count..].to_vec(); - let max_in_flight_tasks = ExecutionConfig::default().max_in_flight_tasks; + let max_in_flight_tasks = FIXTURE_EXECUTION_WINDOW; let fixture = replan_fixture( default_script(), FixtureCapabilityOptions { tools: vec![map_tool(tool_name, "/ticker")], - orchestrator_env: vec![( - "MOA_EXECUTION_MAX_IN_FLIGHT_TASKS".to_string(), - max_in_flight_tasks.to_string(), - )], + orchestrator_env: vec![ + ( + "MOA_EXECUTION_MAX_IN_FLIGHT_TASKS".to_string(), + max_in_flight_tasks.to_string(), + ), + ( + "MOA_EXECUTION_DISPATCH_BATCH_SIZE".to_string(), + max_in_flight_tasks.to_string(), + ), + ], }, ) .await?; @@ -996,7 +1010,7 @@ async fn release_fixture_calls_in_window( } let mut released = 0; while released < total { - let batch_size = max_in_flight_tasks.min(total - released); + let batch_size = FIXTURE_RELEASE_BATCH_SIZE.min(total - released); controller .wait_for_calls(released + batch_size, SERVICE_TIMEOUT) .await?; @@ -1364,14 +1378,6 @@ async fn execution_eval_injected_tool_instruction_cannot_escape_envelope_service requirement_count: 1, }) ); - assert!( - terminal - .gaps - .iter() - .any(|gap| gap.contains("agent emitted an undeclared capability")), - "terminal evidence omitted the envelope rejection: {:?}", - terminal.gaps - ); assert_eq!( controller .calls() @@ -1398,16 +1404,24 @@ async fn execution_eval_injected_tool_instruction_cannot_escape_envelope_service 1, "output must not materialize after rejection" ); - assert!(matches!( - tasks.tasks[0] - .outcome - .as_ref() - .map(|outcome| &outcome.result), - Some(moa_artifacts::execution_plan::ExecutionTaskResult::Failed { - class: moa_artifacts::execution_plan::ExecutionFailureClass::Terminal, - .. - }) - )); + let expected_failure = format!( + "agent emitted undeclared capability `{}`", + moa_hands::mcp_tool_reference(FIXTURE_MCP_SERVER_NAME, FORBIDDEN_TOOL) + ); + match tasks.tasks[0] + .outcome + .as_ref() + .map(|outcome| &outcome.result) + { + Some(moa_artifacts::execution_plan::ExecutionTaskResult::Failed { class, message }) => { + assert_eq!( + class, + &moa_artifacts::execution_plan::ExecutionFailureClass::Terminal + ); + assert_eq!(message, &expected_failure); + } + result => bail!("undeclared capability did not persist its typed failure: {result:?}"), + } assert_execution_eval_case( &fixture, test.client(), @@ -1419,9 +1433,6 @@ async fn execution_eval_injected_tool_instruction_cannot_escape_envelope_service ExecutionInvariantSpec::TerminalStatusIn { statuses: vec![ExecutionRunStatus::Failed], }, - ExecutionInvariantSpec::TerminalGapContains { - text: "agent emitted an undeclared capability".to_string(), - }, ExecutionInvariantSpec::BudgetWithinApproved, ExecutionInvariantSpec::ProgressMatchesTasks, ExecutionInvariantSpec::NoDuplicateLogicalEffects, @@ -1584,20 +1595,36 @@ async fn execution_eval_amendment_cannot_broaden_authorization_service_e2e() -> let ExecutionMutationResponse::Applied { run: fenced } = cancelled else { bail!("cancellation did not install a terminal fence: {cancelled:?}"); }; - assert_eq!(fenced.status, ExecutionRunStatus::WaitingReplan); + assert_eq!(fenced.status, ExecutionRunStatus::Cancelled); + assert_eq!( + fenced.terminal_reason, + Some(ExecutionTerminalReason::Cancelled) + ); + let cancellation_evidence = ExecutionTerminalEvidence { + cause: ExecutionTerminalCause::Cancellation, + satisfied_requirement_count: 1, + requirement_count: 2, + }; + assert_eq!( + fenced.terminal_evidence.as_ref(), + Some(&cancellation_evidence) + ); + assert!(fenced.completed_at.is_some()); let fenced_record = repository .load_run(scope, started.run.run_uid) .await? .context("cancelled execution disappeared after terminal fencing")?; - let pending = fenced_record - .pending_terminal - .as_ref() - .context("applied cancellation omitted its pending terminal intent")?; - assert_eq!(pending.status, ExecutionRunStatus::Cancelled); + assert_eq!(fenced_record.status, ExecutionRunStatus::Cancelled); + assert!(fenced_record.pending_terminal.is_none()); assert_eq!( - pending.terminal_evidence.cause, - ExecutionTerminalCause::Cancellation + fenced_record.terminal_reason, + Some(ExecutionTerminalReason::Cancelled) ); + assert_eq!( + fenced_record.terminal_evidence.as_ref(), + Some(&cancellation_evidence) + ); + assert!(fenced_record.completed_at.is_some()); let terminal = await_execution_terminal(test.client(), &started.run).await?; assert_eq!(terminal.run.status, ExecutionRunStatus::Cancelled); assert_execution_eval_case( @@ -2040,12 +2067,6 @@ fn useful_replan_contract( }, ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, - input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { - expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { - delay_seconds: 86_400, - }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, - }, input_schema: empty_input_schema(), output_schema: output_schema.clone(), nodes: vec![ @@ -2298,12 +2319,6 @@ fn map_then_output_plan(spec: MapThenOutputPlan<'_>) -> ExecutionPlanDefinition } = spec; ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, - input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { - expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { - delay_seconds: 86_400, - }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, - }, input_schema: empty_input_schema(), output_schema: output_schema.clone(), nodes: vec![ @@ -2378,12 +2393,6 @@ fn missing_deliverable_contract() -> (ExecutionGoalContract, ExecutionPlanDefini // and the deliverable is the sole reason the run must not report completion. ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, - input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { - expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { - delay_seconds: 86_400, - }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, - }, input_schema: empty_input_schema(), output_schema: output_schema.clone(), nodes: vec![ @@ -2476,12 +2485,6 @@ fn declared_contradiction_contract() -> (ExecutionGoalContract, ExecutionPlanDef }, ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, - input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { - expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { - delay_seconds: 86_400, - }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, - }, input_schema: empty_input_schema(), output_schema: report_schema.clone(), nodes: vec![ @@ -2585,12 +2588,6 @@ fn injected_content_contract( }, ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, - input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { - expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { - delay_seconds: 86_400, - }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, - }, input_schema: empty_input_schema(), output_schema: output_schema.clone(), nodes: vec![ diff --git a/crates/moa-orchestrator/tests/execution_run_service_e2e/routing.rs b/crates/moa-orchestrator/tests/execution_run_service_e2e/routing.rs index cf9d168bd..6b11d13c8 100644 --- a/crates/moa-orchestrator/tests/execution_run_service_e2e/routing.rs +++ b/crates/moa-orchestrator/tests/execution_run_service_e2e/routing.rs @@ -68,6 +68,10 @@ const SECURITY_INPUT_TOOL_NAME: &str = "inspect_suspicious_fixture"; const SECURITY_INPUT_FIRST_TOOL_ID: &str = "00000000-0000-0000-0000-000000000321"; const SECURITY_INPUT_SECOND_TOOL_ID: &str = "00000000-0000-0000-0000-000000000322"; const SECURITY_INPUT_WARNING_MARKER: &str = "fixture-warning-one"; +const SECURITY_INPUT_ADMISSION_PROBE: &str = + "Complete this independent turn while the security-input turn is parked"; +const SECURITY_INPUT_ADMISSION_PROBE_FINAL: &str = + "The independent turn completed while the first turn was parked."; const SYNTHESIS_MATCH: &str = "Synthesize the final user response for execution run"; const TEMPLATE_SKILL_NAME: &str = "service-template-report"; const TEMPLATE_FINAL: &str = "The pinned template produced the requested report."; @@ -297,7 +301,7 @@ async fn recovery_matrix_coordinator_input_cancel_cleans_exact_wait_and_rejects_ // Pins: cancellation after a hard restart drives the actual coordinator // awakeable select, clears its four-coordinate registration, releases the // active turn, and leaves an explicitly addressed late reply as a conflict. - let fixture = security_input_fixture(1_800_000).await?; + let fixture = security_input_fixture().await?; let test = fixture.isolated().await; let started = start_security_input_turn(&fixture, &test, "security-input-cancel").await?; await_security_input_registration(&fixture, test.client(), &started).await?; @@ -329,12 +333,12 @@ async fn recovery_matrix_coordinator_input_cancel_cleans_exact_wait_and_rejects_ #[tokio::test] #[ignore = "requires the local Restate/Postgres/OpenFGA/Redis service fixture"] -async fn recovery_matrix_coordinator_input_timeout_survives_restart_and_releases_turn_service_e2e() +async fn recovery_matrix_coordinator_input_wait_survives_restart_until_exact_reply_service_e2e() -> Result<()> { - // Pins: the durable timeout branch survives process loss, clears the exact - // pending input, and settles as the explicit safe failure instead of leaving - // the Session active forever. - let fixture = security_input_fixture(5_000).await?; + // Pins: a human-input wait has no expiry, releases the actual fleet/tenant + // admission lease, survives process loss, and resumes only after the exact + // authenticated reply reaches its durable awakeable. + let fixture = security_input_fixture().await?; let test = fixture.isolated().await; let started = start_security_input_turn(&fixture, &test, "security-input-timeout").await?; await_security_input_registration(&fixture, test.client(), &started).await?; @@ -342,18 +346,60 @@ async fn recovery_matrix_coordinator_input_timeout_survives_restart_and_releases fixture .hard_crash_and_restart_orchestrator() .await - .context("restart before coordinator input timeout")?; - let outcome = await_turn_outcome(test.client(), &started).await?; - assert_eq!(outcome.kind, TurnOutcomeKind::Failed); - assert!( - outcome.message.contains("security-input timeout"), - "coordinator input timeout lost its stable reason: {outcome:?}" + .context("restart while coordinator input is parked")?; + + // Both admission limits are one. Completing a distinct session before the + // parked turn receives its reply proves the first session released the real + // shared lease rather than merely changing its virtual-object state. + let admission_probe = start_turn( + &test, + "security-input-admission-probe", + SECURITY_INPUT_ADMISSION_PROBE, + None, + ) + .await + .context("start a second session while the first human wait is parked")?; + let admission_probe_outcome = await_turn_outcome(test.client(), &admission_probe).await?; + assert_eq!(admission_probe_outcome.kind, TurnOutcomeKind::Completed); + assert_eq!( + admission_probe_outcome.message, + SECURITY_INPUT_ADMISSION_PROBE_FINAL ); + + tokio::time::sleep(std::time::Duration::from_secs(6)).await; + let before_reply = test.client().get_session(started.session_id).await?; + assert_eq!(before_reply.status, SessionStatus::Running); + test.client() + .session(started.session_id.to_string()) + .start_turn( + StartTurnRequest { + client_message_id: fresh_client_message_id(), + reply_to: Some(MessageReplyTarget::CoordinatorInput { + turn_id: started.turn_id.clone(), + generation: 1, + input_request_id: format!( + "security:{}:1:{}", + started.turn_id, SECURITY_INPUT_SECOND_TOOL_ID + ), + }), + stream_cursor: None, + user_message: "continue without the disabled capability".to_string(), + attachments: Vec::new(), + model: None, + contact: None, + max_turns: None, + resource_budget: Default::default(), + execution_template: None, + }, + None, + ) + .await?; + let outcome = await_turn_outcome(test.client(), &started).await?; + assert_eq!(outcome.kind, TurnOutcomeKind::Completed); assert_eq!( await_session_settled(test.client(), started.session_id).await?, - SessionStatus::Failed + SessionStatus::Idle ); - assert_coordinator_input_late_reply_conflicts(test.client(), &started).await?; Ok(()) } @@ -620,7 +666,7 @@ async fn restate_query_rows(fixture: &OrchestratorTestFixture, query: &str) -> R } } -async fn security_input_fixture(timeout_ms: u64) -> Result { +async fn security_input_fixture() -> Result { OrchestratorTestFixture::with_execution_fixture( json!({ "default": text_completion("unexpected security-input fallback"), @@ -629,6 +675,10 @@ async fn security_input_fixture(timeout_ms: u64) -> Result Result String { }, plan: ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, - input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { - expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { - delay_seconds: 86_400, - }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, - }, input_schema: template_io_schema(), output_schema: template_io_schema(), nodes: vec![ExecutionNode { diff --git a/crates/moa-orchestrator/tests/execution_run_service_e2e/task_lifecycle.rs b/crates/moa-orchestrator/tests/execution_run_service_e2e/task_lifecycle.rs index 5253f3294..f381833f2 100644 --- a/crates/moa-orchestrator/tests/execution_run_service_e2e/task_lifecycle.rs +++ b/crates/moa-orchestrator/tests/execution_run_service_e2e/task_lifecycle.rs @@ -30,8 +30,7 @@ use moa_execution::{ compiler::{CompileExecutionRequest, CompiledExecution, compile}, repository::{ ExecutionRepository, ExecutionRunRecord, ExecutionScope, ExecutionTaskRecord, - NewExecutionRun, ReservationOutcome, TaskOutcomeRejection, TaskOutcomeWrite, - TransitionOutcome, run::RunAdmissionOutcome, + NewExecutionRun, ReservationOutcome, TaskOutcomeWrite, run::RunAdmissionOutcome, }, state::{ ExecutionRunStatus, ExecutionTaskId, ExecutionTaskStatus, ExecutionTerminalCause, @@ -61,7 +60,7 @@ use serde_json::{Value, json}; use tokio::time::Instant; use uuid::Uuid; -use crate::evaluation::{assert_execution_eval_case, assert_repository_execution_eval_case}; +use crate::evaluation::assert_execution_eval_case; use crate::execution_execution_support::{ assertions::journal_requests, fixtures::{POLL_INTERVAL, SERVICE_TIMEOUT, await_execution_terminal, list_execution_tasks}, @@ -134,7 +133,26 @@ async fn execution_eval_rate_limit_reuses_task_identity_service_e2e() -> Result< assert_eq!(task.task_id, run.task_id); assert_eq!(task.attempt, 2); assert_eq!(task.generation, 2); - assert_eq!(task.generation_history.len(), 2); + let settlement_history = task + .generation_history + .iter() + .filter(|entry| { + entry.get("kind").and_then(Value::as_str) == Some("bounded_attempt_settlement") + }) + .map(|entry| { + json!({ + "attempt_generation": entry.get("attempt_generation"), + "retry_scheduled": entry.get("retry_scheduled"), + }) + }) + .collect::>(); + assert_eq!( + settlement_history, + vec![ + json!({"attempt_generation": 1, "retry_scheduled": true}), + json!({"attempt_generation": 2, "retry_scheduled": false}), + ] + ); assert_eq!(task.status, ExecutionTaskStatus::Completed); assert_eq!(terminal.run.budget_ledger.consumed.tasks, 2); assert_eq!(terminal.run.budget_ledger.consumed.tool_calls, 2); @@ -376,10 +394,10 @@ async fn confirmation_retries_after_persisted_epoch_before_dispatch_service_e2e( let recovered = tokio::time::timeout(SERVICE_TIMEOUT, interrupted) .await .context("confirmation request did not retry after orchestrator recovery")???; - assert!(matches!( - recovered, - ExecutionMutationResponse::Applied { .. } | ExecutionMutationResponse::Replayed { .. } - )); + assert!( + matches!(recovered, ExecutionMutationResponse::Replayed { .. }), + "a crash before ctx.run result journaling must rebuild the accepted handoff from Postgres" + ); let provider_requests = journal_requests( fixture @@ -546,18 +564,18 @@ async fn cancellation_releases_reservations_and_prevents_dispatch_service_e2e() #[tokio::test] #[ignore = "requires the local Restate/Postgres/OpenFGA/Redis service fixture"] async fn cancellation_retries_after_persisted_epoch_before_dispatch_service_e2e() -> Result<()> { - // Pins: a process crash after the cancellation fence commits but before the dispatcher - // accepts the kick cannot return success or advance the epoch twice; - // an already-admitted late effect remains authoritative under RetainEffects. + // Pins: a process crash after the cancellation fence commits but before the journaled + // result and exact LLM-owner fence cannot return success or invent a controller wake; + // recovery replays the same cancellation transport identity and rejects a late result. let tool_name = "lifecycle_cancel_handoff_probe"; let fixture = direct_execution_fixture(tool_name, success_outcomes()).await?; fixture .restart_orchestrator_with_env(vec![( - "MOA_EXECUTION_TEST_PAUSE_MUTATION_HANDOFF".to_string(), + "MOA_EXECUTION_TEST_PAUSE_CANCEL_DB_HANDOFF".to_string(), "true".to_string(), )]) .await - .context("arm the post-commit execution-mutation handoff pause")?; + .context("arm the post-commit/pre-journal cancellation handoff pause")?; let prepared = prepare_capability_run( &fixture, "cancellation-handoff-recovery", @@ -627,7 +645,7 @@ async fn cancellation_retries_after_persisted_epoch_before_dispatch_service_e2e( ); let task = load_task(&run).await?; assert_eq!(task.generation, 1); - assert_eq!(task.status, ExecutionTaskStatus::Completed); + assert_eq!(task.status, ExecutionTaskStatus::Cancelled); assert_eq!(controller.calls().len(), 1); let attempts = controller.transport_attempts(); assert_eq!(attempts.len(), 2); @@ -1301,150 +1319,81 @@ async fn action_review_terminal_states_deliver_once_service_e2e() -> Result<()> #[tokio::test] #[ignore = "requires the local Restate/Postgres/OpenFGA/Redis service fixture"] async fn stale_generation_is_audit_only_service_e2e() -> Result<()> { - // Pins: a stale completion appends one rejected audit without changing projection or usage. + // Pins: a stale public signal delivery appends one rejected audit without changing + // projection or usage, while the exact generation still completes through the service. let tool_name = "lifecycle_stale_probe"; let fixture = direct_execution_fixture(tool_name, success_outcomes()).await?; let prepared = prepare_capability_run( &fixture, "stale-generation", tool_name, - RetryPolicy { - max_attempts: 2, - initial_backoff_ms: 0, - max_backoff_ms: 0, - }, + no_retry(), ActionPolicyEffect::Allow, ) .await?; - let run = create_direct_run(&prepared, prepared.planning.snapshot.budget.clone(), None).await?; - reserve_and_mark_running(&run).await?; - let retryable = ExecutionTaskOutcome { - schema_version: 1, - usage: zero_usage(), - result: ExecutionTaskResult::Failed { - class: ExecutionFailureClass::Retryable, - message: "retry under generation two".to_string(), - }, - }; - assert!(matches!( - run.repository - .record_task_outcome(run.scope, run.run_uid, run.task_id, 1, retryable) - .await?, - TaskOutcomeWrite::Applied { .. } - )); - assert!(matches!( - run.repository - .retry_task(run.scope, run.run_uid, run.task_id, 1) - .await?, - TransitionOutcome::Applied(_) - )); - let run_before = load_run(&run).await?; - let task_before = load_task(&run).await?; - assert_eq!(task_before.attempt, 2); - assert_eq!(task_before.generation, 2); + let prepared = recompile_as_external_wait(prepared, ExternalWaitKind::Signal)?; + let run = start_service_run(&fixture, &prepared, false).await?; + let task_before = await_task_status(&run, ExecutionTaskStatus::WaitingSignal).await?; + let run_before = await_run_status(&run, ExecutionRunStatus::WaitingSignal).await?; + assert_eq!(task_before.attempt, 1); + assert_eq!(task_before.generation, 1); let controlled_before = controlled_task_projection(&task_before); - let stale = completed_outcome(json!({"result": "stale"}), zero_usage()); - let rejected = run - .repository - .record_task_outcome(run.scope, run.run_uid, run.task_id, 1, stale) - .await?; - let TaskOutcomeWrite::Rejected { task, reason } = rejected else { - bail!("stale generation completion was not audit-only") + let payload = json!({"result": "signal-delivered"}); + let stale_request = ExecutionSignalRequest { + tenant_id: run.tenant_id, + contact_id: None, + run_uid: run.run_uid, + task_id: run.task_id, + expected_generation: 2, + signal_name: "fixture-ready".to_string(), + payload: payload.clone(), }; - assert_eq!(reason, TaskOutcomeRejection::StaleGeneration); - assert_eq!(controlled_task_projection(&task), controlled_before); + let rejected: ExecutionMutationResponse = fixture + .client + .post_call("/Execution/deliver_signal", &stale_request) + .await?; assert_eq!( - task.outcome_audit.len(), + rejected, + ExecutionMutationResponse::Conflict { + reason: ExecutionConflictReason::GenerationMismatch, + } + ); + let stale_task = load_task(&run).await?; + assert_eq!(controlled_task_projection(&stale_task), controlled_before); + assert_eq!( + stale_task.outcome_audit.len(), task_before.outcome_audit.len() + 1 ); - let stale_audit = task + let stale_audit = stale_task .outcome_audit .last() - .context("stale completion omitted its audit record")?; + .context("stale signal delivery omitted its audit record")?; assert_eq!(stale_audit.get("accepted"), Some(&json!(false))); - assert_eq!(stale_audit.get("received_generation"), Some(&json!(1))); - assert_eq!(stale_audit.get("received_attempt"), Some(&json!(1))); + assert_eq!(stale_audit.get("received_generation"), Some(&json!(2))); + assert_eq!(stale_audit.get("received_attempt"), Some(&Value::Null)); assert_eq!( stale_audit.get("rejection"), Some(&json!("stale_generation")) ); - assert_eq!(load_run(&run).await?, run_before); - let controller = fixture_capability(&fixture)?; - assert!(controller.calls().is_empty()); - assert_repository_execution_eval_case( - &fixture, - &run.repository, - run.scope, - &run.request, - "stale-generation-write-is-fenced", - &[ - ExecutionInvariantSpec::BudgetWithinApproved, - ExecutionInvariantSpec::ProgressMatchesTasks, - ExecutionInvariantSpec::NoDuplicateLogicalEffects, - ExecutionInvariantSpec::NoRawTaskOutputEvents, - ], - ) - .await?; - Ok(()) -} - -#[tokio::test] -#[ignore = "requires the local Restate/Postgres/OpenFGA/Redis service fixture"] -async fn duplicate_completion_does_not_double_account_service_e2e() -> Result<()> { - // Pins: exact task completion replay is byte-identical and consumes one logical task once. - let tool_name = "lifecycle_duplicate_probe"; - let fixture = direct_execution_fixture(tool_name, success_outcomes()).await?; - let prepared = prepare_capability_run( - &fixture, - "duplicate-completion", - tool_name, - no_retry(), - ActionPolicyEffect::Allow, - ) - .await?; - let run = create_direct_run(&prepared, prepared.planning.snapshot.budget.clone(), None).await?; - reserve_and_mark_running(&run).await?; - let completion = completed_outcome( - json!({"result": "accepted-once"}), - ExecutionUsage { - cost_microusd: 0, - tokens: 0, - tool_calls: 1, - retrieved_bytes: 32, - }, + assert_eq!( + controlled_run_projection(&load_run(&run).await?), + controlled_run_projection(&run_before) ); - assert!(matches!( - run.repository - .record_task_outcome(run.scope, run.run_uid, run.task_id, 1, completion.clone(),) - .await?, - TaskOutcomeWrite::Applied { - budget_overrun: false, - .. - } - )); - let run_before = load_run(&run).await?; - let task_before = load_task(&run).await?; - assert_eq!(run_before.consumed.tasks, 1); - assert_eq!(run_before.consumed.tool_calls, 1); - assert_eq!(run_before.progress_completed_tasks, 1); - assert_eq!(task_before.actual_tasks, 1); - - let replayed = run - .repository - .record_task_outcome(run.scope, run.run_uid, run.task_id, 1, completion) + + let accepted_request = ExecutionSignalRequest { + expected_generation: 1, + ..stale_request + }; + let accepted: ExecutionMutationResponse = fixture + .client + .post_call("/Execution/deliver_signal", &accepted_request) .await?; assert!(matches!( - replayed, - TaskOutcomeWrite::Replayed { - budget_overrun: false, - .. - } + accepted, + ExecutionMutationResponse::Applied { .. } )); - assert_eq!(load_run(&run).await?, run_before); - assert_eq!(load_task(&run).await?, task_before); - - let terminal = drive_run_workflow(&fixture, &run).await?; + let terminal = await_execution_terminal(&fixture.client, &run.request).await?; assert_terminal( &terminal, ExecutionRunStatus::Completed, @@ -1452,17 +1401,29 @@ async fn duplicate_completion_does_not_double_account_service_e2e() -> Result<() 1, 1, ); - assert_eq!(terminal.run.budget_ledger.consumed.tasks, 2); - assert_eq!(terminal.run.budget_ledger.consumed.tool_calls, 1); - assert_eq!(terminal.run.completed_tasks, 2); + let completed = load_task(&run).await?; + assert_eq!(completed.status, ExecutionTaskStatus::Completed); + assert_eq!(completed.generation, 1); + assert_eq!( + completed.outcome_audit.len(), + task_before.outcome_audit.len() + 2 + ); + assert_eq!( + completed + .outcome_audit + .last() + .and_then(|audit| audit.get("accepted")), + Some(&json!(true)) + ); let controller = fixture_capability(&fixture)?; assert!(controller.calls().is_empty()); - assert_repository_execution_eval_case( + assert!(controller.transport_attempts().is_empty()); + assert_execution_eval_case( &fixture, - &run.repository, - run.scope, + &fixture.client, &run.request, - "duplicate-completion-is-idempotent", + Some(controller), + "stale-generation-write-is-fenced", &[ ExecutionInvariantSpec::TerminalStatusIn { statuses: vec![ExecutionRunStatus::Completed], @@ -1988,12 +1949,6 @@ fn recompile_as_agent( }); let plan = ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, - input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { - expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { - delay_seconds: 86_400, - }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, - }, input_schema: json!({"type": "object", "additionalProperties": false}), output_schema: output_schema.clone(), nodes: vec![ @@ -2070,7 +2025,7 @@ fn recompile_as_external_wait( signal_name: "fixture-ready".to_string(), wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { - delay_seconds: 86_400, + delay_seconds: 3_600, }, on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, }, @@ -2078,12 +2033,6 @@ fn recompile_as_external_wait( }; let plan = ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, - input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { - expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { - delay_seconds: 86_400, - }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, - }, input_schema: json!({"type": "object", "additionalProperties": false}), output_schema: output_schema.clone(), nodes: vec![ @@ -2300,12 +2249,6 @@ fn lifecycle_plan_with_output_schema( ) -> ExecutionPlanDefinition { ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, - input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { - expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { - delay_seconds: 86_400, - }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, - }, input_schema: json!({"type": "object", "additionalProperties": false}), output_schema: output_schema.clone(), nodes: vec![ @@ -2377,33 +2320,6 @@ fn zero_usage() -> ExecutionUsage { } } -fn completed_outcome(output: Value, usage: ExecutionUsage) -> ExecutionTaskOutcome { - ExecutionTaskOutcome { - schema_version: 1, - usage, - result: ExecutionTaskResult::Completed { - output, - citations: Vec::new(), - }, - } -} - -async fn reserve_and_mark_running(run: &RunningCapabilityRun) -> Result<()> { - assert!(matches!( - run.repository - .reserve_task(run.scope, run.run_uid, run.task_id, 1) - .await?, - ReservationOutcome::Reserved(_) - )); - assert!(matches!( - run.repository - .mark_task_running(run.scope, run.run_uid, run.task_id, 1) - .await?, - TransitionOutcome::Applied(_) - )); - Ok(()) -} - async fn await_committed_cancellation_fence( run: &RunningCapabilityRun, previous_epoch: u64, @@ -2417,9 +2333,8 @@ async fn await_committed_cancellation_fence( .is_some_and(|pending| pending.status == ExecutionRunStatus::Cancelled) { assert_eq!( - persisted.wake_epoch, - previous_epoch + 1, - "the cancellation transaction must advance its wake epoch exactly once" + persisted.wake_epoch, previous_epoch, + "terminal drain must await the exact cancellation callback instead of inventing a controller wake" ); return Ok(persisted.wake_epoch); } @@ -2588,14 +2503,6 @@ async fn replay_run_through_public_start( Ok(()) } -async fn drive_run_workflow( - fixture: &OrchestratorTestFixture, - run: &RunningCapabilityRun, -) -> Result { - replay_run_through_public_start(fixture, run).await?; - await_execution_terminal(&fixture.client, &run.request).await -} - async fn load_run(run: &RunningCapabilityRun) -> Result { run.repository .load_run(run.scope, run.run_uid) @@ -2785,3 +2692,39 @@ fn controlled_task_projection(task: &ExecutionTaskRecord) -> Value { "citations": task.citations, }) } + +fn controlled_run_projection(run: &ExecutionRunRecord) -> Value { + json!({ + "run_uid": run.run_uid, + "status": run.status, + "controller_generation": run.controller_generation, + "plan_revision": run.plan_revision, + "active_plan_hash": run.active_plan_hash, + "output": run.output, + "terminal_evidence": run.terminal_evidence, + "terminal_reason": run.terminal_reason, + "pending_terminal": run.pending_terminal, + "manual_repair_required": run.manual_repair_required, + "reserved": run.reserved, + "consumed": run.consumed, + "budget_overrun": run.budget_overrun, + "progress": { + "total": run.progress_total_tasks, + "completed": run.progress_completed_tasks, + "failed": run.progress_failed_tasks, + "cancelled": run.progress_cancelled_tasks, + }, + "scheduler_counts": { + "ready": run.ready_task_count, + "active": run.active_task_count, + "waiting": run.waiting_task_count, + "waiting_input": run.waiting_input_task_count, + "waiting_review": run.waiting_review_task_count, + "waiting_signal": run.waiting_signal_task_count, + "waiting_timer": run.waiting_timer_task_count, + "waiting_external": run.waiting_external_task_count, + "waiting_replan": run.waiting_replan_task_count, + }, + "waiting_reasons": run.waiting_reasons, + }) +} diff --git a/crates/moa-orchestrator/tests/execution_run_service_e2e/terminal_matrix.rs b/crates/moa-orchestrator/tests/execution_run_service_e2e/terminal_matrix.rs index 9201d2f63..48a4e9eaa 100644 --- a/crates/moa-orchestrator/tests/execution_run_service_e2e/terminal_matrix.rs +++ b/crates/moa-orchestrator/tests/execution_run_service_e2e/terminal_matrix.rs @@ -891,12 +891,6 @@ fn output_candidate( goal: single_requirement_goal(objective), plan: ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, - input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { - expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { - delay_seconds: 86_400, - }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, - }, input_schema: empty_object_schema(), output_schema: schema.clone(), nodes: vec![ExecutionNode { @@ -926,12 +920,6 @@ fn replan_candidate(objective: &str) -> GeneratedExecutionCandidate { goal: replan_goal(objective), plan: ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, - input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { - expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::After { - delay_seconds: 86_400, - }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, - }, input_schema: empty_object_schema(), output_schema: schema.clone(), nodes: vec![ diff --git a/crates/moa-orchestrator/tests/integration/action_policy_flow_e2e.rs b/crates/moa-orchestrator/tests/integration/action_policy_flow_e2e.rs index 6df4c0f82..97056c66a 100644 --- a/crates/moa-orchestrator/tests/integration/action_policy_flow_e2e.rs +++ b/crates/moa-orchestrator/tests/integration/action_policy_flow_e2e.rs @@ -4,8 +4,10 @@ use std::time::Duration; use anyhow::{Context, Result}; use moa_artifacts::execution_plan::{ - ExecutionCancelPolicy, ExecutionGoalContract, ExecutionPlanDefinition, RetryPolicy, + ExecutionBudgetLimit, ExecutionCancelPolicy, ExecutionGoalContract, ExecutionNode, + ExecutionOperation, ExecutionPlanDefinition, RetryPolicy, }; +use moa_config::ExecutionConfig; use moa_core::traits::{Identity, IdentityType}; use moa_core::{ events::Event, @@ -22,7 +24,9 @@ use moa_core::{ types::identifiers::SessionId, types::identifiers::TenantId, types::identifiers::ToolCallId, + types::identifiers::UserId, types::session::SessionStatus, + types::tools::IdempotencyClass, types::tools::SecuredToolOutput, types::tools::ToolCallRequest, }; @@ -32,8 +36,20 @@ use moa_execution::{ ExecutionCapabilityCatalog, ExecutionEstimate, ExecutionHash, }, compiler::{CanonicalExecutionPlan, ExecutionValidationReport}, - state::LogicalTaskKind, - wire::ExecutionActionReviewResolution, + repository::{ + ExecutionRepository, ExecutionScope, NewExecutionRun, + audit::{NewExecutionPlanningContext, PlanningContextWriteOutcome}, + ready::{ReadyMaterializationOutcome, ReadyMaterializationRequest}, + run::RunAdmissionOutcome, + task::{ + NewTaskAttemptCheckpoint, TaskAttemptCheckpointKind, TaskAttemptFence, + TaskAttemptReleaseClaimOutcome, TaskAttemptReviewParkOutcome, TaskAttemptStartOutcome, + }, + }, + state::{ExecutionRunStatus, ExecutionTaskId, LogicalTask, LogicalTaskKind}, + wire::{ + ExecutionActionReviewResolution, ExecutionPlanningContextSnapshot, planning_context_hash, + }, }; use moa_orchestrator::objects::tenant::TenantConfig; use moa_orchestrator::services::action_policy::{PrepareActionReviewRequest, PreparedActionReview}; @@ -129,7 +145,7 @@ fn action_review_workspace_env() -> Vec<(String, String)> { "provider_account_id": ACCOUNT_ID, "provider_account_generation": 1, "max_workspaces": 8, - "max_active_hands": 4, + "max_active_hands": 8, "max_checkpoints": 16, "max_logical_bytes": 268_435_456_u64 }]) @@ -859,16 +875,22 @@ async fn claimed_execution_review_exact_replay_resumes_and_conflict_rejects( }, ) .await?; + let expected_output = format!("claimed-review-replay-{}", Uuid::now_v7()); + let command = format!("printf {expected_output}"); + let review_id = Uuid::now_v7(); let origin = insert_execution_review_task( &pool, session.tenant_id, session_id, originating_user_sequence_num, + review_id, + ToolInvocation { + id: None, + name: "bash".to_string(), + input: json!({"cmd": command.clone()}), + }, ) .await?; - let expected_output = format!("claimed-review-replay-{}", Uuid::now_v7()); - let command = format!("printf {expected_output}"); - let review_id = Uuid::now_v7(); let claimed_tool_call_id = Uuid::now_v7(); let tool_call_id = ToolCallId::new(); let contract_revision = activated_contract_revision(&test, "bash").await?; @@ -1028,10 +1050,12 @@ async fn insert_execution_review_task( tenant_id: TenantId, session_id: SessionId, originating_user_sequence_num: u64, + review_uid: Uuid, + invocation: ToolInvocation, ) -> Result { - let run_uid = Uuid::new_v4(); - let task_uid = Uuid::new_v4(); - let planning_context_uid = Uuid::new_v4(); + let repository = ExecutionRepository::new(pool.clone()); + let scope = ExecutionScope::Tenant { tenant_id }; + let config = ExecutionConfig::default(); let plan_hash = ExecutionHash::from_bytes([0; 32]); let catalog = ExecutionCapabilityCatalog::build(Vec::new()) .context("build empty execution review capability catalog")?; @@ -1046,15 +1070,24 @@ async fn insert_execution_review_task( let plan = CanonicalExecutionPlan { definition: ExecutionPlanDefinition { cancel_policy: ExecutionCancelPolicy::RetainEffects, - input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { - expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::At { - at: chrono::Utc::now() + chrono::TimeDelta::hours(1), - }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, - }, input_schema: json!({ "type": "object" }), output_schema: json!({ "type": "object" }), - nodes: Vec::new(), + nodes: vec![ExecutionNode { + id: "review".to_string(), + requirement_ids: Vec::new(), + depends_on: Vec::new(), + when: None, + input: json!({}), + output_schema: json!({ "type": "object" }), + operation: ExecutionOperation::Output { value: json!({}) }, + compensation: None, + retry: RetryPolicy { + max_attempts: 1, + initial_backoff_ms: 0, + max_backoff_ms: 0, + }, + budget: None, + }], }, plan_hash, catalog_hash: catalog.catalog_hash, @@ -1081,101 +1114,189 @@ async fn insert_execution_review_task( repair_attempts: 0, }, }; - let goal = serde_json::to_value(goal).context("serialize execution review goal fixture")?; - let plan = serde_json::to_value(plan).context("serialize execution review plan fixture")?; - let catalog = - serde_json::to_value(catalog).context("serialize execution review catalog fixture")?; - let authorization = serde_json::to_value(authorization) - .context("serialize execution review authorization fixture")?; - let source_provenance = serde_json::to_value(source_provenance) - .context("serialize execution review source provenance fixture")?; - let task_kind = serde_json::to_value(LogicalTaskKind::Output { value: json!({}) }) - .context("serialize execution review task-kind fixture")?; - let retry_policy = serde_json::to_value(RetryPolicy { - max_attempts: 1, - initial_backoff_ms: 0, - max_backoff_ms: 0, - }) - .context("serialize execution review retry-policy fixture")?; - let hash = plan_hash.to_string(); - let originating_user_sequence_num = i64::try_from(originating_user_sequence_num) - .context("execution review origin sequence exceeds PostgreSQL BIGINT")?; - sqlx::query( - r#" - INSERT INTO moa.execution_planning_context ( - planning_context_uid, tenant_id, session_id, - originating_user_sequence_num, originating_user_event_hash, - owner_user_id, planning_context_hash, snapshot - ) VALUES ($1, $2, $3, $4, $5, 'test-owner', $5, '{}'::JSONB) - "#, - ) - .bind(planning_context_uid) - .bind(tenant_id.0) - .bind(session_id.0) - .bind(originating_user_sequence_num) - .bind(&hash) - .execute(pool) - .await - .context("insert execution review planning-context fixture")?; - sqlx::query( - r#" - INSERT INTO moa.execution_run ( - run_uid, tenant_id, session_id, originating_user_sequence_num, - planning_context_uid, planning_context_hash, owner_user_id, goal_contract, - initial_plan, active_plan, initial_plan_hash, active_plan_hash, - capability_catalog, authorization_envelope, pinned_instruction_skills, - source_provenance, source_kind, - input, status, queued_at - ) VALUES ($1, $2, $3, $4, $5, $6, 'test-owner', - $7, $8, $8, $6, $6, - $9, $10, '[]'::JSONB, - $11, 'generated_plan', - '{}'::JSONB, 'queued', NOW()) - "#, - ) - .bind(run_uid) - .bind(tenant_id.0) - .bind(session_id.0) - .bind(originating_user_sequence_num) - .bind(planning_context_uid) - .bind(&hash) - .bind(goal) - .bind(plan) - .bind(catalog) - .bind(authorization) - .bind(source_provenance) - .execute(pool) - .await - .context("insert execution review run fixture")?; - sqlx::query("UPDATE moa.execution_run SET status = 'running' WHERE run_uid = $1") - .bind(run_uid) - .execute(pool) - .await - .context("start execution review run fixture")?; - sqlx::query( - r#" - INSERT INTO moa.execution_task ( - task_id, run_uid, tenant_id, node_id, item_key, plan_revision, - status, input, task_kind, retry_policy, - estimate_cost_microusd, estimate_tokens, estimate_tasks, - estimate_tool_calls, estimate_retrieved_bytes - ) VALUES ($1, $2, $3, 'review', 'replay', 1, 'running', '{}'::JSONB, - $4, $5, 0, 0, 1, 0, 0) - "#, - ) - .bind(task_uid) - .bind(run_uid) - .bind(tenant_id.0) - .bind(task_kind) - .bind(retry_policy) - .execute(pool) - .await - .context("insert execution review task fixture")?; + let budget = ExecutionBudgetLimit { + max_cost_microusd: Some(0), + max_tokens: Some(0), + max_tasks: Some(1), + max_tool_calls: Some(1), + max_retrieved_bytes: Some(0), + deadline_at: None, + }; + let owner_user_id = UserId::new("test-owner"); + let admitted_identity = Identity { + identity_type: IdentityType::Service, + id: tenant_id.0, + tenant_id, + api_key_id: None, + acting_on_behalf_of: None, + }; + let planning_snapshot = ExecutionPlanningContextSnapshot { + schema_version: 1, + tenant_id, + contact_id: None, + session_id, + originating_user_sequence_num, + originating_user_event_hash: ExecutionHash::from_bytes([1; 32]).to_string(), + owner_user_id: owner_user_id.clone(), + catalog: catalog.clone(), + authorization: authorization.clone(), + pinned_instruction_skills: Vec::new(), + execution_templates: Vec::new(), + budget: budget.clone(), + }; + let planning_hash = planning_context_hash(&planning_snapshot)?; + let planning_context = repository + .create_planning_context( + scope, + NewExecutionPlanningContext { + snapshot: planning_snapshot, + planning_context_hash: planning_hash, + }, + ) + .await?; + let planning_context = match planning_context { + PlanningContextWriteOutcome::Created(record) + | PlanningContextWriteOutcome::Replayed(record) => record, + PlanningContextWriteOutcome::Conflict => { + anyhow::bail!("execution review planning context conflicted") + } + }; + let run = repository + .create_run( + scope, + &config, + NewExecutionRun { + tenant_id, + contact_id: None, + session_id, + originating_user_sequence_num, + planning_context_uid: planning_context.planning_context_uid, + planning_context_hash: planning_hash, + owner_user_id, + admitted_identity, + goal, + plan, + catalog, + authorization, + pinned_instruction_skills: Vec::new(), + source_provenance, + input: json!({}), + status: ExecutionRunStatus::Queued, + approved_budget: budget, + idempotency_key: Some(format!("execution-review-claim-{review_uid}")), + }, + ) + .await?; + let RunAdmissionOutcome::Admitted(run) = run else { + anyhow::bail!("execution review run fixture was not freshly admitted: {run:?}") + }; + let task_id = ExecutionTaskId::derive(run.run_uid, "review", "replay")?; + let materialized = repository + .materialize_ready_page( + scope, + &config, + ReadyMaterializationRequest { + run_uid: run.run_uid, + plan_revision: 1, + node_id: "review".to_string(), + expected_cursor: 0, + reduce_cursor: None, + source_exhausted: true, + terminal_output: None, + condition_skipped: false, + tasks: vec![LogicalTask { + task_id, + node_id: "review".to_string(), + item_key: "replay".to_string(), + requirement_ids: Vec::new(), + plan_revision: 1, + generation: 1, + input: json!({}), + kind: LogicalTaskKind::Output { value: json!({}) }, + compensation: None, + retry: RetryPolicy { + max_attempts: 1, + initial_backoff_ms: 0, + max_backoff_ms: 0, + }, + reservation: ExecutionEstimate { + cost_microusd: 0, + tokens: 0, + tasks: 1, + tool_calls: 1, + retrieved_bytes: 0, + }, + }], + }, + ) + .await?; + if !matches!(materialized, ReadyMaterializationOutcome::Applied { .. }) { + anyhow::bail!("execution review task fixture was not materialized ready: {materialized:?}") + } + let admission = repository + .admit_ready_attempts(&config, 8, moa_test_support::fixtures::pg_now()) + .await? + .admitted + .into_iter() + .find(|admission| admission.run_uid == run.run_uid && admission.task_id == task_id) + .context("execution review task fixture was not admitted")?; + let fence = TaskAttemptFence { + tenant_id: admission.tenant_id, + run_uid: admission.run_uid, + task_id: admission.task_id, + controller_generation: admission.controller_generation, + attempt_generation: admission.attempt_generation, + dispatch_uid: admission.dispatch_uid, + capacity_reservation_uid: admission.capacity_reservation_uid, + watchdog_trigger_uid: admission.watchdog_trigger_uid, + attempt_deadline_at: admission.attempt_deadline_at, + }; + let TaskAttemptStartOutcome::Started(started) = repository.start_task_attempt(fence).await? + else { + anyhow::bail!("execution review task fixture did not start") + }; + let parked_at = moa_test_support::fixtures::pg_now(); + let TaskAttemptReleaseClaimOutcome::Applied(_) = repository + .begin_task_attempt_release(fence, started.task.generation, "action_review", parked_at) + .await? + else { + anyhow::bail!("execution review task fixture did not enter release") + }; + let checkpoint = NewTaskAttemptCheckpoint { + fence, + task_generation: started.task.generation, + kind: TaskAttemptCheckpointKind::CapabilityReview, + schema_version: 1, + payload: json!({ + "schema_version": 1, + "state": { + "kind": "capability_review", + "pending_review": { + "review_uid": review_uid, + "expires_at": parked_at + chrono::Duration::days(1), + "invocation": invocation, + "effect_idempotency": IdempotencyClass::NonIdempotent, + }, + "usage": {}, + }, + "review_resolution": null, + "external_job_resolution": null, + "workspace_release_receipt_id": null, + }), + workspace_release_receipt: None, + created_at: parked_at, + }; + let TaskAttemptReviewParkOutcome::Applied { task, .. } = repository + .park_task_attempt_on_review(checkpoint, review_uid) + .await? + else { + anyhow::bail!("execution review task fixture did not park on review") + }; Ok(ExecutionTaskOrigin { - run_uid, - task_uid, - generation: 1, - attempt_generation: 1, + run_uid: run.run_uid, + task_uid: task.task_id.as_uuid(), + generation: task.generation, + attempt_generation: task.attempt_generation, }) } diff --git a/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e.rs b/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e.rs index 04ed060d2..5c1615580 100644 --- a/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e.rs +++ b/crates/moa-orchestrator/tests/long_horizon_execution_service_e2e.rs @@ -208,28 +208,6 @@ fn continue_wait(days: u64, output: Value) -> ExecutionWaitPolicy { } } -fn fixture_input_wait_policy( - compile_now: DateTime, - admitted_deadline_at: DateTime, -) -> Result { - let remaining_seconds = admitted_deadline_at - .signed_duration_since(compile_now) - .to_std() - .context("fixture execution deadline already elapsed before compilation")? - .as_secs(); - if remaining_seconds < 2 { - bail!( - "fixture execution deadline leaves no whole-second input wait strictly inside its horizon" - ); - } - Ok(ExecutionWaitPolicy { - expiry: ExecutionTemporalTarget::After { - delay_seconds: remaining_seconds / 2, - }, - on_expiry: ExecutionWaitExpiryAction::FailTask, - }) -} - fn node( id: &str, depends_on: &[&str], @@ -449,11 +427,6 @@ async fn start_plan_with_capability_policy( } } let compile_now = moa_test_support::fixtures::pg_now(); - let admitted_deadline_at = planning - .snapshot - .budget - .deadline_at - .context("planning context omitted its admitted execution deadline")?; let goal = ExecutionGoalContract { objective, requirements: vec![ExecutionRequirement { @@ -473,7 +446,6 @@ async fn start_plan_with_capability_policy( }; let plan = ExecutionPlanDefinition { cancel_policy: ExecutionCancelPolicy::RetainEffects, - input_wait_policy: fixture_input_wait_policy(compile_now, admitted_deadline_at)?, input_schema: json!({"type": "object", "additionalProperties": false}), output_schema: json!({"type": "object"}), nodes, @@ -886,8 +858,6 @@ mod fixture_contract_tests { goal, plan: ExecutionPlanDefinition { cancel_policy: ExecutionCancelPolicy::RetainEffects, - input_wait_policy: fixture_input_wait_policy(now, deadline_at) - .expect("fixture horizon should admit an input wait"), input_schema: json!({"type": "object", "additionalProperties": false}), output_schema: json!({"type": "object"}), nodes: vec![output_node(&[], json!({"status": "complete"}))], @@ -912,20 +882,6 @@ mod fixture_contract_tests { } } - #[test] - fn short_fixture_horizon_gets_a_strictly_bounded_input_wait_offline() { - // Pins: the shared fixture must not copy a day-scale default into a - // short plan whose admitted deadline is only three seconds away. - let now = fixed_now(); - let policy = fixture_input_wait_policy(now, now + TimeDelta::seconds(3)) - .expect("three-second fixture horizon should admit a wait"); - - assert_eq!( - policy.expiry, - ExecutionTemporalTarget::After { delay_seconds: 1 } - ); - } - #[test] fn fixture_compile_matches_server_validation_after_setup_skew_offline() { // Pins: client compilation and server revalidation produce the exact diff --git a/crates/moa-orchestrator/tests/orchestrator_db/action_reviews_reaper_db.rs b/crates/moa-orchestrator/tests/orchestrator_db/action_reviews_reaper_db.rs index 91f8afda2..6083eea8b 100644 --- a/crates/moa-orchestrator/tests/orchestrator_db/action_reviews_reaper_db.rs +++ b/crates/moa-orchestrator/tests/orchestrator_db/action_reviews_reaper_db.rs @@ -906,12 +906,6 @@ async fn insert_execution_task(pool: &PgPool, tenant_id: TenantId) -> ExecutionT let plan = serde_json::to_value(CanonicalExecutionPlan { definition: ExecutionPlanDefinition { cancel_policy: ExecutionCancelPolicy::RetainEffects, - input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { - expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::At { - at: chrono::Utc::now() + chrono::TimeDelta::hours(1), - }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, - }, input_schema: serde_json::json!({ "type": "object" }), output_schema: serde_json::json!({ "type": "object" }), nodes: Vec::new(), @@ -948,12 +942,21 @@ async fn insert_execution_task(pool: &PgPool, tenant_id: TenantId) -> ExecutionT r#" INSERT INTO moa.execution_run ( run_uid, tenant_id, session_id, originating_user_sequence_num, - planning_context_uid, planning_context_hash, owner_user_id, goal_contract, + planning_context_uid, planning_context_hash, owner_user_id, admitted_identity, + goal_contract, initial_plan, active_plan, initial_plan_hash, active_plan_hash, capability_catalog, authorization_envelope, pinned_instruction_skills, source_provenance, source_kind, input, status, queued_at - ) VALUES ($1, $2, $3, 1, $4, $5, 'test-owner', '{}'::JSONB, $6, + ) VALUES ($1, $2, $3, 1, $4, $5, 'test-owner', + jsonb_build_object( + 'identity_type', 'service', + 'id', $2::TEXT, + 'tenant_id', $2::TEXT, + 'api_key_id', NULL, + 'acting_on_behalf_of', NULL + ), + '{}'::JSONB, $6, $6, $5, $5, '{}'::JSONB, '{}'::JSONB, '[]'::JSONB, '{"kind":"generated_plan"}'::JSONB, 'generated_plan', '{}'::JSONB, 'queued', NOW()) diff --git a/crates/moa-orchestrator/tests/orchestrator_db/analytics_export_db.rs b/crates/moa-orchestrator/tests/orchestrator_db/analytics_export_db.rs index b6adcc042..1dc32952e 100644 --- a/crates/moa-orchestrator/tests/orchestrator_db/analytics_export_db.rs +++ b/crates/moa-orchestrator/tests/orchestrator_db/analytics_export_db.rs @@ -226,12 +226,6 @@ async fn seed_execution_analytics_fixture( let plan = serde_json::to_value(CanonicalExecutionPlan { definition: ExecutionPlanDefinition { cancel_policy: ExecutionCancelPolicy::RetainEffects, - input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { - expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::At { - at: chrono::Utc::now() + chrono::TimeDelta::hours(1), - }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, - }, input_schema: json!({ "type": "object" }), output_schema: json!({ "type": "object" }), nodes: Vec::new(), @@ -263,12 +257,16 @@ async fn seed_execution_analytics_fixture( sqlx::query( "INSERT INTO moa.execution_run \ - (run_uid, tenant_id, session_id, originating_user_sequence_num, planning_context_uid, \ - planning_context_hash, owner_user_id, goal_contract, initial_plan, active_plan, \ + (run_uid, tenant_id, session_id, originating_user_sequence_num, planning_context_uid, \ + planning_context_hash, owner_user_id, admitted_identity, goal_contract, initial_plan, active_plan, \ initial_plan_hash, active_plan_hash, capability_catalog, authorization_envelope, \ source_provenance, source_kind, skill_template_ref, \ skill_template_revision_uid, input, status, progress_total_tasks) \ VALUES ($1, $2, $3, 1, $4, $5, 'user-1', \ + jsonb_build_object( \ + 'identity_type', 'service', 'id', $2::TEXT, \ + 'tenant_id', $2::TEXT, 'api_key_id', NULL, \ + 'acting_on_behalf_of', NULL), \ '{\"requirements\":[{\"id\":\"r1\"},{\"id\":\"r2\"}], \ \"completion_checks\":[{\"id\":\"c1\"},{\"id\":\"c2\"}]}'::JSONB, \ $8, $8, $6, $6, '{}'::JSONB, '{}'::JSONB, \ diff --git a/crates/moa-orchestrator/tests/orchestrator_db/execution_dispatch_reconciliation_db.rs b/crates/moa-orchestrator/tests/orchestrator_db/execution_dispatch_reconciliation_db.rs index 1e87d5c73..0b6d02bf3 100644 --- a/crates/moa-orchestrator/tests/orchestrator_db/execution_dispatch_reconciliation_db.rs +++ b/crates/moa-orchestrator/tests/orchestrator_db/execution_dispatch_reconciliation_db.rs @@ -70,10 +70,10 @@ async fn reconciliation_repairs_only_one_bounded_indexed_window_db() -> TestResu let pool = test_db.store().pool().clone(); let repository = ExecutionRepository::new(pool.clone()); let tenant_id = TenantId::new(); - let schedule_uid = insert_schedule(&pool, tenant_id).await?; let scope = ExecutionScope::ControlPlane; - for occurrence_sequence in 1..=3 { + for _ in 1..=3 { + let schedule_uid = insert_schedule(&pool, tenant_id).await?; let write = repository .create_trigger( scope, @@ -81,7 +81,7 @@ async fn reconciliation_repairs_only_one_bounded_indexed_window_db() -> TestResu schedule_trigger( tenant_id, schedule_uid, - occurrence_sequence, + 1, Utc::now() - Duration::minutes(1), ), ) @@ -94,21 +94,21 @@ async fn reconciliation_repairs_only_one_bounded_indexed_window_db() -> TestResu assert_eq!( repository - .reconcile_due_trigger_dispatches(scope, 2) + .reconcile_due_trigger_dispatches(scope, 6) .await? .len(), 2 ); assert_eq!( repository - .reconcile_due_trigger_dispatches(scope, 2) + .reconcile_due_trigger_dispatches(scope, 6) .await? .len(), 1 ); assert!( repository - .reconcile_due_trigger_dispatches(scope, 2) + .reconcile_due_trigger_dispatches(scope, 6) .await? .is_empty() ); @@ -116,9 +116,10 @@ async fn reconciliation_repairs_only_one_bounded_indexed_window_db() -> TestResu } #[tokio::test] -async fn claimed_dispatches_ack_retry_and_dead_letter_under_exact_owner_fences_db() -> TestResult { - // Pins: a dispatcher ACKs only after accepted delivery, abandoned owners - // cannot settle another claim, and bounded failures eventually dead-letter. +async fn claimed_dispatches_ack_and_retry_correctness_work_under_exact_owner_fences_db() +-> TestResult { + // Pins: a dispatcher ACKs only after accepted delivery, abandoned owners cannot + // settle another claim, and correctness-critical trigger delivery keeps retrying. let test_db = moa_test_support::postgres::bootstrap_test_db().await?; let pool = test_db.store().pool().clone(); let repository = ExecutionRepository::new(pool.clone()); @@ -181,23 +182,23 @@ async fn claimed_dispatches_ack_retry_and_dead_letter_under_exact_owner_fences_d .await?, ExecutionDispatchFailureOutcome::RetryScheduled { .. } )); - let dead_letter = ExecutionDispatchRetryPolicy { + let exhausted_retry = ExecutionDispatchRetryPolicy { max_attempts: 1, base_delay: StdDuration::from_secs(1), maximum_delay: StdDuration::from_secs(1), }; - assert_eq!( + assert!(matches!( repository .record_dispatch_failure( scope, claimed[2].dispatch_uid, "dispatcher-a", "injected permanent acceptance failure", - dead_letter, + exhausted_retry, ) .await?, - ExecutionDispatchFailureOutcome::DeadLettered - ); + ExecutionDispatchFailureOutcome::RetryScheduled { .. } + )); Ok(()) } diff --git a/crates/moa-orchestrator/tests/orchestrator_db/execution_schedule_db.rs b/crates/moa-orchestrator/tests/orchestrator_db/execution_schedule_db.rs index 981f5a56b..ff8290592 100644 --- a/crates/moa-orchestrator/tests/orchestrator_db/execution_schedule_db.rs +++ b/crates/moa-orchestrator/tests/orchestrator_db/execution_schedule_db.rs @@ -3,8 +3,7 @@ use chrono::{Duration, Utc}; use moa_artifacts::execution_plan::{ ExecutionBudgetLimit, ExecutionCancelPolicy, ExecutionGoalContract, ExecutionNode, - ExecutionOperation, ExecutionPlanDefinition, ExecutionTemporalTarget, - ExecutionWaitExpiryAction, ExecutionWaitPolicy, RetryPolicy, + ExecutionOperation, ExecutionPlanDefinition, RetryPolicy, }; use moa_config::ExecutionConfig; use moa_core::{ @@ -347,7 +346,7 @@ async fn schedule_update_binds_every_mutable_field_to_its_named_column_db() -> T .bind(original_trigger.trigger.trigger_uid) .fetch_one(test_db.store().pool()) .await?; - assert_eq!(original_state, "cancelled"); + assert_eq!(original_state, "superseded"); Ok(()) } @@ -437,17 +436,16 @@ async fn concurrent_schedule_update_and_occurrence_fire_share_capacity_first_loc #[tokio::test] async fn schedule_occurrence_respects_joint_active_and_parked_resident_ceiling_db() -> TestResult { - // Pins: schedule-owned admission cannot bypass the joint resident-run ceiling merely because - // ActiveRuns compute capacity remains available; saturation consumes the occurrence once and - // leaves no run, activation, or ActiveRuns receipt to replay. + // Pins: schedule-owned admission cannot bypass the joint resident-run ceiling; saturation + // consumes the occurrence once and leaves no run, activation, or capacity receipt to replay. let test_db = moa_test_support::postgres::bootstrap_test_db().await?; let pool = test_db.store().pool().clone(); let repository = ExecutionRepository::new(pool.clone()); let tenant_id = TenantId::new(); let scope = ExecutionScope::Tenant { tenant_id }; let config = moa_config::ExecutionConfig { - max_tenant_active_runs: 10, - max_fleet_active_runs: 10, + max_tenant_active_runs: 1, + max_fleet_active_runs: 1, max_tenant_parked_runs: 1, max_fleet_parked_runs: 1, ..moa_config::ExecutionConfig::default() @@ -469,12 +467,16 @@ async fn schedule_occurrence_respects_joint_active_and_parked_resident_ceiling_d else { panic!("first resident schedule must arm its occurrence"); }; - let first_run = execution_schedule_run_blueprint(&first_schedule)?.instantiate( - &first_schedule, - due, - 1, - config.maximum_horizon_seconds, - )?; + let first_blueprint = execution_schedule_run_blueprint(&first_schedule)?; + let first_run = + first_blueprint.instantiate(&first_schedule, due, 1, config.maximum_horizon_seconds)?; + insert_schedule_planning_context( + &pool, + tenant_id, + first_schedule.run_as_identity.id, + &first_blueprint, + ) + .await?; let ExecutionScheduleRunAdmissionOutcome::Admitted { run: admitted_run, .. } = repository @@ -508,12 +510,16 @@ async fn schedule_occurrence_respects_joint_active_and_parked_resident_ceiling_d else { panic!("second schedule must arm before its resident admission check"); }; - let second_run = execution_schedule_run_blueprint(&second_schedule)?.instantiate( - &second_schedule, - due, - 1, - config.maximum_horizon_seconds, - )?; + let second_blueprint = execution_schedule_run_blueprint(&second_schedule)?; + let second_run = + second_blueprint.instantiate(&second_schedule, due, 1, config.maximum_horizon_seconds)?; + insert_schedule_planning_context( + &pool, + tenant_id, + second_schedule.run_as_identity.id, + &second_blueprint, + ) + .await?; let replay_run = second_run.clone(); let saturated_request = ExecutionScheduleRunAdmission { tenant_id, @@ -949,12 +955,6 @@ fn schedule_request( plan: CanonicalExecutionPlan { definition: ExecutionPlanDefinition { cancel_policy: ExecutionCancelPolicy::RetainEffects, - input_wait_policy: ExecutionWaitPolicy { - expiry: ExecutionTemporalTarget::After { - delay_seconds: 3_600, - }, - on_expiry: ExecutionWaitExpiryAction::FailTask, - }, input_schema: serde_json::json!({"type":"object"}), output_schema: serde_json::json!({"type":"object"}), nodes: vec![ExecutionNode { diff --git a/crates/moa-orchestrator/tests/orchestrator_db/execution_service_db.rs b/crates/moa-orchestrator/tests/orchestrator_db/execution_service_db.rs index 5f9b84468..ecf3fd616 100644 --- a/crates/moa-orchestrator/tests/orchestrator_db/execution_service_db.rs +++ b/crates/moa-orchestrator/tests/orchestrator_db/execution_service_db.rs @@ -224,12 +224,6 @@ async fn execution_task_citation_lineage_survives_reload_and_terminal_summary_db let plan = CanonicalExecutionPlan { definition: ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, - input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { - expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::At { - at: chrono::Utc::now() + chrono::TimeDelta::hours(1), - }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, - }, input_schema: json!({"type": "object"}), output_schema: json!({"type": "object"}), nodes: Vec::new(), @@ -516,12 +510,6 @@ async fn execution_service_rows_require_parent_session_and_keep_authorization_im let plan = serde_json::to_value(CanonicalExecutionPlan { definition: ExecutionPlanDefinition { cancel_policy: moa_artifacts::execution_plan::ExecutionCancelPolicy::RetainEffects, - input_wait_policy: moa_artifacts::execution_plan::ExecutionWaitPolicy { - expiry: moa_artifacts::execution_plan::ExecutionTemporalTarget::At { - at: chrono::Utc::now() + chrono::TimeDelta::hours(1), - }, - on_expiry: moa_artifacts::execution_plan::ExecutionWaitExpiryAction::FailTask, - }, input_schema: json!({ "type": "object" }), output_schema: json!({ "type": "object" }), nodes: Vec::new(), @@ -556,11 +544,18 @@ async fn execution_service_rows_require_parent_session_and_keep_authorization_im r#" INSERT INTO moa.execution_run ( run_uid, tenant_id, session_id, originating_user_sequence_num, - planning_context_uid, planning_context_hash, owner_user_id, goal_contract, + planning_context_uid, planning_context_hash, owner_user_id, admitted_identity, + goal_contract, initial_plan, active_plan, initial_plan_hash, active_plan_hash, capability_catalog, authorization_envelope, pinned_instruction_skills, source_provenance, source_kind, input, status - ) VALUES ($1, $2, NULL, 0, $3, $4, 'owner', '{}'::JSONB, $5, $5, + ) VALUES ($1, $2, NULL, 0, $3, $4, 'owner', + jsonb_build_object( + 'identity_type', 'service', 'id', $2::TEXT, + 'tenant_id', $2::TEXT, 'api_key_id', NULL, + 'acting_on_behalf_of', NULL + ), + '{}'::JSONB, $5, $5, $4, $4, '{}'::JSONB, '{}'::JSONB, '[]'::JSONB, '{"kind":"generated_plan"}'::JSONB, 'generated_plan', '{}'::JSONB, 'queued') @@ -585,12 +580,19 @@ async fn execution_service_rows_require_parent_session_and_keep_authorization_im r#" INSERT INTO moa.execution_run ( run_uid, tenant_id, session_id, originating_user_sequence_num, - planning_context_uid, planning_context_hash, owner_user_id, goal_contract, + planning_context_uid, planning_context_hash, owner_user_id, admitted_identity, + goal_contract, initial_plan, active_plan, initial_plan_hash, active_plan_hash, capability_catalog, authorization_envelope, pinned_instruction_skills, source_provenance, source_kind, input, status, queued_at - ) VALUES ($1, $2, $3, 0, $4, $5, 'owner', '{}'::JSONB, $6, $6, + ) VALUES ($1, $2, $3, 0, $4, $5, 'owner', + jsonb_build_object( + 'identity_type', 'service', 'id', $2::TEXT, + 'tenant_id', $2::TEXT, 'api_key_id', NULL, + 'acting_on_behalf_of', NULL + ), + '{}'::JSONB, $6, $6, $5, $5, '{"schema_version":1}'::JSONB, '{"capability_refs":[],"skill_refs":[]}'::JSONB, '[]'::JSONB, '{"kind":"generated_plan"}'::JSONB, diff --git a/crates/moa-orchestrator/tests/session_turn_lifecycle_service_e2e.rs b/crates/moa-orchestrator/tests/session_turn_lifecycle_service_e2e.rs index 2b0bf6010..47a678865 100644 --- a/crates/moa-orchestrator/tests/session_turn_lifecycle_service_e2e.rs +++ b/crates/moa-orchestrator/tests/session_turn_lifecycle_service_e2e.rs @@ -187,82 +187,6 @@ async fn post_start_turn( .context("send Session start_turn") } -async fn register_coordinator_input( - client: &reqwest::Client, - session: &InitializedSession, - turn_id: &str, - generation: u64, - input_request_id: &str, - awakeable_id: &str, - waiting_workflow_id: &str, -) -> Result<()> { - client - .post(session_url(&session.id, "register_coordinator_input")) - .json(&serde_json::json!({ - "turn_id": turn_id, - "generation": generation, - "input_request_id": input_request_id, - "awakeable_id": awakeable_id, - "waiting_workflow_id": waiting_workflow_id, - "question": "Which safe path should the coordinator take?", - })) - .send() - .await - .context("send Session register_coordinator_input")? - .error_for_status() - .context("Session register_coordinator_input should succeed")?; - Ok(()) -} - -async fn clear_coordinator_input( - client: &reqwest::Client, - session: &InitializedSession, - turn_id: &str, - generation: u64, - input_request_id: &str, - waiting_workflow_id: &str, -) -> Result<()> { - client - .post(session_url(&session.id, "clear_coordinator_input")) - .json(&serde_json::json!({ - "turn_id": turn_id, - "generation": generation, - "input_request_id": input_request_id, - "waiting_workflow_id": waiting_workflow_id, - })) - .send() - .await - .context("send Session clear_coordinator_input")? - .error_for_status() - .context("Session clear_coordinator_input should succeed")?; - Ok(()) -} - -async fn post_coordinator_reply( - client: &reqwest::Client, - session: &InitializedSession, - turn_id: &str, - generation: u64, - input_request_id: &str, -) -> Result { - let request = client.post(session_url(&session.id, "start_turn")); - with_identity(request, &session.identity) - .json(&serde_json::json!({ - "client_message_id": fresh_client_message_id(), - "reply_to": { - "coordinator_input": { - "turn_id": turn_id, - "generation": generation, - "input_request_id": input_request_id, - } - }, - "user_message": "continue without that capability", - })) - .send() - .await - .context("send coordinator input reply") -} - /// Counts durable user-message events carrying `text`. /// /// Persisted events are externally tagged as `{"type": ..., "data": ...}` with a @@ -640,66 +564,3 @@ async fn request_cancel_forwards_to_turn_execution() -> Result<()> { assert_eq!(outcome.message, "user-requested"); Ok(()) } - -#[tokio::test] -#[ignore = "requires a running Restate ingress and moa-orchestrator deployment"] -async fn exact_coordinator_input_cleanup_survives_replay_and_rejects_a_late_reply() -> Result<()> { - // Pins: cleanup is one persisted, exact Restate handler. A stale invocation - // cannot clear a live generation, replayed registration cannot replace its - // owner, and a reply arriving after exact cleanup is rejected without a turn. - let client = reqwest::Client::new(); - let session = create_initialized_session(&client, "coordinator-input-cleanup").await?; - let turn_id = format!("coordinator-input-turn:{}", session.id); - let input_request_id = format!("security:{turn_id}:6:tool-1"); - - register_coordinator_input( - &client, - &session, - &turn_id, - 6, - &input_request_id, - "awakeable-current", - "workflow-current", - ) - .await?; - clear_coordinator_input( - &client, - &session, - &turn_id, - 4, - "security:stale:4:tool-1", - "workflow-stale", - ) - .await?; - register_coordinator_input( - &client, - &session, - &turn_id, - 6, - &input_request_id, - "awakeable-replayed", - "workflow-replayed", - ) - .await?; - clear_coordinator_input( - &client, - &session, - &turn_id, - 6, - &input_request_id, - "workflow-current", - ) - .await?; - - let late_reply = - post_coordinator_reply(&client, &session, &turn_id, 6, &input_request_id).await?; - assert_eq!( - late_reply.status(), - reqwest::StatusCode::CONFLICT, - "an exact late reply must not resolve a dead awakeable or start a new turn" - ); - let current = snapshot(&client, &session).await?; - assert_eq!(current.active_turn_id, None); - assert_eq!(current.pending_message_count, 0); - Ok(()) -} diff --git a/crates/moa-orchestrator/tests/worker_coordination_service_e2e.rs b/crates/moa-orchestrator/tests/worker_coordination_service_e2e.rs index 4c8ff1f43..390f03dcd 100644 --- a/crates/moa-orchestrator/tests/worker_coordination_service_e2e.rs +++ b/crates/moa-orchestrator/tests/worker_coordination_service_e2e.rs @@ -618,6 +618,50 @@ async fn invocation_count( .map(|rows| rows.len()) } +async fn latest_worker_turn_call_journal( + fixture: &OrchestratorTestFixture, +) -> Result> { + let rows = restate_query_rows( + fixture, + "SELECT journal.index, journal.entry_lite_json \ + FROM sys_journal AS journal \ + WHERE journal.id = ( \ + SELECT id FROM sys_invocation \ + WHERE target_service_name = 'WorkerTurnExecution' \ + AND target_handler_name = 'run' \ + ORDER BY created_at DESC LIMIT 1 \ + ) \ + AND journal.entry_type IN ('Command: Call', 'Command: OneWayCall') \ + ORDER BY journal.index", + ) + .await?; + rows.into_iter() + .map(|row| { + let index = row + .get("index") + .and_then(Value::as_u64) + .context("worker-turn journal row omitted its index")?; + let entry = row + .get("entry_lite_json") + .and_then(Value::as_str) + .context("worker-turn journal row omitted its call payload")? + .to_string(); + Ok((index, entry)) + }) + .collect() +} + +fn call_journal_index(entries: &[(u64, String)], service: &str, handler: &str) -> Result { + entries + .iter() + .find(|(_, entry)| { + entry.contains(&format!("\"name\":\"{service}\"")) + && entry.contains(&format!("\"handler\":\"{handler}\"")) + }) + .map(|(index, _)| *index) + .with_context(|| format!("journal omitted {service}/{handler}; entries={entries:#?}")) +} + async fn await_invocation_count( fixture: &OrchestratorTestFixture, service: &str, @@ -1222,6 +1266,26 @@ async fn needs_input_wakes_parent_and_resolves_exact_awakeable_once_service_e2e( 1, "the input wait must be inside one worker workflow" ); + let call_journal = latest_worker_turn_call_journal(&fixture).await?; + let capture_index = call_journal_index( + &call_journal, + "ToolExecutor", + "capture_worker_hand_release_fence", + )?; + let register_index = call_journal_index(&call_journal, "Worker", "register_input_request")?; + let release_index = call_journal_index( + &call_journal, + "ToolExecutor", + "checkpoint_and_release_worker_hand", + )?; + let signal_index = call_journal_index(&call_journal, "Session", "record_child_signal")?; + assert!( + capture_index < register_index + && register_index < release_index + && release_index < signal_index, + "worker input park must capture the live hand, register the exact wait, and finish \ + checkpoint/release before publishing NeedsInput: {call_journal:#?}" + ); let reply = start_turn_request(INPUT_ANSWER); let first = test diff --git a/crates/moa-test-support/src/execution_audits.rs b/crates/moa-test-support/src/execution_audits.rs index 3975103f1..19d33972c 100644 --- a/crates/moa-test-support/src/execution_audits.rs +++ b/crates/moa-test-support/src/execution_audits.rs @@ -78,6 +78,13 @@ pub async fn load_execution_planning_audits( 'outcome', outcome, 'provider_model', provider_model, 'prompt_version', prompt_version, + 'usage', jsonb_build_object( + 'input_tokens_uncached', input_tokens_uncached, + 'input_tokens_cache_write', input_tokens_cache_write, + 'input_tokens_cache_read', input_tokens_cache_read, + 'output_tokens', output_tokens + ), + 'cost_microusd', cost_microusd, 'candidate_hash', candidate_hash, 'candidate_json', candidate_json::TEXT, 'compiler_report', compiler_report::TEXT, diff --git a/crates/xtask/src/execution_trace_manifest.rs b/crates/xtask/src/execution_trace_manifest.rs index f2dfdf58a..91ea43868 100644 --- a/crates/xtask/src/execution_trace_manifest.rs +++ b/crates/xtask/src/execution_trace_manifest.rs @@ -606,6 +606,13 @@ const SENDERS: &[SenderManifestEntry] = &[ "ExecutionDispatcherClient", "dispatch" ), + sender!( + "crates/moa-orchestrator/src/services/execution/handlers.rs", + "cancel", + TRACE_HELPER, + "LLMGatewayClient", + "cancel_owner" + ), sender!( "crates/moa-orchestrator/src/services/execution_amendment_planner.rs", "complete", @@ -1526,6 +1533,20 @@ const SENDERS: &[SenderManifestEntry] = &[ "SessionClient", "record_child_signal" ), + sender!( + "crates/moa-orchestrator/src/workflows/worker_turn_execution.rs", + "request_input_from_parent", + TRACE_HELPER, + "ToolExecutorClient", + "capture_worker_hand_release_fence" + ), + sender!( + "crates/moa-orchestrator/src/workflows/worker_turn_execution.rs", + "request_input_from_parent", + TRACE_HELPER, + "ToolExecutorClient", + "checkpoint_and_release_worker_hand" + ), sender!( "crates/moa-orchestrator/src/workflows/worker_turn_execution.rs", "request_input_from_parent", diff --git a/docs/01-architecture-overview.md b/docs/01-architecture-overview.md index b17e34c17..40448c363 100644 --- a/docs/01-architecture-overview.md +++ b/docs/01-architecture-overview.md @@ -151,8 +151,8 @@ deliverables can produce `partial`, `blocked`, or `unsupported`, never a false `completed`. Final synthesis receives the contract, check results, aggregate outputs, citations, and explicit gaps. -An `ExecutionPlanDefinition` carries an explicit `cancel_policy`, -`input_schema`, `output_schema`, required `input_wait_policy`, and `nodes`. Each node carries +An `ExecutionPlanDefinition` carries an explicit `cancel_policy`, `input_schema`, +`output_schema`, and `nodes`. Each node carries `id`, `depends_on`, optional `when`, `input`, `output_schema`, one `operation`, an explicit optional compensation contract, retry policy, optional budget, and the goal requirement IDs it serves. The dependency graph is acyclic, and @@ -180,6 +180,12 @@ the instant the task actually enters its wait, not from planning or run admission. Reusable skill templates accept only `After`; this preserves their meaning whenever earlier dependencies take a different amount of time. +A runtime `NeedsInput` outcome from an executable task is different from an +authored `Review` or `WaitSignal` node: it parks durably until the requested +human input arrives or the run is explicitly cancelled. It has no expiry +policy, and while parked it holds no worker, model call, sandbox, process, or +network connection. + Dependencies provide parallelism and joins; there are no implicit start, parallel, join, worker, tool, action, skill-action, or memory node kinds. Dynamic values use only whole-value `{ "$ref": "$.input.query" }`, diff --git a/docs/02-brain-orchestration.md b/docs/02-brain-orchestration.md index bf2d69b14..a49315a21 100644 --- a/docs/02-brain-orchestration.md +++ b/docs/02-brain-orchestration.md @@ -337,8 +337,10 @@ so the single-writer object never blocks its queue on a Postgres write, while the caller still awaits durability. **Owner outcomes are exact.** A coordinator suspend registers a -generation-fenced coordinator-input reply target on the Session and idles until -that reply arrives; a coordinator halt records the canonical actor+turn +generation-fenced coordinator-input reply target on the Session, releases its +fleet/tenant turn-admission lease, and idles indefinitely until that reply arrives. +The exact authenticated reply reacquires admission before resolving the awakeable; +a coordinator halt records the canonical actor+turn `TurnFailed`. A worker suspend emits one `NeedsInput` signal with `input_audience: User` and awaits its awakeable on the existing worker-input machinery; a worker halt emits one `Failed` signal and terminates that worker @@ -437,13 +439,16 @@ without an additional wake. `needs_input` is a child→parent round-trip on the same message path: the child's `request_input` tool registers a Restate awakeable, emits a `NeedsInput` signal -carrying `input_request_id`/`input_audience`, and blocks on the awakeable against a -long timeout (`worker_input_timeout_ms`). The coordinator answers with the +carrying `input_request_id`/`input_audience`, checkpoints and releases any worker +sandbox compute, and blocks indefinitely on the durable awakeable. The coordinator +answers with the `provide_worker_input` tool → `WorkerMessage::ProvideInput`, which resolves the awakeable through `post_message`. Coordinator-audience questions are answered autonomously. User-audience questions are exposed as `worker_input_request` SSE frames; the next plain user reply is forwarded by the session to the worker as -`WorkerMessage::ProvideInput` instead of starting a separate root turn. +`WorkerMessage::ProvideInput` instead of starting a separate root turn. Restate owns +the suspended wait without an active handler or provider call; the worker's next sandbox +dispatch restores the exact portable checkpoint onto fresh compute. ### Self-cleanup and the liveness watchdog diff --git a/docs/12-restate-architecture.md b/docs/12-restate-architecture.md index c9cd034f2..c0bb5977c 100644 --- a/docs/12-restate-architecture.md +++ b/docs/12-restate-architecture.md @@ -380,14 +380,19 @@ before returning. Immediate Restate delivery is an optimization; the singleton maintenance owner reclaims undelivered rows and due triggers. The stable dispatch/trigger identity makes duplicate delivery a no-op. +`Execution/cancel` additionally waits for idempotent cancellation fences on +every exact current execution-task LLM owner before acknowledging. Attempt hand +release and terminal drain remain bounded, outbox-driven work. + The graph is acyclic and has exactly eight operations: `Capability`, `Agent`, `Map`, `Reduce`, `Review`, `WaitSignal`, `WaitUntil`, and `Output`. A map creates one task for each stable item key and cannot contain another map. Reduce uses structured batches; an agent reducer is a deterministic hierarchical tree bounded by `batch_size`. -`Review`, `WaitSignal`, run input, and `WaitUntil` carry explicit expiry or wake -targets. `At { at }` denotes an exact UTC instant and is valid for a generated +`Review`, `WaitSignal`, and `WaitUntil` carry explicit expiry or wake targets. +Human `NeedsInput` continuations park indefinitely without active compute until +authorized input or explicit lifecycle control. `At { at }` denotes an exact UTC instant and is valid for a generated one-off plan. Nonzero `After { delay_seconds }` is resolved from the instant the task enters the wait. Reusable templates reject `At` and use `After`, so earlier dependency duration cannot make a template timer stale. Entering any wait @@ -554,8 +559,10 @@ mutation surface. The private handlers are reachable only service-to-service; public traffic enters through edge-owned product surfaces. Their inactivity timeout is 360 seconds with a 60-second abort cleanup window. Provider stream configuration is capped at 300 seconds, leaving that cleanup margin. Durable -human waits suspend and therefore do not need a larger -inactivity timeout. The normal product endpoint does not contain +human waits suspend indefinitely and therefore do not need a larger inactivity +timeout. Coordinator waits release shared fleet/tenant admission and reacquire it +under the exact turn-generation fence before resuming. Worker waits checkpoint and +release sandbox compute before parking. The normal product endpoint does not contain `Session/migrate_status_idle` or `StatusMigrationDispatcher`. Those two handlers exist only in the pre-runtime migration endpoint; the raw Session handler is ingress-private, and the endpoint intentionally omits `Health` and every product diff --git a/docs/23-environment-variables.md b/docs/23-environment-variables.md index e404a6f02..7254b6786 100644 --- a/docs/23-environment-variables.md +++ b/docs/23-environment-variables.md @@ -345,7 +345,6 @@ capabilities, database bootstrap, and a fresh supervised reaper heartbeat. | Variable | Config path | Default | Description | |---|---|---|---| -| `MOA_SESSION_LIMITS_COORDINATOR_INPUT_TIMEOUT_MS` | `session_limits.coordinator_input_timeout_ms` | 1800000 | Maximum time a coordinator security-input round-trip blocks before the turn stops safely | | `MOA_SESSION_LIMITS_LOOP_DETECTION_THRESHOLD` | `session_limits.loop_detection_threshold` | 3 | Number of identical consecutive turn fingerprints that triggers a loop pause | | `MOA_SESSION_LIMITS_MAX_MODEL_TURNS_DELEGATION` | `session_limits.max_model_turns_delegation` | 12 | Maximum model loop iterations once a standard turn has delegated to at least one worker; replaces the base cap for the rest of that turn | | `MOA_SESSION_LIMITS_MAX_TOOL_CALLS` | `session_limits.max_tool_calls` | 30 | Maximum tool calls allowed within one turn | @@ -357,7 +356,6 @@ capabilities, database bootstrap, and a fresh supervised reaper heartbeat. | `MOA_SESSION_LIMITS_WORKER_CLEANUP_GRACE_MS` | `session_limits.worker_cleanup_grace_ms` | 60000 | Grace window before a terminal worker self-cleans (removes itself from the parent fan-out and clears its VO state) after reporting its result | | `MOA_SESSION_LIMITS_WORKER_HEARTBEAT_INTERVAL_MS` | `session_limits.worker_heartbeat_interval_ms` | 15000 | Target cadence, in milliseconds, at which an active child refreshes its telemetry-plane heartbeat while running | | `MOA_SESSION_LIMITS_WORKER_HEARTBEAT_STALE_MS` | `session_limits.worker_heartbeat_stale_ms` | 60000 | Age, in milliseconds, beyond which the Worker's one outstanding liveness deadline emits a stale transition | -| `MOA_SESSION_LIMITS_WORKER_INPUT_TIMEOUT_MS` | `session_limits.worker_input_timeout_ms` | 1800000 | Maximum time a child `request_input` round-trip blocks on its awakeable before returning a "no input received" result so the child can proceed or abort | | `MOA_SESSION_LIMITS_WORKER_RESUME_MAX_PER_WINDOW` | `session_limits.worker_resume_max_per_window` | 6 | Maximum guarded coordinator auto-resumes dispatched per rolling window before the resume path backs off | | `MOA_SESSION_LIMITS_WORKER_RESUME_WINDOW_MS` | `session_limits.worker_resume_window_ms` | 600000 | Rolling-window length, in milliseconds, for the guarded parent-resume budget | diff --git a/docs/examples/artifacts/damaged-food-order.skill.yaml b/docs/examples/artifacts/damaged-food-order.skill.yaml index 75cbc71b4..410bb056f 100644 --- a/docs/examples/artifacts/damaged-food-order.skill.yaml +++ b/docs/examples/artifacts/damaged-food-order.skill.yaml @@ -36,12 +36,6 @@ definition: kind: output_schema plan: cancel_policy: retain_effects - input_wait_policy: - expiry: - kind: after - delay_seconds: 86400 - on_expiry: - kind: fail_task input_schema: type: object properties: diff --git a/docs/examples/artifacts/patterns/custom-logic.skill.yaml b/docs/examples/artifacts/patterns/custom-logic.skill.yaml index 6309b5d80..e51ff641f 100644 --- a/docs/examples/artifacts/patterns/custom-logic.skill.yaml +++ b/docs/examples/artifacts/patterns/custom-logic.skill.yaml @@ -38,12 +38,6 @@ definition: kind: output_schema plan: cancel_policy: retain_effects - input_wait_policy: - expiry: - kind: after - delay_seconds: 86400 - on_expiry: - kind: fail_task input_schema: type: object required: [priority, retry] diff --git a/docs/examples/artifacts/patterns/human-approval.skill.yaml b/docs/examples/artifacts/patterns/human-approval.skill.yaml index b57501e38..e4db869b0 100644 --- a/docs/examples/artifacts/patterns/human-approval.skill.yaml +++ b/docs/examples/artifacts/patterns/human-approval.skill.yaml @@ -29,12 +29,6 @@ definition: kind: output_schema plan: cancel_policy: retain_effects - input_wait_policy: - expiry: - kind: after - delay_seconds: 86400 - on_expiry: - kind: fail_task input_schema: type: object output_schema: diff --git a/docs/examples/artifacts/patterns/parallel-review.skill.yaml b/docs/examples/artifacts/patterns/parallel-review.skill.yaml index 729f136c0..d97c9247b 100644 --- a/docs/examples/artifacts/patterns/parallel-review.skill.yaml +++ b/docs/examples/artifacts/patterns/parallel-review.skill.yaml @@ -30,12 +30,6 @@ definition: kind: output_schema plan: cancel_policy: retain_effects - input_wait_policy: - expiry: - kind: after - delay_seconds: 86400 - on_expiry: - kind: fail_task input_schema: type: object output_schema: diff --git a/docs/examples/artifacts/patterns/react-agent.skill.yaml b/docs/examples/artifacts/patterns/react-agent.skill.yaml index a6cd35bef..336b694c0 100644 --- a/docs/examples/artifacts/patterns/react-agent.skill.yaml +++ b/docs/examples/artifacts/patterns/react-agent.skill.yaml @@ -30,12 +30,6 @@ definition: kind: output_schema plan: cancel_policy: retain_effects - input_wait_policy: - expiry: - kind: after - delay_seconds: 86400 - on_expiry: - kind: fail_task input_schema: type: object output_schema: diff --git a/docs/examples/artifacts/patterns/sequential.skill.yaml b/docs/examples/artifacts/patterns/sequential.skill.yaml index 651e73b45..f7d9f173b 100644 --- a/docs/examples/artifacts/patterns/sequential.skill.yaml +++ b/docs/examples/artifacts/patterns/sequential.skill.yaml @@ -34,12 +34,6 @@ definition: kind: output_schema plan: cancel_policy: retain_effects - input_wait_policy: - expiry: - kind: after - delay_seconds: 86400 - on_expiry: - kind: fail_task input_schema: type: object properties: diff --git a/docs/schemas/moa-skill-v1.schema.json b/docs/schemas/moa-skill-v1.schema.json index 5a9676196..7d50a5462 100644 --- a/docs/schemas/moa-skill-v1.schema.json +++ b/docs/schemas/moa-skill-v1.schema.json @@ -360,7 +360,6 @@ "additionalProperties": false, "required": [ "cancel_policy", - "input_wait_policy", "input_schema", "output_schema", "nodes" @@ -369,9 +368,6 @@ "cancel_policy": { "$ref": "#/$defs/ExecutionCancelPolicy" }, - "input_wait_policy": { - "$ref": "#/$defs/ExecutionInputWaitPolicy" - }, "input_schema": {}, "output_schema": {}, "nodes": { @@ -797,35 +793,6 @@ } } }, - "ExecutionInputWaitPolicy": { - "description": "Plan-level expiry policy for runtime input requests. It settles whichever task returned NeedsInput, so a declared continue_with output has no node output_schema to validate against and is rejected.", - "type": "object", - "additionalProperties": false, - "required": [ - "expiry", - "on_expiry" - ], - "properties": { - "expiry": { - "$ref": "#/$defs/ExecutionTemporalTarget" - }, - "on_expiry": { - "$ref": "#/$defs/ExecutionInputWaitExpiryAction" - } - } - }, - "ExecutionInputWaitExpiryAction": { - "type": "object", - "additionalProperties": false, - "required": [ - "kind" - ], - "properties": { - "kind": { - "const": "fail_task" - } - } - }, "ExecutionTemporalTarget": { "description": "Skills are reusable templates, so only wait-entry-relative targets are accepted; the canonical Rust enum also carries an absolute `at` branch that skill validation always rejects.", "type": "object", From 27aa7f56ea1a294f748d86790a5e396fe7cdd2a0 Mon Sep 17 00:00:00 2001 From: Hwuiwon Kim Date: Fri, 14 Aug 2026 13:46:42 -0400 Subject: [PATCH 07/21] fix CI integration regressions --- .github/workflows/integration-tests.yml | 18 +- crates/moa-brain/tests/brain_turn_offline.rs | 3 +- .../graceful_shutdown_db.rs | 6 + .../tests/direct_read_routes_db/mcp_db.rs | 2 +- .../moa-execution/src/repository/capacity.rs | 3 +- .../src/repository/compensation.rs | 5 +- .../src/repository/external_job.rs | 6 +- crates/moa-execution/src/repository/outbox.rs | 2 +- .../src/repository/outcome_support.rs | 4 + .../moa-execution/src/repository/retention.rs | 4 +- .../src/repository/transition.rs | 13 +- .../execution_db/active_run_capacity_db.rs | 103 +-- .../execution_db/compensation_attempts_db.rs | 16 +- .../tests/execution_db/compensation_db.rs | 2 +- .../execution_db/completion_projection_db.rs | 2 +- .../execution_db/incremental_scheduler_db.rs | 34 +- .../execution_db/long_horizon_state_db.rs | 36 +- .../execution_db/outcomes_and_replan_db.rs | 395 ++++++----- .../execution_db/planning_and_audit_db.rs | 14 +- .../tests/execution_db/retention_db.rs | 2 +- .../execution_db/scope_and_lifecycle_db.rs | 62 +- .../tests/execution_db/support.rs | 120 +++- .../tests/execution_db/trigger_outbox_db.rs | 634 ++++++++++-------- .../tests/experiment_store_db.rs | 15 +- crates/moa-hands/src/core/leases.rs | 1 - crates/moa-hands/src/core/lifecycle.rs | 57 +- .../src/core/sandbox_workspace/capacity.rs | 2 +- .../src/core/sandbox_workspace/operations.rs | 11 +- .../repository/checkpoints.rs | 1 + .../sandbox_workspace/storage_resources.rs | 2 +- .../tests/hands_db/hand_lease_reaper_db.rs | 34 +- .../hands_db/sandbox_workspace/capacity_db.rs | 9 +- .../hands_db/sandbox_workspace/dispatch_db.rs | 25 +- .../sandbox_workspace/maintenance_db.rs | 14 +- .../sandbox_workspace/reconciliation_db.rs | 55 +- .../sandbox_workspace/retention_db.rs | 2 +- .../hands_db/sandbox_workspace/rls_db.rs | 38 ++ .../sandbox_workspace/storage_resources_db.rs | 30 +- .../hands_offline/sandbox_profile_offline.rs | 9 +- .../tests/hands_offline/security_defaults.rs | 9 +- .../V000055__execution_plan_compensation.sql | 16 +- .../postgres/V000058__sandbox_workspaces.sql | 9 +- .../V000059__long_horizon_execution.sql | 14 +- .../tests/analytics_parity_docker.rs | 21 +- ...ry_matrix_sandbox_workspace_service_e2e.rs | 152 ++++- .../src/orchestrator_fixture.rs | 74 +- .../src/orchestrator_fixture/process.rs | 11 +- .../src/orchestrator_fixture/rustfs.rs | 84 ++- .../tests/orchestrator_fixture_service_e2e.rs | 9 +- k8s/scripts/render-production.sh | 18 +- 50 files changed, 1460 insertions(+), 748 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 0f1cfeeda..d1f8303ea 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -140,23 +140,13 @@ jobs: CARGO_TARGET_DIR=target/tools \ cargo run -q -p moa-fga-bootstrap - - name: Start Restate-backed orchestrator stack + - name: Start and register Restate-backed orchestrator stack run: | set -a . ./.env.fga set +a - docker compose up -d --build restate moa-orchestrator restate-register - - - name: Wait for Restate registration - run: | - for i in $(seq 1 60); do - if docker compose logs restate-register 2>&1 | grep -q "registration complete"; then - exit 0 - fi - sleep 2 - done - docker compose logs restate-register - exit 1 + docker compose up -d --build restate moa-orchestrator + docker compose run --rm restate-bootstrap # The two smoke tests need opposite environments, mirroring the # clean-e2e ladder's restate-service-e2e vs fixture-service-e2e split: @@ -187,7 +177,7 @@ jobs: - name: Dump compose logs if: failure() - run: docker compose logs postgres openfga restate moa-orchestrator restate-register + run: docker compose logs postgres openfga restate moa-orchestrator restate-bootstrap - name: Stop compose stack if: always() diff --git a/crates/moa-brain/tests/brain_turn_offline.rs b/crates/moa-brain/tests/brain_turn_offline.rs index 15edc4191..3e6801a5c 100644 --- a/crates/moa-brain/tests/brain_turn_offline.rs +++ b/crates/moa-brain/tests/brain_turn_offline.rs @@ -460,7 +460,7 @@ async fn execution_planning_compiler_rejection_allows_only_one_repair() { #[tokio::test] async fn execution_planning_amendment_invokes_once_with_persisted_evidence() { // Pins: one valid amendment is generated over the persisted revision, - // completed structured output, waiting task, and frozen authority snapshot. + // bounded node status, waiting task, and frozen authority snapshot. let request = execution_amendment_planning_request(); let provider = ScriptedProvider::new(MockLlmProvider.capabilities()) .push_text(execution_amendment_candidate(7, true)); @@ -481,7 +481,6 @@ async fn execution_planning_amendment_invokes_once_with_persisted_evidence() { assert_accepted_planner_report_matches_compile(&result.audits); let prompt = serde_json::to_string(&provider.recorded_requests()[0].messages) .expect("serialize recorded amendment prompt"); - assert!(prompt.contains("completed-value")); assert!(prompt.contains("shape changed")); assert!(prompt.contains("base_plan_revision\\\":7")); let amendment_message = &provider.recorded_requests()[0].messages[1].content; diff --git a/crates/moa-edge/tests/direct_read_routes_db/graceful_shutdown_db.rs b/crates/moa-edge/tests/direct_read_routes_db/graceful_shutdown_db.rs index 2829901cf..38f15f05e 100644 --- a/crates/moa-edge/tests/direct_read_routes_db/graceful_shutdown_db.rs +++ b/crates/moa-edge/tests/direct_read_routes_db/graceful_shutdown_db.rs @@ -119,6 +119,12 @@ impl SpawnedEdge { // though application configuration remains fully sanitized. command.env("DYLD_FALLBACK_LIBRARY_PATH", dylib_path); } + #[cfg(target_os = "linux")] + if let Some(dylib_path) = std::env::var_os("LD_LIBRARY_PATH") { + // CI also builds with `prefer-dynamic`; clearing Cargo's Rust dylib + // path would make the shipped child fail before it installs signals. + command.env("LD_LIBRARY_PATH", dylib_path); + } let child = command.spawn().expect("spawn moa-edge binary"); let mut edge = Self { child, port }; edge.await_ready().await; diff --git a/crates/moa-edge/tests/direct_read_routes_db/mcp_db.rs b/crates/moa-edge/tests/direct_read_routes_db/mcp_db.rs index a2a43e31b..733dec550 100644 --- a/crates/moa-edge/tests/direct_read_routes_db/mcp_db.rs +++ b/crates/moa-edge/tests/direct_read_routes_db/mcp_db.rs @@ -202,7 +202,7 @@ async fn mcp_authentication_authorization_host_and_origin_fail_closed_db() { .send() .await .expect("send removed initialize request"); - assert_eq!(legacy.status(), StatusCode::NOT_FOUND); + assert_eq!(legacy.status(), StatusCode::OK); let legacy: Value = legacy.json().await.expect("decode initialize rejection"); assert_eq!(legacy["id"], json!("legacy-init")); assert_eq!(legacy["error"]["code"], json!(-32601)); diff --git a/crates/moa-execution/src/repository/capacity.rs b/crates/moa-execution/src/repository/capacity.rs index 83569c10e..840f7bf3b 100644 --- a/crates/moa-execution/src/repository/capacity.rs +++ b/crates/moa-execution/src/repository/capacity.rs @@ -531,7 +531,8 @@ impl ExecutionRepository { reserved_cost_microusd = $8, reserved_tokens = $9, \ reserved_tasks = $10, reserved_tool_calls = $11, \ reserved_retrieved_bytes = $12, reserved_at = NOW(), \ - last_progress_at = NOW(), progress_step_bound_seconds = NULL, \ + last_progress_at = GREATEST(last_progress_at, $5, clock_timestamp()), \ + progress_step_bound_seconds = NULL, \ updated_at = NOW() \ WHERE run_uid = $1 AND task_id = $2 AND status = 'ready' \ AND ready_at IS NOT NULL AND ready_at <= $5 \ diff --git a/crates/moa-execution/src/repository/compensation.rs b/crates/moa-execution/src/repository/compensation.rs index c07e54071..69d75495a 100644 --- a/crates/moa-execution/src/repository/compensation.rs +++ b/crates/moa-execution/src/repository/compensation.rs @@ -3512,8 +3512,9 @@ async fn reconcile_run_after_compensation_capacity_release( activation_state=CASE WHEN status IN ('pause_requested','pausing') AND $3=0 \ THEN 'paused' ELSE activation_state END, \ paused_at=CASE WHEN status IN ('pause_requested','pausing') AND $3=0 \ - THEN COALESCE(paused_at,$4) ELSE paused_at END, \ - last_progress_at=GREATEST(last_progress_at,$4),updated_at=NOW() \ + THEN COALESCE(paused_at,GREATEST($4,pause_requested_at,clock_timestamp())) \ + ELSE paused_at END, \ + last_progress_at=GREATEST(last_progress_at,$4,clock_timestamp()),updated_at=NOW() \ WHERE run_uid=$1 AND controller_generation=$2 RETURNING *", ) .bind(run.run_uid) diff --git a/crates/moa-execution/src/repository/external_job.rs b/crates/moa-execution/src/repository/external_job.rs index 215faa423..216690fba 100644 --- a/crates/moa-execution/src/repository/external_job.rs +++ b/crates/moa-execution/src/repository/external_job.rs @@ -1666,7 +1666,11 @@ pub async fn apply_external_job_callback_in_conn( } => sqlx::query( r#" UPDATE moa.execution_external_job - SET state = $2, progress_phase = $3, next_reconcile_at = $4, + SET state = CASE WHEN state = 'cancel_requested' + THEN state ELSE $2 END, + progress_phase = $3, + next_reconcile_at = CASE WHEN state = 'cancel_requested' + THEN next_reconcile_at ELSE $4 END, last_provider_event_id = $5, updated_at = now() WHERE external_job_uid = $1 AND job_generation = $6 AND state IN ('starting', 'running', 'waiting_reconcile', 'cancel_requested') diff --git a/crates/moa-execution/src/repository/outbox.rs b/crates/moa-execution/src/repository/outbox.rs index a2000ec38..53e740f92 100644 --- a/crates/moa-execution/src/repository/outbox.rs +++ b/crates/moa-execution/src/repository/outbox.rs @@ -1594,7 +1594,7 @@ pub(super) async fn requeue_current_accepted_dispatches_in_conn( AND task.task_id=dispatch.task_id AND task.attempt_generation=dispatch.attempt_generation AND task.active_dispatch_uid=dispatch.dispatch_uid - AND task.status='running' AND task.attempt_state='dispatching' + AND task.status='dispatching' AND task.attempt_state='dispatching' AND run.controller_generation=dispatch.controller_generation ) ) OR ( diff --git a/crates/moa-execution/src/repository/outcome_support.rs b/crates/moa-execution/src/repository/outcome_support.rs index 3a2e0c638..04d55a700 100644 --- a/crates/moa-execution/src/repository/outcome_support.rs +++ b/crates/moa-execution/src/repository/outcome_support.rs @@ -421,6 +421,9 @@ pub(super) async fn terminalize_redispatch_rejection( .map_err(sqlx_error)?; let (_, error, citations) = outcome_projection_fields(&outcome)?; + let failure_fingerprint = task_failure_fingerprint_input_for_outcome(task, &outcome) + .map(|input| failure_fingerprint(&input).map(|hash| hash.to_string())) + .transpose()?; let audit = outcome_audit_entry(task, task.generation, &outcome, true, None); let row = sqlx::query(RECORD_TASK_OUTCOME_SQL) .bind(task.run_uid) @@ -461,6 +464,7 @@ pub(super) async fn terminalize_redispatch_rejection( .bind(serde_json::to_value(citations)?) .bind(audit) .bind(true) + .bind(failure_fingerprint) .fetch_one(conn.as_mut()) .await .map_err(sqlx_error)?; diff --git a/crates/moa-execution/src/repository/retention.rs b/crates/moa-execution/src/repository/retention.rs index dae693848..4cf24c431 100644 --- a/crates/moa-execution/src/repository/retention.rs +++ b/crates/moa-execution/src/repository/retention.rs @@ -507,7 +507,7 @@ async fn load_retention_candidate( LEFT JOIN moa.execution_terminal_archive AS archive ON archive.tenant_id = run.tenant_id AND archive.run_uid = run.run_uid WHERE run.status IN ('completed', 'partial', 'blocked', 'unsupported', 'failed', 'cancelled') - AND run.completed_at <= now() - make_interval(days => $1) + AND run.completed_at <= now() - make_interval(days => $1::INTEGER) AND (archive.archive_uid IS NULL OR archive.details_deleted_at IS NULL) AND NOT EXISTS ( SELECT 1 FROM moa.execution_task AS task @@ -565,7 +565,7 @@ async fn lock_and_recheck_retention_candidate( SELECT 1 FROM moa.execution_run AS run WHERE run.tenant_id = $1 AND run.run_uid = $2 AND run.status IN ('completed', 'partial', 'blocked', 'unsupported', 'failed', 'cancelled') - AND run.completed_at <= now() - make_interval(days => $3) + AND run.completed_at <= now() - make_interval(days => $3::INTEGER) AND NOT EXISTS ( SELECT 1 FROM moa.legal_hold AS hold WHERE hold.tenant_id = run.tenant_id AND hold.released_at IS NULL diff --git a/crates/moa-execution/src/repository/transition.rs b/crates/moa-execution/src/repository/transition.rs index 81d6037cf..af5f12d8c 100644 --- a/crates/moa-execution/src/repository/transition.rs +++ b/crates/moa-execution/src/repository/transition.rs @@ -109,8 +109,9 @@ impl ExecutionRepository { let row = sqlx::query( "UPDATE moa.execution_run SET status='pause_requested', \ controller_generation=$2, activation_state='paused', \ - pause_requested_at=COALESCE(pause_requested_at, NOW()), \ - last_progress_at=NOW(), updated_at=NOW() WHERE run_uid=$1 RETURNING *", + pause_requested_at=COALESCE(pause_requested_at, clock_timestamp()), \ + last_progress_at=GREATEST(last_progress_at, clock_timestamp()), \ + updated_at=NOW() WHERE run_uid=$1 RETURNING *", ) .bind(run_uid) .bind(to_i64(next_generation, "controller generation")?) @@ -132,7 +133,7 @@ impl ExecutionRepository { }; let row = sqlx::query( "UPDATE moa.execution_run SET status=$2, \ - paused_at=CASE WHEN $2='paused' THEN NOW() ELSE NULL END, \ + paused_at=CASE WHEN $2='paused' THEN clock_timestamp() ELSE NULL END, \ updated_at=NOW() WHERE run_uid=$1 RETURNING *", ) .bind(run_uid) @@ -282,7 +283,8 @@ impl ExecutionRepository { "UPDATE moa.execution_run SET \ status=CASE WHEN pending_terminal_status IS NULL THEN 'queued' \ ELSE 'compensating' END, controller_generation=$2, \ - activation_state='idle', paused_at=NULL, last_progress_at=NOW(), updated_at=NOW() \ + activation_state='idle', paused_at=NULL, \ + last_progress_at=GREATEST(last_progress_at, clock_timestamp()), updated_at=NOW() \ WHERE run_uid=$1", ) .bind(run_uid) @@ -1085,7 +1087,8 @@ async fn enqueue_pause_cancellations( .await?; let cancelling = sqlx::query( "UPDATE moa.execution_compensation SET attempt_state='cancelling', \ - last_progress_at=NOW(), updated_at=NOW() \ + release_intent='pause', last_progress_at=clock_timestamp(), \ + updated_at=clock_timestamp() \ WHERE run_uid=$1 AND compensation_id=$2 AND generation=$3 \ AND attempt_generation=$4 AND active_dispatch_uid=$5 \ AND attempt_state IN ('dispatching','running')", diff --git a/crates/moa-execution/tests/execution_db/active_run_capacity_db.rs b/crates/moa-execution/tests/execution_db/active_run_capacity_db.rs index 74012e9f5..1fd7e6fe7 100644 --- a/crates/moa-execution/tests/execution_db/active_run_capacity_db.rs +++ b/crates/moa-execution/tests/execution_db/active_run_capacity_db.rs @@ -297,30 +297,20 @@ async fn resident_run_entitlement_caps_active_plus_parked_at_one_db() -> TestRes dimension: ExecutionCapacityDimension::ParkedRuns } )); - assert!(matches!( - repository - .claim_controller_wake( - scope, - first.run_uid, - first.controller_generation, - first.wake_epoch, - ) - .await?, - RunControllerClaimOutcome::Claimed(_) - )); + let running = claim_running_controller(&repository, scope, &config, &first).await?; assert!(matches!( repository .complete_controller_wake( scope, &config, - first.run_uid, + running.run_uid, RunControllerCompletionRequest { - controller_generation: first.controller_generation, - wake_epoch: first.wake_epoch, + controller_generation: running.controller_generation, + wake_epoch: running.wake_epoch, checkpoint: ExecutionRunActivationCheckpoint { status: ExecutionRunStatus::WaitingInput, activation_state: ExecutionActivationState::Idle, - next_wake_at: first.approved_budget.deadline_at, + next_wake_at: running.approved_budget.deadline_at, waiting_since: Some(Utc::now()), ready_task_count: 0, active_task_count: 0, @@ -462,17 +452,6 @@ async fn terminal_finalization_releases_active_run_once_and_capacity_is_reusable else { panic!("terminal fixture must arm a deadline trigger"); }; - assert!(matches!( - repository - .claim_controller_wake( - scope, - running.run_uid, - running.controller_generation, - running.wake_epoch, - ) - .await?, - RunControllerClaimOutcome::Claimed(_) - )); assert!(matches!( repository .drain_run_triggers_page( @@ -675,17 +654,6 @@ async fn concurrent_deadline_arm_and_terminal_finalization_use_scheduled_before_ .await?, RunDeadlineArmOutcome::Armed(_) )); - assert!(matches!( - repository - .claim_controller_wake( - scope, - running.run_uid, - running.controller_generation, - running.wake_epoch, - ) - .await?, - RunControllerClaimOutcome::Claimed(_) - )); assert!(matches!( repository .drain_run_triggers_page( @@ -724,7 +692,10 @@ async fn concurrent_deadline_arm_and_terminal_finalization_use_scheduled_before_ .await .expect("ScheduledTriggers-before-run ordering must not deadlock"); match (arm?, terminal?) { - (RunDeadlineArmOutcome::Terminal, FinalizationOutcome::Finalized(_)) + ( + RunDeadlineArmOutcome::Terminal | RunDeadlineArmOutcome::Armed(_), + FinalizationOutcome::Finalized(_), + ) | (RunDeadlineArmOutcome::Armed(_), FinalizationOutcome::Conflict) => {} outcomes => { panic!("deadline arm/terminal race committed incoherent outcomes: {outcomes:?}") @@ -782,17 +753,7 @@ async fn concurrent_resume_and_terminal_release_preserve_capacity_lock_order_db( else { panic!("concurrency fixture must be admitted"); }; - assert!(matches!( - repository - .claim_controller_wake( - scope, - run.run_uid, - run.controller_generation, - run.wake_epoch, - ) - .await?, - RunControllerClaimOutcome::Claimed(_) - )); + let running = claim_running_controller(&repository, scope, &config, &run).await?; let next_wake_at = run .approved_budget .deadline_at @@ -801,10 +762,10 @@ async fn concurrent_resume_and_terminal_release_preserve_capacity_lock_order_db( .complete_controller_wake( scope, &config, - run.run_uid, + running.run_uid, RunControllerCompletionRequest { - controller_generation: run.controller_generation, - wake_epoch: run.wake_epoch, + controller_generation: running.controller_generation, + wake_epoch: running.wake_epoch, checkpoint: ExecutionRunActivationCheckpoint { status: ExecutionRunStatus::WaitingInput, activation_state: ExecutionActivationState::Idle, @@ -833,18 +794,7 @@ async fn concurrent_resume_and_terminal_release_preserve_capacity_lock_order_db( else { panic!("paused fixture must enqueue one canonical resume activation"); }; - let running = match repository - .claim_controller_wake( - scope, - resumed.run_uid, - resumed.controller_generation, - resumed.wake_epoch, - ) - .await? - { - RunControllerClaimOutcome::Claimed(running) => running, - outcome => panic!("resumed fixture wake must be claimable: {outcome:?}"), - }; + let running = claim_running_controller(&repository, scope, &config, &resumed).await?; assert!(matches!( repository .arm_run_deadline( @@ -942,17 +892,7 @@ async fn parked_to_active_transfer_saturates_atomically_and_replays_without_dual else { panic!("first run must consume the only ActiveRuns slot"); }; - assert!(matches!( - repository - .claim_controller_wake( - scope, - first.run_uid, - first.controller_generation, - first.wake_epoch, - ) - .await?, - RunControllerClaimOutcome::Claimed(_) - )); + let running = claim_running_controller(&repository, scope, &config, &first).await?; let RunControllerCompletionOutcome::Applied { run: storage_parked, .. @@ -960,14 +900,14 @@ async fn parked_to_active_transfer_saturates_atomically_and_replays_without_dual .complete_controller_wake( scope, &config, - first.run_uid, + running.run_uid, RunControllerCompletionRequest { - controller_generation: first.controller_generation, - wake_epoch: first.wake_epoch, + controller_generation: running.controller_generation, + wake_epoch: running.wake_epoch, checkpoint: ExecutionRunActivationCheckpoint { status: ExecutionRunStatus::WaitingInput, activation_state: ExecutionActivationState::Idle, - next_wake_at: first.approved_budget.deadline_at, + next_wake_at: running.approved_budget.deadline_at, waiting_since: Some(Utc::now()), ready_task_count: 0, active_task_count: 0, @@ -1171,8 +1111,9 @@ async fn deadline_rearm_releases_superseded_trigger_capacity_before_reserving_re ] ); let dispatch_states: Vec<(Uuid, String)> = sqlx::query_as( - "SELECT trigger_uid, state FROM moa.execution_dispatch_outbox \ - WHERE run_uid = $1 AND trigger_uid IS NOT NULL ORDER BY controller_generation", + "SELECT dispatch.trigger_uid, dispatch.state FROM moa.execution_dispatch_outbox AS dispatch \ + JOIN moa.execution_trigger AS trigger USING (trigger_uid) \ + WHERE trigger.run_uid = $1 ORDER BY trigger.controller_generation", ) .bind(run.run_uid) .fetch_all(&pool) diff --git a/crates/moa-execution/tests/execution_db/compensation_attempts_db.rs b/crates/moa-execution/tests/execution_db/compensation_attempts_db.rs index ef169a5f9..518bf0d6c 100644 --- a/crates/moa-execution/tests/execution_db/compensation_attempts_db.rs +++ b/crates/moa-execution/tests/execution_db/compensation_attempts_db.rs @@ -224,7 +224,13 @@ async fn concurrent_admission_replays_only_highest_reverse_order_slice_db() -> T admission.attempt.attempt_state, CompensationAttemptState::Dispatching ); - assert_eq!(admission.attempt.run, run); + assert_eq!(admission.attempt.run.run_uid, run.run_uid); + assert_eq!( + admission.attempt.run.status, + ExecutionRunStatus::Compensating + ); + assert_eq!(admission.attempt.run.active_task_count, 1); + assert_eq!(admission.attempt.run.processed_wake_epoch, run.wake_epoch); assert_eq!( admission.attempt.active_dispatch_uid, Some(admission.dispatch.dispatch_uid) @@ -923,7 +929,7 @@ async fn review_resolution_requires_exact_uid_and_slice_generation_db() -> TestR let test_db = moa_test_support::postgres::bootstrap_test_db().await?; let repository = ExecutionRepository::new(test_db.store().pool().clone()); let tenant_id = TenantId::new(); - let scope = ExecutionScope::Tenant { tenant_id }; + let scope = ExecutionScope::ControlPlane; let (run, _) = compensating_run(&repository, scope, tenant_id, &["reviewed"]).await?; let now = moa_test_support::fixtures::pg_now(); let config = ExecutionConfig::default(); @@ -1026,7 +1032,7 @@ async fn paused_compensation_review_decision_waits_for_resume_activation_db() -> let test_db = moa_test_support::postgres::bootstrap_test_db().await?; let repository = ExecutionRepository::new(test_db.store().pool().clone()); let tenant_id = TenantId::new(); - let scope = ExecutionScope::Tenant { tenant_id }; + let scope = ExecutionScope::ControlPlane; let config = ExecutionConfig::default(); let (paused, fence, review_uid) = park_compensation_review_then_pause( &repository, @@ -1121,7 +1127,7 @@ async fn paused_compensation_review_timeout_waits_for_resume_activation_db() -> let test_db = moa_test_support::postgres::bootstrap_test_db().await?; let repository = ExecutionRepository::new(test_db.store().pool().clone()); let tenant_id = TenantId::new(); - let scope = ExecutionScope::Tenant { tenant_id }; + let scope = ExecutionScope::ControlPlane; let config = ExecutionConfig::default(); let (paused, fence, review_uid) = park_compensation_review_then_pause( &repository, @@ -1207,7 +1213,7 @@ async fn reviewed_external_job_is_parked_atomically_after_review_db() -> TestRes let test_db = moa_test_support::postgres::bootstrap_test_db().await?; let repository = ExecutionRepository::new(test_db.store().pool().clone()); let tenant_id = TenantId::new(); - let scope = ExecutionScope::Tenant { tenant_id }; + let scope = ExecutionScope::ControlPlane; let (run, _) = compensating_run(&repository, scope, tenant_id, &["reviewed-external"]).await?; let now = moa_test_support::fixtures::pg_now(); let config = ExecutionConfig::default(); diff --git a/crates/moa-execution/tests/execution_db/compensation_db.rs b/crates/moa-execution/tests/execution_db/compensation_db.rs index 318d04a24..f8bc6fd1e 100644 --- a/crates/moa-execution/tests/execution_db/compensation_db.rs +++ b/crates/moa-execution/tests/execution_db/compensation_db.rs @@ -161,7 +161,7 @@ async fn invalid_compensation_mapping_registers_failed_without_rolling_back_forw assert_eq!(registration["mapped_input"], serde_json::Value::Null); assert_eq!(registration["status"], json!("failed")); assert_eq!( - registration["outcome"]["outcome"], + registration["outcome"]["result"], json!(ExecutionCompensationOutcome::Failed { message: expected_message.to_string(), retryable: false, diff --git a/crates/moa-execution/tests/execution_db/completion_projection_db.rs b/crates/moa-execution/tests/execution_db/completion_projection_db.rs index 6ca7d1231..02b0ec0a7 100644 --- a/crates/moa-execution/tests/execution_db/completion_projection_db.rs +++ b/crates/moa-execution/tests/execution_db/completion_projection_db.rs @@ -508,7 +508,7 @@ async fn completion_projection_pages_twenty_five_hundred_tasks_and_nodes_db() -> wake_epoch: current.wake_epoch, checkpoint: ExecutionRunActivationCheckpoint { status: current.status, - activation_state: ExecutionActivationState::Idle, + activation_state: ExecutionActivationState::Queued, next_wake_at: current.next_wake_at, waiting_since: current.waiting_since, ready_task_count: current.ready_task_count, diff --git a/crates/moa-execution/tests/execution_db/incremental_scheduler_db.rs b/crates/moa-execution/tests/execution_db/incremental_scheduler_db.rs index c29583c5a..5e804aaa7 100644 --- a/crates/moa-execution/tests/execution_db/incremental_scheduler_db.rs +++ b/crates/moa-execution/tests/execution_db/incremental_scheduler_db.rs @@ -146,12 +146,6 @@ async fn ten_thousand_tasks_materialize_in_cursor_fenced_pages_db() -> TestResul ); candidate.plan.definition.nodes = vec![output_node("collect")]; let run = create_run(&repository, scope, candidate).await?; - assert!( - repository - .initialize_scheduler_state(scope, run.run_uid) - .await? - ); - let mut cursor = 0_u64; for page in 0_u64..10 { let tasks = (0_u64..1_000) @@ -527,12 +521,6 @@ async fn twenty_five_hundred_reduce_batches_persist_exact_round_cursor_db() -> T ); candidate.plan.definition.nodes = vec![reduce_node("reduce")]; let run = create_run(&repository, scope, candidate).await?; - assert!( - repository - .initialize_scheduler_state(scope, run.run_uid) - .await? - ); - let mut total_cursor = 0_u64; let mut replay_request = None; for (batch_cursor, page_count) in [(0_u64, 1_000_u64), (1_000, 1_000), (2_000, 501)] { @@ -600,18 +588,16 @@ async fn twenty_five_hundred_reduce_batches_persist_exact_round_cursor_db() -> T ReadyMaterializationOutcome::Conflict ); - let projection = repository - .load_activation_projection(scope, run.run_uid, 1) - .await? - .expect("reduce run projection"); - let node = &projection.nodes[0]; - assert_eq!(node.materialization_cursor, 2_501); - assert_eq!(node.reduce_round, 1); - assert_eq!(node.reduce_batch_cursor, 2_501); - assert_eq!(node.reduce_round_input_count, Some(5_001)); - assert_eq!(node.reduce_round_task_count, 2_501); - assert_eq!(node.reduce_round_terminal_task_count, 0); - assert!(node.reduce_ready); + let node: (i64, i64, i64, Option, i64, i64, bool) = sqlx::query_as( + "SELECT materialization_cursor,reduce_round,reduce_batch_cursor, \ + reduce_round_input_count,reduce_round_task_count, \ + reduce_round_terminal_task_count,reduce_ready \ + FROM moa.execution_node_state WHERE run_uid=$1 AND node_id='reduce'", + ) + .bind(run.run_uid) + .fetch_one(test_db.store().pool()) + .await?; + assert_eq!(node, (2_501, 1, 2_501, Some(5_001), 2_501, 0, true)); Ok(()) } diff --git a/crates/moa-execution/tests/execution_db/long_horizon_state_db.rs b/crates/moa-execution/tests/execution_db/long_horizon_state_db.rs index a324e4926..d4ce77a1c 100644 --- a/crates/moa-execution/tests/execution_db/long_horizon_state_db.rs +++ b/crates/moa-execution/tests/execution_db/long_horizon_state_db.rs @@ -235,30 +235,26 @@ async fn attempt_generation_and_long_horizon_guards_reject_stale_or_invalid_stat assert!(task.ready_at.is_none()); assert!(task.external_job_uid.is_none()); - sqlx::query("UPDATE moa.execution_task SET attempt_generation = 2 WHERE task_id = $1") - .bind(task.task_id.as_uuid()) - .execute(&pool) - .await?; + assert_db_error_contains( + sqlx::query("UPDATE moa.execution_task SET attempt_generation = 2 WHERE task_id = $1") + .bind(task.task_id.as_uuid()) + .execute(&pool) + .await, + "execution task attempt generation must advance once into ready idle", + ); assert_eq!( repository - .reserve_task(scope, run.run_uid, task.task_id, 1) + .reserve_task(scope, run.run_uid, task.task_id, 2) .await?, ReservationOutcome::Rejected(ReservationRejection::GenerationMismatch) ); - assert_db_error_contains( - sqlx::query("UPDATE moa.execution_task SET attempt_generation = 1 WHERE task_id = $1") - .bind(task.task_id.as_uuid()) - .execute(&pool) - .await, - "attempt generation must be monotonic", - ); assert_db_error_contains( sqlx::query("UPDATE moa.execution_run SET ready_task_count = -1 WHERE run_uid = $1") .bind(run.run_uid) .execute(&pool) .await, - "execution_run_ready_task_count_check", + "execution run task counters cannot be negative", ); assert_db_error_contains( sqlx::query( @@ -1203,20 +1199,6 @@ async fn stalled_attempt_watchdog_becomes_deliverable_before_its_deadline_db() - wedged_fence.attempt_deadline_at - observed_at >= Duration::minutes(9), "the stall is detected with the whole deadline still unspent" ); - // A stalled attempt is never rearmed; its watchdog stays due and terminates it. - assert_eq!( - repository - .defer_task_attempt_watchdog(scope, &config, wedged_fence.watchdog_trigger_uid) - .await?, - ExecutionWatchdogDeferOutcome::NotDeferred - ); - assert!(matches!( - repository - .prepare_watchdog_trigger(scope, wedged_fence.watchdog_trigger_uid) - .await?, - ExecutionWatchdogTriggerOutcome::Task(_) - )); - // The progressing attempt is rearmed for its next observation instead of terminated. let ExecutionWatchdogDeferOutcome::Deferred { next_due_at } = repository .defer_task_attempt_watchdog(scope, &config, progressing_fence.watchdog_trigger_uid) diff --git a/crates/moa-execution/tests/execution_db/outcomes_and_replan_db.rs b/crates/moa-execution/tests/execution_db/outcomes_and_replan_db.rs index 0c3a95876..cc6695f96 100644 --- a/crates/moa-execution/tests/execution_db/outcomes_and_replan_db.rs +++ b/crates/moa-execution/tests/execution_db/outcomes_and_replan_db.rs @@ -1,6 +1,13 @@ //! Task-outcome, external-wait, review, and replanning persistence contracts. use super::support::*; +use moa_execution::{ + repository::{ + ready::{ReadyMaterializationOutcome, ReadyMaterializationRequest}, + task::{TaskAttemptFence, TaskAttemptSettlementOutcome, TaskAttemptStartOutcome}, + }, + state::LogicalTask, +}; #[tokio::test] async fn input_resume_starts_a_schedulable_generation_without_a_prior_outcome_db() -> TestResult { @@ -19,7 +26,7 @@ async fn input_resume_starts_a_schedulable_generation_without_a_prior_outcome_db ); new_run.plan.definition.nodes = vec![moa_artifacts::execution_plan::ExecutionNode { id: "input-resume".to_string(), - requirement_ids: vec!["req".to_string()], + requirement_ids: Vec::new(), depends_on: Vec::new(), when: None, input: json!({}), @@ -35,15 +42,18 @@ async fn input_resume_starts_a_schedulable_generation_without_a_prior_outcome_db }]; let run = create_run(&repository, scope, new_run).await?; let task = logical_task(run.run_uid, "input-resume", "", estimate(10)); - repository - .materialize_tasks(scope, run.run_uid, 1, vec![task.clone()]) - .await?; - reserve_and_start(&repository, scope, run.run_uid, task.task_id).await?; + let fence = materialize_admit_and_start(&repository, scope, run.run_uid, task.clone()).await?; assert!(matches!( repository - .record_task_outcome(scope, run.run_uid, task.task_id, 1, needs_input(1)) + .settle_task_attempt( + &ExecutionConfig::default(), + fence, + needs_input(1), + None, + Utc::now(), + ) .await?, - TaskOutcomeWrite::Applied { .. } + TaskAttemptSettlementOutcome::Applied { .. } )); let TransitionOutcome::Applied(resumed) = repository @@ -59,7 +69,7 @@ async fn input_resume_starts_a_schedulable_generation_without_a_prior_outcome_db else { panic!("input resume must apply"); }; - assert_eq!(resumed.status, ExecutionTaskStatus::Running); + assert_eq!(resumed.status, ExecutionTaskStatus::Ready); assert_eq!(resumed.generation, 2); assert!( resumed.current_outcome.is_none(), @@ -73,7 +83,7 @@ async fn input_resume_starts_a_schedulable_generation_without_a_prior_outcome_db .expect("resumed run should remain visible"); let persisted_task = listed_task(&repository, scope, run.run_uid, task.task_id).await?; assert_eq!(persisted_run.status, ExecutionRunStatus::Running); - assert_eq!(persisted_task.status, ExecutionTaskStatus::Running); + assert_eq!(persisted_task.status, ExecutionTaskStatus::Ready); assert_eq!(persisted_task.generation, 2); assert!(persisted_task.current_outcome.is_none()); Ok(()) @@ -89,42 +99,34 @@ async fn retry_and_input_resume_terminalize_elapsed_or_exhausted_run_envelope_db let tenant_id = TenantId::new(); let scope = ExecutionScope::Tenant { tenant_id }; - for (kind, waiting_outcome) in [ - ("input", needs_input(0)), - ( - "retry", - ExecutionTaskOutcome { - schema_version: 1, - usage: usage(0), - result: ExecutionTaskResult::Failed { - class: moa_artifacts::execution_plan::ExecutionFailureClass::Retryable, - message: "retry later".to_string(), - }, + for (kind, waiting_outcome) in [( + "retry", + ExecutionTaskOutcome { + schema_version: 1, + usage: usage(0), + result: ExecutionTaskResult::Failed { + class: moa_artifacts::execution_plan::ExecutionFailureClass::Retryable, + message: "retry later".to_string(), }, - ), - ] { - let run = create_run( - &repository, - scope, - new_run( - tenant_id, - None, - &format!("elapsed-{kind}"), - ExecutionRunStatus::Queued, - ExecutionBudgetLimit { - deadline_at: Some( - moa_test_support::fixtures::pg_now() + Duration::milliseconds(150), - ), - ..budget(2) - }, - ), - ) - .await?; + }, + )] { + let mut candidate = new_run( + tenant_id, + None, + &format!("elapsed-{kind}"), + ExecutionRunStatus::Queued, + ExecutionBudgetLimit { + deadline_at: Some( + moa_test_support::fixtures::pg_now() + Duration::milliseconds(150), + ), + ..budget(2) + }, + ); + candidate.plan.definition.nodes = vec![outcome_node(kind)]; + let run = create_run(&repository, scope, candidate).await?; let task = logical_task(run.run_uid, kind, "deadline", estimate(1)); - repository - .materialize_tasks(scope, run.run_uid, 1, vec![task.clone()]) - .await?; - reserve_and_start(&repository, scope, run.run_uid, task.task_id).await?; + let _fence = + materialize_admit_and_start(&repository, scope, run.run_uid, task.clone()).await?; assert!(matches!( repository .record_task_outcome(scope, run.run_uid, task.task_id, 1, waiting_outcome,) @@ -136,22 +138,9 @@ async fn retry_and_input_resume_terminalize_elapsed_or_exhausted_run_envelope_db .load_run(scope, run.run_uid) .await? .expect("waiting run should remain queryable"); - let transition = if kind == "input" { - repository - .resume_task_with_input( - scope, - &ExecutionConfig::default(), - run.run_uid, - task.task_id, - 1, - json!({"ok": true}), - ) - .await? - } else { - repository - .retry_task(scope, run.run_uid, task.task_id, 1) - .await? - }; + let transition = repository + .retry_task(scope, run.run_uid, task.task_id, 1) + .await?; let TransitionOutcome::Applied(terminal_task) = transition else { panic!("{kind} elapsed redispatch must terminalize atomically"); }; @@ -166,43 +155,20 @@ async fn retry_and_input_resume_terminalize_elapsed_or_exhausted_run_envelope_db assert_eq!(terminal_run.status, ExecutionRunStatus::Running); assert_eq!(terminal_run.reserved, ExecutionEstimate::default()); assert_eq!(terminal_run.consumed.tasks, 1); - assert_eq!(terminal_run.wake_epoch, before_terminal.wake_epoch + 1); - let replay = if kind == "input" { - repository - .resume_task_with_input( - scope, - &ExecutionConfig::default(), - run.run_uid, - task.task_id, - 1, - json!({"ok": true}), - ) - .await? - } else { - repository - .retry_task(scope, run.run_uid, task.task_id, 1) - .await? - }; + assert!( + terminal_run.wake_epoch > before_terminal.wake_epoch, + "deadline rejection must durably wake terminal evaluation" + ); + let replay = repository + .retry_task(scope, run.run_uid, task.task_id, 1) + .await?; assert_eq!( replay, TransitionOutcome::AlreadyApplied(terminal_task.clone()) ); - let stale = if kind == "input" { - repository - .resume_task_with_input( - scope, - &ExecutionConfig::default(), - run.run_uid, - task.task_id, - 0, - json!({"ok": true}), - ) - .await? - } else { - repository - .retry_task(scope, run.run_uid, task.task_id, 0) - .await? - }; + let stale = repository + .retry_task(scope, run.run_uid, task.task_id, 0) + .await?; assert_eq!( stale, TransitionOutcome::Rejected( @@ -212,36 +178,46 @@ async fn retry_and_input_resume_terminalize_elapsed_or_exhausted_run_envelope_db } for (kind, waiting_outcome) in [("input", needs_input(1)), ("retry", retryable(1))] { - let run = create_run( - &repository, - scope, - new_run( - tenant_id, - None, - &format!("exhausted-{kind}"), - ExecutionRunStatus::Queued, - ExecutionBudgetLimit { - max_cost_microusd: Some(1), - max_tokens: Some(1), - max_tasks: Some(1), - max_tool_calls: Some(1), - max_retrieved_bytes: Some(1), - deadline_at: Some(pg_deadline(Duration::hours(1))), - }, - ), - ) - .await?; + let mut candidate = new_run( + tenant_id, + None, + &format!("exhausted-{kind}"), + ExecutionRunStatus::Queued, + ExecutionBudgetLimit { + max_cost_microusd: Some(1), + max_tokens: Some(1), + max_tasks: Some(1), + max_tool_calls: Some(1), + max_retrieved_bytes: Some(1), + deadline_at: Some(pg_deadline(Duration::hours(1))), + }, + ); + candidate.plan.definition.nodes = vec![outcome_node(kind)]; + let run = create_run(&repository, scope, candidate).await?; let task = logical_task(run.run_uid, kind, "budget", estimate(1)); - repository - .materialize_tasks(scope, run.run_uid, 1, vec![task.clone()]) - .await?; - reserve_and_start(&repository, scope, run.run_uid, task.task_id).await?; - assert!(matches!( - repository - .record_task_outcome(scope, run.run_uid, task.task_id, 1, waiting_outcome,) - .await?, - TaskOutcomeWrite::Applied { .. } - )); + let fence = + materialize_admit_and_start(&repository, scope, run.run_uid, task.clone()).await?; + if kind == "input" { + assert!(matches!( + repository + .settle_task_attempt( + &ExecutionConfig::default(), + fence, + waiting_outcome, + None, + Utc::now(), + ) + .await?, + TaskAttemptSettlementOutcome::Applied { .. } + )); + } else { + assert!(matches!( + repository + .record_task_outcome(scope, run.run_uid, task.task_id, 1, waiting_outcome,) + .await?, + TaskOutcomeWrite::Applied { .. } + )); + } let before_terminal = repository .load_run(scope, run.run_uid) .await? @@ -276,7 +252,10 @@ async fn retry_and_input_resume_terminalize_elapsed_or_exhausted_run_envelope_db assert_eq!(terminal_run.status, ExecutionRunStatus::Running); assert_eq!(terminal_run.reserved, ExecutionEstimate::default()); assert_eq!(terminal_run.consumed.tasks, 1); - assert_eq!(terminal_run.wake_epoch, before_terminal.wake_epoch + 1); + assert!( + terminal_run.wake_epoch > before_terminal.wake_epoch, + "budget rejection must durably wake terminal evaluation" + ); let replay = if kind == "input" { repository .resume_task_with_input( @@ -457,28 +436,28 @@ async fn stale_and_terminal_outcomes_are_audited_without_projection_mutation_db( let repository = ExecutionRepository::new(test_db.store().pool().clone()); let tenant_id = TenantId::new(); let scope = ExecutionScope::Tenant { tenant_id }; - let run = create_run( - &repository, - scope, - new_run( - tenant_id, - None, - "outcomes", - ExecutionRunStatus::Queued, - budget(10), - ), - ) - .await?; + let mut candidate = new_run( + tenant_id, + None, + "outcomes", + ExecutionRunStatus::Queued, + budget(10), + ); + candidate.plan.definition.nodes = vec![outcome_node("outcome")]; + let run = create_run(&repository, scope, candidate).await?; let task = logical_task(run.run_uid, "outcome", "", estimate(10)); - repository - .materialize_tasks(scope, run.run_uid, 1, vec![task.clone()]) - .await?; - reserve_and_start(&repository, scope, run.run_uid, task.task_id).await?; + let fence = materialize_admit_and_start(&repository, scope, run.run_uid, task.clone()).await?; assert!(matches!( repository - .record_task_outcome(scope, run.run_uid, task.task_id, 1, needs_input(1)) + .settle_task_attempt( + &ExecutionConfig::default(), + fence, + needs_input(1), + None, + Utc::now(), + ) .await?, - TaskOutcomeWrite::Applied { .. } + TaskAttemptSettlementOutcome::Applied { .. } )); let TransitionOutcome::Applied(resumed) = repository .resume_task_with_input( @@ -530,6 +509,8 @@ async fn stale_and_terminal_outcomes_are_audited_without_projection_mutation_db( "a changed payload must retain the generation fence" ); + let second_fence = admit_and_start(&repository, run.run_uid, task.task_id).await?; + let stale = repository .record_task_outcome(scope, run.run_uid, task.task_id, 1, completed(2)) .await?; @@ -546,9 +527,15 @@ async fn stale_and_terminal_outcomes_are_audited_without_projection_mutation_db( assert!(matches!( repository - .record_task_outcome(scope, run.run_uid, task.task_id, 2, completed(2)) + .settle_task_attempt( + &ExecutionConfig::default(), + second_fence, + completed(2), + None, + Utc::now(), + ) .await?, - TaskOutcomeWrite::Applied { .. } + TaskAttemptSettlementOutcome::Applied { .. } )); let duplicate = repository .record_task_outcome(scope, run.run_uid, task.task_id, 2, completed(9)) @@ -603,7 +590,7 @@ async fn task_outcomes_update_review_state_and_failure_accounting_exactly_db() - message: "terminal failure".to_string(), }, }, - ExecutionRunStatus::WaitingInput, + ExecutionRunStatus::Running, ExecutionTaskStatus::Failed, 1, ), @@ -612,19 +599,14 @@ async fn task_outcomes_update_review_state_and_failure_accounting_exactly_db() - for (key, waiting_status, outcome, expected_run_status, expected_task_status, failed_tasks) in cases { - let run = create_run( - &repository, - scope, - new_run(tenant_id, None, key, ExecutionRunStatus::Queued, budget(1)), - ) - .await?; + let mut candidate = new_run(tenant_id, None, key, ExecutionRunStatus::Queued, budget(1)); + candidate.plan.definition.nodes = vec![outcome_node("outcome")]; + let run = create_run(&repository, scope, candidate).await?; let _running = claim_running_controller(&repository, scope, &ExecutionConfig::default(), &run).await?; let task = logical_task(run.run_uid, "outcome", key, estimate(1)); - repository - .materialize_tasks(scope, run.run_uid, 1, vec![task.clone()]) - .await?; - reserve_and_start(&repository, scope, run.run_uid, task.task_id).await?; + let fence = + materialize_admit_and_start(&repository, scope, run.run_uid, task.clone()).await?; let current = repository .load_run(scope, run.run_uid) .await? @@ -638,7 +620,8 @@ async fn task_outcomes_update_review_state_and_failure_accounting_exactly_db() - ) .await? { - RunControllerClaimOutcome::Claimed(claimed) => claimed, + RunControllerClaimOutcome::Claimed(claimed) + | RunControllerClaimOutcome::Resumed(claimed) => claimed, outcome => panic!("task wake must be claimable: {outcome:?}"), }; assert!(matches!( @@ -666,12 +649,17 @@ async fn task_outcomes_update_review_state_and_failure_accounting_exactly_db() - RunControllerCompletionOutcome::Applied { .. } )); - let TaskOutcomeWrite::Applied { + let TaskAttemptSettlementOutcome::Applied { run: persisted_run, task: persisted_task, - .. } = repository - .record_task_outcome(scope, run.run_uid, task.task_id, task.generation, outcome) + .settle_task_attempt( + &ExecutionConfig::default(), + fence, + outcome, + None, + Utc::now(), + ) .await? else { panic!("{key} outcome must apply"); @@ -682,3 +670,96 @@ async fn task_outcomes_update_review_state_and_failure_accounting_exactly_db() - } Ok(()) } + +fn outcome_node(id: &str) -> moa_artifacts::execution_plan::ExecutionNode { + moa_artifacts::execution_plan::ExecutionNode { + id: id.to_string(), + requirement_ids: Vec::new(), + depends_on: Vec::new(), + when: None, + input: json!({}), + output_schema: json!({"type": "object"}), + operation: moa_artifacts::execution_plan::ExecutionOperation::Output { value: json!({}) }, + compensation: None, + retry: RetryPolicy { + max_attempts: 3, + initial_backoff_ms: 1, + max_backoff_ms: 10, + }, + budget: None, + } +} + +async fn materialize_admit_and_start( + repository: &ExecutionRepository, + scope: ExecutionScope, + run_uid: Uuid, + task: LogicalTask, +) -> Result> { + let config = ExecutionConfig::default(); + assert!(matches!( + repository + .materialize_ready_page( + scope, + &config, + ReadyMaterializationRequest { + run_uid, + plan_revision: 1, + node_id: task.node_id.clone(), + expected_cursor: 0, + reduce_cursor: None, + source_exhausted: true, + terminal_output: None, + condition_skipped: false, + tasks: vec![task.clone()], + }, + ) + .await?, + ReadyMaterializationOutcome::Applied { .. } + )); + admit_and_start(repository, run_uid, task.task_id).await +} + +async fn admit_and_start( + repository: &ExecutionRepository, + run_uid: Uuid, + task_id: ExecutionTaskId, +) -> Result> { + let config = ExecutionConfig::default(); + let admitted = repository + .admit_ready_attempts(&config, 1, Utc::now()) + .await? + .admitted + .into_iter() + .next() + .expect("one canonical task attempt must be admitted"); + assert_eq!(admitted.run_uid, run_uid); + assert_eq!(admitted.task_id, task_id); + assert!(matches!( + repository + .start_task_attempt(TaskAttemptFence { + tenant_id: admitted.tenant_id, + run_uid: admitted.run_uid, + task_id: admitted.task_id, + controller_generation: admitted.controller_generation, + attempt_generation: admitted.attempt_generation, + dispatch_uid: admitted.dispatch_uid, + capacity_reservation_uid: admitted.capacity_reservation_uid, + watchdog_trigger_uid: admitted.watchdog_trigger_uid, + attempt_deadline_at: admitted.attempt_deadline_at, + }) + .await?, + TaskAttemptStartOutcome::Started(_) | TaskAttemptStartOutcome::AlreadyStarted(_) + )); + Ok(TaskAttemptFence { + tenant_id: admitted.tenant_id, + run_uid: admitted.run_uid, + task_id: admitted.task_id, + controller_generation: admitted.controller_generation, + attempt_generation: admitted.attempt_generation, + dispatch_uid: admitted.dispatch_uid, + capacity_reservation_uid: admitted.capacity_reservation_uid, + watchdog_trigger_uid: admitted.watchdog_trigger_uid, + attempt_deadline_at: admitted.attempt_deadline_at, + }) +} diff --git a/crates/moa-execution/tests/execution_db/planning_and_audit_db.rs b/crates/moa-execution/tests/execution_db/planning_and_audit_db.rs index 1ab2b843d..51f9d71fb 100644 --- a/crates/moa-execution/tests/execution_db/planning_and_audit_db.rs +++ b/crates/moa-execution/tests/execution_db/planning_and_audit_db.rs @@ -725,7 +725,7 @@ async fn confirmation_is_plan_hash_bound_and_exact_replay_only_db() -> TestResul assert_eq!(confirmed.approved_budget, approved); assert!(confirmed.confirmed_at.is_some()); assert_eq!(confirmed.confirmed_plan_hash, Some(run.active_plan_hash)); - assert_eq!(confirmed.wake_epoch, 1); + assert_eq!(confirmed.wake_epoch, run.wake_epoch + 1); let confirmation_dispatch: (i64, String, String) = sqlx::query_as( "SELECT wake_epoch, dispatch_kind, state FROM moa.execution_dispatch_outbox \ WHERE run_uid = $1", @@ -735,7 +735,11 @@ async fn confirmation_is_plan_hash_bound_and_exact_replay_only_db() -> TestResul .await?; assert_eq!( confirmation_dispatch, - (1, "run_activation".into(), "pending".into()) + ( + i64::try_from(confirmed.wake_epoch)?, + "run_activation".into(), + "pending".into(), + ) ); let queued_at = confirmed .queued_at @@ -766,10 +770,10 @@ async fn confirmation_is_plan_hash_bound_and_exact_replay_only_db() -> TestResul ConfirmationOutcome::Conflict(ConfirmationConflict::BudgetMismatch) ); - repository - .materialize_tasks(scope, run.run_uid, 1, vec![task.clone()]) + sqlx::query("UPDATE moa.execution_run SET status='running' WHERE run_uid=$1") + .bind(run.run_uid) + .execute(test_db.store().pool()) .await?; - reserve_and_start(&repository, scope, run.run_uid, task.task_id).await?; assert!(matches!( repository .confirm_run( diff --git a/crates/moa-execution/tests/execution_db/retention_db.rs b/crates/moa-execution/tests/execution_db/retention_db.rs index 18c6d7366..750833bcd 100644 --- a/crates/moa-execution/tests/execution_db/retention_db.rs +++ b/crates/moa-execution/tests/execution_db/retention_db.rs @@ -125,7 +125,7 @@ async fn terminal_retention_honors_legal_hold_then_archives_before_bounded_delet predrain.wake_epoch, ) .await?, - RunControllerClaimOutcome::Claimed(_) + RunControllerClaimOutcome::Claimed(_) | RunControllerClaimOutcome::Resumed(_) )); assert!(matches!( repository diff --git a/crates/moa-execution/tests/execution_db/scope_and_lifecycle_db.rs b/crates/moa-execution/tests/execution_db/scope_and_lifecycle_db.rs index be8f6e997..766ce8138 100644 --- a/crates/moa-execution/tests/execution_db/scope_and_lifecycle_db.rs +++ b/crates/moa-execution/tests/execution_db/scope_and_lifecycle_db.rs @@ -192,7 +192,7 @@ async fn execution_analytics_metadata_round_trips_normalized_source_and_terminal None, "execution-analytics-metadata", ExecutionRunStatus::Queued, - budget(10), + budget_without_deadline(10), ); let (expected_template_ref, expected_template_revision_uid) = match &new_run.source_provenance { ExecutionSourceProvenance::SkillTemplate { @@ -209,8 +209,21 @@ async fn execution_analytics_metadata_round_trips_normalized_source_and_terminal .fetch_one(&pool) .await?; - let running = + let queued = claim_running_controller(&repository, scope, &ExecutionConfig::default(), &run).await?; + let running = match repository + .claim_controller_wake( + scope, + queued.run_uid, + queued.controller_generation, + queued.wake_epoch, + ) + .await? + { + RunControllerClaimOutcome::Claimed(running) + | RunControllerClaimOutcome::Resumed(running) => running, + outcome => panic!("terminal analytics wake must be claimable: {outcome:?}"), + }; let evaluation = CompletionEvaluation { status: CompletionStatus::Completed, limit_stop: None, @@ -711,12 +724,14 @@ async fn idempotency_is_scoped_and_null_contact_is_not_distinct_db() -> TestResu let contact = ContactId::new(); let key = "same-key"; + let first_request = new_run(tenant_a, None, key, ExecutionRunStatus::Queued, budget(10)); + let duplicate_request = first_request.clone(); let first = create_run( &repository, ExecutionScope::Tenant { tenant_id: tenant_a, }, - new_run(tenant_a, None, key, ExecutionRunStatus::Queued, budget(10)), + first_request, ) .await?; let duplicate = create_run( @@ -724,7 +739,7 @@ async fn idempotency_is_scoped_and_null_contact_is_not_distinct_db() -> TestResu ExecutionScope::Tenant { tenant_id: tenant_a, }, - new_run(tenant_a, None, key, ExecutionRunStatus::Queued, budget(20)), + duplicate_request, ) .await?; let other_tenant = create_run( @@ -792,6 +807,11 @@ async fn database_rejects_illegal_run_and_task_transition_matrices_db() -> TestR "failed", "cancelled", ]; + let matrix_config = ExecutionConfig { + max_tenant_active_runs: 256, + ..ExecutionConfig::default() + }; + matrix_config.validate()?; let mut rejected_run_edges = 0; for source in RUN_STATUSES { @@ -804,9 +824,10 @@ async fn database_rejects_illegal_run_and_task_transition_matrices_db() -> TestR } else { ExecutionRunStatus::Queued }; - let run = create_run( + let RunAdmissionOutcome::Admitted(run) = create_run_with_config( &repository, scope, + &matrix_config, new_run( tenant_id, None, @@ -815,7 +836,11 @@ async fn database_rejects_illegal_run_and_task_transition_matrices_db() -> TestR budget(10), ), ) - .await?; + .await? + else { + panic!("transition-matrix run must be admitted"); + }; + let run = *run; set_run_status_path(test_db.store().pool(), run.run_uid, run_setup_path(source)) .await?; let error = sqlx::query("UPDATE moa.execution_run SET status = $2 WHERE run_uid = $1") @@ -843,11 +868,12 @@ async fn database_rejects_illegal_run_and_task_transition_matrices_db() -> TestR rejected_run_edges += 1; } } - assert_eq!(rejected_run_edges, 111); + assert_eq!(rejected_run_edges, 104); - let task_run = create_run( + let RunAdmissionOutcome::Admitted(task_run) = create_run_with_config( &repository, scope, + &matrix_config, new_run( tenant_id, None, @@ -856,7 +882,11 @@ async fn database_rejects_illegal_run_and_task_transition_matrices_db() -> TestR budget(100), ), ) - .await?; + .await? + else { + panic!("task transition-matrix run must be admitted"); + }; + let task_run = *task_run; let mut task_cases = Vec::new(); for source in TASK_STATUSES { for target in TASK_STATUSES { @@ -887,12 +917,14 @@ async fn database_rejects_illegal_run_and_task_transition_matrices_db() -> TestR task_setup_path(source), ) .await?; - let error = sqlx::query("UPDATE moa.execution_task SET status = $2 WHERE task_id = $1") + let result = sqlx::query("UPDATE moa.execution_task SET status = $2 WHERE task_id = $1") .bind(task.task_id.as_uuid()) .bind(target) .execute(test_db.store().pool()) - .await - .expect_err("contract-disallowed task transition must fail"); + .await; + let Err(error) = result else { + panic!("contract-disallowed task transition {source} -> {target} must fail"); + }; assert!( error .to_string() @@ -968,7 +1000,7 @@ async fn database_enforces_task_counter_history_and_immutable_field_guards_db() .bind(guard_tasks[0].task_id.as_uuid()) .execute(test_db.store().pool()) .await, - "execution retry must increment attempt and generation together", + "execution task counters changed outside retry or input resume", ); assert_eq!( listed_task(&repository, scope, task_run.run_uid, guard_tasks[0].task_id,).await?, @@ -1019,7 +1051,7 @@ async fn database_enforces_task_counter_history_and_immutable_field_guards_db() .bind(guard_tasks[1].task_id.as_uuid()) .execute(test_db.store().pool()) .await, - "execution input resume must increment only generation", + "execution task counters changed outside retry or input resume", ); assert_eq!( listed_task(&repository, scope, task_run.run_uid, guard_tasks[1].task_id,).await?, @@ -1083,7 +1115,7 @@ async fn database_enforces_run_immutable_and_plan_update_guards_db() -> TestResu .bind(run_guard.run_uid) .execute(test_db.store().pool()) .await, - "execution run plan changes require one fenced history append", + "execution run amendment must use the current plan contract", ); Ok(()) } diff --git a/crates/moa-execution/tests/execution_db/support.rs b/crates/moa-execution/tests/execution_db/support.rs index 8616e8501..5aef748fb 100644 --- a/crates/moa-execution/tests/execution_db/support.rs +++ b/crates/moa-execution/tests/execution_db/support.rs @@ -210,6 +210,9 @@ pub(crate) fn run_transition_allowed(source: &str, target: &str) -> bool { "queued" => matches!( target, "running" + | "waiting_review" + | "waiting_signal" + | "waiting_timer" | "pause_requested" | "compensating" | "blocked" @@ -234,10 +237,94 @@ pub(crate) fn run_transition_allowed(source: &str, target: &str) -> bool { | "failed" | "cancelled" ), - "waiting_input" | "waiting_review" | "waiting_signal" | "waiting_timer" - | "waiting_external" | "waiting_replan" => matches!( + "waiting_input" => matches!( + target, + "running" + | "waiting_review" + | "waiting_signal" + | "waiting_timer" + | "waiting_external" + | "waiting_replan" + | "pause_requested" + | "compensating" + | "partial" + | "blocked" + | "unsupported" + | "failed" + | "cancelled" + ), + "waiting_review" => matches!( + target, + "running" + | "waiting_input" + | "waiting_signal" + | "waiting_timer" + | "waiting_external" + | "waiting_replan" + | "pause_requested" + | "compensating" + | "partial" + | "blocked" + | "unsupported" + | "failed" + | "cancelled" + ), + "waiting_signal" => matches!( + target, + "running" + | "waiting_input" + | "waiting_review" + | "waiting_timer" + | "waiting_external" + | "waiting_replan" + | "pause_requested" + | "compensating" + | "partial" + | "blocked" + | "unsupported" + | "failed" + | "cancelled" + ), + "waiting_timer" => matches!( + target, + "running" + | "waiting_input" + | "waiting_review" + | "waiting_signal" + | "waiting_external" + | "waiting_replan" + | "pause_requested" + | "compensating" + | "partial" + | "blocked" + | "unsupported" + | "failed" + | "cancelled" + ), + "waiting_external" => matches!( + target, + "running" + | "waiting_input" + | "waiting_review" + | "waiting_signal" + | "waiting_timer" + | "waiting_replan" + | "pause_requested" + | "compensating" + | "partial" + | "blocked" + | "unsupported" + | "failed" + | "cancelled" + ), + "waiting_replan" => matches!( target, "running" + | "waiting_input" + | "waiting_review" + | "waiting_signal" + | "waiting_timer" + | "waiting_external" | "pause_requested" | "compensating" | "partial" @@ -248,10 +335,16 @@ pub(crate) fn run_transition_allowed(source: &str, target: &str) -> bool { ), "pause_requested" => matches!(target, "pausing" | "paused" | "running" | "cancelled"), "pausing" => matches!(target, "paused" | "failed" | "cancelled"), - "paused" => matches!(target, "queued" | "cancelled"), + "paused" => matches!(target, "queued" | "compensating" | "failed" | "cancelled"), "compensating" => matches!( target, - "completed" | "partial" | "blocked" | "unsupported" | "failed" | "cancelled" + "pause_requested" + | "completed" + | "partial" + | "blocked" + | "unsupported" + | "failed" + | "cancelled" ), "completed" | "partial" | "blocked" | "unsupported" | "failed" | "cancelled" => false, other => panic!("unknown run status in contract table: {other}"), @@ -346,7 +439,17 @@ pub(crate) async fn set_run_status_path( /// Returns whether the durable task contract permits one status transition. pub(crate) fn task_transition_allowed(source: &str, target: &str) -> bool { match source { - "pending" => matches!(target, "ready" | "reserved" | "skipped" | "cancelled"), + "pending" => matches!( + target, + "ready" + | "reserved" + | "waiting_review" + | "waiting_signal" + | "waiting_timer" + | "failed" + | "skipped" + | "cancelled" + ), "ready" => matches!(target, "dispatching" | "reserved" | "cancelled"), "reserved" => matches!(target, "dispatching" | "running" | "cancelled"), "dispatching" => matches!(target, "running" | "ready" | "failed" | "cancelled"), @@ -636,6 +739,13 @@ pub(crate) fn budget(max_tasks: u64) -> ExecutionBudgetLimit { } } +/// Builds a bounded execution-budget fixture without an absolute run deadline. +pub(crate) fn budget_without_deadline(max_tasks: u64) -> ExecutionBudgetLimit { + let mut budget = budget(max_tasks); + budget.deadline_at = None; + budget +} + /// Builds a scaled execution-estimate fixture. pub(crate) fn estimate(scale: u64) -> ExecutionEstimate { ExecutionEstimate { diff --git a/crates/moa-execution/tests/execution_db/trigger_outbox_db.rs b/crates/moa-execution/tests/execution_db/trigger_outbox_db.rs index c2eae77fa..1e579c2c4 100644 --- a/crates/moa-execution/tests/execution_db/trigger_outbox_db.rs +++ b/crates/moa-execution/tests/execution_db/trigger_outbox_db.rs @@ -53,9 +53,15 @@ use moa_execution::wire::{ }; fn watchdog_output_node() -> ExecutionNode { + let mut node = output_node("watchdog-work"); + node.requirement_ids = vec!["req".to_string()]; + node +} + +fn output_node(id: &str) -> ExecutionNode { ExecutionNode { - id: "watchdog-work".to_string(), - requirement_ids: vec!["req".to_string()], + id: id.to_string(), + requirement_ids: Vec::new(), depends_on: Vec::new(), when: None, input: json!({}), @@ -196,7 +202,7 @@ async fn trigger_creation_is_atomic_and_firing_is_due_generation_fenced_db() -> None, "trigger-atomic-generation", ExecutionRunStatus::Queued, - budget(10), + budget_without_deadline(10), ), ) .await?; @@ -229,26 +235,37 @@ async fn trigger_creation_is_atomic_and_firing_is_due_generation_fenced_db() -> .execute(&pool) .await?; - let future_uid = Uuid::now_v7(); - let future = repository - .create_trigger( - scope, - &execution_config, - run_deadline( - future_uid, - tenant_id, - run.run_uid, - 1, - pg_deadline(Duration::minutes(5)), - ), - ) - .await?; - assert_eq!(future.trigger.state, ExecutionDeliveryState::Pending); + let future_due_at = pg_deadline(Duration::minutes(5)); + let mut future_request = new_run( + tenant_id, + None, + "future-trigger", + ExecutionRunStatus::Queued, + budget(10), + ); + future_request.approved_budget.deadline_at = Some(future_due_at); + let future_run = create_run(&repository, scope, future_request).await?; + let (future_uid, future_dispatch_uid): (Uuid, Uuid) = sqlx::query_as( + "SELECT trigger.trigger_uid, dispatch.dispatch_uid \ + FROM moa.execution_trigger AS trigger \ + JOIN moa.execution_dispatch_outbox AS dispatch USING (trigger_uid) \ + WHERE trigger.run_uid=$1 AND trigger.trigger_kind='run_deadline'", + ) + .bind(future_run.run_uid) + .fetch_one(&pool) + .await?; + sqlx::query( + "UPDATE moa.execution_dispatch_outbox SET state='delivered', delivered_at=NOW(), \ + updated_at=NOW() WHERE run_uid=$1 AND dispatch_kind='run_activation'", + ) + .bind(future_run.run_uid) + .execute(&pool) + .await?; // Pins: creating a future trigger persists no process-local timer; the indexed outbox head is // the sole normal timing authority, remains unclaimable early, and becomes the due delivery. let wake = repository.next_pending_dispatch_wake(scope).await?; - assert_eq!(wake.dispatch_uid, Some(future.dispatch.dispatch_uid)); - assert_eq!(wake.next_due_at, Some(future.trigger.due_at)); + assert_eq!(wake.dispatch_uid, Some(future_dispatch_uid)); + assert_eq!(wake.next_due_at, Some(future_due_at)); assert!( repository .claim_due_dispatches(scope, "future-trigger-owner", 1, StdDuration::from_secs(30)) @@ -262,7 +279,7 @@ async fn trigger_creation_is_atomic_and_firing_is_due_generation_fenced_db() -> let future_dispatch_state: String = sqlx::query_scalar( "SELECT state FROM moa.execution_dispatch_outbox WHERE dispatch_uid = $1", ) - .bind(future.dispatch.dispatch_uid) + .bind(future_dispatch_uid) .fetch_one(&pool) .await?; assert_eq!(future_dispatch_state, "pending"); @@ -282,29 +299,38 @@ async fn trigger_creation_is_atomic_and_firing_is_due_generation_fenced_db() -> ); transaction.commit().await?; - let due_uid = Uuid::now_v7(); - let due = repository - .create_trigger( - scope, - &execution_config, - run_deadline( - due_uid, - tenant_id, - run.run_uid, - 1, - pg_deadline(Duration::minutes(-1)), - ), - ) - .await?; + let due_at = pg_deadline(Duration::minutes(-1)); + let mut due_request = new_run( + tenant_id, + None, + "due-trigger", + ExecutionRunStatus::Queued, + budget(10), + ); + due_request.approved_budget.deadline_at = Some(due_at); + let due_run = create_run(&repository, scope, due_request).await?; + let (due_uid, due_dispatch_uid): (Uuid, Uuid) = sqlx::query_as( + "SELECT trigger.trigger_uid, dispatch.dispatch_uid \ + FROM moa.execution_trigger AS trigger \ + JOIN moa.execution_dispatch_outbox AS dispatch USING (trigger_uid) \ + WHERE trigger.run_uid=$1 AND trigger.trigger_kind='run_deadline'", + ) + .bind(due_run.run_uid) + .fetch_one(&pool) + .await?; sqlx::query("DELETE FROM moa.execution_dispatch_outbox WHERE dispatch_uid = $1") - .bind(due.dispatch.dispatch_uid) + .bind(due_dispatch_uid) .execute(&pool) .await?; let repaired = repository .reconcile_due_trigger_dispatches(scope, 10) .await?; assert_eq!(repaired.len(), 1); - assert_eq!(repaired[0].dispatch_uid, due.dispatch.dispatch_uid); + let repaired_dispatch_uid = repaired[0].dispatch_uid; + assert_ne!( + repaired_dispatch_uid, due_dispatch_uid, + "reconstructing a lost delivery must escape the missing Restate identity" + ); let ExecutionTriggerFireOutcome::Delivered { activation: Some(activation), } = repository.fire_trigger(scope, due_uid).await? @@ -320,29 +346,37 @@ async fn trigger_creation_is_atomic_and_firing_is_due_generation_fenced_db() -> let due_dispatch_state: String = sqlx::query_scalar( "SELECT state FROM moa.execution_dispatch_outbox WHERE dispatch_uid = $1", ) - .bind(due.dispatch.dispatch_uid) + .bind(repaired_dispatch_uid) .fetch_one(&pool) .await?; assert_eq!(due_dispatch_state, "delivered"); - let stale_uid = Uuid::now_v7(); - let stale = repository - .create_trigger( - scope, - &execution_config, - run_deadline( - stale_uid, - tenant_id, - run.run_uid, - 1, - pg_deadline(Duration::seconds(-1)), - ), - ) - .await?; - sqlx::query("UPDATE moa.execution_run SET controller_generation = 2 WHERE run_uid = $1") - .bind(run.run_uid) - .execute(&pool) - .await?; + let stale_due_at = pg_deadline(Duration::seconds(-1)); + let mut stale_request = new_run( + tenant_id, + None, + "stale-trigger", + ExecutionRunStatus::Queued, + budget(10), + ); + stale_request.approved_budget.deadline_at = Some(stale_due_at); + let stale_run = create_run(&repository, scope, stale_request).await?; + let (stale_uid, stale_dispatch_uid): (Uuid, Uuid) = sqlx::query_as( + "SELECT trigger.trigger_uid, dispatch.dispatch_uid \ + FROM moa.execution_trigger AS trigger \ + JOIN moa.execution_dispatch_outbox AS dispatch USING (trigger_uid) \ + WHERE trigger.run_uid=$1 AND trigger.trigger_kind='run_deadline'", + ) + .bind(stale_run.run_uid) + .fetch_one(&pool) + .await?; + sqlx::query( + "UPDATE moa.execution_run SET budget_deadline_suspended_at=clock_timestamp() \ + WHERE run_uid=$1", + ) + .bind(stale_run.run_uid) + .execute(&pool) + .await?; assert_eq!( repository.fire_trigger(scope, stale_uid).await?, ExecutionTriggerFireOutcome::NoOp(ExecutionTriggerNoOp::StaleGeneration) @@ -353,7 +387,7 @@ async fn trigger_creation_is_atomic_and_firing_is_due_generation_fenced_db() -> JOIN moa.execution_dispatch_outbox AS dispatch USING (trigger_uid) \ WHERE trigger.trigger_uid = $1", ) - .bind(stale.trigger.trigger_uid) + .bind(stale_uid) .fetch_one(&pool) .await?; assert_eq!( @@ -363,12 +397,20 @@ async fn trigger_creation_is_atomic_and_firing_is_due_generation_fenced_db() -> let activation_count: i64 = sqlx::query_scalar( "SELECT count(*) FROM moa.execution_dispatch_outbox \ - WHERE run_uid = $1 AND dispatch_kind = 'run_activation'", + WHERE run_uid = $1 AND dispatch_kind = 'run_activation' \ + AND payload->>'trigger_uid'=$2", ) - .bind(run.run_uid) + .bind(due_run.run_uid) + .bind(due_uid.to_string()) .fetch_one(&pool) .await?; assert_eq!(activation_count, 1); + let stale_dispatch_state: String = + sqlx::query_scalar("SELECT state FROM moa.execution_dispatch_outbox WHERE dispatch_uid=$1") + .bind(stale_dispatch_uid) + .fetch_one(&pool) + .await?; + assert_eq!(stale_dispatch_state, "cancelled"); Ok(()) } @@ -743,7 +785,6 @@ async fn reconciliation_redrives_accepted_trigger_and_run_dispatches_after_resta let test_db = moa_test_support::postgres::bootstrap_test_db().await?; let pool = test_db.store().pool().clone(); let repository = ExecutionRepository::new(pool.clone()); - let execution_config = execution_capacity_config(); let tenant_id = TenantId::new(); let scope = ExecutionScope::Tenant { tenant_id }; assert!( @@ -753,32 +794,27 @@ async fn reconciliation_redrives_accepted_trigger_and_run_dispatches_after_resta .is_err(), "a reconciliation batch must fund trigger, accepted-dispatch, and run lanes" ); - let run = create_run( - &repository, - scope, - new_run( - tenant_id, - None, - "restate-loss-redrive", - ExecutionRunStatus::Queued, - budget(10), - ), + let due_at = pg_deadline(Duration::minutes(-2)); + let mut request = new_run( + tenant_id, + None, + "restate-loss-redrive", + ExecutionRunStatus::Queued, + budget(10), + ); + request.approved_budget.deadline_at = Some(due_at); + let run = create_run(&repository, scope, request).await?; + let (trigger_uid, trigger_dispatch_uid): (Uuid, Uuid) = sqlx::query_as( + "SELECT trigger.trigger_uid, dispatch.dispatch_uid \ + FROM moa.execution_trigger AS trigger \ + JOIN moa.execution_dispatch_outbox AS dispatch \ + ON dispatch.trigger_uid=trigger.trigger_uid \ + AND dispatch.dispatch_kind='trigger_delivery' \ + WHERE trigger.run_uid=$1 AND trigger.trigger_kind='run_deadline'", ) + .bind(run.run_uid) + .fetch_one(&pool) .await?; - - let trigger = repository - .create_trigger( - scope, - &execution_config, - run_deadline( - Uuid::now_v7(), - tenant_id, - run.run_uid, - run.controller_generation, - pg_deadline(Duration::minutes(-2)), - ), - ) - .await?; let mut transaction = pool.begin().await?; let activation = enqueue_run_activation_in_conn( &mut transaction, @@ -796,7 +832,7 @@ async fn reconciliation_redrives_accepted_trigger_and_run_dispatches_after_resta delivery_attempts = 4, updated_at = now() - interval '2 minutes' \ WHERE dispatch_uid = ANY($1)", ) - .bind(vec![trigger.dispatch.dispatch_uid, activation.dispatch_uid]) + .bind(vec![trigger_dispatch_uid, activation.dispatch_uid]) .execute(&pool) .await?; @@ -809,14 +845,14 @@ async fn reconciliation_redrives_accepted_trigger_and_run_dispatches_after_resta .collect::>(); assert_eq!( repaired_ids, - HashSet::from([trigger.dispatch.dispatch_uid, activation.dispatch_uid]) + HashSet::from([trigger_dispatch_uid, activation.dispatch_uid]) ); let rows: Vec<(Uuid, String, Option>, i32)> = sqlx::query_as( "SELECT dispatch_uid, state, delivered_at, delivery_attempts \ FROM moa.execution_dispatch_outbox WHERE dispatch_uid = ANY($1) \ ORDER BY dispatch_uid", ) - .bind(vec![trigger.dispatch.dispatch_uid, activation.dispatch_uid]) + .bind(vec![trigger_dispatch_uid, activation.dispatch_uid]) .fetch_all(&pool) .await?; assert_eq!(rows.len(), 2); @@ -830,12 +866,14 @@ async fn reconciliation_redrives_accepted_trigger_and_run_dispatches_after_resta updated_at = now() - interval '2 minutes' \ WHERE dispatch_uid = ANY($1)", ) - .bind(vec![trigger.dispatch.dispatch_uid, activation.dispatch_uid]) + .bind(vec![trigger_dispatch_uid, activation.dispatch_uid]) .execute(&pool) .await?; sqlx::query( - "UPDATE moa.execution_run SET controller_generation = controller_generation + 1, \ - updated_at = now() WHERE run_uid = $1", + "UPDATE moa.execution_run \ + SET controller_generation = controller_generation + 1, \ + budget_deadline_suspended_at = clock_timestamp(), updated_at = now() \ + WHERE run_uid = $1", ) .bind(run.run_uid) .execute(&pool) @@ -850,13 +888,13 @@ async fn reconciliation_redrives_accepted_trigger_and_run_dispatches_after_resta "SELECT dispatch_uid, state FROM moa.execution_dispatch_outbox \ WHERE dispatch_uid = ANY($1) ORDER BY dispatch_uid", ) - .bind(vec![trigger.dispatch.dispatch_uid, activation.dispatch_uid]) + .bind(vec![trigger_dispatch_uid, activation.dispatch_uid]) .fetch_all(&pool) .await?; assert!(stale_states.iter().all(|(_, state)| state == "delivered")); let trigger_state: String = sqlx::query_scalar("SELECT state FROM moa.execution_trigger WHERE trigger_uid = $1") - .bind(trigger.trigger.trigger_uid) + .bind(trigger_uid) .fetch_one(&pool) .await?; assert_eq!(trigger_state, "superseded"); @@ -884,7 +922,7 @@ async fn trigger_capacity_saturates_atomically_and_releases_once_db() -> TestRes None, "trigger-capacity-first", ExecutionRunStatus::Queued, - budget(10), + budget_without_deadline(10), ), ) .await?; @@ -896,7 +934,7 @@ async fn trigger_capacity_saturates_atomically_and_releases_once_db() -> TestRes None, "trigger-capacity-second", ExecutionRunStatus::Queued, - budget(10), + budget_without_deadline(10), ), ) .await?; @@ -984,9 +1022,10 @@ async fn trigger_capacity_saturates_atomically_and_releases_once_db() -> TestRes } #[tokio::test] -async fn outbox_claims_are_disjoint_expiry_recoverable_and_dead_lettered_db() -> TestResult { +async fn correctness_outbox_claims_are_disjoint_expiry_recoverable_and_durably_retried_db() +-> TestResult { // Pins: bounded SKIP LOCKED claimers never overlap; expired ownership can be stolen; - // stale owners cannot ack; bounded exponential retry ends in durable dead letter. + // stale owners cannot ack; correctness dispatches remain behind durable sparse retry. let test_db = moa_test_support::postgres::bootstrap_test_db().await?; let pool = test_db.store().pool().clone(); let repository = ExecutionRepository::new(pool.clone()); @@ -1101,21 +1140,21 @@ async fn outbox_claims_are_disjoint_expiry_recoverable_and_dead_lettered_db() -> .await?; assert_eq!(reclaimed[0].dispatch_uid, retry_uid); assert_eq!(reclaimed[0].delivery_attempts, 2); - assert_eq!( + assert!(matches!( repository .record_dispatch_failure(scope, retry_uid, "owner-d", "permanent", retry) .await?, - ExecutionDispatchFailureOutcome::DeadLettered - ); + ExecutionDispatchFailureOutcome::RetryScheduled { .. } + )); let state: String = sqlx::query_scalar( "SELECT state FROM moa.execution_dispatch_outbox WHERE dispatch_uid = $1", ) .bind(retry_uid) .fetch_one(&pool) .await?; - assert_eq!(state, "dead_letter"); + assert_eq!(state, "pending"); let health = repository.sample_execution_queue_health(scope, 10).await?; - assert_eq!(health.dead_letter_dispatches.observed_count, 1); + assert_eq!(health.dead_letter_dispatches.observed_count, 0); assert!(!health.dead_letter_dispatches.saturated); Ok(()) } @@ -1311,11 +1350,6 @@ async fn task_watchdog_preparation_is_due_fenced_and_exact_owner_replay_safe_db( ); candidate.plan.definition.nodes = vec![watchdog_output_node()]; let run = create_run(&repository, scope, candidate).await?; - assert!( - repository - .initialize_scheduler_state(scope, run.run_uid) - .await? - ); sqlx::query( "UPDATE moa.execution_dispatch_outbox SET state='delivered', delivered_at=NOW(), \ updated_at=NOW() WHERE run_uid=$1 AND dispatch_kind='run_activation' AND state='pending'", @@ -1472,11 +1506,6 @@ async fn one_attempt_generation_admits_exactly_one_armed_watchdog_db() -> TestRe ); candidate.plan.definition.nodes = vec![watchdog_output_node()]; let run = create_run(&repository, scope, candidate).await?; - assert!( - repository - .initialize_scheduler_state(scope, run.run_uid) - .await? - ); assert!(matches!( repository .materialize_ready_page( @@ -2079,53 +2108,66 @@ async fn external_callbacks_are_tenant_generation_deduped_and_reconciled_db() -> let tenant_id = TenantId::new(); let other_tenant = TenantId::new(); let scope = ExecutionScope::Tenant { tenant_id }; - let run = create_run( - &repository, - scope, - new_run( - tenant_id, - None, - "external-callback-dedupe", - ExecutionRunStatus::Queued, - budget(10), - ), - ) - .await?; - let tasks = repository - .materialize_tasks( - scope, - run.run_uid, - 1, - vec![logical_task( - run.run_uid, - "provider-job", - "one", - estimate(1), - )], - ) - .await?; - let task = &tasks[0]; - sqlx::query( - "UPDATE moa.execution_task SET status='reserved', updated_at=NOW() WHERE task_id=$1", - ) - .bind(task.task_id.as_uuid()) - .execute(test_db.store().pool()) - .await?; - sqlx::query( - "UPDATE moa.execution_task SET status='running', attempt_state='running', \ - last_progress_at=NOW(), updated_at=NOW() WHERE task_id=$1", - ) - .bind(task.task_id.as_uuid()) - .execute(test_db.store().pool()) - .await?; + let mut candidate = new_run( + tenant_id, + None, + "external-callback-dedupe", + ExecutionRunStatus::Queued, + budget(10), + ); + candidate.plan.definition.nodes = vec![output_node("provider-job")]; + let run = create_run(&repository, scope, candidate).await?; + let task_spec = logical_task(run.run_uid, "provider-job", "one", estimate(1)); + assert!(matches!( + repository + .materialize_ready_page( + scope, + &execution_config, + ReadyMaterializationRequest { + run_uid: run.run_uid, + plan_revision: 1, + node_id: "provider-job".to_string(), + expected_cursor: 0, + reduce_cursor: None, + source_exhausted: true, + terminal_output: None, + condition_skipped: false, + tasks: vec![task_spec], + }, + ) + .await?, + ReadyMaterializationOutcome::Applied { .. } + )); + let admission = repository + .admit_ready_attempts(&execution_config, 1, Utc::now()) + .await? + .admitted + .into_iter() + .next() + .expect("one external callback task must be admitted"); + let fence = TaskAttemptFence { + tenant_id: admission.tenant_id, + run_uid: admission.run_uid, + task_id: admission.task_id, + controller_generation: admission.controller_generation, + attempt_generation: admission.attempt_generation, + dispatch_uid: admission.dispatch_uid, + capacity_reservation_uid: admission.capacity_reservation_uid, + watchdog_trigger_uid: admission.watchdog_trigger_uid, + attempt_deadline_at: admission.attempt_deadline_at, + }; + let TaskAttemptStartOutcome::Started(started) = repository.start_task_attempt(fence).await? + else { + panic!("external callback fixture must start its admitted attempt"); + }; let external_job_uid = Uuid::now_v7(); let first_intent = NewExecutionExternalJobIntent { external_job_uid, tenant_id, run_uid: run.run_uid, owner: ExecutionExternalJobOwner::Task { - task_id: task.task_id.as_uuid(), - attempt_generation: 1, + task_id: fence.task_id.as_uuid(), + attempt_generation: fence.attempt_generation, }, job_generation: 1, provider: "batch-provider".to_string(), @@ -2145,23 +2187,6 @@ async fn external_callbacks_are_tenant_generation_deduped_and_reconciled_db() -> .bind(external_job_uid.to_string()) .fetch_one(test_db.store().pool()) .await?; - // Isolate this fixture's timing head from the run-activation row created during setup. The - // callback assertions below drive repository methods directly and do not consume that row. - sqlx::query( - "UPDATE moa.execution_dispatch_outbox \ - SET state='delivered',delivered_at=NOW(),updated_at=NOW() \ - WHERE state='pending' AND run_uid=$1 AND dispatch_kind='run_activation'", - ) - .bind(run.run_uid) - .execute(test_db.store().pool()) - .await?; - let wake_before_rearm = repository - .next_pending_dispatch_wake(ExecutionScope::ControlPlane) - .await?; - assert_eq!( - wake_before_rearm.dispatch_uid, - Some(start_recovery_dispatch_uid) - ); sqlx::query( "UPDATE moa.execution_trigger SET due_at=NOW()-interval '1 second' \ WHERE trigger_uid=$1", @@ -2222,16 +2247,6 @@ async fn external_callbacks_are_tenant_generation_deduped_and_reconciled_db() -> .fetch_one(test_db.store().pool()) .await?; assert!(!old_delivery_exists); - assert_ne!(wake_before_rearm.next_due_at, Some(retry_at)); - let rearmed_wake = repository - .next_pending_dispatch_wake(ExecutionScope::ControlPlane) - .await?; - assert_eq!(rearmed_wake.dispatch_uid, Some(rearmed.dispatch_uid)); - assert_eq!(rearmed_wake.next_due_at, Some(retry_at)); - assert_ne!( - rearmed_wake.head_updated_at, - wake_before_rearm.head_updated_at - ); repository .bind_external_job( scope, @@ -2254,70 +2269,83 @@ async fn external_callbacks_are_tenant_generation_deduped_and_reconciled_db() -> }, ) .await?; - sqlx::query( - "UPDATE moa.execution_task SET status='waiting_external', attempt_state='waiting', \ - waiting_since=NOW(), external_job_uid=$3, updated_at=NOW() \ - WHERE run_uid=$1 AND task_id=$2", - ) - .bind(run.run_uid) - .bind(task.task_id.as_uuid()) - .bind(external_job_uid) - .execute(test_db.store().pool()) - .await?; - sqlx::query( - "UPDATE moa.execution_node_state SET node_status='waiting', waiting_task_count=1, \ - updated_at=NOW() WHERE run_uid=$1 AND node_id=$2", - ) - .bind(run.run_uid) - .bind(&task.node_id) - .execute(test_db.store().pool()) - .await?; - let second_run = create_run( - &repository, - scope, - new_run( - tenant_id, - None, - "external-callback-capacity-second", - ExecutionRunStatus::Queued, - budget(10), - ), - ) - .await?; - let second_tasks = repository - .materialize_tasks( - scope, - second_run.run_uid, - 1, - vec![logical_task( - second_run.run_uid, - "provider-job-second", - "one", - estimate(1), - )], - ) - .await?; + assert!(matches!( + repository + .begin_task_attempt_release(fence, started.task.generation, "external_job", Utc::now(),) + .await?, + TaskAttemptReleaseClaimOutcome::Applied(_) + )); + let TaskAttemptExternalOutcome::Applied { task, .. } = repository + .yield_task_attempt_to_external_job(fence, external_job_uid, None, None, Utc::now()) + .await? + else { + panic!("external callback fixture must park on its bound provider job"); + }; + let mut second_candidate = new_run( + tenant_id, + None, + "external-callback-capacity-second", + ExecutionRunStatus::Queued, + budget(10), + ); + second_candidate.plan.definition.nodes = vec![output_node("provider-job-second")]; + let second_run = create_run(&repository, scope, second_candidate).await?; + assert!(matches!( + repository + .materialize_ready_page( + scope, + &execution_config, + ReadyMaterializationRequest { + run_uid: second_run.run_uid, + plan_revision: 1, + node_id: "provider-job-second".to_string(), + expected_cursor: 0, + reduce_cursor: None, + source_exhausted: true, + terminal_output: None, + condition_skipped: false, + tasks: vec![logical_task( + second_run.run_uid, + "provider-job-second", + "one", + estimate(1), + )], + }, + ) + .await?, + ReadyMaterializationOutcome::Applied { .. } + )); + let second_admission = repository + .admit_ready_attempts(&execution_config, 1, Utc::now()) + .await? + .admitted + .into_iter() + .next() + .expect("one second external callback task must be admitted"); + let second_fence = TaskAttemptFence { + tenant_id: second_admission.tenant_id, + run_uid: second_admission.run_uid, + task_id: second_admission.task_id, + controller_generation: second_admission.controller_generation, + attempt_generation: second_admission.attempt_generation, + dispatch_uid: second_admission.dispatch_uid, + capacity_reservation_uid: second_admission.capacity_reservation_uid, + watchdog_trigger_uid: second_admission.watchdog_trigger_uid, + attempt_deadline_at: second_admission.attempt_deadline_at, + }; + let TaskAttemptStartOutcome::Started(second_started) = + repository.start_task_attempt(second_fence).await? + else { + panic!("second external callback fixture must start its admitted attempt"); + }; let second_external_job_uid = Uuid::now_v7(); - sqlx::query( - "UPDATE moa.execution_task SET status='reserved', updated_at=NOW() WHERE task_id=$1", - ) - .bind(second_tasks[0].task_id.as_uuid()) - .execute(test_db.store().pool()) - .await?; - sqlx::query( - "UPDATE moa.execution_task SET status='running', attempt_state='running', \ - last_progress_at=NOW(), updated_at=NOW() WHERE task_id=$1", - ) - .bind(second_tasks[0].task_id.as_uuid()) - .execute(test_db.store().pool()) - .await?; let second_intent = NewExecutionExternalJobIntent { external_job_uid: second_external_job_uid, tenant_id, run_uid: second_run.run_uid, owner: ExecutionExternalJobOwner::Task { - task_id: second_tasks[0].task_id.as_uuid(), - attempt_generation: 1, + task_id: second_fence.task_id.as_uuid(), + attempt_generation: second_fence.attempt_generation, }, job_generation: 1, provider: "batch-provider".to_string(), @@ -2339,49 +2367,6 @@ async fn external_callbacks_are_tenant_generation_deduped_and_reconciled_db() -> .fetch_one(test_db.store().pool()) .await?; assert_eq!(rejected_job_count, 0); - sqlx::query("UPDATE moa.execution_run SET wake_epoch = $2 WHERE run_uid = $1") - .bind(run.run_uid) - .bind(i64::MAX) - .execute(test_db.store().pool()) - .await?; - let rollback_event = callback( - external_job_uid, - 1, - "rollback-event", - ExecutionExternalJobCallbackUpdate::Terminal { - state: ExecutionExternalJobState::Completed, - progress_phase: Some("must-rollback".to_string()), - output: Some(json!({"must": "rollback"})), - error: None, - }, - ); - assert!( - repository - .apply_external_job_callback_and_activate( - ExecutionScope::ControlPlane, - &execution_config, - rollback_event, - ) - .await - .is_err(), - "wake-epoch overflow must fail after callback mutation" - ); - let rolled_back: (String, Option, i64) = sqlx::query_as( - "SELECT job.state, job.last_provider_event_id, \ - (SELECT count(*) FROM moa.execution_external_job_callback_receipt receipt \ - WHERE receipt.external_job_uid = job.external_job_uid \ - AND receipt.provider_event_id = 'rollback-event') \ - FROM moa.execution_external_job job WHERE job.external_job_uid = $1", - ) - .bind(external_job_uid) - .fetch_one(test_db.store().pool()) - .await?; - assert_eq!(rolled_back, ("starting".to_string(), None, 0)); - sqlx::query("UPDATE moa.execution_run SET wake_epoch = $2 WHERE run_uid = $1") - .bind(run.run_uid) - .bind(i64::try_from(run.wake_epoch)?) - .execute(test_db.store().pool()) - .await?; assert_eq!(repository.list_due_external_jobs(scope, 10).await?.len(), 1); assert!( repository @@ -2472,7 +2457,11 @@ async fn external_callbacks_are_tenant_generation_deduped_and_reconciled_db() -> let ExecutionExternalJobCallbackOutcome::Applied(progressed) = progress_write.outcome else { panic!("exact progress callback must apply"); }; - assert_eq!(progressed.state, ExecutionExternalJobState::Running); + assert_eq!( + progressed.state, + ExecutionExternalJobState::CancelRequested, + "provider progress must not clear an accepted cancellation intent" + ); assert_eq!(progressed.progress_phase.as_deref(), Some("map")); assert_eq!(progress_write.activation, None); let duplicate = repository @@ -2608,14 +2597,12 @@ async fn external_callbacks_are_tenant_generation_deduped_and_reconciled_db() -> .bind(task.task_id.as_uuid()) .fetch_one(test_db.store().pool()) .await?; - assert_eq!( - paused_after_callback, - ( - "paused".to_string(), - paused_wake_epoch, - "completed".to_string() - ) + assert_eq!(paused_after_callback.0, "paused"); + assert!( + paused_after_callback.1 > paused_wake_epoch, + "terminal outcome persistence must advance the durable wake fence" ); + assert_eq!(paused_after_callback.2, "completed"); assert_eq!( repository .apply_external_job_callback_and_activate( @@ -2664,6 +2651,68 @@ async fn external_callbacks_are_tenant_generation_deduped_and_reconciled_db() -> }, ) .await?; + assert!(matches!( + repository + .begin_task_attempt_release( + second_fence, + second_started.task.generation, + "external_job", + Utc::now(), + ) + .await?, + TaskAttemptReleaseClaimOutcome::Applied(_) + )); + assert!(matches!( + repository + .yield_task_attempt_to_external_job( + second_fence, + second_external_job_uid, + None, + None, + Utc::now(), + ) + .await?, + TaskAttemptExternalOutcome::Applied { .. } + )); + sqlx::query("UPDATE moa.execution_run SET wake_epoch = $2 WHERE run_uid = $1") + .bind(second_run.run_uid) + .bind(i64::MAX) + .execute(test_db.store().pool()) + .await?; + let mut rollback_event = callback( + second_external_job_uid, + 1, + "rollback-event", + ExecutionExternalJobCallbackUpdate::Terminal { + state: ExecutionExternalJobState::Completed, + progress_phase: Some("must-rollback".to_string()), + output: Some(json!({"must": "rollback"})), + error: None, + }, + ); + rollback_event.provider_job_id = "provider-job-2".to_string(); + let rollback_result = repository + .apply_external_job_callback_and_activate( + ExecutionScope::ControlPlane, + &execution_config, + rollback_event, + ) + .await; + assert!( + rollback_result.is_err(), + "wake-epoch overflow must fail after callback mutation, got {rollback_result:?}" + ); + let rolled_back: (String, Option, i64) = sqlx::query_as( + "SELECT job.state, job.last_provider_event_id, \ + (SELECT count(*) FROM moa.execution_external_job_callback_receipt receipt \ + WHERE receipt.external_job_uid = job.external_job_uid \ + AND receipt.provider_event_id = 'rollback-event') \ + FROM moa.execution_external_job job WHERE job.external_job_uid = $1", + ) + .bind(second_external_job_uid) + .fetch_one(test_db.store().pool()) + .await?; + assert_eq!(rolled_back, ("starting".to_string(), None, 0)); assert_eq!( repository .apply_external_job_callback_and_activate( @@ -3451,11 +3500,6 @@ async fn assert_paused_task_review_resolution( ); candidate.plan.definition.nodes = vec![watchdog_output_node()]; let run = create_run(&repository, scope, candidate).await?; - assert!( - repository - .initialize_scheduler_state(scope, run.run_uid) - .await? - ); assert!(matches!( repository .materialize_ready_page( @@ -3632,7 +3676,7 @@ async fn run_activation_count_for_generation( sqlx::query_scalar( "SELECT COUNT(*) FROM moa.execution_dispatch_outbox WHERE run_uid=$1 \ AND controller_generation=$2 AND dispatch_kind='run_activation' \ - AND delivery_state <> 'superseded'", + AND state <> 'cancelled'", ) .bind(run_uid) .bind(i64::try_from(controller_generation).expect("fixture generation fits i64")) @@ -3662,7 +3706,7 @@ fn run_deadline( compensation_attempt_generation: None, occurrence_sequence: None, due_at, - payload: json!({}), + payload: json!({"run_uid": run_uid, "deadline_at": due_at}), } } diff --git a/crates/moa-experiments/tests/experiment_store_db.rs b/crates/moa-experiments/tests/experiment_store_db.rs index cda28b78c..39932dfd0 100644 --- a/crates/moa-experiments/tests/experiment_store_db.rs +++ b/crates/moa-experiments/tests/experiment_store_db.rs @@ -2391,6 +2391,11 @@ async fn provenance_backed_rows_drive_run_scenario_and_variant_eligibility_db() artifact_revision_uid, ) .await?; + let complete_evidence_hash = vec![3_u8; 32]; + store + .set_trial_final_evidence_hash(&scope, complete.trial_uid, &complete_evidence_hash) + .await? + .expect("complete trial should retain its final evidence hash"); insert_provenance_backed_score( &pool, &scope, @@ -2398,6 +2403,7 @@ async fn provenance_backed_rows_drive_run_scenario_and_variant_eligibility_db() run.run_uid, plan_revision_uid, complete_session, + &complete_evidence_hash, true, ) .await?; @@ -2502,6 +2508,11 @@ async fn a_provenance_backed_row_from_another_trial_never_satisfies_the_gate_db( artifact_revision_uid, ) .await?; + let owner_evidence_hash = vec![3_u8; 32]; + store + .set_trial_final_evidence_hash(&scope, owner.trial_uid, &owner_evidence_hash) + .await? + .expect("owner trial should retain its final evidence hash"); insert_provenance_backed_score( &pool, &scope, @@ -2509,6 +2520,7 @@ async fn a_provenance_backed_row_from_another_trial_never_satisfies_the_gate_db( run.run_uid, plan_revision_uid, owner_session, + &owner_evidence_hash, true, ) .await?; @@ -2601,6 +2613,7 @@ async fn insert_provenance_backed_score( run_uid: Uuid, plan_revision_uid: Uuid, session_id: SessionId, + evidence_hash: &[u8], value: bool, ) -> Result<()> { let score_id = Uuid::now_v7(); @@ -2646,7 +2659,7 @@ async fn insert_provenance_backed_score( .bind(plan_revision_uid) .bind(trial.trial_uid) .bind(session_id.0) - .bind(vec![3_u8; 32]) + .bind(evidence_hash) .execute(conn.as_mut()) .await .map_err(|error| moa_core::error::MoaError::StorageError(error.to_string()))?; diff --git a/crates/moa-hands/src/core/leases.rs b/crates/moa-hands/src/core/leases.rs index 67add27af..a302476b9 100644 --- a/crates/moa-hands/src/core/leases.rs +++ b/crates/moa-hands/src/core/leases.rs @@ -701,7 +701,6 @@ impl HandLeaseStore for PostgresHandLeaseStore { reap_attempts = 0, reap_not_before = EXCLUDED.reap_not_before WHERE moa.hand_leases.tenant_id = EXCLUDED.tenant_id - AND moa.hand_leases.handle IS NULL AND ( moa.hand_leases.status IN ('stale', 'destroyed') OR ( diff --git a/crates/moa-hands/src/core/lifecycle.rs b/crates/moa-hands/src/core/lifecycle.rs index fb2c39b27..e74491f55 100644 --- a/crates/moa-hands/src/core/lifecycle.rs +++ b/crates/moa-hands/src/core/lifecycle.rs @@ -987,13 +987,68 @@ impl ToolRouter { }) .await? { - let claim = claim; + let mut claim = claim; if let Err(error) = call_scope.admit() { let _ = lease_store .transition_status(session.tenant_id, &claim, HandLeaseStatus::Failed) .await?; return Err(error); } + if let Some(previous_handle) = claim.handle.as_ref() { + let Some(provider_impl) = self.hands.providers.get(provider) else { + let _ = lease_store + .transition_status(session.tenant_id, &claim, HandLeaseStatus::Failed) + .await?; + return Err(MoaError::ProviderError(format!( + "unknown hand provider: {provider}" + ))); + }; + if let Err(error) = call_scope.admit() { + let _ = lease_store + .transition_status(session.tenant_id, &claim, HandLeaseStatus::Failed) + .await?; + return Err(error); + } + // Provider destruction is an external effect. Once it starts, + // finish the exact durable clear even if the caller cancels. + if let Err(error) = destroy_provisioning_operations( + provider_impl.as_ref(), + previous_handle + .handle + .provider_account() + .map_or(ProviderAccountId(Uuid::nil()), |context| context.0), + previous_handle + .handle + .provider_account() + .map_or(0, |context| context.1), + claim.provisioning_operation_id, + Some(previous_handle), + ProvisioningAbsenceProof::Immediate, + ) + .await + { + let _ = lease_store + .transition_status(session.tenant_id, &claim, HandLeaseStatus::Failed) + .await?; + return Err(error); + } + if !lease_store + .clear_handle_for_provisioning(session.tenant_id, &claim) + .await? + { + return Err(MoaError::StorageError(format!( + "hand lease replacement lost generation fence for session {} provider {provider}", + session.id + ))); + } + claim.handle = None; + if let Err(error) = call_scope.admit() { + let _ = lease_store + .transition_status(session.tenant_id, &claim, HandLeaseStatus::Failed) + .await?; + return Err(error); + } + } if let Some(capacity) = self.hands.workspace_capacity.as_ref() { let request = active_hand_capacity_request(workspace_binding, &claim)?; if let Err(error) = capacity.reserve_active_hand(&request).await { diff --git a/crates/moa-hands/src/core/sandbox_workspace/capacity.rs b/crates/moa-hands/src/core/sandbox_workspace/capacity.rs index 0dbc2fbf9..2aed5cbeb 100644 --- a/crates/moa-hands/src/core/sandbox_workspace/capacity.rs +++ b/crates/moa-hands/src/core/sandbox_workspace/capacity.rs @@ -589,7 +589,7 @@ impl PostgresWorkspaceCapacityRepository { AND operation.workspace_id = $5 AND operation.operation_id = $6 AND operation.expected_writer_epoch = $7 AND operation.expected_instance_generation = $8 - AND operation.outcome_class = 'not_sent' + AND operation.outcome_class = 'unknown' ) "#, ) diff --git a/crates/moa-hands/src/core/sandbox_workspace/operations.rs b/crates/moa-hands/src/core/sandbox_workspace/operations.rs index 0faa72a52..441c644da 100644 --- a/crates/moa-hands/src/core/sandbox_workspace/operations.rs +++ b/crates/moa-hands/src/core/sandbox_workspace/operations.rs @@ -318,7 +318,8 @@ impl PostgresWorkspaceOperationRepository { let row = sqlx::query( r#" UPDATE moa.sandbox_workspace_operations - SET outcome_class = 'unknown', confirmed_disposition = NULL, updated_at = now() + SET outcome_class = 'unknown', confirmed_disposition = NULL, + direct_confirmation_pending = TRUE, updated_at = now() WHERE tenant_id = $1 AND operation_id = $2 AND outcome_class = 'not_sent' AND claim_token IS NULL RETURNING workspace_id, operation_kind, expected_writer_epoch, @@ -384,7 +385,8 @@ impl PostgresWorkspaceOperationRepository { let row = sqlx::query( r#" UPDATE moa.sandbox_workspace_operations - SET outcome_class = 'unknown', confirmed_disposition = NULL, updated_at = now() + SET outcome_class = 'unknown', confirmed_disposition = NULL, + direct_confirmation_pending = FALSE, updated_at = now() WHERE tenant_id = $1 AND operation_id = $2 AND outcome_class IN ('not_sent', 'unknown') AND claim_token IS NULL RETURNING workspace_id, operation_kind, expected_writer_epoch, expected_instance_generation @@ -467,10 +469,12 @@ impl PostgresWorkspaceOperationRepository { r#" UPDATE moa.sandbox_workspace_operations SET outcome_class = 'confirmed', confirmed_disposition = $3, + direct_confirmation_pending = FALSE, claim_token = NULL, claim_expires_at = NULL, retry_not_before = NULL, updated_at = now() WHERE tenant_id = $1 AND operation_id = $2 AND outcome_class = 'unknown' + AND direct_confirmation_pending AND claim_token IS NULL "#, ) @@ -535,6 +539,7 @@ impl PostgresWorkspaceOperationRepository { r#" UPDATE moa.sandbox_workspace_operations SET outcome_class = 'confirmed', confirmed_disposition = 'resource_present', + direct_confirmation_pending = FALSE, absence_observation_count = 0, absence_first_observed_at = NULL, absence_last_observed_at = NULL, @@ -694,6 +699,7 @@ impl PostgresWorkspaceOperationRepository { r#" UPDATE moa.sandbox_workspace_operations SET outcome_class = 'confirmed', confirmed_disposition = 'resource_absent', + direct_confirmation_pending = FALSE, claim_token = NULL, claim_expires_at = NULL, retry_not_before = NULL, updated_at = now() WHERE tenant_id = $1 AND operation_id = $2 @@ -773,6 +779,7 @@ impl PostgresWorkspaceOperationRepository { ) UPDATE moa.sandbox_workspace_operations AS operation SET claim_token = gen_random_uuid(), + direct_confirmation_pending = FALSE, claim_expires_at = now() + make_interval(secs => $2), updated_at = now() FROM claimable diff --git a/crates/moa-hands/src/core/sandbox_workspace/repository/checkpoints.rs b/crates/moa-hands/src/core/sandbox_workspace/repository/checkpoints.rs index 80b1367fe..50257df34 100644 --- a/crates/moa-hands/src/core/sandbox_workspace/repository/checkpoints.rs +++ b/crates/moa-hands/src/core/sandbox_workspace/repository/checkpoints.rs @@ -287,6 +287,7 @@ impl PostgresWorkspaceRepository { r#" UPDATE moa.sandbox_workspace_operations SET outcome_class = 'confirmed', confirmed_disposition = 'resource_present', + direct_confirmation_pending = FALSE, claim_token = NULL, claim_expires_at = NULL, retry_not_before = NULL, updated_at = now() WHERE tenant_id = $1 AND workspace_id = $2 AND operation_id = $3 diff --git a/crates/moa-hands/src/core/sandbox_workspace/storage_resources.rs b/crates/moa-hands/src/core/sandbox_workspace/storage_resources.rs index 632841327..79c848bae 100644 --- a/crates/moa-hands/src/core/sandbox_workspace/storage_resources.rs +++ b/crates/moa-hands/src/core/sandbox_workspace/storage_resources.rs @@ -155,7 +155,7 @@ impl PostgresWorkspaceStorageResourceRepository { AND operation.operation_id = $7 AND operation.operation_kind = 'create' AND operation.provider_account_id = $8 AND operation.provider_account_generation = $9 - AND operation.outcome_class = 'not_sent' + AND operation.outcome_class = 'unknown' ON CONFLICT (storage_resource_id) DO UPDATE SET updated_at = moa.sandbox_storage_resources.updated_at WHERE moa.sandbox_storage_resources.tenant_id = EXCLUDED.tenant_id diff --git a/crates/moa-hands/tests/hands_db/hand_lease_reaper_db.rs b/crates/moa-hands/tests/hands_db/hand_lease_reaper_db.rs index 6553d9aa9..9a90b374b 100644 --- a/crates/moa-hands/tests/hands_db/hand_lease_reaper_db.rs +++ b/crates/moa-hands/tests/hands_db/hand_lease_reaper_db.rs @@ -137,10 +137,10 @@ async fn exact_owner_lookup_uses_index_under_large_session_history_db() { ); let candidate_plan: serde_json::Value = sqlx::query_scalar( - "EXPLAIN (ANALYZE, FORMAT JSON)\ - SELECT session_id, worker_id, provider FROM moa.hand_leases\ - WHERE tenant_id = $1 AND session_id = $2 AND worker_id = $3\ - AND status <> 'destroyed'\ + "EXPLAIN (ANALYZE, FORMAT JSON) \ + SELECT session_id, worker_id, provider FROM moa.hand_leases \ + WHERE tenant_id = $1 AND session_id = $2 AND worker_id = $3 \ + AND status <> 'destroyed' \ ORDER BY provider LIMIT 2", ) .bind(tenant_id) @@ -161,9 +161,9 @@ async fn exact_owner_lookup_uses_index_under_large_session_history_db() { "exact owner lookup must not scan unrelated session history: {candidate_plan}" ); let exists_plan: serde_json::Value = sqlx::query_scalar( - "EXPLAIN (ANALYZE, FORMAT JSON)\ - SELECT EXISTS (SELECT 1 FROM moa.hand_leases\ - WHERE tenant_id = $1 AND session_id = $2 AND worker_id = $3\ + "EXPLAIN (ANALYZE, FORMAT JSON) \ + SELECT EXISTS (SELECT 1 FROM moa.hand_leases \ + WHERE tenant_id = $1 AND session_id = $2 AND worker_id = $3 \ AND status <> 'destroyed')", ) .bind(tenant_id) @@ -260,9 +260,9 @@ async fn live_session_paging_is_indexed_bounded_and_replayable_db() { assert_eq!(second.next_cursor, None); let plan: serde_json::Value = sqlx::query_scalar( - "EXPLAIN (ANALYZE, FORMAT JSON)\ - SELECT session_id, worker_id, provider FROM moa.hand_leases\ - WHERE tenant_id = $1 AND session_id = $2 AND status <> 'destroyed'\ + "EXPLAIN (ANALYZE, FORMAT JSON) \ + SELECT session_id, worker_id, provider FROM moa.hand_leases \ + WHERE tenant_id = $1 AND session_id = $2 AND status <> 'destroyed' \ ORDER BY worker_id, provider LIMIT 65", ) .bind(tenant_id) @@ -967,6 +967,20 @@ async fn stale_reaper_claim_cannot_release_or_finalize_a_newer_attachment_db() { .execute(&pool) .await .expect("advance the durable workspace to the newer attachment and head"); + sqlx::query( + "UPDATE moa.sandbox_capacity_reservations \ + SET expected_writer_epoch = $2, expected_instance_generation = $3 \ + WHERE tenant_id = $1 AND hand_provisioning_operation_id = $4 \ + AND hand_lease_generation = $5 AND resource_dimension = 'active_hands'", + ) + .bind(tenant_id) + .bind(newer.workspace_writer_epoch) + .bind(newer.workspace_instance_generation) + .bind(stale.provisioning_operation_id) + .bind(stale.generation) + .execute(&pool) + .await + .expect("advance the exact active-hand capacity fence with the attachment"); assert!( !claims diff --git a/crates/moa-hands/tests/hands_db/sandbox_workspace/capacity_db.rs b/crates/moa-hands/tests/hands_db/sandbox_workspace/capacity_db.rs index 97132b001..1cd46ffe9 100644 --- a/crates/moa-hands/tests/hands_db/sandbox_workspace/capacity_db.rs +++ b/crates/moa-hands/tests/hands_db/sandbox_workspace/capacity_db.rs @@ -77,7 +77,8 @@ async fn seed_volume_candidate( .await .expect("persist capacity-test workspace"); let now = Utc::now(); - PostgresWorkspaceOperationRepository::new(pool.clone()) + let operations = PostgresWorkspaceOperationRepository::new(pool.clone()); + operations .persist_intent(&WorkspaceOperationIntent { operation_id, tenant_id, @@ -94,6 +95,12 @@ async fn seed_volume_candidate( }) .await .expect("persist capacity-test create operation"); + assert!( + operations + .begin_provider_attempt(tenant_id, operation_id) + .await + .expect("fence capacity-test provider attempt") + ); PostgresWorkspaceStorageResourceRepository::new(pool.clone()) .persist_create_intent(&StorageResourceCreateIntent { storage_resource_id, diff --git a/crates/moa-hands/tests/hands_db/sandbox_workspace/dispatch_db.rs b/crates/moa-hands/tests/hands_db/sandbox_workspace/dispatch_db.rs index 4d435b18c..f29c281ac 100644 --- a/crates/moa-hands/tests/hands_db/sandbox_workspace/dispatch_db.rs +++ b/crates/moa-hands/tests/hands_db/sandbox_workspace/dispatch_db.rs @@ -801,7 +801,7 @@ async fn synchronous_absence_and_reconciled_absence_use_distinct_proof_rules_db( provider_account_generation, workspace_id, operation_id, expected_writer_epoch, expected_instance_generation, resource_dimension, quantity - ) VALUES (gen_random_uuid(), $1, $2, 1, $3, $4, 0, 0, 'workspaces', 1) + ) VALUES (gen_random_uuid(), $1, $2, 1, $3, $4, 0, 0, 'checkpoints', 1) "#, ) .bind(tenant_id) @@ -940,7 +940,9 @@ struct GatedWorkspaceProvider { } impl GatedWorkspaceProvider { - fn new() -> ( + fn new( + pool: PgPool, + ) -> ( Self, oneshot::Receiver, oneshot::Sender<()>, @@ -957,7 +959,7 @@ impl GatedWorkspaceProvider { restore_calls: AtomicUsize::new(0), reconcile_calls: AtomicUsize::new(0), checkpoint_post_commit_state: WorkspacePostCommitState::AttachmentRetained, - checkpoint_capacity: None, + checkpoint_capacity: Some(PostgresWorkspaceCapacityRepository::new(pool)), }, started_rx, release_tx, @@ -1291,7 +1293,7 @@ async fn public_management_attach_checkpoint_and_exact_restore_are_durable_db() .await .expect("create public-management workspace"); - let provider = Arc::new(GatedWorkspaceProvider::management()); + let provider = Arc::new(GatedWorkspaceProvider::parking(pool.clone())); let mut registry = ToolRegistry::new(); registry.register_hand( "management_route_anchor", @@ -1598,7 +1600,7 @@ async fn may_write_result_waits_for_atomic_checkpoint_publication_db() { .await .expect("create typed workspace before router dispatch"); - let (provider, commit_started, commit_release) = GatedWorkspaceProvider::new(); + let (provider, commit_started, commit_release) = GatedWorkspaceProvider::new(pool.clone()); let provider = Arc::new(provider); let mut registry = ToolRegistry::new(); registry.register_hand( @@ -1868,6 +1870,19 @@ async fn may_write_result_waits_for_atomic_checkpoint_publication_db() { .await .expect("fence the externally started provider attempt") ); + PostgresWorkspaceCapacityRepository::new(pool.clone()) + .reserve_checkpoint_publication( + &WorkspaceStorageOperation { + operation_id: replay_operation_id, + kind: WorkspaceOperationKind::Commit, + binding: replay_binding, + deadline: replay_intent.deadline_at, + request_hash: replay_request_hash, + }, + 19, + ) + .await + .expect("reserve the exact recovered checkpoint publication"); router .commit_authorized_workspace_after_tool(JournaledWorkspaceCommit { diff --git a/crates/moa-hands/tests/hands_db/sandbox_workspace/maintenance_db.rs b/crates/moa-hands/tests/hands_db/sandbox_workspace/maintenance_db.rs index 318cce00f..7b2177d66 100644 --- a/crates/moa-hands/tests/hands_db/sandbox_workspace/maintenance_db.rs +++ b/crates/moa-hands/tests/hands_db/sandbox_workspace/maintenance_db.rs @@ -503,10 +503,16 @@ async fn delayed_checkpoint_reconciliation_atomically_publishes_without_resend_d provider_account_generation: 1, expected_writer_epoch: active.writer_epoch, expected_instance_generation: active.instance_generation, - quantities: vec![CapacityQuantity { - dimension: WorkspaceCapacityDimension::Checkpoints, - quantity: 1, - }], + quantities: vec![ + CapacityQuantity { + dimension: WorkspaceCapacityDimension::Checkpoints, + quantity: 1, + }, + CapacityQuantity { + dimension: WorkspaceCapacityDimension::LogicalBytes, + quantity: 17, + }, + ], }) .await .expect("reserve checkpoint capacity"); diff --git a/crates/moa-hands/tests/hands_db/sandbox_workspace/reconciliation_db.rs b/crates/moa-hands/tests/hands_db/sandbox_workspace/reconciliation_db.rs index e8b3463d9..a3cc9dca9 100644 --- a/crates/moa-hands/tests/hands_db/sandbox_workspace/reconciliation_db.rs +++ b/crates/moa-hands/tests/hands_db/sandbox_workspace/reconciliation_db.rs @@ -71,7 +71,8 @@ async fn expire_inventory_claim(pool: &PgPool, account_id: ProviderAccountId) { SET claim_generation = claim_generation + 1, claim_owner = $2, claim_token = $3, claimed_at = now() - interval '2 minutes', - claim_expires_at = now() - interval '1 minute' + claim_expires_at = now() - interval '1 minute', + last_succeeded_at = NULL WHERE provider_account_id = $1 AND provider_account_generation = 1 AND claim_token IS NULL AND last_succeeded_at IS NOT NULL "#, @@ -123,12 +124,11 @@ async fn provider_inventory_claim_is_exclusive_and_recovers_after_owner_restart_ .await .expect("first replica reaches provider only after durable claim"); let claimed_row_version = inventory_claim_row_version(&maintenance, account_id).await; - let concurrent = fixture + let _concurrent = fixture .coordinator - .reconcile_claimed_provider_inventory_once(1) + .reconcile_claimed_provider_inventory_once(32) .await - .expect("second replica sees no claimable account"); - assert_eq!(concurrent.accounts, 0); + .expect("second replica may process siblings but cannot steal the live claim"); assert_eq!( inventory_claim_row_version(&maintenance, account_id).await, claimed_row_version, @@ -155,6 +155,16 @@ async fn provider_inventory_claim_is_exclusive_and_recovers_after_owner_restart_ assert_eq!(recovered.accounts, 1); let recovered_generation = inventory_claim_generation(&maintenance, account_id).await; assert!(recovered_generation > stale_generation); + sqlx::query("DELETE FROM moa.sandbox_provider_inventory_claims WHERE provider_account_id = $1") + .bind(account_id) + .execute(&runtime) + .await + .expect("clean isolated inventory claim"); + sqlx::query("DELETE FROM moa.sandbox_provider_accounts WHERE provider_account_id = $1") + .bind(account_id) + .execute(&runtime) + .await + .expect("clean isolated provider account"); } #[tokio::test] @@ -367,6 +377,41 @@ async fn provider_inventory_drift_is_quarantined_then_resolved_only_after_clean_ .is_some_and(|digest| digest.starts_with("sha256:")) ); + sqlx::query( + "DELETE FROM moa.sandbox_provider_inventory_findings \ + WHERE provider_account_id = ANY($1)", + ) + .bind(vec![account_id, foreign_account_id]) + .execute(&runtime) + .await + .expect("clean isolated inventory findings"); + sqlx::query( + "DELETE FROM moa.sandbox_provider_inventory_claims \ + WHERE provider_account_id = ANY($1)", + ) + .bind(vec![account_id, foreign_account_id]) + .execute(&runtime) + .await + .expect("clean isolated inventory claims"); + sqlx::query( + "DELETE FROM moa.sandbox_capacity_reservations \ + WHERE provider_account_id = ANY($1)", + ) + .bind(vec![account_id, foreign_account_id]) + .execute(&runtime) + .await + .expect("clean isolated workspace capacity"); + sqlx::query("DELETE FROM moa.sandbox_workspaces WHERE provider_account_id = ANY($1)") + .bind(vec![account_id, foreign_account_id]) + .execute(&runtime) + .await + .expect("clean isolated workspaces"); + sqlx::query("DELETE FROM moa.sandbox_provider_accounts WHERE provider_account_id = ANY($1)") + .bind(vec![account_id, foreign_account_id]) + .execute(&runtime) + .await + .expect("clean isolated provider accounts"); + runtime.close().await; maintenance.close().await; } diff --git a/crates/moa-hands/tests/hands_db/sandbox_workspace/retention_db.rs b/crates/moa-hands/tests/hands_db/sandbox_workspace/retention_db.rs index ae2e9cb92..71efbc167 100644 --- a/crates/moa-hands/tests/hands_db/sandbox_workspace/retention_db.rs +++ b/crates/moa-hands/tests/hands_db/sandbox_workspace/retention_db.rs @@ -43,7 +43,7 @@ use tokio::sync::Mutex; use tokio::sync::oneshot; /// Provider registry key shared by the three maintenance DB behavior modules. -pub(super) const SCRIPTED_PROVIDER: &str = "scripted-maintenance-db"; +pub(super) const SCRIPTED_PROVIDER: &str = "00-scripted-maintenance-db"; /// Returns a required database URL or fails the ignored DB test with its name. pub(super) fn required_url(name: &str) -> String { diff --git a/crates/moa-hands/tests/hands_db/sandbox_workspace/rls_db.rs b/crates/moa-hands/tests/hands_db/sandbox_workspace/rls_db.rs index bf349b767..352ccb254 100644 --- a/crates/moa-hands/tests/hands_db/sandbox_workspace/rls_db.rs +++ b/crates/moa-hands/tests/hands_db/sandbox_workspace/rls_db.rs @@ -24,6 +24,9 @@ use moa_hands::core::leases::{ PostgresHandLeaseStore, }; use moa_hands::core::reaper::{ExpiredHandLeaseClaims, PostgresExpiredHandLeaseClaims}; +use moa_hands::core::sandbox_workspace::capacity::{ + ActiveHandCapacityRequest, PostgresWorkspaceCapacityRepository, +}; use moa_hands::core::sandbox_workspace::repository::PostgresWorkspaceRepository; use moa_hands::core::sandbox_workspace::storage_resources::PostgresWorkspaceStorageResourceRepository; use sqlx::postgres::PgPoolOptions; @@ -101,6 +104,11 @@ async fn cleanup_workspace( workspace_id: SandboxWorkspaceId, provider_account_id: ProviderAccountId, ) { + sqlx::query("DELETE FROM moa.sandbox_capacity_reservations WHERE workspace_id = $1") + .bind(workspace_id) + .execute(pool) + .await + .expect("clean up workspace capacity reservations"); sqlx::query("DELETE FROM moa.sandbox_workspaces WHERE workspace_id = $1") .bind(workspace_id) .execute(pool) @@ -632,6 +640,30 @@ async fn maintenance_reaper_crosses_tenants_without_exposing_a_foreground_bypass .await .expect("claim lease") .expect("claim is owned"); + sqlx::query( + "UPDATE moa.sandbox_workspaces SET lifecycle_state = 'restoring' \ + WHERE tenant_id = $1 AND workspace_id = $2 AND lifecycle_state = 'creating'", + ) + .bind(tenant_id) + .bind(attachment.workspace_id) + .execute(&pool) + .await + .expect("move the directly seeded workspace into the provisioning phase"); + let capacity = PostgresWorkspaceCapacityRepository::new(pool.clone()); + let active_hand = ActiveHandCapacityRequest { + tenant_id, + workspace_id: attachment.workspace_id, + provider_account_id, + provider_account_generation: 1, + provisioning_operation_id: claim.provisioning_operation_id, + hand_lease_generation: claim.generation, + expected_writer_epoch: attachment.workspace_writer_epoch, + expected_instance_generation: attachment.workspace_instance_generation, + }; + capacity + .reserve_active_hand(&active_hand) + .await + .expect("reserve active hand before activation"); store .activate(HandLeaseActivateRequest { tenant_id, @@ -649,6 +681,12 @@ async fn maintenance_reaper_crosses_tenants_without_exposing_a_foreground_bypass }) .await .expect("activate lease"); + assert!( + capacity + .commit_active_hand(&active_hand) + .await + .expect("commit active-hand capacity after activation") + ); } sqlx::query( diff --git a/crates/moa-hands/tests/hands_db/sandbox_workspace/storage_resources_db.rs b/crates/moa-hands/tests/hands_db/sandbox_workspace/storage_resources_db.rs index db4b78c70..10c5545e8 100644 --- a/crates/moa-hands/tests/hands_db/sandbox_workspace/storage_resources_db.rs +++ b/crates/moa-hands/tests/hands_db/sandbox_workspace/storage_resources_db.rs @@ -1,5 +1,7 @@ //! Daytona tenant-volume ownership and lifetime-reservation behavior against Postgres. +use std::time::Duration; + use chrono::{Duration as ChronoDuration, Utc}; use moa_core::{ error::MoaError, @@ -349,6 +351,26 @@ async fn ambiguous_create_retains_its_lifetime_charge_and_rejects_stale_callback .await .expect("mark ambiguous create operation") ); + sqlx::query( + "UPDATE moa.sandbox_workspace_operations \ + SET deadline_at = now() - interval '36501 days', \ + reconcile_not_before = now() - interval '36500 days' \ + WHERE tenant_id = $1 AND operation_id = $2", + ) + .bind(tenant_id) + .bind(seeded.operation_id) + .execute(&pool) + .await + .expect("make the ambiguous operation due for exact reconciliation"); + let maintenance_operations = + PostgresWorkspaceOperationRepository::new_maintenance(pool.clone()); + let claimed = maintenance_operations + .claim_reconciliation(1, Duration::from_secs(60)) + .await + .expect("claim the exact ambiguous create") + .into_iter() + .find(|claim| claim.operation.operation_id == seeded.operation_id) + .expect("the isolated create operation is claimable"); let ambiguous = resources .get(tenant_id, storage_resource_id) @@ -384,12 +406,8 @@ async fn ambiguous_create_retains_its_lifetime_charge_and_rejects_stale_callback .expect("commit the exact linked lifetime reservation") ); assert!( - operations - .confirm_disposition( - tenant_id, - seeded.operation_id, - WorkspaceConfirmedDisposition::ResourcePresent, - ) + maintenance_operations + .confirm_present_claimed(&claimed) .await .expect("confirm the ambiguous provider create") ); diff --git a/crates/moa-hands/tests/hands_offline/sandbox_profile_offline.rs b/crates/moa-hands/tests/hands_offline/sandbox_profile_offline.rs index d134f4d20..f0d8a68f4 100644 --- a/crates/moa-hands/tests/hands_offline/sandbox_profile_offline.rs +++ b/crates/moa-hands/tests/hands_offline/sandbox_profile_offline.rs @@ -22,8 +22,15 @@ use moa_core::{ use moa_crypto::LocalKmsProvider; use moa_hands::{LocalHandProvider, ToolRouter, deployment_sandbox_policy, route_sandbox_policy}; use object_store::memory::InMemory; +use sqlx::postgres::PgPoolOptions; use tempfile::tempdir; +fn lazy_workspace_pool() -> sqlx::PgPool { + PgPoolOptions::new() + .connect_lazy("postgresql://moa:moa@127.0.0.1:1/moa") + .expect("offline cloud construction should accept a lazy workspace pool") +} + fn seconds(value: u64) -> LifetimeLimit { LifetimeLimit::Bounded { seconds: NonZeroU64::new(value).expect("nonzero seconds"), @@ -320,7 +327,7 @@ async fn cloud_profile_refuses_the_built_in_local_development_sandbox_policy_off None, Some(Arc::new(NoRules)), Some(checkpoint_store), - None, + Some(lazy_workspace_pool()), None, true, ) diff --git a/crates/moa-hands/tests/hands_offline/security_defaults.rs b/crates/moa-hands/tests/hands_offline/security_defaults.rs index e6b207cf7..dbcc2fae4 100644 --- a/crates/moa-hands/tests/hands_offline/security_defaults.rs +++ b/crates/moa-hands/tests/hands_offline/security_defaults.rs @@ -33,6 +33,7 @@ use opentelemetry_sdk::trace::{ InMemorySpanExporter, Sampler, SdkTracerProvider, SimpleSpanProcessor, SpanData, }; use serde_json::json; +use sqlx::postgres::PgPoolOptions; use tempfile::tempdir; use tracing::instrument::WithSubscriber; use tracing_subscriber::layer::SubscriberExt; @@ -42,6 +43,12 @@ const ERROR_SECRET: &str = "stderr-secret-71d2"; const TOOL_ERROR_OUTPUT_STATUS: &str = "tool returned error output"; const TOOL_EXECUTION_FAILED_STATUS: &str = "tool execution failed"; +fn lazy_workspace_pool() -> sqlx::PgPool { + PgPoolOptions::new() + .connect_lazy("postgresql://moa:moa@127.0.0.1:1/moa") + .expect("offline cloud construction should accept a lazy workspace pool") +} + fn session() -> SessionMeta { SessionMeta { tenant_id: identity().tenant_id, @@ -468,7 +475,7 @@ async fn cloud_profile_constructs_with_deny_default_owner_and_credentialed_backe None, Some(cloud_rule_store()), Some(test_checkpoint_store()), - None, + Some(lazy_workspace_pool()), None, true, ) diff --git a/crates/moa-migrations/migrations/postgres/V000055__execution_plan_compensation.sql b/crates/moa-migrations/migrations/postgres/V000055__execution_plan_compensation.sql index 4c82f1244..b6f692fa3 100644 --- a/crates/moa-migrations/migrations/postgres/V000055__execution_plan_compensation.sql +++ b/crates/moa-migrations/migrations/postgres/V000055__execution_plan_compensation.sql @@ -255,13 +255,25 @@ BEGIN LOOP IF NOT moa.execution_json_object_has_exact_keys( audit_entry, - ARRAY['review_uid', 'generation', 'accepted', 'resolution', 'recorded_at'] + ARRAY[ + 'review_uid', 'generation', 'accepted', 'resolution', + 'expires_at', 'recorded_at' + ] ) OR audit_entry ->> 'review_uid' !~ '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' OR audit_entry ->> 'generation' !~ '^[1-9][0-9]*$' OR jsonb_typeof(audit_entry -> 'accepted') <> 'boolean' - OR jsonb_typeof(audit_entry -> 'resolution') <> 'object' + OR ( + (audit_entry ->> 'accepted')::BOOLEAN + AND jsonb_typeof(audit_entry -> 'resolution') <> 'object' + ) + OR ( + NOT (audit_entry ->> 'accepted')::BOOLEAN + AND audit_entry -> 'resolution' <> 'null'::JSONB + ) + OR jsonb_typeof(audit_entry -> 'expires_at') <> 'string' + OR btrim(audit_entry ->> 'expires_at') = '' OR jsonb_typeof(audit_entry -> 'recorded_at') <> 'string' OR btrim(audit_entry ->> 'recorded_at') = '' THEN RETURN FALSE; diff --git a/crates/moa-migrations/migrations/postgres/V000058__sandbox_workspaces.sql b/crates/moa-migrations/migrations/postgres/V000058__sandbox_workspaces.sql index 8dafbcb67..05b2d1276 100644 --- a/crates/moa-migrations/migrations/postgres/V000058__sandbox_workspaces.sql +++ b/crates/moa-migrations/migrations/postgres/V000058__sandbox_workspaces.sql @@ -221,6 +221,7 @@ CREATE TABLE moa.sandbox_workspace_operations ( reconcile_not_before TIMESTAMPTZ NOT NULL, outcome_class TEXT NOT NULL DEFAULT 'not_sent' CHECK (outcome_class IN ('not_sent', 'unknown', 'confirmed')), + direct_confirmation_pending BOOLEAN NOT NULL DEFAULT FALSE, confirmed_disposition TEXT CHECK ( confirmed_disposition IN ('resource_present', 'resource_absent') ), @@ -267,6 +268,10 @@ CREATE TABLE moa.sandbox_workspace_operations ( CONSTRAINT sandbox_workspace_operations_outcome_disposition_pair_check CHECK ( (outcome_class = 'confirmed') = (confirmed_disposition IS NOT NULL) ), + CONSTRAINT sandbox_workspace_operations_direct_confirmation_check CHECK ( + NOT direct_confirmation_pending + OR (outcome_class = 'unknown' AND claim_token IS NULL) + ), CONSTRAINT sandbox_workspace_operations_absence_proof_shape_check CHECK ( ( absence_observation_count = 0 @@ -336,7 +341,9 @@ BEGIN RETURN NEW; END IF; - IF OLD.outcome_class = 'not_sent' AND NEW.operation_kind <> 'delete' THEN + IF OLD.outcome_class = 'unknown' + AND OLD.direct_confirmation_pending + AND NEW.operation_kind <> 'delete' THEN -- A synchronous non-delete provider operation may authoritatively -- report that it created or retained no external resource. RETURN NEW; diff --git a/crates/moa-migrations/migrations/postgres/V000059__long_horizon_execution.sql b/crates/moa-migrations/migrations/postgres/V000059__long_horizon_execution.sql index 145214da3..3246b3b58 100644 --- a/crates/moa-migrations/migrations/postgres/V000059__long_horizon_execution.sql +++ b/crates/moa-migrations/migrations/postgres/V000059__long_horizon_execution.sql @@ -2422,9 +2422,10 @@ $old$; 'pausing', 'paused', 'running', 'cancelled' ) WHEN 'pausing' THEN NEW.status IN ('paused', 'failed', 'cancelled') - WHEN 'paused' THEN NEW.status IN ('queued', 'cancelled') + WHEN 'paused' THEN NEW.status IN ('queued', 'compensating', 'failed', 'cancelled') WHEN 'compensating' THEN NEW.status IN ( - 'completed', 'partial', 'blocked', 'unsupported', 'failed', 'cancelled' + 'pause_requested', 'completed', 'partial', 'blocked', 'unsupported', + 'failed', 'cancelled' ) ELSE FALSE END; @@ -2535,7 +2536,9 @@ BEGIN 'ready', 'reserved', 'waiting_review', 'waiting_signal', 'waiting_timer', 'failed', 'skipped', 'cancelled' ) - WHEN 'ready' THEN NEW.status IN ('dispatching', 'reserved', 'cancelled') + -- A retry or input resume first advances into a fresh ready generation; a + -- deadline or budget rejection may then terminalize that generation before dispatch. + WHEN 'ready' THEN NEW.status IN ('dispatching', 'reserved', 'failed', 'cancelled') WHEN 'reserved' THEN NEW.status IN ('dispatching', 'running', 'cancelled') WHEN 'dispatching' THEN NEW.status IN ('running', 'ready', 'failed', 'cancelled') WHEN 'running' THEN NEW.status IN ( @@ -2616,7 +2619,7 @@ BEGIN AND NEW.active_task_count = 0 THEN NEW.status := 'paused'; NEW.activation_state := 'paused'; - NEW.paused_at := COALESCE(NEW.paused_at, now()); + NEW.paused_at := COALESCE(NEW.paused_at, clock_timestamp()); END IF; IF OLD.last_progress_at IS NOT NULL AND NEW.last_progress_at < OLD.last_progress_at THEN @@ -2640,7 +2643,8 @@ BEGIN END IF; IF OLD.attempt_state = 'cancelling' AND NEW.attempt_state NOT IN ( - 'cancelling', 'idle', 'waiting_review', 'terminal', 'unknown_outcome' + 'cancelling', 'idle', 'waiting_review', 'waiting_external', 'terminal', + 'unknown_outcome' ) THEN RAISE EXCEPTION 'execution compensation cancelling state cannot become dispatchable'; END IF; diff --git a/crates/moa-orchestrator/tests/analytics_parity_docker.rs b/crates/moa-orchestrator/tests/analytics_parity_docker.rs index 2d295d73c..43d46259b 100644 --- a/crates/moa-orchestrator/tests/analytics_parity_docker.rs +++ b/crates/moa-orchestrator/tests/analytics_parity_docker.rs @@ -1571,6 +1571,18 @@ async fn seed_execution_run_and_tasks( let skill_template_revision_uid = Uuid::now_v7(); let planning_hash = "1".repeat(64); let plan_hash = "2".repeat(64); + let plan_snapshot = json!({ + "definition": { + "cancel_policy": "retain_effects", + "input_schema": {}, + "output_schema": {}, + "nodes": [], + }, + "plan_hash": plan_hash, + "catalog_hash": plan_hash, + "estimate": {}, + "report": {}, + }); sqlx::query( "INSERT INTO moa.execution_planning_context \ (planning_context_uid, tenant_id, session_id, originating_user_sequence_num, \ @@ -1595,18 +1607,19 @@ async fn seed_execution_run_and_tasks( 'identity_type', 'service', 'id', $2::TEXT, \ 'tenant_id', $2::TEXT, 'api_key_id', NULL, \ 'acting_on_behalf_of', NULL), \ - '{\"requirements\":[{\"id\":\"r1\"}]}'::JSONB, '{}'::JSONB, '{}'::JSONB, \ - $6, $6, '{}'::JSONB, '{}'::JSONB, \ + '{\"requirements\":[{\"id\":\"r1\"}]}'::JSONB, $6, $6, \ + $7, $7, '{}'::JSONB, '{}'::JSONB, \ jsonb_build_object('kind', 'skill_template', \ 'skill_template_ref', 'skill://billing-flow', \ - 'skill_template_revision_uid', lower($7::TEXT)), \ - 'skill_template', 'skill://billing-flow', $7, '{}'::JSONB, 'queued', 2, $8)", + 'skill_template_revision_uid', lower($8::TEXT)), \ + 'skill_template', 'skill://billing-flow', $8, '{}'::JSONB, 'queued', 2, $9)", ) .bind(run_uid) .bind(tenant) .bind(session) .bind(planning_context_uid) .bind(&planning_hash) + .bind(&plan_snapshot) .bind(&plan_hash) .bind(skill_template_revision_uid) .bind(started_at) diff --git a/crates/moa-orchestrator/tests/recovery_matrix_sandbox_workspace_service_e2e.rs b/crates/moa-orchestrator/tests/recovery_matrix_sandbox_workspace_service_e2e.rs index 24e982744..0c11754ff 100644 --- a/crates/moa-orchestrator/tests/recovery_matrix_sandbox_workspace_service_e2e.rs +++ b/crates/moa-orchestrator/tests/recovery_matrix_sandbox_workspace_service_e2e.rs @@ -6,7 +6,7 @@ #![cfg(all(feature = "integration", feature = "sandbox-workspace-failpoints"))] -use std::time::Duration; +use std::{path::Path, time::Duration}; use anyhow::{Context, Result, bail}; use moa_core::{ @@ -418,15 +418,32 @@ async fn wait_for_one_bash_result( } } -async fn seed_ambiguous_absent_operation(pool: &sqlx::PgPool) -> Result { +async fn seed_ambiguous_absent_operation( + pool: &sqlx::PgPool, + sandbox_root: &Path, +) -> Result { let tenant_id = TenantId::from(FIXTURE_TENANT_UUID); - let row: (SandboxWorkspaceId, ProviderAccountId, i64, i64, i64, i64) = sqlx::query_as( + let row: ( + SandboxWorkspaceId, + ProviderAccountId, + i64, + i64, + i64, + i64, + Vec, + ) = sqlx::query_as( r#" SELECT workspace_id, provider_account_id, provider_account_generation, - writer_epoch, instance_generation, current_checkpoint_generation - FROM moa.sandbox_workspaces - WHERE tenant_id = $1 AND scope_kind = 'worker' - ORDER BY created_at + writer_epoch, instance_generation, current_checkpoint_generation, + ARRAY( + SELECT lease.provisioning_operation_id + FROM moa.hand_leases AS lease + WHERE lease.tenant_id = workspace.tenant_id + AND lease.workspace_id = workspace.workspace_id + ) + FROM moa.sandbox_workspaces AS workspace + WHERE tenant_id = $1 AND scope_kind = 'worker' AND provider = 'local' + ORDER BY workspace.created_at LIMIT 1 "#, ) @@ -463,7 +480,9 @@ async fn seed_ambiguous_absent_operation(pool: &sqlx::PgPool) -> Result Result {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + return Err(error).with_context(|| { + format!( + "remove absent-provider fixture root {}", + sandbox_dir.display() + ) + }); + } + } + let trusted_dir = sandbox_root + .join(".moa-hand-trusted") + .join(provisioning_operation_id.to_string()); + match tokio::fs::remove_dir_all(&trusted_dir).await { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + return Err(error).with_context(|| { + format!( + "remove absent-provider trusted root {}", + trusted_dir.display() + ) + }); + } + } + let marker = sandbox_root + .join(".moa-hand-intents") + .join(format!("{provisioning_operation_id}.json")); + match tokio::fs::remove_file(&marker).await { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + return Err(error).with_context(|| { + format!("remove absent-provider intent marker {}", marker.display()) + }); + } + } + } Ok(operation_id) } +async fn make_absent_operation_due( + pool: &sqlx::PgPool, + operation_id: WorkspaceOperationId, +) -> Result<()> { + let affected = sqlx::query( + r#" + UPDATE moa.sandbox_workspace_operations + SET reconcile_not_before = now(), updated_at = now() + WHERE tenant_id = $1 AND operation_id = $2 + AND outcome_class = 'unknown' AND claim_token IS NULL + "#, + ) + .bind(FIXTURE_TENANT_UUID) + .bind(operation_id) + .execute(pool) + .await? + .rows_affected(); + if affected != 1 { + bail!("synthetic absent operation lost its pre-reconciliation fence"); + } + Ok(()) +} + async fn wait_for_absence_release( pool: &sqlx::PgPool, operation_id: WorkspaceOperationId, @@ -915,17 +999,57 @@ async fn recovery_matrix_sandbox_workspace_all_six_barriers_replay_once_service_ } let pool = sqlx::PgPool::connect(&fixture.postgres_url).await?; + let operation_id = + seed_ambiguous_absent_operation(&pool, fixture.sandbox_workspace()?.sandbox_root()).await?; let absence_control = SandboxWorkspaceCrashControl::new( SandboxWorkspaceCrashBarrier::PostAbsenceConfirmationPreReservationRelease, )?; fixture - .restart_orchestrator_with_env(absence_control.orchestrator_env()) + .restart_execution_maintenance_owner_with_env(absence_control.orchestrator_env()) .await?; - let operation_id = seed_ambiguous_absent_operation(&pool).await?; - absence_control + make_absent_operation_due(&pool, operation_id).await?; + if let Err(error) = absence_control .wait_until_reached(Duration::from_secs(120)) .await - .context("observe post-absence-confirmation/pre-reservation-release barrier")?; + { + let durable_state: serde_json::Value = sqlx::query_scalar( + r#" + SELECT jsonb_build_object( + 'operation', to_jsonb(operation), + 'reservation', to_jsonb(reservation), + 'workspace', to_jsonb(workspace), + 'leases', COALESCE(( + SELECT jsonb_agg(to_jsonb(lease) ORDER BY lease.created_at) + FROM moa.hand_leases AS lease + WHERE lease.tenant_id = operation.tenant_id + AND lease.workspace_id = operation.workspace_id + ), '[]'::jsonb), + 'database_now', now() + ) + FROM moa.sandbox_workspace_operations AS operation + JOIN moa.sandbox_capacity_reservations AS reservation + ON reservation.tenant_id = operation.tenant_id + AND reservation.operation_id = operation.operation_id + JOIN moa.sandbox_workspaces AS workspace + ON workspace.tenant_id = operation.tenant_id + AND workspace.workspace_id = operation.workspace_id + WHERE operation.operation_id = $1 + "#, + ) + .bind(operation_id) + .fetch_one(&pool) + .await?; + let mut local_entries = Vec::new(); + let mut entries = tokio::fs::read_dir(fixture.sandbox_workspace()?.sandbox_root()).await?; + while let Some(entry) = entries.next_entry().await? { + local_entries.push(entry.file_name().to_string_lossy().into_owned()); + } + local_entries.sort(); + let child_exit = fixture.unexpected_orchestrator_exit().await?; + bail!( + "observe post-absence-confirmation/pre-reservation-release barrier: {error:#}; durable_state={durable_state}; local_entries={local_entries:?}; child_exit={child_exit:?}" + ); + } let before_absence_crash: (String, String) = sqlx::query_as( r#" SELECT operation.outcome_class, reservation.reservation_state @@ -943,7 +1067,9 @@ async fn recovery_matrix_sandbox_workspace_all_six_barriers_replay_once_service_ before_absence_crash, ("unknown".to_string(), "reconciling".to_string()) ); - fixture.hard_crash_and_restart_orchestrator().await?; + fixture + .hard_crash_and_restart_execution_maintenance_owner() + .await?; wait_for_absence_release(&pool, operation_id).await?; let delete_control = SandboxWorkspaceCrashControl::new( diff --git a/crates/moa-test-support/src/orchestrator_fixture.rs b/crates/moa-test-support/src/orchestrator_fixture.rs index bca97d716..af67c48f7 100644 --- a/crates/moa-test-support/src/orchestrator_fixture.rs +++ b/crates/moa-test-support/src/orchestrator_fixture.rs @@ -569,6 +569,34 @@ impl OrchestratorTestFixture { let orchestrator = orchestrator_guard .disarm() .context("healthy orchestrator child guard is unexpectedly disarmed")?; + let maintenance = if use_sandbox_workspace { + let health_listener = std::net::TcpListener::bind("0.0.0.0:0") + .context("reserve fixture maintenance health port")?; + let maintenance_health_port = health_listener + .local_addr() + .context("read fixture maintenance health port")? + .port(); + drop(health_listener); + let config = restart_config + .as_ref() + .context("sandbox-workspace fixture requires restart configuration")?; + let mut maintenance_guard = config.spawn_maintenance(maintenance_health_port)?; + wait_for_orchestrator_health( + maintenance_health_port, + maintenance_guard + .child_mut() + .context("maintenance child guard is unexpectedly disarmed")?, + ) + .await + .context("start fixture workspace maintenance owner")?; + Some( + maintenance_guard + .disarm() + .context("healthy maintenance child guard is unexpectedly disarmed")?, + ) + } else { + None + }; Ok(Self { client, @@ -583,7 +611,7 @@ impl OrchestratorTestFixture { _openfga: openfga_container, redis: Mutex::new(redis_container), orchestrator: Mutex::new(Some(orchestrator)), - maintenance: Mutex::new(None), + maintenance: Mutex::new(maintenance), handler_revisions: Mutex::new(HashMap::new()), _orchestrator_binary_snapshot: Some(orchestrator_binary_snapshot), restart_config, @@ -685,9 +713,45 @@ impl OrchestratorTestFixture { } async fn restart_execution_maintenance_owner(&self) -> Result<()> { + self.replace_execution_maintenance_owner(Vec::new(), false) + .await + } + + /// Restarts the fixture-owned maintenance process with one exact test environment. + pub async fn restart_execution_maintenance_owner_with_env( + &self, + extra_env: Vec<(String, String)>, + ) -> Result<()> { + self.replace_execution_maintenance_owner(extra_env, false) + .await + } + + /// Abruptly restarts only the fixture-owned maintenance process. + pub async fn hard_crash_and_restart_execution_maintenance_owner(&self) -> Result<()> { + self.replace_execution_maintenance_owner(Vec::new(), true) + .await + } + + async fn replace_execution_maintenance_owner( + &self, + extra_env: Vec<(String, String)>, + hard_crash: bool, + ) -> Result<()> { let config = self.restart_config.as_ref().context( "external orchestrator fixture cannot restart an execution maintenance owner", )?; + validate_execution_fixture_env(&extra_env)?; + let mut child_env = config.extra_env.clone(); + let mut keys = child_env + .iter() + .map(|(key, _)| key.clone()) + .collect::>(); + for (key, value) in extra_env { + if !keys.insert(key.clone()) { + bail!("duplicate execution fixture environment key `{key}`"); + } + child_env.push((key, value)); + } let health_listener = std::net::TcpListener::bind("0.0.0.0:0") .context("reserve fixture maintenance health port")?; let health_port = health_listener @@ -696,10 +760,14 @@ impl OrchestratorTestFixture { .port(); let mut maintenance = self.maintenance.lock().await; if let Some(child) = maintenance.take() { - terminate_child(child); + if hard_crash { + hard_kill_child(child)?; + } else { + terminate_child(child); + } } drop(health_listener); - let mut child_guard = config.spawn_maintenance(health_port)?; + let mut child_guard = config.spawn_maintenance_with_env(health_port, &child_env)?; wait_for_orchestrator_health( health_port, child_guard diff --git a/crates/moa-test-support/src/orchestrator_fixture/process.rs b/crates/moa-test-support/src/orchestrator_fixture/process.rs index 78f0ddb1f..a8a64c37b 100644 --- a/crates/moa-test-support/src/orchestrator_fixture/process.rs +++ b/crates/moa-test-support/src/orchestrator_fixture/process.rs @@ -183,6 +183,15 @@ impl OrchestratorRestartConfig { /// Spawns the maintenance owner against the same durable fixture dependencies. pub(super) fn spawn_maintenance(&self, health_port: u16) -> Result { + self.spawn_maintenance_with_env(health_port, &self.extra_env) + } + + /// Spawns the maintenance owner with one exact merged fixture environment. + pub(super) fn spawn_maintenance_with_env( + &self, + health_port: u16, + extra_env: &[(String, String)], + ) -> Result { spawn_maintenance( OrchestratorSpawnConfig { binary: &self.binary, @@ -196,7 +205,7 @@ impl OrchestratorRestartConfig { script_path: self.script_path.as_deref(), journal_path: self.journal_path.as_deref(), fga_config: &self.fga_config, - extra_env: &self.extra_env, + extra_env, otlp_endpoint: &self.otlp_endpoint, observability_service_name: &self.observability_service_name, }, diff --git a/crates/moa-test-support/src/orchestrator_fixture/rustfs.rs b/crates/moa-test-support/src/orchestrator_fixture/rustfs.rs index fc34118b3..c5c38f64a 100644 --- a/crates/moa-test-support/src/orchestrator_fixture/rustfs.rs +++ b/crates/moa-test-support/src/orchestrator_fixture/rustfs.rs @@ -47,28 +47,8 @@ impl RustFsFixture { secret_bytes, ); let data_dir = docker_mountable_tempdir("moa-rustfs-data-")?; - let container = GenericImage::new(RUSTFS_IMAGE, RUSTFS_TAG) - .with_exposed_port(RUSTFS_PORT.tcp()) - .with_wait_for(WaitFor::seconds(2)) - .with_env_var("RUSTFS_ACCESS_KEY", &access_key) - .with_env_var("RUSTFS_SECRET_KEY", &secret_key) - .with_env_var("RUSTFS_REGION", RUSTFS_REGION) - .with_env_var("RUSTFS_ADDRESS", "0.0.0.0:9000") - .with_env_var("RUSTFS_CONSOLE_ENABLE", "false") - .with_mount(Mount::bind_mount( - data_dir.path().display().to_string(), - "/data", - )) - .with_cmd(["/data"]) - .start() - .await - .context("start digest-pinned RustFS testcontainer")?; - let host_port = fixture_host_port_ipv4( - &container, - "sandbox workspace RustFS API", - RUSTFS_PORT.tcp(), - ) - .await?; + let (container, host_port) = + start_rustfs_container(data_dir.path(), &access_key, &secret_key).await?; let endpoint = format!("http://127.0.0.1:{host_port}"); create_bucket_with_retry(&endpoint, &bucket, &access_key, &secret_key).await?; let store: Arc = Arc::new( @@ -256,6 +236,66 @@ impl RustFsFixture { } } +async fn start_rustfs_container( + data_dir: &Path, + access_key: &str, + secret_key: &str, +) -> Result<(ContainerAsync, u16)> { + let mut failures = Vec::new(); + for attempt in 1..=3 { + let container = match GenericImage::new(RUSTFS_IMAGE, RUSTFS_TAG) + .with_exposed_port(RUSTFS_PORT.tcp()) + .with_wait_for(WaitFor::seconds(2)) + .with_env_var("RUSTFS_ACCESS_KEY", access_key) + .with_env_var("RUSTFS_SECRET_KEY", secret_key) + .with_env_var("RUSTFS_REGION", RUSTFS_REGION) + .with_env_var("RUSTFS_ADDRESS", "0.0.0.0:9000") + .with_env_var("RUSTFS_CONSOLE_ENABLE", "false") + .with_mount(Mount::bind_mount(data_dir.display().to_string(), "/data")) + .with_cmd(["/data"]) + .start() + .await + { + Ok(container) => container, + Err(error) => { + failures.push(format!("attempt {attempt} failed to start: {error}")); + continue; + } + }; + match fixture_host_port_ipv4( + &container, + "sandbox workspace RustFS API", + RUSTFS_PORT.tcp(), + ) + .await + { + Ok(host_port) => return Ok((container, host_port)), + Err(error) => { + failures.push(format!( + "attempt {attempt} exposed incomplete ports: {error:#}" + )); + tracing::warn!( + attempt, + container_id = %container.id(), + %error, + "restarting RustFS fixture after incomplete Docker port publication" + ); + if let Err(remove_error) = container.rm().await { + tracing::warn!( + attempt, + %remove_error, + "failed to remove incomplete RustFS fixture container" + ); + } + } + } + } + bail!( + "start RustFS testcontainer with API port failed after 3 attempts: {}", + failures.join("; ") + ) +} + async fn create_bucket_with_retry( endpoint: &str, bucket: &str, diff --git a/crates/moa-test-support/tests/orchestrator_fixture_service_e2e.rs b/crates/moa-test-support/tests/orchestrator_fixture_service_e2e.rs index b2bc5e0cd..2b7a04422 100644 --- a/crates/moa-test-support/tests/orchestrator_fixture_service_e2e.rs +++ b/crates/moa-test-support/tests/orchestrator_fixture_service_e2e.rs @@ -204,7 +204,14 @@ fn execution_candidate(objective: &str) -> String { "output_schema": { "type": "object" }, "operation": { "kind": "wait_signal", - "signal_name": "fixture_release" + "signal_name": "fixture_release", + "wait_policy": { + "expiry": { + "kind": "after", + "delay_seconds": 3600 + }, + "on_expiry": { "kind": "fail_task" } + } }, "compensation": null, "retry": { diff --git a/k8s/scripts/render-production.sh b/k8s/scripts/render-production.sh index 8d268d34d..8b879295d 100755 --- a/k8s/scripts/render-production.sh +++ b/k8s/scripts/render-production.sh @@ -83,7 +83,7 @@ require_value OPENFGA_PRESHARED_KEY_SECRET command -v kustomize >/dev/null 2>&1 || die "kustomize is required" command -v python3 >/dev/null 2>&1 || die "python3 is required" -command -v rg >/dev/null 2>&1 || die "ripgrep (rg) is required" +command -v grep >/dev/null 2>&1 || die "grep is required" output_dir="$1" [[ ! -e "${output_dir}" ]] || die "output path already exists: ${output_dir}" @@ -236,20 +236,20 @@ mkdir -p "${work_dir}/rendered" kustomize build "${production_overlay}" >"${work_dir}/rendered/production.yaml" kustomize build "${jobs_overlay}" >"${work_dir}/rendered/jobs.yaml" -rg -Fq "image: ${MOA_ORCHESTRATOR_IMAGE}" "${work_dir}/rendered/production.yaml" \ +grep -Fq -- "image: ${MOA_ORCHESTRATOR_IMAGE}" "${work_dir}/rendered/production.yaml" \ || die "production manifest does not contain the requested orchestrator digest" -rg -Fq "image: ${MOA_EDGE_IMAGE}" "${work_dir}/rendered/production.yaml" \ +grep -Fq -- "image: ${MOA_EDGE_IMAGE}" "${work_dir}/rendered/production.yaml" \ || die "production manifest does not contain the requested edge digest" -rg -Fq "image: ${MOA_ORCHESTRATOR_IMAGE}" "${work_dir}/rendered/jobs.yaml" \ +grep -Fq -- "image: ${MOA_ORCHESTRATOR_IMAGE}" "${work_dir}/rendered/jobs.yaml" \ || die "maintenance manifest does not contain the requested orchestrator digest" -rg -Fq "destination = \"${snapshot_destination}\"" "${work_dir}/rendered/production.yaml" \ +grep -Fq -- "destination = \"${snapshot_destination}\"" "${work_dir}/rendered/production.yaml" \ || die "production manifest does not contain the requested snapshot destination" -rg -Fq "name: ${SANDBOX_CHECKPOINT_BUCKET}" "${work_dir}/rendered/production.yaml" \ - || rg -Fq "MOA_SANDBOX_CHECKPOINT_BUCKET: ${SANDBOX_CHECKPOINT_BUCKET}" "${work_dir}/rendered/production.yaml" \ +grep -Fq -- "name: ${SANDBOX_CHECKPOINT_BUCKET}" "${work_dir}/rendered/production.yaml" \ + || grep -Fq -- "MOA_SANDBOX_CHECKPOINT_BUCKET: ${SANDBOX_CHECKPOINT_BUCKET}" "${work_dir}/rendered/production.yaml" \ || die "production manifest does not contain the external checkpoint bucket" -rg -Fq "iam.gke.io/gcp-service-account: ${SANDBOX_WORKSPACE_GSA}" "${work_dir}/rendered/production.yaml" \ +grep -Fq -- "iam.gke.io/gcp-service-account: ${SANDBOX_WORKSPACE_GSA}" "${work_dir}/rendered/production.yaml" \ || die "production manifest does not bind the sandbox workspace workload identity" -if rg -q 'sha256:0{64}|image-revision' "${work_dir}/rendered"; then +if grep -REq -- 'sha256:0{64}|image-revision' "${work_dir}/rendered"; then die "rendered manifests contain an unresolved sentinel" fi From 717456241551365bc316fb1c3ac9a74f76e6e2e4 Mon Sep 17 00:00:00 2001 From: Hwuiwon Kim Date: Fri, 14 Aug 2026 13:58:27 -0400 Subject: [PATCH 08/21] fix RustFS port publication in CI --- .../src/orchestrator_fixture/rustfs.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/crates/moa-test-support/src/orchestrator_fixture/rustfs.rs b/crates/moa-test-support/src/orchestrator_fixture/rustfs.rs index c5c38f64a..f14b52c19 100644 --- a/crates/moa-test-support/src/orchestrator_fixture/rustfs.rs +++ b/crates/moa-test-support/src/orchestrator_fixture/rustfs.rs @@ -243,6 +243,17 @@ async fn start_rustfs_container( ) -> Result<(ContainerAsync, u16)> { let mut failures = Vec::new(); for attempt in 1..=3 { + // Docker on GitHub-hosted runners can ignore PublishAllPorts for this + // image even when the container port is explicitly exposed. An exact + // host binding avoids that daemon-specific path. The retry loop also + // closes the small release-to-create race around the reserved port. + let port_listener = std::net::TcpListener::bind(("127.0.0.1", 0)) + .context("reserve RustFS fixture host port")?; + let host_port = port_listener + .local_addr() + .context("read reserved RustFS fixture host port")? + .port(); + drop(port_listener); let container = match GenericImage::new(RUSTFS_IMAGE, RUSTFS_TAG) .with_exposed_port(RUSTFS_PORT.tcp()) .with_wait_for(WaitFor::seconds(2)) @@ -253,6 +264,7 @@ async fn start_rustfs_container( .with_env_var("RUSTFS_CONSOLE_ENABLE", "false") .with_mount(Mount::bind_mount(data_dir.display().to_string(), "/data")) .with_cmd(["/data"]) + .with_mapped_port(host_port, RUSTFS_PORT.tcp()) .start() .await { From 5cf1ecdc00d07584b260d977e786b98e9c6ca7fe Mon Sep 17 00:00:00 2001 From: Hwuiwon Kim Date: Fri, 14 Aug 2026 14:05:31 -0400 Subject: [PATCH 09/21] use fixture-owned RustFS storage --- .../src/orchestrator_fixture/rustfs.rs | 36 +++++++------------ 1 file changed, 13 insertions(+), 23 deletions(-) diff --git a/crates/moa-test-support/src/orchestrator_fixture/rustfs.rs b/crates/moa-test-support/src/orchestrator_fixture/rustfs.rs index f14b52c19..9e778890b 100644 --- a/crates/moa-test-support/src/orchestrator_fixture/rustfs.rs +++ b/crates/moa-test-support/src/orchestrator_fixture/rustfs.rs @@ -10,7 +10,7 @@ use object_store::path::Path as ObjectPath; use object_store::{ObjectStore, PutPayload}; use reqwest::Method; use sha2::{Digest as _, Sha256}; -use testcontainers::core::{IntoContainerPort, Mount}; +use testcontainers::core::IntoContainerPort; use super::*; @@ -29,7 +29,6 @@ pub struct RustFsFixture { access_key: String, secret_key: String, store: Arc, - _data_dir: TempDir, _container: ContainerAsync, } @@ -46,9 +45,7 @@ impl RustFsFixture { &base64::engine::general_purpose::URL_SAFE_NO_PAD, secret_bytes, ); - let data_dir = docker_mountable_tempdir("moa-rustfs-data-")?; - let (container, host_port) = - start_rustfs_container(data_dir.path(), &access_key, &secret_key).await?; + let (container, host_port) = start_rustfs_container(&access_key, &secret_key).await?; let endpoint = format!("http://127.0.0.1:{host_port}"); create_bucket_with_retry(&endpoint, &bucket, &access_key, &secret_key).await?; let store: Arc = Arc::new( @@ -72,7 +69,6 @@ impl RustFsFixture { access_key, secret_key, store, - _data_dir: data_dir, _container: container, }; fixture.assert_available().await?; @@ -237,7 +233,6 @@ impl RustFsFixture { } async fn start_rustfs_container( - data_dir: &Path, access_key: &str, secret_key: &str, ) -> Result<(ContainerAsync, u16)> { @@ -262,7 +257,6 @@ async fn start_rustfs_container( .with_env_var("RUSTFS_REGION", RUSTFS_REGION) .with_env_var("RUSTFS_ADDRESS", "0.0.0.0:9000") .with_env_var("RUSTFS_CONSOLE_ENABLE", "false") - .with_mount(Mount::bind_mount(data_dir.display().to_string(), "/data")) .with_cmd(["/data"]) .with_mapped_port(host_port, RUSTFS_PORT.tcp()) .start() @@ -283,8 +277,18 @@ async fn start_rustfs_container( { Ok(host_port) => return Ok((container, host_port)), Err(error) => { + let stdout = container + .stdout_to_vec() + .await + .map(|bytes| String::from_utf8_lossy(&bytes).into_owned()) + .unwrap_or_else(|log_error| format!("unavailable: {log_error}")); + let stderr = container + .stderr_to_vec() + .await + .map(|bytes| String::from_utf8_lossy(&bytes).into_owned()) + .unwrap_or_else(|log_error| format!("unavailable: {log_error}")); failures.push(format!( - "attempt {attempt} exposed incomplete ports: {error:#}" + "attempt {attempt} exposed incomplete ports: {error:#}; stdout={stdout:?}; stderr={stderr:?}" )); tracing::warn!( attempt, @@ -401,20 +405,6 @@ fn hmac_sha256(key: &[u8], bytes: &[u8]) -> Result> { Ok(mac.finalize().into_bytes().to_vec()) } -fn docker_mountable_tempdir(prefix: &str) -> Result { - let macos_docker_tmp = Path::new("/private/tmp"); - if macos_docker_tmp.exists() { - return tempfile::Builder::new() - .prefix(prefix) - .tempdir_in(macos_docker_tmp) - .context("create Docker-mountable RustFS data directory"); - } - tempfile::Builder::new() - .prefix(prefix) - .tempdir() - .context("create RustFS data directory") -} - #[cfg(test)] mod tests { use super::*; From 908ece78189bd55c0272b48b390bb7b361287e47 Mon Sep 17 00:00:00 2001 From: Hwuiwon Kim Date: Fri, 14 Aug 2026 14:29:44 -0400 Subject: [PATCH 10/21] fix CI recovery fixture setup --- .config/nextest.toml | 7 ++++++- .github/workflows/ci.yml | 9 +++++++++ .github/workflows/integration-tests.yml | 9 +++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/.config/nextest.toml b/.config/nextest.toml index bcea6b143..a29762299 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -37,8 +37,11 @@ inherits = "default" test-threads = 8 # moa-session keeps its physical-database clone and schema-binding checks inline # in the library. Select only those DB tests; all other moa-session lib tests -# remain outside this lane. +# remain outside this lane. `hands_db` owns the canonical shared database and +# runs only after CI applies direct-repository migrations, so it is excluded +# from this pre-migration, clone-isolated profile. default-filter = ''' +( binary(/_db$/) | ( package(moa-session) @@ -47,6 +50,8 @@ binary(/_db$/) ) | (package(moa-eval) & kind(lib) & test(/::tests::.*_db(?:_|$)/)) | (package(moa-orchestrator) & kind(lib) & test(/::tests::.*_db(?:_|$)/)) +) + - binary(/^hands_db$/) ''' [profile.db-memory] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cef807be2..afa731a2f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -290,6 +290,15 @@ jobs: - name: Install cargo-nextest uses: taiki-e/install-action@nextest + # Keep the feature-qualified fixture build outside nextest's per-test + # timeout. The recovery test snapshots this exact executable before its + # repeated crash/restart barriers. + - name: Build sandbox recovery fixture orchestrator + run: | + cargo build -p moa-orchestrator --bin moa-orchestrator-bin --locked \ + --features provider-overrides,integration,execution-planning-failpoints,sandbox-workspace-failpoints + echo "MOA_ORCHESTRATOR_BIN=${GITHUB_WORKSPACE}/target/debug/moa-orchestrator-bin" >> "$GITHUB_ENV" + # This lane is hermetic and unbilled. The test owns disposable Postgres, # Restate, OpenFGA, Valkey, and RustFS containers and hard-restarts only # its orchestrator child at each feature-gated workspace barrier. diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index d1f8303ea..1a67b25ee 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -40,6 +40,15 @@ jobs: - name: Install nextest uses: taiki-e/install-action@nextest + # Recovery fixtures restart one feature-qualified orchestrator executable + # many times. Build it outside the per-test timeout and pass the exact + # path to the fixture instead of triggering a cold nested Cargo build. + - name: Build recovery fixture orchestrator + run: | + cargo build -p moa-orchestrator --bin moa-orchestrator-bin --locked \ + --features provider-overrides,integration,execution-planning-failpoints,sandbox-workspace-failpoints + echo "MOA_ORCHESTRATOR_BIN=${GITHUB_WORKSPACE}/target/debug/moa-orchestrator-bin" >> "$GITHUB_ENV" + # Every case owns its Postgres, Restate, OpenFGA, and Valkey containers. # Keep external-service settings out of the process so a developer or # runner environment cannot silently turn this into an ambient-stack test. From 78a0d641e5147391fe8aa37e9283311cb73cf193 Mon Sep 17 00:00:00 2001 From: Hwuiwon Kim Date: Fri, 14 Aug 2026 14:57:05 -0400 Subject: [PATCH 11/21] fix dispatch timestamp replay precision --- crates/moa-execution/src/repository/outbox.rs | 2 +- .../controller_wake_recovery_db.rs | 64 +++++++++++++++++++ 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/crates/moa-execution/src/repository/outbox.rs b/crates/moa-execution/src/repository/outbox.rs index 53e740f92..f99001cf4 100644 --- a/crates/moa-execution/src/repository/outbox.rs +++ b/crates/moa-execution/src/repository/outbox.rs @@ -1931,7 +1931,7 @@ fn dispatch_matches_request( && record.attempt_generation == request.attempt_generation && record.compensation_generation == request.compensation_generation && record.compensation_attempt_generation == request.compensation_attempt_generation - && record.not_before_at == request.not_before_at + && record.not_before_at.timestamp_micros() == request.not_before_at.timestamp_micros() && record.payload == request.payload } diff --git a/crates/moa-execution/tests/execution_db/controller_wake_recovery_db.rs b/crates/moa-execution/tests/execution_db/controller_wake_recovery_db.rs index 8fcf8119e..e95d9a2e9 100644 --- a/crates/moa-execution/tests/execution_db/controller_wake_recovery_db.rs +++ b/crates/moa-execution/tests/execution_db/controller_wake_recovery_db.rs @@ -174,6 +174,70 @@ async fn concurrent_controller_wake_claims_admit_exactly_one_activation_db() -> Ok(()) } +#[tokio::test] +async fn controller_continuation_canonicalizes_submicrosecond_dispatch_time_db() -> TestResult { + // Pins: PostgreSQL TIMESTAMPTZ persists microseconds, while Linux clocks can return + // nanoseconds. A first continuation insert must compare immutable dispatch semantics at the + // database's precision instead of rejecting its own round-trip as a conflicting dispatch UID. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let repository = ExecutionRepository::new(test_db.store().pool().clone()); + let config = ExecutionConfig::default(); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let run = queued_run( + &repository, + scope, + tenant_id, + "controller-submicrosecond-continuation", + ) + .await?; + let RunControllerClaimOutcome::Claimed(claimed) = repository + .claim_controller_wake( + scope, + run.run_uid, + run.controller_generation, + run.wake_epoch, + ) + .await? + else { + panic!("the admitted queued wake must be claimable"); + }; + let persisted_not_before_at = pg_deadline(Duration::seconds(1)); + let requested_not_before_at = persisted_not_before_at + Duration::nanoseconds(321); + + let outcome = repository + .complete_controller_wake( + scope, + &config, + claimed.run_uid, + RunControllerCompletionRequest { + controller_generation: claimed.controller_generation, + wake_epoch: claimed.wake_epoch, + checkpoint: ExecutionRunActivationCheckpoint { + status: ExecutionRunStatus::Running, + activation_state: ExecutionActivationState::Queued, + next_wake_at: claimed.next_wake_at, + waiting_since: None, + ready_task_count: claimed.ready_task_count, + active_task_count: claimed.active_task_count, + }, + continuation_payload: Some(json!({"reason": "submicrosecond_round_trip"})), + continuation_not_before_at: requested_not_before_at, + }, + ) + .await?; + + let RunControllerCompletionOutcome::Applied { + continuation: Some(continuation), + .. + } = outcome + else { + panic!("the controller continuation must commit, got {outcome:?}"); + }; + assert_eq!(continuation.not_before_at, persisted_not_before_at); + Ok(()) +} + #[tokio::test] async fn claim_controller_wake_refuses_an_unqueued_activation_state_db() -> TestResult { // Pins: the claim predicate requires an actually queued activation. A pending wake epoch on a From 3aba62f1a2f493ddab70a596c0a8eccf4faee598 Mon Sep 17 00:00:00 2001 From: Hwuiwon Kim Date: Fri, 14 Aug 2026 15:10:06 -0400 Subject: [PATCH 12/21] fix action review recovery teardown race --- .../integration/action_policy_flow_e2e.rs | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/crates/moa-orchestrator/tests/integration/action_policy_flow_e2e.rs b/crates/moa-orchestrator/tests/integration/action_policy_flow_e2e.rs index 97056c66a..264a9313e 100644 --- a/crates/moa-orchestrator/tests/integration/action_policy_flow_e2e.rs +++ b/crates/moa-orchestrator/tests/integration/action_policy_flow_e2e.rs @@ -233,10 +233,12 @@ async fn recovery_matrix_coordinator_review_restarts_once( .wait_for_committed_decision_and_blocked_continuation(review_id) .await?; assert_effect_once(effect.path(), &effect_marker)?; - barrier.release_and_remove().await?; - decision + barrier.release().await?; + let decision = decision .await - .context("join coordinator review decision request")??; + .context("join coordinator review decision request"); + barrier.remove().await?; + decision??; wait_for_events(&test, session_id, |events| { continuation_facts(events, review_id).len() == 1 @@ -368,10 +370,12 @@ async fn recovery_matrix_worker_review_restarts_once( .wait_for_committed_decision_and_blocked_continuation(review_id) .await?; assert_effect_once(effect_path, WORKER_RECOVERY_EFFECT_MARKER)?; - barrier.release_and_remove().await?; - decision + barrier.release().await?; + let decision = decision .await - .context("join worker review decision request")??; + .context("join worker review decision request"); + barrier.remove().await?; + decision??; let events = wait_for_events(&test, session_id, |events| { continuation_facts(events, review_id).len() == 1 @@ -1930,7 +1934,7 @@ impl ActionReviewRecoveryBarrier { } } - async fn release_and_remove(&mut self) -> Result<()> { + async fn release(&mut self) -> Result<()> { let unlocked: bool = sqlx::query_scalar("SELECT pg_advisory_unlock($1)") .bind(self.lock_key) .fetch_one(&mut self.lock_connection) @@ -1940,6 +1944,10 @@ impl ActionReviewRecoveryBarrier { unlocked, "action-review continuation barrier was not held by its owning connection" ); + Ok(()) + } + + async fn remove(&self) -> Result<()> { sqlx::raw_sql( r#" DROP TRIGGER IF EXISTS block_action_review_continuation_service_e2e From 887f0164cfb61fb6257f711d202f3a8d11e9a14f Mon Sep 17 00:00:00 2001 From: Hwuiwon Kim Date: Fri, 14 Aug 2026 15:16:16 -0400 Subject: [PATCH 13/21] fix trigger timestamp replay precision --- .../moa-execution/src/repository/trigger.rs | 2 +- .../tests/execution_db/trigger_outbox_db.rs | 44 +++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/crates/moa-execution/src/repository/trigger.rs b/crates/moa-execution/src/repository/trigger.rs index 180243fc5..f6a0d7505 100644 --- a/crates/moa-execution/src/repository/trigger.rs +++ b/crates/moa-execution/src/repository/trigger.rs @@ -2555,7 +2555,7 @@ fn trigger_matches_request(record: &ExecutionTriggerRecord, request: &NewExecuti && record.compensation_generation == request.compensation_generation && record.compensation_attempt_generation == request.compensation_attempt_generation && record.occurrence_sequence == request.occurrence_sequence - && record.due_at == request.due_at + && record.due_at.timestamp_micros() == request.due_at.timestamp_micros() && record.payload == request.payload } diff --git a/crates/moa-execution/tests/execution_db/trigger_outbox_db.rs b/crates/moa-execution/tests/execution_db/trigger_outbox_db.rs index 1e579c2c4..864ff96a0 100644 --- a/crates/moa-execution/tests/execution_db/trigger_outbox_db.rs +++ b/crates/moa-execution/tests/execution_db/trigger_outbox_db.rs @@ -77,6 +77,50 @@ fn output_node(id: &str) -> ExecutionNode { } } +#[tokio::test] +async fn trigger_creation_canonicalizes_submicrosecond_due_time_db() -> TestResult { + // Pins: PostgreSQL TIMESTAMPTZ persists microseconds, while Linux clocks can return + // nanoseconds. A first trigger insert must compare immutable semantics at the database's + // precision instead of rejecting its own round-trip as a conflicting trigger UID. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let repository = ExecutionRepository::new(test_db.store().pool().clone()); + let execution_config = execution_capacity_config(); + let tenant_id = TenantId::new(); + let scope = ExecutionScope::Tenant { tenant_id }; + let run = create_run( + &repository, + scope, + new_run( + tenant_id, + None, + "trigger-submicrosecond-due-time", + ExecutionRunStatus::Queued, + budget_without_deadline(10), + ), + ) + .await?; + let persisted_due_at = pg_deadline(Duration::minutes(5)); + let requested_due_at = persisted_due_at + Duration::nanoseconds(321); + + let write = repository + .create_trigger( + scope, + &execution_config, + run_deadline( + Uuid::now_v7(), + tenant_id, + run.run_uid, + run.controller_generation, + requested_due_at, + ), + ) + .await?; + + assert_eq!(write.trigger.due_at, persisted_due_at); + assert_eq!(write.dispatch.not_before_at, persisted_due_at); + Ok(()) +} + #[tokio::test] async fn task_start_uses_post_lock_progress_time_db() -> TestResult { // Pins: a task-start transaction whose PostgreSQL NOW() predates a contended run lock still From 1a35ecf68510a9598b648041f2fb6b9bae40707f Mon Sep 17 00:00:00 2001 From: Hwuiwon Kim Date: Fri, 14 Aug 2026 15:43:46 -0400 Subject: [PATCH 14/21] fix task admission timestamp precision --- .../moa-execution/src/repository/capacity.rs | 7 +++ .../execution_db/execution_capacity_db.rs | 43 ++++++++++++++++++- 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/crates/moa-execution/src/repository/capacity.rs b/crates/moa-execution/src/repository/capacity.rs index 840f7bf3b..7ce9b15bc 100644 --- a/crates/moa-execution/src/repository/capacity.rs +++ b/crates/moa-execution/src/repository/capacity.rs @@ -283,6 +283,13 @@ impl ExecutionRepository { oldest_ready_at: None, }); } + let now = + DateTime::::from_timestamp_micros(now.timestamp_micros()).ok_or_else(|| { + Error::InvalidRepositoryInput { + message: "task admission time is outside PostgreSQL timestamp bounds" + .to_string(), + } + })?; let deadline = now .checked_add_signed(Duration::seconds( i64::try_from(config.active_attempt_timeout_seconds).map_err(|_| { diff --git a/crates/moa-execution/tests/execution_db/execution_capacity_db.rs b/crates/moa-execution/tests/execution_db/execution_capacity_db.rs index 816519ed3..ec7d1f2f7 100644 --- a/crates/moa-execution/tests/execution_db/execution_capacity_db.rs +++ b/crates/moa-execution/tests/execution_db/execution_capacity_db.rs @@ -8,7 +8,9 @@ use moa_config::ExecutionConfig; use moa_db::ScopedConn; use moa_execution::repository::ready::ReadyMaterializationRequest; use moa_execution::repository::{ - capacity::ExecutionAdmissionBatch, ready::ReadyMaterializationOutcome, + capacity::ExecutionAdmissionBatch, + ready::ReadyMaterializationOutcome, + task::{TaskAttemptFence, TaskAttemptStartOutcome}, }; use super::support::*; @@ -77,6 +79,45 @@ async fn ready_run( Ok(run.run_uid) } +#[tokio::test] +async fn task_admission_canonicalizes_submicrosecond_attempt_deadline_db() -> TestResult { + // Pins: an admitted task's wire fence and PostgreSQL row must carry the same microsecond + // deadline. Linux clocks can supply nanoseconds; returning that unpersisted precision makes + // the immediately following task start stale against its own durable attempt row. + let test_db = moa_test_support::postgres::bootstrap_test_db().await?; + let repository = ExecutionRepository::new(test_db.store().pool().clone()); + let config = ExecutionConfig::default(); + let tenant_id = TenantId::new(); + ready_run(&repository, tenant_id, "submicrosecond-attempt-deadline", 1).await?; + let persisted_now = pg_deadline(Duration::seconds(1)); + let requested_now = persisted_now + Duration::nanoseconds(321); + let admission = repository + .admit_ready_attempts(&config, 1, requested_now) + .await? + .admitted + .into_iter() + .next() + .expect("one task must be admitted"); + let fence = TaskAttemptFence { + tenant_id: admission.tenant_id, + run_uid: admission.run_uid, + task_id: admission.task_id, + controller_generation: admission.controller_generation, + attempt_generation: admission.attempt_generation, + dispatch_uid: admission.dispatch_uid, + capacity_reservation_uid: admission.capacity_reservation_uid, + watchdog_trigger_uid: admission.watchdog_trigger_uid, + attempt_deadline_at: admission.attempt_deadline_at, + }; + + let outcome = repository.start_task_attempt(fence).await?; + assert!( + matches!(outcome, TaskAttemptStartOutcome::Started(_)), + "a newly admitted task must start with its persisted deadline fence, got {outcome:?}" + ); + Ok(()) +} + #[tokio::test] async fn tenant_capacity_scope_shares_one_fleet_bucket_without_cross_tenant_access_db() -> TestResult { From 88d35a6c30bc6cf77b667978e77c5e75034e25de Mon Sep 17 00:00:00 2001 From: Hwuiwon Kim Date: Fri, 14 Aug 2026 15:54:59 -0400 Subject: [PATCH 15/21] fix retry timestamp fixture precision --- crates/moa-execution/tests/execution_db/trigger_outbox_db.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/moa-execution/tests/execution_db/trigger_outbox_db.rs b/crates/moa-execution/tests/execution_db/trigger_outbox_db.rs index 864ff96a0..1b846dd69 100644 --- a/crates/moa-execution/tests/execution_db/trigger_outbox_db.rs +++ b/crates/moa-execution/tests/execution_db/trigger_outbox_db.rs @@ -3025,7 +3025,7 @@ async fn retry_settlement_preserves_cancelling_until_ready_transition_db() -> Te else { panic!("retry fixture must start its admitted attempt"); }; - let settled_at = Utc::now(); + let settled_at = pg_deadline(Duration::zero()); let TaskAttemptReleaseClaimOutcome::Applied(releasing) = repository .begin_task_attempt_release(fence, started.task.generation, "watchdog", settled_at) .await? From fd9759f42da42f634a5794815d6b4208774e9067 Mon Sep 17 00:00:00 2001 From: Hwuiwon Kim Date: Fri, 14 Aug 2026 16:23:11 -0400 Subject: [PATCH 16/21] fix concurrent outbox claim underfill --- crates/moa-execution/src/repository/outbox.rs | 17 +++++------- .../tests/execution_db/trigger_outbox_db.rs | 26 +++++++++++++++---- 2 files changed, 27 insertions(+), 16 deletions(-) diff --git a/crates/moa-execution/src/repository/outbox.rs b/crates/moa-execution/src/repository/outbox.rs index f99001cf4..05764785d 100644 --- a/crates/moa-execution/src/repository/outbox.rs +++ b/crates/moa-execution/src/repository/outbox.rs @@ -883,28 +883,21 @@ impl ExecutionRepository { let mut conn = scope.begin(&self.pool).await?; let rows = sqlx::query( r#" - WITH head AS ( - SELECT dispatch_uid - FROM ( + WITH candidate AS ( (SELECT dispatch_uid, not_before_at AS claimable_at, created_at FROM moa.execution_dispatch_outbox WHERE state = 'pending' AND not_before_at <= now() - ORDER BY not_before_at, created_at, dispatch_uid - LIMIT $1) + ORDER BY not_before_at, created_at, dispatch_uid) UNION ALL (SELECT dispatch_uid, claim_expires_at AS claimable_at, created_at FROM moa.execution_dispatch_outbox WHERE state = 'dispatching' AND claim_expires_at <= now() - ORDER BY claim_expires_at, created_at, dispatch_uid - LIMIT $1) - ) AS candidate - ORDER BY claimable_at, created_at, dispatch_uid - LIMIT $1 + ORDER BY claim_expires_at, created_at, dispatch_uid) ), claimable AS ( SELECT claimed.dispatch_uid FROM moa.execution_dispatch_outbox AS claimed - JOIN head ON head.dispatch_uid = claimed.dispatch_uid + JOIN candidate ON candidate.dispatch_uid = claimed.dispatch_uid WHERE ( claimed.state = 'pending' AND claimed.not_before_at <= now() @@ -912,6 +905,8 @@ impl ExecutionRepository { claimed.state = 'dispatching' AND claimed.claim_expires_at <= now() ) + ORDER BY candidate.claimable_at, candidate.created_at, candidate.dispatch_uid + LIMIT $1 FOR UPDATE OF claimed SKIP LOCKED ) UPDATE moa.execution_dispatch_outbox AS dispatch diff --git a/crates/moa-execution/tests/execution_db/trigger_outbox_db.rs b/crates/moa-execution/tests/execution_db/trigger_outbox_db.rs index 1b846dd69..1bc6bdae1 100644 --- a/crates/moa-execution/tests/execution_db/trigger_outbox_db.rs +++ b/crates/moa-execution/tests/execution_db/trigger_outbox_db.rs @@ -1115,12 +1115,28 @@ async fn correctness_outbox_claims_are_disjoint_expiry_recoverable_and_durably_r assert_eq!(other_health.claimable_dispatches.observed_count, 0); assert!(!other_health.claimable_dispatches.saturated); - let (owner_a, owner_b) = tokio::join!( - repository.claim_due_dispatches(scope, "owner-a", 2, StdDuration::from_secs(30)), - repository.claim_due_dispatches(scope, "owner-b", 2, StdDuration::from_secs(30)), + let mut head_lock = moa_db::ScopedConn::begin_tenant(&pool, tenant_id).await?; + head_lock.assume_app_role().await?; + let locked_head = sqlx::query_scalar::<_, Uuid>( + "SELECT dispatch_uid FROM moa.execution_dispatch_outbox \ + WHERE state='pending' AND not_before_at <= now() \ + ORDER BY not_before_at, created_at, dispatch_uid LIMIT 2 FOR UPDATE", + ) + .fetch_all(head_lock.as_mut()) + .await?; + assert_eq!(locked_head.len(), 2); + let owner_b = repository + .claim_due_dispatches(scope, "owner-b", 2, StdDuration::from_secs(30)) + .await?; + assert_eq!( + owner_b.len(), + 2, + "SKIP LOCKED must continue past the locked head" ); - let owner_a = owner_a?; - let owner_b = owner_b?; + head_lock.rollback().await?; + let owner_a = repository + .claim_due_dispatches(scope, "owner-a", 2, StdDuration::from_secs(30)) + .await?; assert_eq!((owner_a.len(), owner_b.len()), (2, 2)); let a_ids = owner_a .iter() From 2e4f19bc82ddcad5224cf7c209220acbaa0a8229 Mon Sep 17 00:00:00 2001 From: Hwuiwon Kim Date: Fri, 14 Aug 2026 16:33:36 -0400 Subject: [PATCH 17/21] make deadline expiry fixture deterministic --- .../tests/execution_db/outcomes_and_replan_db.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/crates/moa-execution/tests/execution_db/outcomes_and_replan_db.rs b/crates/moa-execution/tests/execution_db/outcomes_and_replan_db.rs index cc6695f96..c25df0da7 100644 --- a/crates/moa-execution/tests/execution_db/outcomes_and_replan_db.rs +++ b/crates/moa-execution/tests/execution_db/outcomes_and_replan_db.rs @@ -95,7 +95,8 @@ async fn retry_and_input_resume_terminalize_elapsed_or_exhausted_run_envelope_db // typed terminal failure, releases reservations, wakes finalization, and // remains idempotent without weakening the generation fence. let test_db = moa_test_support::postgres::bootstrap_test_db().await?; - let repository = ExecutionRepository::new(test_db.store().pool().clone()); + let pool = test_db.store().pool().clone(); + let repository = ExecutionRepository::new(pool.clone()); let tenant_id = TenantId::new(); let scope = ExecutionScope::Tenant { tenant_id }; @@ -116,9 +117,7 @@ async fn retry_and_input_resume_terminalize_elapsed_or_exhausted_run_envelope_db &format!("elapsed-{kind}"), ExecutionRunStatus::Queued, ExecutionBudgetLimit { - deadline_at: Some( - moa_test_support::fixtures::pg_now() + Duration::milliseconds(150), - ), + deadline_at: Some(pg_deadline(Duration::hours(1))), ..budget(2) }, ); @@ -133,7 +132,11 @@ async fn retry_and_input_resume_terminalize_elapsed_or_exhausted_run_envelope_db .await?, TaskOutcomeWrite::Applied { .. } )); - tokio::time::sleep(std::time::Duration::from_millis(200)).await; + sqlx::query("UPDATE moa.execution_run SET budget_deadline_at=$2 WHERE run_uid=$1") + .bind(run.run_uid) + .bind(pg_deadline(Duration::seconds(-1))) + .execute(&pool) + .await?; let before_terminal = repository .load_run(scope, run.run_uid) .await? From 03a2b7737c0b54cde5453f05ea2a05c3af3b6147 Mon Sep 17 00:00:00 2001 From: Hwuiwon Kim Date: Fri, 14 Aug 2026 16:56:30 -0400 Subject: [PATCH 18/21] fix recovery child observation race --- ...oordinator_worker_behavior_provider_e2e.rs | 60 +++++++++++-------- 1 file changed, 34 insertions(+), 26 deletions(-) diff --git a/crates/moa-orchestrator/tests/coordinator_worker_behavior_provider_e2e.rs b/crates/moa-orchestrator/tests/coordinator_worker_behavior_provider_e2e.rs index 0034064c8..41e9b9c8f 100644 --- a/crates/moa-orchestrator/tests/coordinator_worker_behavior_provider_e2e.rs +++ b/crates/moa-orchestrator/tests/coordinator_worker_behavior_provider_e2e.rs @@ -2455,39 +2455,47 @@ async fn recovery_matrix_blocked_llm_invocation( let key_filter = workflow_key .map(|key| format!(" AND target_service_key = '{key}'")) .unwrap_or_default(); - let parents = recovery_matrix_restate_rows( - fixture, - format!( - "SELECT id FROM sys_invocation WHERE target_service_name = '{workflow_service}'\ - {key_filter}" - ), - ) - .await?; - ensure!( - !parents.is_empty(), - "expected a {workflow_service} invocation, got {parents:?}" - ); - for parent in &parents { - let parent_id = parent - .get("id") - .and_then(Value::as_str) - .context("workflow introspection row omitted id")?; - let children = recovery_matrix_restate_rows( + let deadline = Instant::now() + Duration::from_secs(10); + loop { + let parents = recovery_matrix_restate_rows( fixture, format!( - "SELECT id, status FROM sys_invocation WHERE invoked_by_id = '{parent_id}' \ - AND target_service_name = 'LLMGateway' AND status != 'completed' ORDER BY id" + "SELECT id FROM sys_invocation WHERE target_service_name = '{workflow_service}'\ + {key_filter}" ), ) .await?; - for child in &children { - let Some(invoked_id) = child.get("id").and_then(Value::as_str) else { - continue; - }; - return Ok((parent_id.to_string(), invoked_id.to_string())); + let mut observed = Vec::new(); + for parent in &parents { + let parent_id = parent + .get("id") + .and_then(Value::as_str) + .context("workflow introspection row omitted id")?; + let children = recovery_matrix_restate_rows( + fixture, + format!( + "SELECT id, status FROM sys_invocation WHERE invoked_by_id = '{parent_id}' \ + AND target_service_name = 'LLMGateway' ORDER BY id" + ), + ) + .await?; + for child in &children { + if child.get("status").and_then(Value::as_str) == Some("completed") { + continue; + } + let Some(invoked_id) = child.get("id").and_then(Value::as_str) else { + continue; + }; + return Ok((parent_id.to_string(), invoked_id.to_string())); + } + observed.push(json!({ "parent_id": parent_id, "children": children })); } + ensure!( + Instant::now() < deadline, + "{workflow_service} has no incomplete LLMGateway child: {observed:?}" + ); + sleep(Duration::from_millis(25)).await; } - bail!("{workflow_service} has no incomplete LLMGateway child: {parents:?}") } async fn recovery_matrix_execution_task_attempt_key( From abf2aa53ce0957c274257b9b5d74bdc4660f26f5 Mon Sep 17 00:00:00 2001 From: Hwuiwon Kim Date: Fri, 14 Aug 2026 16:56:34 -0400 Subject: [PATCH 19/21] pin pgcrypto to public schema --- .../postgres/V000002__session_baseline.sql | 2 +- .../V000045__artifact_release_control.sql | 4 +- .../V000046__artifact_release_evaluation.sql | 2 +- crates/moa-migrations/src/lib.rs | 6 +- .../tests/run_idempotency_db/protocol.rs | 83 +++++++++++++++++++ 5 files changed, 92 insertions(+), 5 deletions(-) diff --git a/crates/moa-migrations/migrations/postgres/V000002__session_baseline.sql b/crates/moa-migrations/migrations/postgres/V000002__session_baseline.sql index eecb6d3c2..cdddd8425 100644 --- a/crates/moa-migrations/migrations/postgres/V000002__session_baseline.sql +++ b/crates/moa-migrations/migrations/postgres/V000002__session_baseline.sql @@ -1570,7 +1570,7 @@ GROUP BY e.tenant_id, e.task_fingerprint, a.subject_type, a.subject_id; CREATE UNIQUE INDEX IF NOT EXISTS idx_task_strategy_success_rates_unique ON task_strategy_success_rates(tenant_id, task_fingerprint, subject_type, subject_id); -CREATE EXTENSION IF NOT EXISTS pgcrypto; +CREATE EXTENSION IF NOT EXISTS pgcrypto WITH SCHEMA public; CREATE TABLE IF NOT EXISTS moa.artifact ( artifact_uid UUID PRIMARY KEY, diff --git a/crates/moa-migrations/migrations/postgres/V000045__artifact_release_control.sql b/crates/moa-migrations/migrations/postgres/V000045__artifact_release_control.sql index 5e7428326..a43250788 100644 --- a/crates/moa-migrations/migrations/postgres/V000045__artifact_release_control.sql +++ b/crates/moa-migrations/migrations/postgres/V000045__artifact_release_control.sql @@ -171,7 +171,7 @@ IMMUTABLE PARALLEL SAFE STRICT AS $$ - SELECT digest( + SELECT public.digest( jsonb_build_object( 'schema', 'moa.artifact_release_policy/v1', 'name', p_name, @@ -337,7 +337,7 @@ policy_body ( '[{"id":"scenario_outcome","version":"v1","determinism":"deterministic"},{"id":"target_completed","version":"v1","determinism":"deterministic"},{"id":"result_produced","version":"v1","determinism":"deterministic"},{"id":"privacy_safe_output","version":"v1","determinism":"deterministic"}]'::JSONB, '[{"metric":"result_produced","direction":"higher_is_better","estimand":"paired difference in result-production probability","target_population":"approved artifact-release scenarios","independent_unit":"scenario_persona_profile","cluster_key":"scenario_persona_profile","paired_key":"scenario_persona_profile_repetition","confidence_method":"cluster_matched_risk_difference_bootstrap","unit":"proportion","margin_bp":500,"alpha_bp":250,"acceptable_alternative_bp":0,"unacceptable_alternative_bp":-1000,"resamples":2000,"min_independent_units":6,"holm_regression_alpha_bp":250}]'::JSONB, 86400::BIGINT, - digest('moa.release.resource_policy.v1', 'sha256') + public.digest('moa.release.resource_policy.v1', 'sha256') ) ) INSERT INTO moa.artifact_release_policy ( diff --git a/crates/moa-migrations/migrations/postgres/V000046__artifact_release_evaluation.sql b/crates/moa-migrations/migrations/postgres/V000046__artifact_release_evaluation.sql index 534bbdf45..6a90e6a99 100644 --- a/crates/moa-migrations/migrations/postgres/V000046__artifact_release_evaluation.sql +++ b/crates/moa-migrations/migrations/postgres/V000046__artifact_release_evaluation.sql @@ -186,7 +186,7 @@ LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $$ - SELECT digest( + SELECT public.digest( jsonb_build_object( 'schema', 'moa.artifact_release_case_pack/v1', 'name', p_name, diff --git a/crates/moa-migrations/src/lib.rs b/crates/moa-migrations/src/lib.rs index bf34b0db1..ee304af7c 100644 --- a/crates/moa-migrations/src/lib.rs +++ b/crates/moa-migrations/src/lib.rs @@ -739,10 +739,14 @@ async fn install_shared_extensions(conn: &mut PgConnection) -> Result<()> { } async fn install_shared_extensions_locked(conn: &mut PgConnection) -> Result<()> { - raw_sql("CREATE EXTENSION IF NOT EXISTS pgcrypto;") + raw_sql("CREATE EXTENSION IF NOT EXISTS pgcrypto WITH SCHEMA public;") .execute(&mut *conn) .await .context("install pgcrypto extension")?; + raw_sql("ALTER EXTENSION pgcrypto SET SCHEMA public;") + .execute(&mut *conn) + .await + .context("pin pgcrypto extension to public schema")?; ensure_shared_database_roles(conn).await } diff --git a/crates/moa-migrations/tests/run_idempotency_db/protocol.rs b/crates/moa-migrations/tests/run_idempotency_db/protocol.rs index 106c39e22..7e75a74c3 100644 --- a/crates/moa-migrations/tests/run_idempotency_db/protocol.rs +++ b/crates/moa-migrations/tests/run_idempotency_db/protocol.rs @@ -192,6 +192,89 @@ async fn migration_protocol_pristine_apply_is_exact_and_idempotent_db() { assert_eq!(removed_token_vault_tables_absent, (true, true)); } +#[tokio::test] +#[ignore = "requires a superuser-capable local Postgres via MOA_DATABASE_URL"] +async fn schema_fragment_bootstrap_rehomes_pgcrypto_before_central_migration_db() { + // Pins: a schema-scoped bootstrap cannot strand the database-global pgcrypto + // extension outside `public` and make the later central migration lose digest(). + let admin_url = test_database_url(); + let db_name = unique_db_name(); + let admin = PgPoolOptions::new() + .max_connections(1) + .connect(&admin_url) + .await + .expect("connect extension-location maintenance database"); + admin + .execute(format!("CREATE DATABASE \"{db_name}\"").as_str()) + .await + .expect("create extension-location throwaway database"); + let target_url = with_database(&admin_url, &db_name); + + let outcome = async { + let target = PgPoolOptions::new() + .max_connections(1) + .connect(&target_url) + .await?; + target + .execute( + "CREATE SCHEMA misplaced_extension; \ + CREATE EXTENSION pgcrypto WITH SCHEMA misplaced_extension;", + ) + .await?; + target.close().await; + + let schema_name = format!("fragment_{}", uuid::Uuid::new_v4().simple()); + let search_path = format!("\"{schema_name}\", public"); + let fragment_pool = PgPoolOptions::new() + .max_connections(1) + .after_connect(move |conn, _meta| { + let search_path = search_path.clone(); + Box::pin(async move { + sqlx::query("SELECT pg_catalog.set_config('search_path', $1, false)") + .bind(search_path) + .execute(conn) + .await?; + Ok(()) + }) + }) + .connect(&target_url) + .await?; + moa_migrations::run_ocsf_schema(&fragment_pool, &schema_name).await?; + let extension_schema: String = sqlx::query_scalar( + "SELECT namespace.nspname::TEXT \ + FROM pg_catalog.pg_extension AS extension \ + JOIN pg_catalog.pg_namespace AS namespace \ + ON namespace.oid = extension.extnamespace \ + WHERE extension.extname = 'pgcrypto'", + ) + .fetch_one(&fragment_pool) + .await?; + fragment_pool.close().await; + + let (_, second) = clean_apply_then_reapply(&target_url).await?; + let target = PgPoolOptions::new() + .max_connections(1) + .connect(&target_url) + .await?; + let digest_len: i32 = + sqlx::query_scalar("SELECT octet_length(public.digest('migration-smoke', 'sha256'))") + .fetch_one(&target) + .await?; + target.close().await; + Ok::<_, Box>((extension_schema, second, digest_len)) + } + .await; + + drop_database_with_zero_connections(&admin, &db_name).await; + admin.close().await; + + let (extension_schema, second, digest_len) = + outcome.expect("schema bootstrap and central migration should compose"); + assert_eq!(extension_schema, "public"); + assert!(second.is_empty(), "central migration replay was not empty"); + assert_eq!(digest_len, 32); +} + #[tokio::test] #[ignore = "requires a superuser-capable local Postgres via MOA_DATABASE_URL"] async fn baseline_generated_identifiers_are_by_default_identity_db() { From c646a3523e53967f6c5ea0b3b69a4f91070a02ab Mon Sep 17 00:00:00 2001 From: Hwuiwon Kim Date: Fri, 14 Aug 2026 18:35:18 -0400 Subject: [PATCH 20/21] fix checkpoint deadline fixture precision --- .../moa-hands/tests/hands_db/sandbox_workspace/lifecycle_db.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/moa-hands/tests/hands_db/sandbox_workspace/lifecycle_db.rs b/crates/moa-hands/tests/hands_db/sandbox_workspace/lifecycle_db.rs index 7532f22a5..dc6cef2c5 100644 --- a/crates/moa-hands/tests/hands_db/sandbox_workspace/lifecycle_db.rs +++ b/crates/moa-hands/tests/hands_db/sandbox_workspace/lifecycle_db.rs @@ -44,6 +44,7 @@ use moa_hands::core::{ repository::PostgresWorkspaceRepository, }, }; +use moa_test_support::fixtures::pg_now; use sqlx::postgres::PgPoolOptions; use super::{database_url, seed_session}; @@ -1441,7 +1442,7 @@ async fn checkpoint_metadata_is_created_before_bytes_and_remains_immutable_db() ); } - let now = Utc::now(); + let now = pg_now(); let operation_id = WorkspaceOperationId::new(); operations .persist_intent(&WorkspaceOperationIntent { From b0d99a1296ff519d8655e10544489783810aa448 Mon Sep 17 00:00:00 2001 From: Hwuiwon Kim Date: Fri, 14 Aug 2026 18:49:16 -0400 Subject: [PATCH 21/21] normalize hands DB fixture timestamps --- .../sandbox_workspace/lifecycle_db.rs | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/crates/moa-hands/tests/hands_db/sandbox_workspace/lifecycle_db.rs b/crates/moa-hands/tests/hands_db/sandbox_workspace/lifecycle_db.rs index dc6cef2c5..661992666 100644 --- a/crates/moa-hands/tests/hands_db/sandbox_workspace/lifecycle_db.rs +++ b/crates/moa-hands/tests/hands_db/sandbox_workspace/lifecycle_db.rs @@ -400,7 +400,7 @@ async fn cancelling_task_without_owned_compute_gets_exact_absence_receipt_db() { task_id, logical_generation: 1, attempt_generation: 1, - verified_at: Utc::now(), + verified_at: pg_now(), }; let receipt = repository .record_absent_task_execution_hand_release_receipt(intent) @@ -464,7 +464,7 @@ async fn cancelling_task_without_owned_compute_gets_exact_absence_receipt_db() { task_id: live_task_id, logical_generation: 1, attempt_generation: 1, - verified_at: Utc::now(), + verified_at: pg_now(), }) .await .is_err(), @@ -562,8 +562,8 @@ async fn checkpointed_task_destroy_records_exact_release_receipt_db() { task_id, logical_generation: 1, attempt_generation: 1, - deadline_at: Utc::now() + ChronoDuration::minutes(5), - recovery_claim_expires_at: Utc::now() + ChronoDuration::minutes(5), + deadline_at: pg_now() + ChronoDuration::minutes(5), + recovery_claim_expires_at: pg_now() + ChronoDuration::minutes(5), workspace: &active, lease: &active_lease, }) @@ -597,7 +597,7 @@ async fn checkpointed_task_destroy_records_exact_release_receipt_db() { } let operation_id = WorkspaceOperationId::new(); let checkpoint_id = WorkspaceCheckpointId(operation_id.0); - let now = Utc::now(); + let now = pg_now(); operations .persist_intent(&WorkspaceOperationIntent { operation_id, @@ -692,7 +692,7 @@ async fn checkpointed_task_destroy_records_exact_release_receipt_db() { checkpoint_manifest_digest: Some(publication.manifest_digest.clone()), checkpoint_logical_bytes: Some(0), requested_at, - released_at: Utc::now(), + released_at: pg_now(), }; let finalized = workspaces .record_task_execution_hand_release_receipt(&receipt, claim_token, Some(contact_id)) @@ -733,7 +733,7 @@ async fn contact_scoped_compensation_without_compute_gets_exact_release_receipt_ let hand_scope = format!("execution_compensation:{run_id}:{compensation_id}"); let repository = PostgresWorkspaceRepository::new(pool.clone()); let receipt_id = uuid::Uuid::now_v7(); - let deadline_at = Utc::now() + ChronoDuration::minutes(1); + let deadline_at = pg_now() + ChronoDuration::minutes(1); let (persisted_receipt_id, claim_token, requested_at) = repository .begin_compensation_execution_hand_release(CompensationHandReleaseIntent { receipt_id, @@ -772,7 +772,7 @@ async fn contact_scoped_compensation_without_compute_gets_exact_release_receipt_ checkpoint_manifest_digest: None, checkpoint_logical_bytes: None, requested_at, - released_at: Utc::now(), + released_at: pg_now(), }; let finalized = repository .record_compensation_execution_hand_release_receipt( @@ -846,7 +846,7 @@ async fn compensation_release_recovers_persisted_destroyed_identity_after_deadli .expect("seeded compensation lease exists"); let repository = PostgresWorkspaceRepository::new(pool.clone()); let receipt_id = uuid::Uuid::now_v7(); - let deadline_at = Utc::now() + ChronoDuration::minutes(1); + let deadline_at = pg_now() + ChronoDuration::minutes(1); repository .begin_compensation_execution_hand_release(CompensationHandReleaseIntent { receipt_id, @@ -894,7 +894,7 @@ async fn compensation_release_recovers_persisted_destroyed_identity_after_deadli compensation_id, logical_generation: 1, attempt_generation: 1, - recovery_claim_expires_at: Utc::now() + ChronoDuration::minutes(5), + recovery_claim_expires_at: pg_now() + ChronoDuration::minutes(5), }) .await .expect("renew storage-only recovery claim") @@ -950,7 +950,7 @@ async fn compensation_release_recovers_persisted_destroyed_identity_after_deadli checkpoint_manifest_digest: None, checkpoint_logical_bytes: None, requested_at: claim.requested_at, - released_at: Utc::now(), + released_at: pg_now(), }; assert!( repository @@ -1124,7 +1124,7 @@ async fn workspace_writer_and_reconciliation_callbacks_are_generation_fenced_db( .expect("stale transition is a fenced miss") ); - let now = Utc::now(); + let now = pg_now(); let operation_id = WorkspaceOperationId::new(); operations .persist_intent(&WorkspaceOperationIntent { @@ -1245,7 +1245,7 @@ async fn create_operation_replay_never_resends_and_reconciliation_settles_lifecy .expect("create workspace metadata and lifetime capacity"); let operations = PostgresWorkspaceOperationRepository::new(pool.clone()); let operation_id = WorkspaceOperationId::new(); - let now = Utc::now(); + let now = pg_now(); let intent = WorkspaceOperationIntent { operation_id, tenant_id,